Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
1532 lines
58 KiB
Python
1532 lines
58 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import functools
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from typing import Any, cast
|
|
|
|
import torch
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.distributed import (
|
|
get_tensor_model_parallel_world_size,
|
|
tensor_model_parallel_all_reduce,
|
|
)
|
|
from vllm.forward_context import get_forward_context
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
|
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
|
from vllm.model_executor.layers.rocm_paged_mxfp4_indexer import (
|
|
RocmSparseAttnIndexer,
|
|
RocmSparseMQAIndexer,
|
|
)
|
|
from vllm.models.deepseek_v41.attention import (
|
|
DeepseekV4Attention,
|
|
DeepseekV4Indexer,
|
|
DeepseekV4IndexerCache,
|
|
_replace_layer_index,
|
|
)
|
|
from vllm.models.deepseek_v41.common.ops import (
|
|
combine_topk_swa_indices,
|
|
dequantize_and_gather_k_cache,
|
|
)
|
|
from vllm.models.deepseek_v41.sparse_mla import (
|
|
DeepseekV4FlashMLAMetadata,
|
|
DeepseekV4SparseMLABackend,
|
|
DeepseekV4SparseMLAMetadataBuilder,
|
|
DeepseekV41SparseSWAMetadataBuilder,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.platforms.rocm import _ON_GFX950
|
|
from vllm.triton_utils import tl, triton
|
|
from vllm.utils.multi_stream_utils import execute_in_parallel
|
|
from vllm.v1.attention.backend import (
|
|
AttentionCGSupport,
|
|
CommonAttentionMetadata,
|
|
MultipleOf,
|
|
)
|
|
from vllm.v1.attention.backends.mla.sparse_swa import (
|
|
DeepseekSparseSWABackend,
|
|
DeepseekSparseSWAMetadata,
|
|
)
|
|
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
|
build_prefill_topk_ragged_indices,
|
|
build_ragged_indices_from_dense,
|
|
rocm_inv_rope_einsum,
|
|
rocm_inverse_rope_mxfp8_rows,
|
|
rocm_inverse_rope_rows_,
|
|
rocm_mxfp8_wo_a_bmm,
|
|
rocm_sparse_attn_decode,
|
|
rocm_sparse_attn_prefill,
|
|
)
|
|
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
|
|
rocm_paged_mxfp4_cache_layout,
|
|
)
|
|
from vllm.v1.kv_cache_interface import KVCacheSpec
|
|
from vllm.v1.worker.workspace import current_workspace_manager
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def _trust_dsv4_extra_cache_nan_free(
|
|
kv_cache_dtype: str,
|
|
has_kv_transfer: bool,
|
|
has_extra_cache: bool,
|
|
) -> bool:
|
|
return (
|
|
_ON_GFX950
|
|
and kv_cache_dtype == "fp8_ds_mla"
|
|
and not has_kv_transfer
|
|
and has_extra_cache
|
|
)
|
|
|
|
|
|
def _aiter_sparse_mla_enabled(has_kv_transfer: bool) -> bool:
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
|
|
if not rocm_aiter_ops.is_triton_sparse_mla_enabled():
|
|
return False
|
|
# The aiter kernel does not scrub NaNs, so like the trust check above it
|
|
# needs a cache only the canonical writer has touched.
|
|
if has_kv_transfer:
|
|
logger.warning_once(
|
|
"VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is ignored with a KV "
|
|
"connector; DeepSeek V4 keeps the Triton sparse attention."
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
|
lengths = lengths.to(dtype=torch.int32).contiguous()
|
|
indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device)
|
|
torch.cumsum(lengths, dim=0, out=indptr[1:])
|
|
return indptr
|
|
|
|
|
|
def apply_pre_quantized_block_scaled_mm(
|
|
linear: torch.nn.Module,
|
|
x_fp8: torch.Tensor,
|
|
x_scale: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
"""Block-scaled fp8 GEMM on pre-quantized activations.
|
|
|
|
The fused q/kv norm kernel writes fp8 qr + per-1x128 scales; this
|
|
drives the linear's block-scaled GEMM directly with them, bypassing
|
|
apply_weights which would re-quantize the fp8 input. Only valid for
|
|
the wq_b-style column/replicated linears: their output is the local
|
|
TP shard, so no all-reduce is needed.
|
|
"""
|
|
from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel import (
|
|
FP8BlockParams,
|
|
)
|
|
|
|
params = FP8BlockParams.from_layer(linear)
|
|
weight_scale = (
|
|
params.weight_scale
|
|
if params.weight_scale_inv is None
|
|
else params.weight_scale_inv
|
|
)
|
|
kernel = linear.quant_method.fp8_linear
|
|
out = kernel.apply_block_scaled_mm(
|
|
A=x_fp8, B=params.weight, As=x_scale, Bs=weight_scale
|
|
)
|
|
return out.to(dtype=kernel.config.out_dtype)
|
|
|
|
|
|
@triton.jit
|
|
def _compute_topk_lens_kernel(
|
|
topk_lens_ptr,
|
|
topk_indices_ptr,
|
|
topk_indices_stride,
|
|
topk,
|
|
is_valid_token_ptr,
|
|
TRITON_BLOCK_SIZE: tl.constexpr,
|
|
):
|
|
token_idx = tl.program_id(0)
|
|
is_valid_token = tl.load(is_valid_token_ptr + token_idx)
|
|
|
|
count = tl.zeros((), dtype=tl.int32)
|
|
for i in range(0, topk, TRITON_BLOCK_SIZE):
|
|
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
|
mask = offset < topk
|
|
local_idx = tl.load(
|
|
topk_indices_ptr + token_idx * topk_indices_stride + offset,
|
|
mask=mask,
|
|
other=-1,
|
|
)
|
|
count += tl.sum((local_idx >= 0).to(tl.int32), axis=0)
|
|
|
|
tl.store(topk_lens_ptr + token_idx, tl.where(is_valid_token, count, 0))
|
|
|
|
|
|
@triton.jit
|
|
def _pack_global_topk_ragged_kernel(
|
|
global_topk_ragged_ptr,
|
|
topk_indptr_ptr,
|
|
topk_indices_ptr,
|
|
topk_indices_stride,
|
|
token_to_req_indices_ptr,
|
|
block_table_ptr,
|
|
block_table_stride,
|
|
block_size,
|
|
topk,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
):
|
|
token_idx = tl.program_id(0)
|
|
block_idx = tl.program_id(1)
|
|
offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
|
|
|
out_start = tl.load(topk_indptr_ptr + token_idx)
|
|
out_end = tl.load(topk_indptr_ptr + token_idx + 1)
|
|
out_len = out_end - out_start
|
|
if block_idx * BLOCK_SIZE >= out_len:
|
|
return
|
|
|
|
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
|
mask = (offset < out_len) & (offset < topk)
|
|
local_idx = tl.load(
|
|
topk_indices_ptr + token_idx * topk_indices_stride + offset,
|
|
mask=mask,
|
|
other=-1,
|
|
)
|
|
valid = mask & (local_idx >= 0)
|
|
block_indices = local_idx // block_size
|
|
block_numbers = tl.load(
|
|
block_table_ptr + req_idx * block_table_stride + block_indices,
|
|
mask=valid,
|
|
other=0,
|
|
)
|
|
block_offsets = local_idx % block_size
|
|
slot_ids = tl.where(valid, block_numbers * block_size + block_offsets, -1)
|
|
tl.store(global_topk_ragged_ptr + out_start + offset, slot_ids, mask=mask)
|
|
|
|
|
|
def compute_global_topk_ragged_indices_and_indptr(
|
|
topk_indices: torch.Tensor,
|
|
token_to_req_indices: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
block_size: int,
|
|
is_valid_token: torch.Tensor,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
|
|
num_tokens = topk_indices.shape[0]
|
|
topk = topk_indices.shape[1]
|
|
|
|
topk_lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device)
|
|
_compute_topk_lens_kernel[(num_tokens,)](
|
|
topk_lens,
|
|
topk_indices,
|
|
topk_indices.stride(0),
|
|
topk,
|
|
is_valid_token,
|
|
TRITON_BLOCK_SIZE=1024,
|
|
)
|
|
|
|
topk_indptr = _build_indptr_from_lengths(topk_lens)
|
|
global_topk_ragged = torch.empty(
|
|
num_tokens * topk,
|
|
dtype=torch.int32,
|
|
device=topk_indices.device,
|
|
)
|
|
if global_topk_ragged.numel() > 0:
|
|
block = 128
|
|
_pack_global_topk_ragged_kernel[(num_tokens, triton.cdiv(topk, block))](
|
|
global_topk_ragged,
|
|
topk_indptr,
|
|
topk_indices,
|
|
topk_indices.stride(0),
|
|
token_to_req_indices,
|
|
block_table,
|
|
block_table.stride(0),
|
|
block_size,
|
|
topk,
|
|
BLOCK_SIZE=block,
|
|
)
|
|
return global_topk_ragged, topk_indptr, topk_lens
|
|
|
|
|
|
def _copy_ragged_to_graph_buffers(
|
|
ragged_indices: torch.Tensor,
|
|
ragged_indptr: torch.Tensor,
|
|
ragged_indices_buffer: torch.Tensor,
|
|
ragged_indptr_buffer: torch.Tensor,
|
|
num_rows: int,
|
|
max_entries_per_row: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Copy dynamic ragged metadata into persistent CUDA graph buffers.
|
|
|
|
FULL decode graphs capture kernel argument addresses. Keep the returned
|
|
tensors backed by stable storage, while indptr continues to bound reads.
|
|
"""
|
|
indptr_out = ragged_indptr_buffer[: num_rows + 1]
|
|
indptr_out.copy_(ragged_indptr, non_blocking=True)
|
|
|
|
max_entries = max(num_rows * max_entries_per_row, 1)
|
|
ragged_out = ragged_indices_buffer[:max_entries]
|
|
source_entries = ragged_indices.numel()
|
|
if source_entries > 0:
|
|
ragged_out[:source_entries].copy_(ragged_indices, non_blocking=True)
|
|
if _ON_GFX950:
|
|
# Preserve the graph-stable base pointer while exposing source capacity
|
|
# to the sync-free split selector; indptr still carries the true NNZ.
|
|
ragged_out = ragged_out[: max(source_entries, 1)]
|
|
return ragged_out, indptr_out
|
|
|
|
|
|
# (ragged_indices, ragged_indptr, lens) as consumed by rocm_sparse_attn_decode.
|
|
_TopkRagged = tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
|
|
|
|
|
@dataclass
|
|
class DeepseekV41ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
|
|
decode_swa_ragged_indices: torch.Tensor | None = None
|
|
decode_swa_ragged_indptr: torch.Tensor | None = None
|
|
# Built only for the aiter prefill, which reads the SWA cache in place.
|
|
prefill_swa_ragged_indices: torch.Tensor | None = None
|
|
prefill_swa_ragged_indptr: torch.Tensor | None = None
|
|
|
|
|
|
class DeepseekV41ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder):
|
|
@classmethod
|
|
def get_cudagraph_support(
|
|
cls,
|
|
vllm_config: VllmConfig,
|
|
kv_cache_spec: KVCacheSpec,
|
|
) -> AttentionCGSupport:
|
|
spec_config = vllm_config.speculative_config
|
|
if spec_config is not None and spec_config.enable_adaptive_verification:
|
|
# Per-token request ownership and compressed slot mappings are built
|
|
# from device query boundaries into persistent buffers.
|
|
return AttentionCGSupport.ALWAYS
|
|
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
|
|
|
|
|
|
class DeepseekV41ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBuilder):
|
|
@classmethod
|
|
def get_cudagraph_support(
|
|
cls,
|
|
vllm_config: VllmConfig,
|
|
kv_cache_spec: KVCacheSpec,
|
|
) -> AttentionCGSupport:
|
|
spec_config = vllm_config.speculative_config
|
|
if spec_config is not None and spec_config.enable_adaptive_verification:
|
|
# SWA indices, lengths, and request ownership are built from device
|
|
# query boundaries and copied into persistent ragged buffers.
|
|
return AttentionCGSupport.ALWAYS
|
|
return super().get_cudagraph_support(vllm_config, kv_cache_spec)
|
|
|
|
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
|
|
# also refreshes the ROCm-specific ragged SWA indices and indptrs.
|
|
supports_draft_decode_metadata_update = False
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
max_tokens = self.vllm_config.scheduler_config.max_num_batched_tokens
|
|
# The non-causal (DSpark draft) path widens each token's SWA index list
|
|
# to ``noncausal_index_width`` (>= window_size), so size the persistent
|
|
# ragged buffer to the wider bound to cover both causal and non-causal.
|
|
swa_index_width = max(self.window_size, self.noncausal_index_width)
|
|
self.decode_swa_ragged_indices_buffer = torch.empty(
|
|
max_tokens * swa_index_width,
|
|
dtype=torch.int32,
|
|
device=self.device,
|
|
)
|
|
self.decode_swa_ragged_indptr_buffer = torch.empty(
|
|
max_tokens + 1,
|
|
dtype=torch.int32,
|
|
device=self.device,
|
|
)
|
|
self._use_aiter_sparse_mla = _aiter_sparse_mla_enabled(
|
|
self.vllm_config.kv_transfer_config is not None
|
|
)
|
|
|
|
def build(
|
|
self,
|
|
common_prefix_len: int,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
fast_build: bool = False,
|
|
replay_start: torch.Tensor | None = None,
|
|
) -> DeepseekV41ROCMAiterSparseSWAMetadata:
|
|
base = super().build(
|
|
common_prefix_len=common_prefix_len,
|
|
common_attn_metadata=common_attn_metadata,
|
|
fast_build=fast_build,
|
|
replay_start=replay_start,
|
|
)
|
|
|
|
prefill_ragged_indices = None
|
|
prefill_ragged_indptr = None
|
|
if (
|
|
self._use_aiter_sparse_mla
|
|
and base.num_prefill_tokens > 0
|
|
and base.prefill_swa_indices is not None
|
|
and base.prefill_swa_lens is not None
|
|
):
|
|
# Paged-coordinate rows, so every layer's prefill shares one build.
|
|
prefill_ragged_indices, prefill_ragged_indptr = (
|
|
build_ragged_indices_from_dense(
|
|
base.prefill_swa_indices.reshape(base.num_prefill_tokens, -1),
|
|
base.prefill_swa_lens,
|
|
)
|
|
)
|
|
|
|
ragged_indices = None
|
|
ragged_indptr = None
|
|
if (
|
|
base.num_decode_tokens > 0
|
|
and base.decode_swa_indices is not None
|
|
and base.decode_swa_lens is not None
|
|
):
|
|
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
|
base.decode_swa_indices.reshape(
|
|
base.num_decode_tokens, base.decode_swa_width
|
|
),
|
|
base.decode_swa_lens,
|
|
)
|
|
ragged_indices, ragged_indptr = _copy_ragged_to_graph_buffers(
|
|
ragged_indices,
|
|
ragged_indptr,
|
|
self.decode_swa_ragged_indices_buffer,
|
|
self.decode_swa_ragged_indptr_buffer,
|
|
base.num_decode_tokens,
|
|
base.decode_swa_width,
|
|
)
|
|
|
|
return DeepseekV41ROCMAiterSparseSWAMetadata(
|
|
**vars(base),
|
|
decode_swa_ragged_indices=ragged_indices,
|
|
decode_swa_ragged_indptr=ragged_indptr,
|
|
prefill_swa_ragged_indices=prefill_ragged_indices,
|
|
prefill_swa_ragged_indptr=prefill_ragged_indptr,
|
|
)
|
|
|
|
|
|
@functools.cache
|
|
def _aiter_indexer_cache_ops() -> tuple[Callable[..., None], Callable[..., tuple]]:
|
|
"""The aiter indexer key writer and query quantizer."""
|
|
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
|
|
k_norm_rope_mxfp4_cache,
|
|
)
|
|
from aiter.ops.triton.rope.q_rope_mxfp4_quant import q_rope_mxfp4_quant
|
|
|
|
return k_norm_rope_mxfp4_cache, q_rope_mxfp4_quant
|
|
|
|
|
|
def rocm_mxfp4_indexer_k_store(
|
|
k_pre: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
cos_sin_cache: torch.Tensor,
|
|
rms_norm_weight: torch.Tensor,
|
|
rms_norm_eps: float,
|
|
k_cache: torch.Tensor,
|
|
kv_slot_mapping: torch.Tensor,
|
|
compress_ratio: int,
|
|
use_fp4_cache: bool,
|
|
*,
|
|
num_heads: int,
|
|
) -> None:
|
|
"""`indexer_k_norm_rope_store` for the ROCm MXFP4 cache: aiter's cache op
|
|
writes the key in the order its MQA-logits kernel reads with ``num_heads``
|
|
query heads."""
|
|
assert use_fp4_cache, "the ROCm indexer cache op writes MXFP4 only"
|
|
layout = rocm_paged_mxfp4_cache_layout(num_heads, k_pre.shape[1], k_cache.shape[1])
|
|
k_norm_rope_mxfp4_cache, _ = _aiter_indexer_cache_ops()
|
|
k_norm_rope_mxfp4_cache(
|
|
k_pre,
|
|
positions,
|
|
cos_sin_cache,
|
|
rms_norm_weight,
|
|
rms_norm_eps,
|
|
k_cache,
|
|
kv_slot_mapping,
|
|
compress_ratio,
|
|
shuffle=layout,
|
|
)
|
|
|
|
|
|
def rocm_mxfp4_indexer_q_quant(
|
|
positions: torch.Tensor,
|
|
index_q: torch.Tensor,
|
|
index_q_cos_sin_cache: torch.Tensor,
|
|
index_weights: torch.Tensor,
|
|
index_weights_softmax_scale: float,
|
|
index_weights_head_scale: float,
|
|
use_fp4: bool = True,
|
|
weights_out_dtype: torch.dtype = torch.float32,
|
|
) -> tuple[tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
|
|
"""`fused_indexer_q_rope_quant` for the ROCm MXFP4 indexer, on aiter's op:
|
|
((packed [T, H, D // 2], e8m0 as one int32 per head [T, H]), fp32
|
|
weights)."""
|
|
assert use_fp4 and weights_out_dtype == torch.float32, (
|
|
"the ROCm indexer quantizes Q to MXFP4 and scores with fp32 weights"
|
|
)
|
|
_, q_rope_mxfp4_quant = _aiter_indexer_cache_ops()
|
|
q_packed, q_scale, weights_out = q_rope_mxfp4_quant(
|
|
index_q,
|
|
positions,
|
|
index_q_cos_sin_cache,
|
|
index_weights,
|
|
index_weights_softmax_scale * index_weights_head_scale,
|
|
)
|
|
return (q_packed, q_scale.view(torch.int32).squeeze(-1)), weights_out
|
|
|
|
|
|
class DeepseekV41RocmMxfp4Indexer(DeepseekV4Indexer):
|
|
"""The indexer on aiter's paged MXFP4 cache: its K store and Q quant write
|
|
in the order aiter's MQA-logits kernel reads, and its layers score with it.
|
|
``_produce_k`` and ``forward_q`` mirror ``DeepseekV4Indexer``'s except for
|
|
those two calls."""
|
|
|
|
mqa_cls = RocmSparseMQAIndexer
|
|
attn_cls = RocmSparseAttnIndexer
|
|
|
|
def _produce_k(
|
|
self,
|
|
latent: torch.Tensor | None,
|
|
positions: torch.Tensor,
|
|
rotary_emb: torch.nn.Module,
|
|
) -> None:
|
|
attn_metadata = get_forward_context().attn_metadata
|
|
if not isinstance(attn_metadata, dict) or latent is None:
|
|
return
|
|
assert self.owns_k
|
|
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
|
|
k_pre, _ = self.wk(latent)
|
|
rocm_mxfp4_indexer_k_store(
|
|
k_pre,
|
|
positions,
|
|
rotary_emb.cos_sin_cache,
|
|
self.k_norm.weight,
|
|
self.k_norm.variance_epsilon,
|
|
self.k_cache.kv_cache,
|
|
indexer_metadata.slot_mapping,
|
|
self.compress_ratio,
|
|
self.use_fp4_kv,
|
|
num_heads=self.n_head,
|
|
)
|
|
|
|
def forward_q(
|
|
self,
|
|
qr: torch.Tensor | QuantizedActivation,
|
|
qr_scale: torch.Tensor | None,
|
|
indexer_weights: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
rotary_emb: torch.nn.Module,
|
|
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
|
q = self._wq_b_proj(qr, qr_scale).view(-1, self.n_head, self.head_dim)
|
|
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
|
|
positions,
|
|
q,
|
|
rotary_emb.cos_sin_cache,
|
|
indexer_weights,
|
|
self.softmax_scale,
|
|
self.n_head**-0.5,
|
|
use_fp4=self.use_fp4_kv,
|
|
weights_out_dtype=self.indexer_weights_dtype,
|
|
)
|
|
return q, q_scale, weights
|
|
|
|
|
|
class DeepseekV41ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend):
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "ROCM_FLASHMLA_SPARSE_DSV41"
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
return [64, 128]
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type[DeepseekV41ROCMAiterMLASparseMetadataBuilder]:
|
|
return DeepseekV41ROCMAiterMLASparseMetadataBuilder
|
|
|
|
|
|
class DeepseekV41ROCMAiterSparseSWABackend(DeepseekSparseSWABackend):
|
|
@staticmethod
|
|
def get_builder_cls() -> type["DeepseekV41ROCMAiterSparseSWAMetadataBuilder"]:
|
|
return DeepseekV41ROCMAiterSparseSWAMetadataBuilder
|
|
|
|
|
|
class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
|
"""ROCm sparse MLA attention layer for DeepSeek V4.1."""
|
|
|
|
backend_cls = DeepseekV41ROCMAiterMLASparseBackend
|
|
swa_backend_cls = DeepseekV41ROCMAiterSparseSWABackend
|
|
_use_aiter_sparse_mla = False
|
|
|
|
def _indexer_cls(
|
|
self, k_cache: DeepseekV4IndexerCache | None
|
|
) -> type[DeepseekV4Indexer]:
|
|
if k_cache is not None and k_cache.rocm_mxfp4:
|
|
return DeepseekV41RocmMxfp4Indexer
|
|
return super()._indexer_cls(k_cache)
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
vllm_config = args[0] if args else kwargs["vllm_config"]
|
|
super().__init__(*args, **kwargs)
|
|
# CUDA executes WO_A with a quantized grouped-BMM kernel. ROCm's
|
|
# correctness path below dequantizes WO_A once and uses torch.einsum,
|
|
# so retain the ordinary MXFP8 linear kernel for post-load processing
|
|
# instead of asking for the CUDA-only BMM kernel.
|
|
self.wo_a.is_bmm = False
|
|
self._has_kv_transfer = vllm_config.kv_transfer_config is not None
|
|
# Block scale for the preshuffled weight; None = not preshuffled.
|
|
self._wqa_wkv_scale: torch.Tensor | None = None
|
|
self._wo_b_scale: torch.Tensor | None = None
|
|
self._fused_compressor_weight: torch.Tensor | None
|
|
self.register_buffer("_fused_compressor_weight", None, persistent=False)
|
|
self._fused_compressor_split_sizes: tuple[int, int] | None = None
|
|
# Decode ragged topk metadata is a pure function of the indices its
|
|
# index source published, so every consumer of a source rebuilds the
|
|
# same thing. The source owns one cache per compress ratio, refreshed
|
|
# when it runs; consumers below it read through.
|
|
self._topk_ragged_cache: dict[int, _TopkRagged] = {}
|
|
self._prefill_topk_ragged_cache: dict[
|
|
str, tuple[torch.Tensor, torch.Tensor]
|
|
] = {}
|
|
self._index_source_prefix: str | None = None
|
|
if self.compress_ratio > 0:
|
|
assert self.index_source_layer_id is not None
|
|
self._index_source_prefix = _replace_layer_index(
|
|
self.prefix, self.index_source_layer_id
|
|
)
|
|
if self._index_source_prefix not in self._static_forward_context:
|
|
raise NotImplementedError(
|
|
f"Index source {self._index_source_prefix} not found on "
|
|
"this rank; PP splits inside a v4.1 index-sharing group "
|
|
"are not supported."
|
|
)
|
|
self._use_aiter_sparse_mla = _aiter_sparse_mla_enabled(self._has_kv_transfer)
|
|
if self.compressor is None and self.indexer is None:
|
|
# Dense layers have nothing to overlap; keep the base serial path.
|
|
self.aux_stream_list = None
|
|
|
|
@classmethod
|
|
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
|
return num_heads
|
|
|
|
def prepare_attn_preshuffle(self) -> None:
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
|
|
if not rocm_aiter_ops.is_enabled():
|
|
return
|
|
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
|
_upcast_e8m0_to_fp32,
|
|
)
|
|
from vllm.model_executor.utils import replace_parameter
|
|
|
|
def _prep(linear) -> torch.Tensor | None:
|
|
w = getattr(linear, "weight", None)
|
|
if w is None or w.dim() != 2:
|
|
return None
|
|
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
|
|
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
|
|
return None
|
|
ws = getattr(linear, "weight_scale_inv", None) # per-block scale
|
|
if ws is None:
|
|
return None
|
|
if ws.dtype == torch.float8_e8m0fnu:
|
|
ws = _upcast_e8m0_to_fp32(ws).contiguous()
|
|
# Shuffle the weight in place (single weight, no unshuffled copy).
|
|
replace_parameter(
|
|
linear,
|
|
"weight",
|
|
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
|
)
|
|
return ws
|
|
|
|
self._wqa_wkv_scale = _prep(self.fused_wqa_wkv)
|
|
self._wo_b_scale = _prep(self.wo_b)
|
|
|
|
def prepare_compressor_gemm_fusion(self) -> bool:
|
|
# V4.1 derives index keys from the source compressor's emitted latent
|
|
# and has no nested ``indexer.compressor``. Keep the projections
|
|
# separate and use the shared linear/PyTorch correctness path.
|
|
return False
|
|
|
|
def _bpre_attn_gemm(
|
|
self,
|
|
weight: torch.Tensor,
|
|
scale: torch.Tensor,
|
|
x: torch.Tensor,
|
|
reduce_tp: bool,
|
|
) -> torch.Tensor:
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
|
|
x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True)
|
|
out = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
|
|
x_fp8, weight, x_scale, scale, output_dtype=x.dtype
|
|
)
|
|
if reduce_tp and get_tensor_model_parallel_world_size() > 1:
|
|
out = tensor_model_parallel_all_reduce(out)
|
|
return out
|
|
|
|
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
|
if self._wqa_wkv_scale is not None and hidden_states.dim() == 2:
|
|
return self._bpre_attn_gemm(
|
|
self.fused_wqa_wkv.weight, self._wqa_wkv_scale, hidden_states, False
|
|
)
|
|
return super()._fused_wqa_wkv_gemm(hidden_states)
|
|
|
|
def _run_parallel_input_projections(
|
|
self, hidden_states: torch.Tensor
|
|
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
|
if self.indexer is not None and self.compressor is None:
|
|
# Serial: qr must sit on the default stream before the q-side
|
|
# fork below can consume it (ROCm).
|
|
aux_streams = self.aux_stream_list
|
|
self.aux_stream_list = None
|
|
try:
|
|
return super()._run_parallel_input_projections(hidden_states)
|
|
finally:
|
|
self.aux_stream_list = aux_streams
|
|
return super()._run_parallel_input_projections(hidden_states)
|
|
|
|
def forward(
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
llama_4_scaling: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
if (
|
|
self.indexer is None
|
|
or self._prepare_and_attn_fn == self._prepare_and_attn_eager
|
|
or not current_platform.enable_multi_stream_overlap(
|
|
self.aux_stream_list, get_forward_context().attn_metadata
|
|
)
|
|
):
|
|
# Sequential fallback: no forks outside the capture region, where
|
|
# HIP event sync is unreliable; MRV1 keeps the input prep in its
|
|
# wide eager region (#51430).
|
|
aux_streams = self.aux_stream_list
|
|
self.aux_stream_list = None
|
|
try:
|
|
return super().forward(positions, hidden_states, llama_4_scaling)
|
|
finally:
|
|
self.aux_stream_list = aux_streams
|
|
|
|
attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states)
|
|
if self.compressor is not None:
|
|
q, kv, index_q, index_q_scale, index_weights = self._forward_csa2_full(
|
|
hidden_states, positions
|
|
)
|
|
else:
|
|
q, kv, index_q, index_q_scale, index_weights = self._forward_csa2_reindex(
|
|
hidden_states, positions
|
|
)
|
|
self._sparse_indexer_and_attn(
|
|
hidden_states,
|
|
index_q,
|
|
index_q_scale,
|
|
index_weights,
|
|
q,
|
|
kv,
|
|
positions,
|
|
attn_out,
|
|
)
|
|
return self._o_proj(attn_out, positions)
|
|
|
|
def _forward_csa2_full(
|
|
self, hidden_states: torch.Tensor, positions: torch.Tensor
|
|
) -> tuple[
|
|
torch.Tensor,
|
|
torch.Tensor,
|
|
torch.Tensor | None,
|
|
torch.Tensor | None,
|
|
torch.Tensor | None,
|
|
]:
|
|
"""CSA layers: one fork replaces the base pipeline's three.
|
|
|
|
Each fork/join is a HIP event pair per side stream on ROCm, so the
|
|
base three-fork pipeline is expensive there; the compressor chain and
|
|
the indexer weights projection run fully in parallel with the default
|
|
chain. The indexer K write and q-side stay serial after the join:
|
|
``wk`` consumes the aux-0 latent and the q-side's wq_b consumes the
|
|
default-stream qr.
|
|
"""
|
|
attn_metadata = get_forward_context().attn_metadata
|
|
compressor = self.compressor
|
|
indexer = self.indexer
|
|
assert compressor is not None and indexer is not None
|
|
aux_streams = self.aux_stream_list
|
|
assert aux_streams is not None and len(aux_streams) >= 2
|
|
|
|
def default_chain():
|
|
qr_kv = self._fused_wqa_wkv_gemm(hidden_states)
|
|
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
|
|
q = self._wq_b_proj(qr, qr_scale).view(
|
|
-1, self.n_local_heads, self.head_dim
|
|
)
|
|
return (
|
|
self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata),
|
|
qr,
|
|
qr_scale,
|
|
kv,
|
|
)
|
|
|
|
def compressor_chain():
|
|
kv_score = torch.mm(
|
|
hidden_states,
|
|
compressor.fused_wkv_wgate.weight.T,
|
|
out_dtype=torch.float32,
|
|
)
|
|
latent = compressor(kv_score, positions)
|
|
compressor.insert_cache(latent, positions, self.rotary_emb)
|
|
return latent
|
|
|
|
def indexer_weights_chain():
|
|
# ReplicatedLinear returns (output, bias); bias is None.
|
|
weights, _ = indexer.weights_proj(hidden_states)
|
|
return weights
|
|
|
|
(q, qr, qr_scale, kv), (latent, indexer_weights) = execute_in_parallel(
|
|
default_chain,
|
|
[compressor_chain, indexer_weights_chain],
|
|
self.ln_events[0],
|
|
self.ln_events[1:3],
|
|
aux_streams[:2],
|
|
enable=True,
|
|
default_first=True,
|
|
)
|
|
|
|
indexer._produce_k(latent, positions, self.indexer_rotary_emb)
|
|
index_q, index_q_scale, index_weights_out = indexer.forward_q(
|
|
qr, qr_scale, indexer_weights, positions, self.indexer_rotary_emb
|
|
)
|
|
return q, kv, index_q, index_q_scale, index_weights_out
|
|
|
|
def _forward_csa2_reindex(
|
|
self, hidden_states: torch.Tensor, positions: torch.Tensor
|
|
) -> tuple[
|
|
torch.Tensor,
|
|
torch.Tensor,
|
|
torch.Tensor | None,
|
|
torch.Tensor | None,
|
|
torch.Tensor | None,
|
|
]:
|
|
"""Indexer-only layers: the q-side overlaps the SWA q path.
|
|
|
|
The input projections run serial first: the q-side consumes qr, so
|
|
it can only overlap the SWA q projection and KV insert.
|
|
"""
|
|
indexer = self.indexer
|
|
assert indexer is not None
|
|
attn_metadata = get_forward_context().attn_metadata
|
|
# Serial on these layers via the _run_parallel_input_projections
|
|
# override: qr must sit on the default stream before the fork.
|
|
qr_kv, _, indexer_weights = self._run_parallel_input_projections(hidden_states)
|
|
assert indexer_weights is not None
|
|
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
|
|
|
|
def swa_q_chain():
|
|
q = self._wq_b_proj(qr, qr_scale).view(
|
|
-1, self.n_local_heads, self.head_dim
|
|
)
|
|
return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
|
|
|
def indexer_q_chain():
|
|
return indexer.forward_q(
|
|
qr, qr_scale, indexer_weights, positions, self.indexer_rotary_emb
|
|
)
|
|
|
|
aux_streams = self.aux_stream_list
|
|
assert aux_streams is not None
|
|
q, aux_results = execute_in_parallel(
|
|
swa_q_chain,
|
|
[indexer_q_chain],
|
|
self.ln_events[0],
|
|
self.ln_events[1:2],
|
|
aux_streams[:1],
|
|
enable=True,
|
|
default_first=True,
|
|
)
|
|
index_q, index_q_scale, index_weights_out = aux_results[0]
|
|
return q, kv, index_q, index_q_scale, index_weights_out
|
|
|
|
@functools.cached_property
|
|
def _wq_b_uses_aiter_block_scaled(self) -> bool:
|
|
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
|
|
|
|
Cached: the linear kernels and the aiter env gates are fixed once
|
|
the model is built, so this is evaluated at the first forward
|
|
only.
|
|
|
|
The fused norm+quant path is only valid if the quant and GEMM it
|
|
replaces are exactly the aiter ones; otherwise fall back to the
|
|
shared path.
|
|
"""
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
from vllm.model_executor.kernels.linear.scaled_mm import (
|
|
Fp8BlockScaledMMLinearKernel,
|
|
)
|
|
|
|
if not rocm_aiter_ops.is_linear_fp8_enabled():
|
|
return False
|
|
|
|
linears = [self.wq_b]
|
|
if self.indexer is not None:
|
|
linears.append(self.indexer.wq_b)
|
|
for linear in linears:
|
|
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
|
|
if not isinstance(kernel, Fp8BlockScaledMMLinearKernel):
|
|
return False
|
|
return True
|
|
|
|
def _split_qkv_and_norm(
|
|
self, qr_kv: torch.Tensor
|
|
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
|
"""Fuse q/kv RMSNorm + per-1x128 fp8 q quant into one aiter kernel.
|
|
|
|
The shared path norms q and kv in one triton kernel and the wq_b
|
|
linears then re-read the bf16 qr to quantize it. The aiter kernel
|
|
computes both RMSNorms (fp32 accumulate) and the fp8 group quant
|
|
in a single pass, writing fp8 qr + group scales directly; both
|
|
wq_b GEMMs (attention and indexer) then consume that pair and
|
|
skip their own input quant. kv stays bf16: the fused insert
|
|
kernel RoPE/quantizes it itself. Falls back to the shared path
|
|
when the aiter linear path is not active.
|
|
"""
|
|
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
|
if not (
|
|
qr.dim() == 2
|
|
and qr.shape[0] > 0
|
|
and self.q_lora_rank % 128 == 0
|
|
and self._wq_b_uses_aiter_block_scaled
|
|
):
|
|
return super()._split_qkv_and_norm(qr_kv)
|
|
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
|
|
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
|
q=qr,
|
|
q_weight=self.q_norm.weight.data,
|
|
q_epsilon=self.eps,
|
|
kv=kv,
|
|
kv_weight=self.kv_norm.weight.data,
|
|
kv_epsilon=self.eps,
|
|
group_size=128,
|
|
transpose_scale=False,
|
|
)
|
|
|
|
def _alloc_attn_out(
|
|
self, num_tokens: int, hidden_states: torch.Tensor
|
|
) -> torch.Tensor | QuantizedActivation:
|
|
if not _ON_GFX950:
|
|
return super()._alloc_attn_out(num_tokens, hidden_states)
|
|
# wo_a's MXFP8 input: the decode reduce writes it directly, prefill
|
|
# rows are rotated and quantized after their bf16 attention.
|
|
width = self.n_local_heads * self.head_dim
|
|
data = torch.empty(
|
|
(num_tokens, width), dtype=torch.float8_e4m3fn, device=hidden_states.device
|
|
)
|
|
scale = torch.empty(
|
|
(num_tokens, width // 32), dtype=torch.uint8, device=hidden_states.device
|
|
)
|
|
return QuantizedActivation(
|
|
data=data,
|
|
scale=scale,
|
|
orig_dtype=hidden_states.dtype,
|
|
orig_shape=data.shape,
|
|
quant_key=kMxfp8Dynamic,
|
|
)
|
|
|
|
def _o_proj(
|
|
self, attn_out: torch.Tensor | QuantizedActivation, positions: torch.Tensor
|
|
) -> torch.Tensor:
|
|
if isinstance(attn_out, QuantizedActivation):
|
|
z = rocm_mxfp8_wo_a_bmm(
|
|
attn_out.data,
|
|
attn_out.scale,
|
|
self.wo_a,
|
|
self.n_local_groups,
|
|
self.o_lora_rank,
|
|
)
|
|
return self._wo_b_after_wo_a(z)
|
|
o = attn_out[:, : self.n_local_heads, :]
|
|
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
|
|
z = rocm_inv_rope_einsum(
|
|
self.rotary_emb,
|
|
o,
|
|
positions,
|
|
self.rope_head_dim,
|
|
self.n_local_groups,
|
|
self.o_lora_rank,
|
|
self.wo_a,
|
|
inverse_rope=False,
|
|
)
|
|
return self._wo_b_after_wo_a(z.flatten(1))
|
|
|
|
def _wo_b_after_wo_a(self, zf: torch.Tensor) -> torch.Tensor:
|
|
if self._wo_b_scale is not None and zf.dim() == 2:
|
|
return self._bpre_attn_gemm(
|
|
self.wo_b.weight, self._wo_b_scale, zf, self.wo_b.reduce_results
|
|
)
|
|
return self.wo_b(zf)
|
|
|
|
def forward_mqa(
|
|
self,
|
|
q: torch.Tensor,
|
|
kv: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
output: torch.Tensor | QuantizedActivation,
|
|
) -> None:
|
|
mxfp8_out = output if isinstance(output, QuantizedActivation) else None
|
|
if mxfp8_out is None:
|
|
assert isinstance(output, torch.Tensor)
|
|
assert output.shape == q.shape, (
|
|
f"output buffer shape {output.shape} must match q shape {q.shape}"
|
|
)
|
|
assert output.dtype == q.dtype, (
|
|
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
|
|
)
|
|
|
|
forward_context = get_forward_context()
|
|
attn_metadata = forward_context.attn_metadata
|
|
|
|
if attn_metadata is None:
|
|
# Warmup dummy run: no real metadata. Reserve the workspace
|
|
# _forward_prefill would; the dequantize / topk / sparse_fwd kernels
|
|
# are skipped this step. The aiter prefill reads the caches in place,
|
|
# so it asks only for the bf16 rows of the MXFP8 output.
|
|
swa_only = self.compress_ratio == 0
|
|
N = (
|
|
0
|
|
if swa_only
|
|
else (self.max_model_len + self.compress_ratio - 1)
|
|
// self.compress_ratio
|
|
)
|
|
M = N + self.window_size + self.max_num_batched_tokens
|
|
shapes = self._prefill_workspace_shapes(
|
|
M,
|
|
self.max_num_batched_tokens,
|
|
q,
|
|
gather=not self._use_aiter_sparse_mla,
|
|
)
|
|
if shapes:
|
|
current_workspace_manager().get_simultaneous(*shapes)
|
|
if mxfp8_out is None:
|
|
output.zero_()
|
|
else:
|
|
mxfp8_out.data.zero_()
|
|
mxfp8_out.scale.zero_()
|
|
return
|
|
|
|
assert isinstance(attn_metadata, dict)
|
|
rocm_metadata = cast(
|
|
DeepseekV4FlashMLAMetadata | None,
|
|
attn_metadata.get(self.compressed_cache_prefix)
|
|
if self.compressed_cache_prefix is not None
|
|
else None,
|
|
)
|
|
swa_metadata = cast(
|
|
DeepseekV41ROCMAiterSparseSWAMetadata | None,
|
|
attn_metadata.get(self.swa_cache_layer.prefix),
|
|
)
|
|
assert swa_metadata is not None
|
|
|
|
swa_only = self.compress_ratio == 0
|
|
self_kv_cache = None if swa_only else self._compressed_kv_cache()
|
|
swa_kv_cache = self.swa_cache_layer.kv_cache
|
|
|
|
num_decodes = swa_metadata.num_decodes
|
|
num_prefills = swa_metadata.num_prefills
|
|
num_decode_tokens = swa_metadata.num_decode_tokens
|
|
|
|
if num_prefills > 0:
|
|
self._forward_prefill(
|
|
q=q[num_decode_tokens:],
|
|
positions=positions[num_decode_tokens:],
|
|
compressed_k_cache=self_kv_cache,
|
|
swa_k_cache=swa_kv_cache,
|
|
output=(
|
|
output[num_decode_tokens:]
|
|
if isinstance(output, torch.Tensor)
|
|
else None
|
|
),
|
|
attn_metadata=rocm_metadata,
|
|
swa_metadata=swa_metadata,
|
|
mxfp8_out=(
|
|
None
|
|
if mxfp8_out is None
|
|
else (
|
|
mxfp8_out.data[num_decode_tokens:],
|
|
mxfp8_out.scale[num_decode_tokens:],
|
|
)
|
|
),
|
|
)
|
|
if mxfp8_out is not None:
|
|
# The reduce epilogue rotates and quantizes every decode row and
|
|
# the prefill path already did its own, so nothing is owed here.
|
|
if num_decodes > 0:
|
|
self._forward_decode(
|
|
q=q[:num_decode_tokens],
|
|
positions=positions[:num_decode_tokens],
|
|
kv_cache=self_kv_cache,
|
|
swa_metadata=swa_metadata,
|
|
attn_metadata=rocm_metadata,
|
|
swa_only=swa_only,
|
|
output=None,
|
|
output_mxfp8=(
|
|
mxfp8_out.data[:num_decode_tokens],
|
|
mxfp8_out.scale[:num_decode_tokens],
|
|
),
|
|
)
|
|
return
|
|
assert isinstance(output, torch.Tensor)
|
|
rotated = 0
|
|
if num_decodes > 0:
|
|
rotated = self._forward_decode(
|
|
q=q[:num_decode_tokens],
|
|
positions=positions[:num_decode_tokens],
|
|
kv_cache=self_kv_cache,
|
|
swa_metadata=swa_metadata,
|
|
attn_metadata=rocm_metadata,
|
|
swa_only=swa_only,
|
|
output=output[:num_decode_tokens],
|
|
)
|
|
# Only the decode reduce rotates its own rows, and only the leading
|
|
# `rotated` of them; prefill rows and any decode path that did not
|
|
# fuse still owe the standalone pass. Settle that here rather than in
|
|
# _o_proj: the split is batch-dependent and _o_proj runs compiled,
|
|
# where such a value freezes at its trace-time value.
|
|
rocm_inverse_rope_rows_(
|
|
output[rotated:, : self.n_local_heads, :],
|
|
positions[rotated:],
|
|
self.rotary_emb.cos_sin_cache,
|
|
self.rope_head_dim,
|
|
)
|
|
|
|
def _decode_topk_ragged(
|
|
self,
|
|
swa_metadata: DeepseekV41ROCMAiterSparseSWAMetadata,
|
|
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
|
num_decodes: int,
|
|
num_decode_tokens: int,
|
|
) -> _TopkRagged:
|
|
"""Ragged form of the topk indices this layer's index source published.
|
|
|
|
The packing depends only on the shared ``topk_indices_buffer`` and
|
|
per-step metadata, apart from the layer's own compress ratio, so the
|
|
source memoizes one result per ratio for all the layers below it.
|
|
"""
|
|
assert attn_metadata is not None
|
|
assert swa_metadata.is_valid_token is not None
|
|
assert self.topk_indices_buffer is not None
|
|
assert self._index_source_prefix is not None
|
|
|
|
source = self._static_forward_context[self._index_source_prefix]
|
|
if source is self:
|
|
# Fresh indices as of this layer; drop what the last step cached.
|
|
source._topk_ragged_cache = {}
|
|
cached = source._topk_ragged_cache.get(self.compress_ratio)
|
|
if cached is not None:
|
|
return cached
|
|
|
|
built = compute_global_topk_ragged_indices_and_indptr(
|
|
self.topk_indices_buffer[:num_decode_tokens],
|
|
swa_metadata.token_to_req_indices,
|
|
attn_metadata.block_table[:num_decodes],
|
|
attn_metadata.block_size // self.compress_ratio,
|
|
swa_metadata.is_valid_token[:num_decode_tokens],
|
|
)
|
|
source._topk_ragged_cache[self.compress_ratio] = built
|
|
return built
|
|
|
|
def _forward_decode(
|
|
self,
|
|
q: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
kv_cache: torch.Tensor | None,
|
|
swa_metadata: DeepseekV41ROCMAiterSparseSWAMetadata,
|
|
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
|
swa_only: bool,
|
|
output: torch.Tensor | None,
|
|
output_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
|
|
) -> int:
|
|
"""Returns how many leading rows the decode epilogue inverse-RoPE'd."""
|
|
num_decodes = swa_metadata.num_decodes
|
|
num_decode_tokens = swa_metadata.num_decode_tokens
|
|
|
|
topk_lens = None
|
|
topk_ragged_indices = None
|
|
topk_ragged_indptr = None
|
|
if not swa_only:
|
|
(
|
|
topk_ragged_indices,
|
|
topk_ragged_indptr,
|
|
topk_lens,
|
|
) = self._decode_topk_ragged(
|
|
swa_metadata=swa_metadata,
|
|
attn_metadata=attn_metadata,
|
|
num_decodes=num_decodes,
|
|
num_decode_tokens=num_decode_tokens,
|
|
)
|
|
|
|
if self._use_aiter_sparse_mla:
|
|
assert swa_metadata.decode_swa_ragged_indices is not None
|
|
assert swa_metadata.decode_swa_ragged_indptr is not None
|
|
out = output
|
|
if output_mxfp8 is not None:
|
|
# The aiter kernel has no MXFP8 epilogue: write bf16 rows, then
|
|
# rotate and quantize them as the decode reduce would have.
|
|
out = self._mxfp8_bf16_rows(q.shape[0], q)
|
|
assert out is not None
|
|
self._aiter_sparse_mla(
|
|
q,
|
|
out,
|
|
self.swa_cache_layer.kv_cache,
|
|
swa_metadata.decode_swa_ragged_indices,
|
|
swa_metadata.decode_swa_ragged_indptr,
|
|
kv_cache,
|
|
topk_ragged_indices,
|
|
topk_ragged_indptr,
|
|
)
|
|
if output_mxfp8 is not None:
|
|
rocm_inverse_rope_mxfp8_rows(
|
|
out,
|
|
positions,
|
|
self.rotary_emb.cos_sin_cache,
|
|
self.rope_head_dim,
|
|
output_mxfp8[0],
|
|
output_mxfp8[1],
|
|
)
|
|
return q.shape[0]
|
|
# The aiter kernel leaves the inverse RoPE to the caller.
|
|
return 0
|
|
|
|
return rocm_sparse_attn_decode(
|
|
q=q,
|
|
kv_cache=kv_cache,
|
|
swa_k_cache=self.swa_cache_layer.kv_cache,
|
|
swa_only=swa_only,
|
|
topk_indices=None,
|
|
topk_lens=topk_lens,
|
|
swa_indices=swa_metadata.decode_swa_indices,
|
|
swa_lens=swa_metadata.decode_swa_lens,
|
|
swa_ragged_indices=swa_metadata.decode_swa_ragged_indices,
|
|
swa_ragged_indptr=swa_metadata.decode_swa_ragged_indptr,
|
|
topk_ragged_indices=topk_ragged_indices,
|
|
topk_ragged_indptr=topk_ragged_indptr,
|
|
attn_sink=self.attn_sink,
|
|
scale=self.scale,
|
|
head_dim=self.head_dim,
|
|
nope_head_dim=self.nope_head_dim,
|
|
rope_head_dim=self.rope_head_dim,
|
|
output=output,
|
|
output_mxfp8=output_mxfp8,
|
|
inv_rope_positions=positions,
|
|
inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache,
|
|
extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free(
|
|
self.kv_cache_dtype,
|
|
self._has_kv_transfer,
|
|
not swa_only and kv_cache is not None,
|
|
),
|
|
)
|
|
|
|
def _forward_prefill(
|
|
self,
|
|
q: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
compressed_k_cache: torch.Tensor | None,
|
|
swa_k_cache: torch.Tensor,
|
|
output: torch.Tensor | None,
|
|
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
|
swa_metadata: DeepseekV41ROCMAiterSparseSWAMetadata,
|
|
mxfp8_out: tuple[torch.Tensor, torch.Tensor] | None = None,
|
|
) -> None:
|
|
"""Sparse prefill into bf16 ``output``, or into ``mxfp8_out``.
|
|
|
|
With ``mxfp8_out`` the rows go to a bf16 workspace first and are then
|
|
inverse-RoPE'd and quantized into it in one pass.
|
|
"""
|
|
if self._use_aiter_sparse_mla:
|
|
self._forward_prefill_aiter(
|
|
q=q,
|
|
positions=positions,
|
|
compressed_k_cache=compressed_k_cache,
|
|
swa_k_cache=swa_k_cache,
|
|
output=output,
|
|
attn_metadata=attn_metadata,
|
|
swa_metadata=swa_metadata,
|
|
mxfp8_out=mxfp8_out,
|
|
)
|
|
return
|
|
|
|
swa_only = attn_metadata is None
|
|
|
|
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
|
num_decodes = swa_metadata.num_decodes
|
|
num_decode_tokens = swa_metadata.num_decode_tokens
|
|
|
|
seq_lens = swa_metadata.prefill_seq_lens
|
|
gather_lens = swa_metadata.prefill_gather_lens
|
|
assert seq_lens is not None
|
|
assert gather_lens is not None
|
|
# CPU copy, so sizing the gather grid from it does not sync.
|
|
seq_lens_cpu = swa_metadata.prefill_seq_lens_cpu
|
|
assert seq_lens_cpu is not None
|
|
|
|
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
|
query_start_loc = swa_metadata.query_start_loc
|
|
assert query_start_loc_cpu is not None
|
|
assert query_start_loc is not None
|
|
prefill_token_base = query_start_loc_cpu[num_decodes]
|
|
|
|
# Local indices filled by the index source; SWA-only layers pass
|
|
# top_k=0 and never read them.
|
|
assert self.topk_indices_buffer is not None
|
|
topk_indices = self.topk_indices_buffer[num_decode_tokens:]
|
|
topk_indices = topk_indices[:num_prefill_tokens]
|
|
if not swa_only:
|
|
top_k = topk_indices.shape[-1]
|
|
N = (self.max_model_len + self.compress_ratio - 1) // self.compress_ratio
|
|
else:
|
|
top_k = 0
|
|
N = 0
|
|
|
|
M = N + self.window_size + self.max_num_batched_tokens
|
|
chunk_plan = swa_metadata.get_prefill_chunk_plan(
|
|
compress_ratio=self.compress_ratio,
|
|
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
|
|
has_compressed=not swa_only,
|
|
)
|
|
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
|
|
|
workspace = current_workspace_manager().get_simultaneous(
|
|
*self._prefill_workspace_shapes(M, num_prefill_tokens, q)
|
|
)
|
|
kv_rows = workspace[0].view(-1, q.shape[-1])
|
|
if mxfp8_out is not None:
|
|
output = workspace[1]
|
|
assert output is not None
|
|
for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan:
|
|
chunk_size = chunk_end - chunk_start
|
|
kv = kv_rows[: chunk_size * chunk_M].view(chunk_size, chunk_M, -1)
|
|
if not swa_only:
|
|
assert attn_metadata is not None
|
|
assert compressed_k_cache is not None
|
|
block_table = attn_metadata.block_table[num_decodes:]
|
|
# compressed_k_cache is OCP on every platform (Triton encoder).
|
|
dequantize_and_gather_k_cache(
|
|
kv[:chunk_size],
|
|
compressed_k_cache,
|
|
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
|
|
gather_lens=None,
|
|
block_table=block_table[chunk_start:chunk_end],
|
|
block_size=attn_metadata.block_size // self.compress_ratio,
|
|
offset=0,
|
|
use_fnuz=False,
|
|
max_gather_len=int(seq_lens_cpu[chunk_start:chunk_end].max())
|
|
// self.compress_ratio,
|
|
)
|
|
|
|
swa_block_table = swa_metadata.block_table[num_decodes:]
|
|
dequantize_and_gather_k_cache(
|
|
kv[:chunk_size],
|
|
swa_k_cache,
|
|
seq_lens=seq_lens[chunk_start:chunk_end],
|
|
gather_lens=gather_lens[chunk_start:chunk_end],
|
|
block_table=swa_block_table[chunk_start:chunk_end],
|
|
block_size=swa_metadata.block_size,
|
|
offset=chunk_N,
|
|
use_fnuz=current_platform.is_fp8_fnuz(),
|
|
)
|
|
|
|
query_start = (
|
|
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
|
|
)
|
|
query_end = (
|
|
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
|
)
|
|
|
|
combined_indices, combined_lens = combine_topk_swa_indices(
|
|
topk_indices[query_start:query_end],
|
|
query_start_loc[
|
|
num_decodes + chunk_start : num_decodes + chunk_end + 1
|
|
],
|
|
seq_lens[chunk_start:chunk_end],
|
|
gather_lens[chunk_start:chunk_end],
|
|
self.window_size,
|
|
self.compress_ratio,
|
|
top_k,
|
|
chunk_M,
|
|
chunk_N,
|
|
)
|
|
rocm_sparse_attn_prefill(
|
|
q=q[query_start:query_end],
|
|
kv=kv.view(-1, 1, q.shape[-1]),
|
|
indices=combined_indices,
|
|
topk_length=combined_lens,
|
|
scale=self.scale,
|
|
head_dim=self.head_dim,
|
|
nope_head_dim=self.nope_head_dim,
|
|
rope_head_dim=self.rope_head_dim,
|
|
attn_sink=self.attn_sink,
|
|
output=output[query_start:query_end],
|
|
)
|
|
if mxfp8_out is not None:
|
|
rocm_inverse_rope_mxfp8_rows(
|
|
output,
|
|
positions[:num_prefill_tokens],
|
|
self.rotary_emb.cos_sin_cache,
|
|
self.rope_head_dim,
|
|
mxfp8_out[0][:num_prefill_tokens],
|
|
mxfp8_out[1][:num_prefill_tokens],
|
|
)
|
|
|
|
def _prefill_workspace_shapes(
|
|
self, M: int, num_prefill_tokens: int, q: torch.Tensor, gather: bool = True
|
|
) -> list[tuple[tuple[int, ...], torch.dtype]]:
|
|
"""The prefill gather buffer, plus bf16 rows for the MXFP8 output.
|
|
|
|
One ``get_simultaneous`` call: separate calls alias the same memory.
|
|
``gather=False`` leaves out the gather buffer, which the aiter kernel,
|
|
reading the caches in place, never needs.
|
|
"""
|
|
shapes: list[tuple[tuple[int, ...], torch.dtype]] = []
|
|
if gather:
|
|
shapes.append(((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16))
|
|
if _ON_GFX950:
|
|
shapes.append(
|
|
(
|
|
(num_prefill_tokens, self.n_local_heads, self.head_dim),
|
|
torch.bfloat16,
|
|
)
|
|
)
|
|
return shapes
|
|
|
|
def _prefill_topk_ragged(
|
|
self,
|
|
swa_metadata: DeepseekV41ROCMAiterSparseSWAMetadata,
|
|
attn_metadata: DeepseekV4FlashMLAMetadata,
|
|
compressed_k_cache: torch.Tensor,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Prefill counterpart of ``_decode_topk_ragged``: layers that share an
|
|
index source and a compressed cache reuse one build per step."""
|
|
assert swa_metadata.token_to_req_indices is not None
|
|
assert swa_metadata.query_start_loc is not None
|
|
assert swa_metadata.seq_lens is not None
|
|
assert swa_metadata.is_valid_token is not None
|
|
assert self.topk_indices_buffer is not None
|
|
assert self._index_source_prefix is not None
|
|
assert self.compressed_cache_prefix is not None
|
|
|
|
source = self._static_forward_context[self._index_source_prefix]
|
|
if source is self:
|
|
source._prefill_topk_ragged_cache = {}
|
|
cached = source._prefill_topk_ragged_cache.get(self.compressed_cache_prefix)
|
|
if cached is not None:
|
|
return cached
|
|
|
|
num_decode_tokens = swa_metadata.num_decode_tokens
|
|
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
|
built = build_prefill_topk_ragged_indices(
|
|
self.topk_indices_buffer[
|
|
num_decode_tokens : num_decode_tokens + num_prefill_tokens
|
|
],
|
|
swa_metadata.token_to_req_indices,
|
|
swa_metadata.query_start_loc,
|
|
swa_metadata.seq_lens,
|
|
swa_metadata.is_valid_token,
|
|
attn_metadata.block_table,
|
|
block_size=attn_metadata.block_size // self.compress_ratio,
|
|
compress_ratio=self.compress_ratio,
|
|
num_compressed=(self.max_model_len + self.compress_ratio - 1)
|
|
// self.compress_ratio,
|
|
token_offset=num_decode_tokens,
|
|
num_rows=compressed_k_cache.shape[0] * compressed_k_cache.shape[1],
|
|
)
|
|
source._prefill_topk_ragged_cache[self.compressed_cache_prefix] = built
|
|
return built
|
|
|
|
def _forward_prefill_aiter(
|
|
self,
|
|
q: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
compressed_k_cache: torch.Tensor | None,
|
|
swa_k_cache: torch.Tensor,
|
|
output: torch.Tensor | None,
|
|
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
|
swa_metadata: DeepseekV41ROCMAiterSparseSWAMetadata,
|
|
mxfp8_out: tuple[torch.Tensor, torch.Tensor] | None = None,
|
|
) -> None:
|
|
"""Prefill straight off the paged caches, with no bf16 gather.
|
|
|
|
With ``mxfp8_out`` the kernel writes bf16 rows to the workspace, which
|
|
are then inverse-RoPE'd and quantized into it as ``_forward_prefill``
|
|
does.
|
|
"""
|
|
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
|
assert swa_metadata.prefill_swa_ragged_indices is not None
|
|
assert swa_metadata.prefill_swa_ragged_indptr is not None
|
|
|
|
topk_indices = None
|
|
topk_indptr = None
|
|
if attn_metadata is None:
|
|
compressed_k_cache = None
|
|
else:
|
|
assert compressed_k_cache is not None
|
|
topk_indices, topk_indptr = self._prefill_topk_ragged(
|
|
swa_metadata, attn_metadata, compressed_k_cache
|
|
)
|
|
|
|
if mxfp8_out is not None:
|
|
output = self._mxfp8_bf16_rows(num_prefill_tokens, q)
|
|
assert output is not None
|
|
self._aiter_sparse_mla(
|
|
q[:num_prefill_tokens],
|
|
output[:num_prefill_tokens],
|
|
swa_k_cache,
|
|
swa_metadata.prefill_swa_ragged_indices,
|
|
swa_metadata.prefill_swa_ragged_indptr,
|
|
compressed_k_cache,
|
|
topk_indices,
|
|
topk_indptr,
|
|
)
|
|
if mxfp8_out is not None:
|
|
rocm_inverse_rope_mxfp8_rows(
|
|
output[:num_prefill_tokens],
|
|
positions[:num_prefill_tokens],
|
|
self.rotary_emb.cos_sin_cache,
|
|
self.rope_head_dim,
|
|
mxfp8_out[0][:num_prefill_tokens],
|
|
mxfp8_out[1][:num_prefill_tokens],
|
|
)
|
|
|
|
def _mxfp8_bf16_rows(self, num_tokens: int, q: torch.Tensor) -> torch.Tensor:
|
|
"""bf16 attention rows for the aiter kernel to write when the layer's
|
|
output is MXFP8, taken from the workspace the warmup reserved."""
|
|
shapes = self._prefill_workspace_shapes(0, num_tokens, q, gather=False)
|
|
assert shapes, "an MXFP8 attention output is gfx950-only"
|
|
return current_workspace_manager().get_simultaneous(*shapes)[0]
|
|
|
|
def _aiter_sparse_mla(
|
|
self,
|
|
q: torch.Tensor,
|
|
output: torch.Tensor,
|
|
swa_k_cache: torch.Tensor,
|
|
swa_indices: torch.Tensor,
|
|
swa_indptr: torch.Tensor,
|
|
compressed_k_cache: torch.Tensor | None,
|
|
topk_indices: torch.Tensor | None,
|
|
topk_indptr: torch.Tensor | None,
|
|
) -> None:
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
|
|
# Both fp8 caches are read in place: the SWA window as the main segment,
|
|
# the top-k compressed tokens as the extra one.
|
|
rocm_aiter_ops.triton_sparse_mla_fwd(
|
|
q,
|
|
swa_k_cache,
|
|
output,
|
|
self.scale,
|
|
swa_indptr,
|
|
swa_indices,
|
|
attn_sink=self.attn_sink[: q.shape[1]],
|
|
extra_kv_buffer=compressed_k_cache,
|
|
extra_kv_indptr=topk_indptr,
|
|
extra_kv_indices=topk_indices,
|
|
)
|