[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -207,7 +207,7 @@ from collections.abc import Callable
from dataclasses import dataclass, replace
from enum import Enum
from math import lcm
from typing import ClassVar, Generic, TypeVar, cast
from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar, cast
import numpy as np
import torch
@@ -284,6 +284,7 @@ from vllm.v1.attention.backend import (
CommonAttentionMetadata,
MLAAttentionImpl,
)
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
from vllm.v1.attention.backends.mla.prefill import (
MLAPrefillBackend,
get_mla_prefill_backend,
@@ -309,6 +310,11 @@ from vllm.v1.kv_cache_interface import (
get_kv_quant_mode,
)
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
)
logger = init_logger(__name__)
_FP8_DTYPE = current_platform.fp8_dtype()
@@ -433,6 +439,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
use_sparse: bool = False,
indexer: object | None = None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
non_causal_multi_token_decode: bool = False,
sliding_window: int | None = None,
prefill_backend_cls: type[MLAPrefillBackend] | None = None,
@@ -538,9 +545,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# explicitly so backbone "skip" layers (indexer=None) still find it.
if use_sparse:
extra_impl_args["topk_indices_buffer"] = topk_indices_buffer
if index_group_builder is not None:
extra_impl_args["index_group_builder"] = index_group_builder
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
impl = impl_cls(
num_heads=self.num_heads,
head_size=self.head_size,
scale=self.scale,
@@ -562,6 +571,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
indexer=indexer,
**extra_impl_args,
)
self.impl = impl # type: ignore[assignment]
index_group = getattr(impl, "index_group", None)
self.hisparse_cache = (
index_group.cache(cast(Any, impl).index_group_index)
if isinstance(index_group, HiSparseMLAIndexGroup)
else None
)
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
# MLA reads this weight directly to build W_UK/W_UV, so a backend must
@@ -723,6 +739,61 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
return self._chunked_prefill_workspace_size
def update_kv_cache(
self,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor | None,
attn_metadata: "MLACommonMetadata | None",
kv_cache_dtype: str,
k_scale: torch.Tensor,
) -> None:
cache = self.hisparse_cache
if slot_mapping is None or (cache is not None and cache.dummy_batch):
return
kv_c_normed, k_pe, slot_mapping = maybe_gather_mla_latent_cache_inputs(
kv_c_normed,
k_pe,
slot_mapping,
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
self.use_pcp,
)
assert slot_mapping is not None
if cache is not None:
kv_cache, slot_mapping, num_rows = cache.write_target(
kv_c_normed.shape[0], slot_mapping.numel()
)
kv_c_normed = kv_c_normed[:num_rows]
k_pe = k_pe[:num_rows]
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_c_normed,
k_pe,
kv_cache,
slot_mapping,
kv_cache_dtype,
k_scale,
)
if cache is not None:
mirror_target = cache.mirror_write_target(kv_c_normed.shape[0])
if mirror_target is not None:
mirror_cache, mirror_slots = mirror_target
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_c_normed,
k_pe,
mirror_cache,
mirror_slots,
kv_cache_dtype,
k_scale,
)
def prepare_kv_cache_update(
self, attn_metadata: "MLACommonMetadata | None"
) -> None:
cache = self.hisparse_cache
if cache is not None and cache.runtime.is_group_leader:
cache.prepare_group_for_batch(attn_metadata)
def forward(
self,
q: torch.Tensor,
@@ -750,25 +821,18 @@ class MLAAttention(nn.Module, AttentionLayerBase):
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
)
layer_slot_mapping = slot_mapping.get(self.layer_name)
kv_for_cache, kpe_for_cache, layer_slot_mapping = (
maybe_gather_mla_latent_cache_inputs(
kv_c_normed,
k_pe,
layer_slot_mapping,
attn_metadata.num_decode_tokens
if attn_metadata is not None
else None,
self.use_pcp,
)
)
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_for_cache,
kpe_for_cache,
self.prepare_kv_cache_update(attn_metadata)
self.update_kv_cache(
kv_c_normed,
k_pe,
self_kv_cache,
layer_slot_mapping,
attn_metadata,
self.kv_cache_dtype,
self._k_scale,
)
if self.hisparse_cache is not None:
self.hisparse_cache.finish_kv_update()
output = torch.empty(output_shape, dtype=q.dtype, device=q.device)
self.forward_impl(
q,
@@ -1113,6 +1177,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
return output_padded
def _use_sparse_mha(self, attn_metadata: "MLACommonMetadata") -> bool:
if self.hisparse_cache is not None:
return False
prefill = attn_metadata.prefill
if prefill is None:
return False
@@ -1296,6 +1362,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
return MLAAttentionSpec(
**common_kwargs,
is_index_group_leader=self.indexer is not None,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
@@ -1349,19 +1416,14 @@ def unified_mla_kv_cache_update(
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
layer_name
)
attn_layer.prepare_kv_cache_update(attn_metadata)
if layer_slot_mapping is not None:
kv_c_normed, k_pe, layer_slot_mapping = maybe_gather_mla_latent_cache_inputs(
kv_c_normed,
k_pe,
layer_slot_mapping,
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
attn_layer.use_pcp,
)
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
attn_layer.update_kv_cache(
kv_c_normed,
k_pe,
kv_cache,
layer_slot_mapping,
attn_metadata,
kv_cache_dtype,
k_scale,
)
@@ -1409,6 +1471,8 @@ def unified_mla_attention_with_output(
del kv_cache_dummy_dep
layer_name = _resolve_layer_name(layer_name)
attn_metadata, layer, kv_cache, _ = get_attention_context(layer_name)
if layer.hisparse_cache is not None:
layer.hisparse_cache.finish_kv_update()
layer.forward_impl(
q,
kv_c_normed,
@@ -1501,6 +1565,7 @@ class MLACommonBackend(AttentionBackend):
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack an inline fp32 scale pair after the
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
spec = super().customize_spec(spec)
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
return spec
@@ -2090,7 +2155,11 @@ def build_mla_chunked_context_metadata(
local_token_offset += num_local_tokens
seq_lens_cpu = _flat_int32(seq_lens_flat)
starts = _flat_int32(starts_flat).to(device, non_blocking=True)
starts_and_context_lens = _flat_int32(starts_flat + context_lens).to(
device, non_blocking=True
)
starts = starts_and_context_lens[: len(starts_flat)]
context_lens_gpu = starts_and_context_lens[len(starts_flat) :]
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
@@ -2145,7 +2214,7 @@ def build_mla_chunked_context_metadata(
chunks.append(chunk)
return MLACommonPrefillMetadata.ChunkedContextMetadata(
context_lens=context_lens_cpu.to(device, non_blocking=True),
context_lens=context_lens_gpu,
workspace=chunked_prefill_workspace,
chunks=chunks,
context_lens_list=context_lens,
@@ -2835,12 +2904,12 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
output_lse = None
for chunk in chunked_context.chunks:
toks = chunk.num_context_tokens
block_table = prefill_metadata.block_table[chunk.request_slice]
chunk_block_table = prefill_metadata.block_table[chunk.request_slice]
if self.kv_cache_dtype == "fp8_ds_mla":
ops.cp_gather_and_upconvert_fp8_kv_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace[:toks],
block_table=block_table,
block_table=chunk_block_table,
workspace_starts=chunk.cu_seq_lens,
batch_size=chunk.num_requests,
seq_starts=chunk.starts,
@@ -2853,7 +2922,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
ops.gather_mla_context_cache_cpu(
src_cache=kv_c_and_k_pe_cache,
dst=workspace[:toks],
block_table=block_table,
block_table=chunk_block_table,
starts=chunk.starts,
cu_seq_lens=chunk.cu_seq_lens,
)
@@ -2861,7 +2930,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=block_table,
block_table=chunk_block_table,
cu_seq_lens=chunk.cu_seq_lens,
token_to_seq=chunk.token_to_seq,
num_tokens=toks,
@@ -2874,7 +2943,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace[:toks],
block_table=block_table,
block_table=chunk_block_table,
cu_seq_lens=chunk.cu_seq_lens,
batch_size=chunk.num_requests,
seq_starts=chunk.starts,
@@ -3,13 +3,15 @@
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
import math
from dataclasses import dataclass
from shutil import which
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
import numpy as np
import torch
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_dcp_group,
get_tensor_model_parallel_world_size,
@@ -17,6 +19,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBaseImpl,
MLACommonDecodeMetadata,
MLACommonMetadata,
MLACommonPrefillMetadata,
accumulate_mla_context_chunk,
@@ -33,11 +36,19 @@ from vllm.utils.torch_utils import (
is_quantized_kv_cache,
np_to_pinned_tensor,
)
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
from vllm.v1.attention.backend import AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroup,
SparseMLAIndexGroupBuilder,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.hisparse.runtime import (
HiSparsePrefillStagingPlan,
build_hisparse_prefill_staging_plan,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -47,8 +58,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
T = TypeVar("T", bound=AttentionMetadata)
GLOBAL_TOPK_MASK_MAX_BYTES = 128 * 1024 * 1024 # 128 MiB
@@ -116,9 +125,45 @@ def _is_masked_mha_available(
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)
def _use_dense_mha_prefill(
vllm_config: "VllmConfig", prefill_max_seq_len: int, topk_tokens: int
) -> bool:
attention_config = vllm_config.attention_config
return (
attention_config.hisparse_config is None
and not attention_config.sparse_mla_force_mqa
and prefill_max_seq_len <= topk_tokens
)
@dataclass
class SparseMLAPrefillMetadata(MLACommonPrefillMetadata):
host_staging_plan: HiSparsePrefillStagingPlan | None = None
@dataclass(kw_only=True)
class SparseMLACommonMetadata(MLACommonMetadata[MLACommonDecodeMetadata]):
block_table: torch.Tensor
req_id_per_token: torch.Tensor
seq_lens: torch.Tensor | None = None
block_size: int = 64
topk_tokens: int = 2048
num_decodes: int = 0
num_prefills: int = 0
num_decode_tokens: int = 0
decode_max_query_len: int = 0
prefill_max_seq_len: int = 0
prefill: SparseMLAPrefillMetadata | None = None
cp_kv_cache_interleave_size: int = 1
T = TypeVar("T", bound=SparseMLACommonMetadata)
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
metadata_cls: type[T]
require_uniform_decodes: ClassVar[bool] = False
hisparse_supports_multi_token_decode: ClassVar[bool] = False
def __init__(
self,
@@ -138,6 +183,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
dtype=torch.int32,
device=device,
)
self._context_lens_cpu = torch.empty(
vllm_config.scheduler_config.max_num_seqs,
dtype=torch.int32,
device="cpu",
pin_memory=PIN_MEMORY,
)
parallel_config = vllm_config.parallel_config
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
try:
@@ -197,6 +248,22 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
)
def _init_reorder_batch_threshold(
self,
reorder_batch_threshold: int | None = 1,
supports_spec_as_decode: bool = False,
supports_dcp_with_varlen: bool = False,
) -> None:
if self.vllm_config.attention_config.hisparse_config is not None:
reorder_batch_threshold = 1
if not self.hisparse_supports_multi_token_decode:
supports_spec_as_decode = False
super()._init_reorder_batch_threshold(
reorder_batch_threshold,
supports_spec_as_decode,
supports_dcp_with_varlen,
)
@staticmethod
def determine_chunked_prefill_workspace_size(vllm_config: "VllmConfig") -> int:
scheduler_config = vllm_config.scheduler_config
@@ -245,9 +312,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
context_lens_cpu = torch.empty(
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
)
context_lens_cpu = self._context_lens_cpu[:num_prefills]
torch.subtract(
seq_lens_cpu[num_decodes : num_decodes + num_prefills],
prefill_query_lens_cpu,
@@ -289,34 +354,66 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
prefill_max_query_len,
prefill_query_lens_cpu,
) = self._build_prefill_fields(common_attn_metadata, num_decodes, num_prefills)
decode_max_query_len = 0
if num_decodes:
if common_attn_metadata.max_logits_per_req is not None:
decode_max_query_len = common_attn_metadata.max_logits_per_req
elif num_prefills == 0:
decode_max_query_len = common_attn_metadata.max_query_len
else:
decode_query_lens_cpu = torch.diff(
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
)
decode_max_query_len = int(decode_query_lens_cpu.max().item())
prefill_max_seq_len = 0
prefill: MLACommonPrefillMetadata | None = None
prefill: SparseMLAPrefillMetadata | None = None
if num_prefills > 0 and self._prefill_backend is not None:
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
prefill_max_seq_len = int(
seq_lens_cpu[num_decodes : num_decodes + num_prefills].max().item()
)
prefill = MLACommonPrefillMetadata(
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
block_table = common_attn_metadata.block_table_tensor[num_decodes:, ...]
chunked_context = self._build_chunked_context_fields(
common_attn_metadata,
num_decodes,
num_prefills,
prefill_query_lens_cpu,
)
staging_plan = None
if self.vllm_config.attention_config.hisparse_config is not None:
prefill_seq_lens_cpu = seq_lens_cpu[
num_decodes : num_decodes + num_prefills
]
staging_block_capacity = int(
(
(prefill_seq_lens_cpu + self.kv_cache_spec.block_size - 1)
// self.kv_cache_spec.block_size
).sum()
)
staging_plan = build_hisparse_prefill_staging_plan(
block_table,
common_attn_metadata.seq_lens[num_decodes:],
self.kv_cache_spec.block_size,
staging_block_capacity,
)
if chunked_context is not None:
block_table = staging_plan.block_table
prefill = SparseMLAPrefillMetadata(
block_table=block_table,
query_start_loc=prefill_query_start_loc,
max_query_len=prefill_max_query_len,
query_lens_cpu=prefill_query_lens_cpu,
chunked_context=self._build_chunked_context_fields(
common_attn_metadata,
num_decodes,
num_prefills,
prefill_query_lens_cpu,
),
chunked_context=chunked_context,
q_data_type=self.model_config.dtype,
output_dtype=self.model_config.dtype,
prefill_backend=self._prefill_backend,
use_dense_mha=(
prefill_max_seq_len <= self.topk_tokens
and not self.vllm_config.attention_config.sparse_mla_force_mqa
use_dense_mha=_use_dense_mha_prefill(
self.vllm_config, prefill_max_seq_len, self.topk_tokens
),
topk_mask_workspace=self.topk_mask_workspace,
host_staging_plan=staging_plan,
)
self._prefill_backend.prepare_metadata(prefill)
@@ -335,6 +432,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
num_decodes=num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decode_tokens,
decode_max_query_len=decode_max_query_len,
prefill_max_seq_len=prefill_max_seq_len,
prefill=prefill,
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
@@ -560,6 +658,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
kv_b_proj: "ColumnParallelLinear",
indexer: object | None = None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
q_pad_num_heads: int | None = None,
) -> None:
super().__init__(
@@ -577,6 +676,21 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
)
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
self.index_group: SparseMLAIndexGroup | None = None
self.index_group_index = 0
if index_group_builder is None and self.topk_indices_buffer is not None:
index_group_builder = SparseMLAIndexGroupBuilder(self.topk_indices_buffer)
if index_group_builder is not None:
vllm_config = get_current_vllm_config()
self.index_group, self.index_group_index = (
index_group_builder.register_layer(
indexer is not None,
vllm_config,
head_size=head_size,
kv_cache_dtype=kv_cache_dtype,
)
)
self._use_flashinfer_concat_mla_k = (
has_flashinfer()
and which("ninja") is not None
@@ -593,6 +707,33 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
kv_cache_dtype=kv_cache_dtype,
)
def record_logical_topk_ready(self) -> None:
if self.index_group is not None:
self.index_group.set_logical_topk_ready(self.index_group_index)
def prepare_for_batch(self, attn_metadata: T | None) -> None:
if self.index_group is not None:
self.index_group.prepare_for_batch(self.index_group_index, attn_metadata)
def _convert_logical_to_physical_topk(
self,
logical_topk_indices: torch.Tensor,
attn_metadata: Any,
*,
block_stride_rows: int | None,
return_valid_counts: bool,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
group = self.index_group
assert group is not None
assert self.dcp_world_size == 1
return group.convert_logical_to_physical_topk(
self.index_group_index,
logical_topk_indices,
attn_metadata,
block_stride_rows=block_stride_rows,
return_valid_counts=return_valid_counts,
)
@staticmethod
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
"""Whether this prefill batch's top-k masks fit the workspace."""
@@ -853,8 +994,10 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
) -> None:
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
topk_tokens = attn_metadata.topk_tokens # type: ignore[attr-defined]
prefill_metadata = attn_metadata.prefill
assert prefill_metadata is not None
prefill_max_seq_len = attn_metadata.prefill_max_seq_len
topk_tokens = attn_metadata.topk_tokens
force_dense = getattr(self, "_sparse_mla_force_dense_mha", False)
force_masked = getattr(self, "_sparse_mla_force_masked_mha", False)
if force_dense or (prefill_max_seq_len <= topk_tokens and not force_masked):
@@ -863,7 +1006,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
kv_c_normed,
k_pe,
kv_c_and_k_pe_cache,
cast(MLACommonMetadata, attn_metadata),
attn_metadata,
k_scale,
output,
output_scale,
@@ -871,13 +1014,11 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
assert output_scale is None
assert self.masked_mha_available
prefill_metadata = attn_metadata.prefill # type: ignore[attr-defined]
assert prefill_metadata is not None
assert prefill_metadata.query_lens_cpu is not None
assert self.topk_indices_buffer is not None
q_lens = prefill_metadata.query_lens_cpu.tolist()
num_decode_tokens = attn_metadata.num_decode_tokens # type: ignore[attr-defined]
num_decode_tokens = attn_metadata.num_decode_tokens
topk_all = self.topk_indices_buffer[
num_decode_tokens : num_decode_tokens + q.shape[0]
]
+10
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
@@ -11,6 +12,11 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
)
@dataclass
class MLAModules:
@@ -30,6 +36,7 @@ class MLAModules:
topk_indices_buffer: torch.Tensor | None
indexer_rotary_emb: torch.nn.Module | None = None
g_proj: torch.nn.Module | None = None
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None
# --8<-- [start:multi_head_latent_attention]
@@ -128,6 +135,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
use_sparse=self.is_sparse,
indexer=self.indexer,
topk_indices_buffer=mla_modules.topk_indices_buffer,
index_group_builder=mla_modules.index_group_builder,
non_causal_multi_token_decode=non_causal_multi_token_decode,
)
indexer_op = getattr(self.indexer, "indexer_op", None)
@@ -223,6 +231,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
if self.indexer and self.is_sparse and not self.skip_topk:
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
if self.is_sparse:
self.mla_attn.impl.record_logical_topk_ready() # type: ignore[attr-defined]
if llama_4_scaling is not None:
q *= llama_4_scaling
@@ -425,7 +425,6 @@ def sparse_attn_indexer(
quant_block_size,
scale_fmt,
)
# The indexer and main MLA may classify the same short extend differently
# because they use independent decode thresholds. Only the main MLA route
# can determine whether the top-k indices will be consumed.
@@ -955,6 +954,7 @@ class SparseAttnIndexer(CustomOp):
self.dcp_rank,
self.dcp_world_size,
self.cp_kv_cache_interleave_size,
False,
candidate_blocks=self.candidate_blocks,
candidate_block_size=self.candidate_block_size,
candidate_write=self.candidate_write,
+24 -1
View File
@@ -103,10 +103,14 @@ from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionBackend, AttentionType
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerBackend,
)
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, SparseCacheRole
from .interfaces import (
MixtureOfExperts,
@@ -656,6 +660,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
num_kv_heads=1,
head_size=self.head_dim,
dtype=self.dtype,
cache_role=SparseCacheRole.INDEXER,
) # Only has one vector instead of K + V
def forward(self): ...
@@ -1012,6 +1017,7 @@ class DeepseekV2MLAAttention(nn.Module):
quant_config: QuantizationConfig | None = None,
prefix: str = "",
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
input_size: int | None = None,
reduce_results: bool = True,
non_causal_multi_token_decode: bool = False,
@@ -1199,6 +1205,7 @@ class DeepseekV2MLAAttention(nn.Module):
indexer_rotary_emb=self.indexer_rope_emb,
is_sparse=self.is_v32,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
)
self.mla_attn = MultiHeadLatentAttentionWrapper(
@@ -1242,6 +1249,7 @@ class DeepseekV2DecoderLayer(nn.Module):
prefix: str,
config: DeepseekV2Config | None = None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
) -> None:
super().__init__()
@@ -1290,6 +1298,11 @@ class DeepseekV2DecoderLayer(nn.Module):
and parallel_config.pipeline_parallel_size == 1
and is_moe_layer
)
attn_kwargs = (
{"index_group_builder": index_group_builder}
if attn_cls is DeepseekV2MLAAttention
else {}
)
self.self_attn = attn_cls(
vllm_config=vllm_config,
config=config,
@@ -1306,6 +1319,7 @@ class DeepseekV2DecoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
reduce_results=not self.use_sequence_parallel_moe,
**attn_kwargs,
)
if is_moe_layer:
@@ -1429,6 +1443,14 @@ class DeepseekV2Model(nn.Module):
)
else:
topk_indices_buffer = None
index_group_builder = (
SparseMLAIndexGroupBuilder(
topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
if topk_indices_buffer is not None
else None
)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
@@ -1445,6 +1467,7 @@ class DeepseekV2Model(nn.Module):
vllm_config=vllm_config,
prefix=prefix,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
),
prefix=f"{prefix}.layers",
)
@@ -67,6 +67,10 @@ from vllm.model_executor.models.glm4_moe import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
@@ -108,6 +112,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
prefix: str,
config: "Glm4MoeLiteConfig | None" = None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
) -> None:
super().__init__()
@@ -136,6 +141,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
attn_cls = Glm4MoeLiteMLAAttention
else:
attn_cls = Glm4MoeLiteAttention
attn_kwargs: dict[str, typing.Any] = (
{"index_group_builder": index_group_builder}
if attn_cls is Glm4MoeLiteMLAAttention
else {}
)
self.self_attn = attn_cls(
vllm_config=vllm_config,
@@ -152,6 +162,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
**attn_kwargs,
)
if (
@@ -234,6 +245,14 @@ class Glm4MoeLiteModel(nn.Module):
)
else:
topk_indices_buffer = None
index_group_builder = (
SparseMLAIndexGroupBuilder(
topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
if topk_indices_buffer is not None
else None
)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
@@ -252,6 +271,7 @@ class Glm4MoeLiteModel(nn.Module):
config=config,
prefix=prefix,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
),
prefix=f"{prefix}.layers",
)
@@ -51,6 +51,10 @@ from vllm.model_executor.model_loader.weight_utils import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .glm4_moe_lite import (
Glm4MixtureOfExperts,
@@ -108,6 +112,14 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
)
else:
topk_indices_buffer = None
index_group_builder = (
SparseMLAIndexGroupBuilder(
topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
if topk_indices_buffer is not None
else None
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
@@ -117,6 +129,7 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
prefix=prefix,
config=self.config,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
)
def forward(
@@ -47,6 +47,35 @@ _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
def autotune_hisparse_flashinfer_attention(runner: "GPUModelRunner") -> None:
"""Autotune each HiSparse FlashInfer sparse-MLA configuration."""
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
)
tuned: set[tuple[object, ...]] = set()
for layer in runner.vllm_config.compilation_config.static_forward_context.values():
impl = getattr(layer, "impl", None)
if not isinstance(impl, FlashInferMLASparseImpl):
continue
if getattr(layer, "hisparse_cache", None) is None:
continue
if impl.topk_indices_buffer is None:
continue
key = (
impl.kv_cache_dtype,
impl.num_heads,
impl.qk_nope_head_dim,
impl.qk_rope_head_dim,
impl.kv_lora_rank,
impl.topk_indices_buffer.shape[1],
)
if key in tuned:
continue
impl.autotune_hisparse_decode(layer)
tuned.add(key)
def _attention_backend_name(backend: object) -> str | None:
get_name = getattr(backend, "get_name", None)
if get_name is None:
+14 -2
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
write_flashinfer_autotune_cache,
)
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
deepseek_v4_sparse_mla_attention_warmup,
flashinfer_sparse_mla_decode_autotune_warmup,
)
@@ -353,9 +354,12 @@ def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ..
return tuple(dict.fromkeys(token_counts))
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
def _run_flashinfer_autotune_dummy_runs(
runner: "GPUModelRunner", *, skip_attn: bool = False
) -> None:
import vllm.utils.flashinfer as fi_utils
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
for num_tokens in _flashinfer_autotune_token_counts(runner):
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
logger.info(
@@ -369,6 +373,7 @@ def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**dummy_run_kwargs,
)
@@ -431,7 +436,14 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
_run_flashinfer_autotune_dummy_runs(runner)
hisparse_enabled = (
runner.vllm_config.attention_config.hisparse_config is not None
)
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
autotune_hisparse_flashinfer_attention(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
finally: