[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -207,7 +207,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import Enum
|
||||
from math import lcm
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -284,6 +284,7 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
MLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.prefill import (
|
||||
MLAPrefillBackend,
|
||||
get_mla_prefill_backend,
|
||||
@@ -309,6 +310,11 @@ from vllm.v1.kv_cache_interface import (
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_FP8_DTYPE = current_platform.fp8_dtype()
|
||||
@@ -433,6 +439,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
sliding_window: int | None = None,
|
||||
prefill_backend_cls: type[MLAPrefillBackend] | None = None,
|
||||
@@ -538,9 +545,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# explicitly so backbone "skip" layers (indexer=None) still find it.
|
||||
if use_sparse:
|
||||
extra_impl_args["topk_indices_buffer"] = topk_indices_buffer
|
||||
if index_group_builder is not None:
|
||||
extra_impl_args["index_group_builder"] = index_group_builder
|
||||
|
||||
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
|
||||
impl = impl_cls(
|
||||
num_heads=self.num_heads,
|
||||
head_size=self.head_size,
|
||||
scale=self.scale,
|
||||
@@ -562,6 +571,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
indexer=indexer,
|
||||
**extra_impl_args,
|
||||
)
|
||||
self.impl = impl # type: ignore[assignment]
|
||||
index_group = getattr(impl, "index_group", None)
|
||||
self.hisparse_cache = (
|
||||
index_group.cache(cast(Any, impl).index_group_index)
|
||||
if isinstance(index_group, HiSparseMLAIndexGroup)
|
||||
else None
|
||||
)
|
||||
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
|
||||
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
|
||||
# MLA reads this weight directly to build W_UK/W_UV, so a backend must
|
||||
@@ -723,6 +739,61 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return self._chunked_prefill_workspace_size
|
||||
|
||||
def update_kv_cache(
|
||||
self,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
attn_metadata: "MLACommonMetadata | None",
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
) -> None:
|
||||
cache = self.hisparse_cache
|
||||
if slot_mapping is None or (cache is not None and cache.dummy_batch):
|
||||
return
|
||||
kv_c_normed, k_pe, slot_mapping = maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
self.use_pcp,
|
||||
)
|
||||
assert slot_mapping is not None
|
||||
if cache is not None:
|
||||
kv_cache, slot_mapping, num_rows = cache.write_target(
|
||||
kv_c_normed.shape[0], slot_mapping.numel()
|
||||
)
|
||||
kv_c_normed = kv_c_normed[:num_rows]
|
||||
k_pe = k_pe[:num_rows]
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
if cache is not None:
|
||||
mirror_target = cache.mirror_write_target(kv_c_normed.shape[0])
|
||||
if mirror_target is not None:
|
||||
mirror_cache, mirror_slots = mirror_target
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
mirror_cache,
|
||||
mirror_slots,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
|
||||
def prepare_kv_cache_update(
|
||||
self, attn_metadata: "MLACommonMetadata | None"
|
||||
) -> None:
|
||||
cache = self.hisparse_cache
|
||||
if cache is not None and cache.runtime.is_group_leader:
|
||||
cache.prepare_group_for_batch(attn_metadata)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -750,25 +821,18 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
|
||||
)
|
||||
layer_slot_mapping = slot_mapping.get(self.layer_name)
|
||||
kv_for_cache, kpe_for_cache, layer_slot_mapping = (
|
||||
maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens
|
||||
if attn_metadata is not None
|
||||
else None,
|
||||
self.use_pcp,
|
||||
)
|
||||
)
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_for_cache,
|
||||
kpe_for_cache,
|
||||
self.prepare_kv_cache_update(attn_metadata)
|
||||
self.update_kv_cache(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
self_kv_cache,
|
||||
layer_slot_mapping,
|
||||
attn_metadata,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
if self.hisparse_cache is not None:
|
||||
self.hisparse_cache.finish_kv_update()
|
||||
output = torch.empty(output_shape, dtype=q.dtype, device=q.device)
|
||||
self.forward_impl(
|
||||
q,
|
||||
@@ -1113,6 +1177,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
return output_padded
|
||||
|
||||
def _use_sparse_mha(self, attn_metadata: "MLACommonMetadata") -> bool:
|
||||
if self.hisparse_cache is not None:
|
||||
return False
|
||||
prefill = attn_metadata.prefill
|
||||
if prefill is None:
|
||||
return False
|
||||
@@ -1296,6 +1362,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
**common_kwargs,
|
||||
is_index_group_leader=self.indexer is not None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
@@ -1349,19 +1416,14 @@ def unified_mla_kv_cache_update(
|
||||
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
layer_name
|
||||
)
|
||||
attn_layer.prepare_kv_cache_update(attn_metadata)
|
||||
if layer_slot_mapping is not None:
|
||||
kv_c_normed, k_pe, layer_slot_mapping = maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
attn_layer.use_pcp,
|
||||
)
|
||||
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
attn_layer.update_kv_cache(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_cache,
|
||||
layer_slot_mapping,
|
||||
attn_metadata,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
@@ -1409,6 +1471,8 @@ def unified_mla_attention_with_output(
|
||||
del kv_cache_dummy_dep
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, layer, kv_cache, _ = get_attention_context(layer_name)
|
||||
if layer.hisparse_cache is not None:
|
||||
layer.hisparse_cache.finish_kv_update()
|
||||
layer.forward_impl(
|
||||
q,
|
||||
kv_c_normed,
|
||||
@@ -1501,6 +1565,7 @@ class MLACommonBackend(AttentionBackend):
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
spec = super().customize_spec(spec)
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
return spec
|
||||
@@ -2090,7 +2155,11 @@ def build_mla_chunked_context_metadata(
|
||||
local_token_offset += num_local_tokens
|
||||
|
||||
seq_lens_cpu = _flat_int32(seq_lens_flat)
|
||||
starts = _flat_int32(starts_flat).to(device, non_blocking=True)
|
||||
starts_and_context_lens = _flat_int32(starts_flat + context_lens).to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
starts = starts_and_context_lens[: len(starts_flat)]
|
||||
context_lens_gpu = starts_and_context_lens[len(starts_flat) :]
|
||||
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
|
||||
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
|
||||
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
|
||||
@@ -2145,7 +2214,7 @@ def build_mla_chunked_context_metadata(
|
||||
chunks.append(chunk)
|
||||
|
||||
return MLACommonPrefillMetadata.ChunkedContextMetadata(
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
context_lens=context_lens_gpu,
|
||||
workspace=chunked_prefill_workspace,
|
||||
chunks=chunks,
|
||||
context_lens_list=context_lens,
|
||||
@@ -2835,12 +2904,12 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
output_lse = None
|
||||
for chunk in chunked_context.chunks:
|
||||
toks = chunk.num_context_tokens
|
||||
block_table = prefill_metadata.block_table[chunk.request_slice]
|
||||
chunk_block_table = prefill_metadata.block_table[chunk.request_slice]
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
workspace_starts=chunk.cu_seq_lens,
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
@@ -2853,7 +2922,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
ops.gather_mla_context_cache_cpu(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
starts=chunk.starts,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
)
|
||||
@@ -2861,7 +2930,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
num_tokens=toks,
|
||||
@@ -2874,7 +2943,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
|
||||
@@ -3,13 +3,15 @@
|
||||
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from shutil import which
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -17,6 +19,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBaseImpl,
|
||||
MLACommonDecodeMetadata,
|
||||
MLACommonMetadata,
|
||||
MLACommonPrefillMetadata,
|
||||
accumulate_mla_context_chunk,
|
||||
@@ -33,11 +36,19 @@ from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroup,
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparsePrefillStagingPlan,
|
||||
build_hisparse_prefill_staging_plan,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -47,8 +58,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES = 128 * 1024 * 1024 # 128 MiB
|
||||
|
||||
|
||||
@@ -116,9 +125,45 @@ def _is_masked_mha_available(
|
||||
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)
|
||||
|
||||
|
||||
def _use_dense_mha_prefill(
|
||||
vllm_config: "VllmConfig", prefill_max_seq_len: int, topk_tokens: int
|
||||
) -> bool:
|
||||
attention_config = vllm_config.attention_config
|
||||
return (
|
||||
attention_config.hisparse_config is None
|
||||
and not attention_config.sparse_mla_force_mqa
|
||||
and prefill_max_seq_len <= topk_tokens
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseMLAPrefillMetadata(MLACommonPrefillMetadata):
|
||||
host_staging_plan: HiSparsePrefillStagingPlan | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class SparseMLACommonMetadata(MLACommonMetadata[MLACommonDecodeMetadata]):
|
||||
block_table: torch.Tensor
|
||||
req_id_per_token: torch.Tensor
|
||||
seq_lens: torch.Tensor | None = None
|
||||
block_size: int = 64
|
||||
topk_tokens: int = 2048
|
||||
num_decodes: int = 0
|
||||
num_prefills: int = 0
|
||||
num_decode_tokens: int = 0
|
||||
decode_max_query_len: int = 0
|
||||
prefill_max_seq_len: int = 0
|
||||
prefill: SparseMLAPrefillMetadata | None = None
|
||||
cp_kv_cache_interleave_size: int = 1
|
||||
|
||||
|
||||
T = TypeVar("T", bound=SparseMLACommonMetadata)
|
||||
|
||||
|
||||
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
metadata_cls: type[T]
|
||||
require_uniform_decodes: ClassVar[bool] = False
|
||||
hisparse_supports_multi_token_decode: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -138,6 +183,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self._context_lens_cpu = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
try:
|
||||
@@ -197,6 +248,22 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
|
||||
)
|
||||
|
||||
def _init_reorder_batch_threshold(
|
||||
self,
|
||||
reorder_batch_threshold: int | None = 1,
|
||||
supports_spec_as_decode: bool = False,
|
||||
supports_dcp_with_varlen: bool = False,
|
||||
) -> None:
|
||||
if self.vllm_config.attention_config.hisparse_config is not None:
|
||||
reorder_batch_threshold = 1
|
||||
if not self.hisparse_supports_multi_token_decode:
|
||||
supports_spec_as_decode = False
|
||||
super()._init_reorder_batch_threshold(
|
||||
reorder_batch_threshold,
|
||||
supports_spec_as_decode,
|
||||
supports_dcp_with_varlen,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def determine_chunked_prefill_workspace_size(vllm_config: "VllmConfig") -> int:
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
@@ -245,9 +312,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
context_lens_cpu = torch.empty(
|
||||
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
|
||||
)
|
||||
context_lens_cpu = self._context_lens_cpu[:num_prefills]
|
||||
torch.subtract(
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills],
|
||||
prefill_query_lens_cpu,
|
||||
@@ -289,34 +354,66 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
prefill_max_query_len,
|
||||
prefill_query_lens_cpu,
|
||||
) = self._build_prefill_fields(common_attn_metadata, num_decodes, num_prefills)
|
||||
decode_max_query_len = 0
|
||||
if num_decodes:
|
||||
if common_attn_metadata.max_logits_per_req is not None:
|
||||
decode_max_query_len = common_attn_metadata.max_logits_per_req
|
||||
elif num_prefills == 0:
|
||||
decode_max_query_len = common_attn_metadata.max_query_len
|
||||
else:
|
||||
decode_query_lens_cpu = torch.diff(
|
||||
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
|
||||
)
|
||||
decode_max_query_len = int(decode_query_lens_cpu.max().item())
|
||||
|
||||
prefill_max_seq_len = 0
|
||||
prefill: MLACommonPrefillMetadata | None = None
|
||||
prefill: SparseMLAPrefillMetadata | None = None
|
||||
if num_prefills > 0 and self._prefill_backend is not None:
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
prefill_max_seq_len = int(
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills].max().item()
|
||||
)
|
||||
prefill = MLACommonPrefillMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
|
||||
block_table = common_attn_metadata.block_table_tensor[num_decodes:, ...]
|
||||
chunked_context = self._build_chunked_context_fields(
|
||||
common_attn_metadata,
|
||||
num_decodes,
|
||||
num_prefills,
|
||||
prefill_query_lens_cpu,
|
||||
)
|
||||
staging_plan = None
|
||||
if self.vllm_config.attention_config.hisparse_config is not None:
|
||||
prefill_seq_lens_cpu = seq_lens_cpu[
|
||||
num_decodes : num_decodes + num_prefills
|
||||
]
|
||||
staging_block_capacity = int(
|
||||
(
|
||||
(prefill_seq_lens_cpu + self.kv_cache_spec.block_size - 1)
|
||||
// self.kv_cache_spec.block_size
|
||||
).sum()
|
||||
)
|
||||
staging_plan = build_hisparse_prefill_staging_plan(
|
||||
block_table,
|
||||
common_attn_metadata.seq_lens[num_decodes:],
|
||||
self.kv_cache_spec.block_size,
|
||||
staging_block_capacity,
|
||||
)
|
||||
if chunked_context is not None:
|
||||
block_table = staging_plan.block_table
|
||||
prefill = SparseMLAPrefillMetadata(
|
||||
block_table=block_table,
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
max_query_len=prefill_max_query_len,
|
||||
query_lens_cpu=prefill_query_lens_cpu,
|
||||
chunked_context=self._build_chunked_context_fields(
|
||||
common_attn_metadata,
|
||||
num_decodes,
|
||||
num_prefills,
|
||||
prefill_query_lens_cpu,
|
||||
),
|
||||
chunked_context=chunked_context,
|
||||
q_data_type=self.model_config.dtype,
|
||||
output_dtype=self.model_config.dtype,
|
||||
prefill_backend=self._prefill_backend,
|
||||
use_dense_mha=(
|
||||
prefill_max_seq_len <= self.topk_tokens
|
||||
and not self.vllm_config.attention_config.sparse_mla_force_mqa
|
||||
use_dense_mha=_use_dense_mha_prefill(
|
||||
self.vllm_config, prefill_max_seq_len, self.topk_tokens
|
||||
),
|
||||
topk_mask_workspace=self.topk_mask_workspace,
|
||||
host_staging_plan=staging_plan,
|
||||
)
|
||||
self._prefill_backend.prepare_metadata(prefill)
|
||||
|
||||
@@ -335,6 +432,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
decode_max_query_len=decode_max_query_len,
|
||||
prefill_max_seq_len=prefill_max_seq_len,
|
||||
prefill=prefill,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
@@ -560,6 +658,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_b_proj: "ColumnParallelLinear",
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
q_pad_num_heads: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
@@ -577,6 +676,21 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
)
|
||||
|
||||
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
|
||||
self.index_group: SparseMLAIndexGroup | None = None
|
||||
self.index_group_index = 0
|
||||
if index_group_builder is None and self.topk_indices_buffer is not None:
|
||||
index_group_builder = SparseMLAIndexGroupBuilder(self.topk_indices_buffer)
|
||||
if index_group_builder is not None:
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.index_group, self.index_group_index = (
|
||||
index_group_builder.register_layer(
|
||||
indexer is not None,
|
||||
vllm_config,
|
||||
head_size=head_size,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and which("ninja") is not None
|
||||
@@ -593,6 +707,33 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
def record_logical_topk_ready(self) -> None:
|
||||
if self.index_group is not None:
|
||||
self.index_group.set_logical_topk_ready(self.index_group_index)
|
||||
|
||||
def prepare_for_batch(self, attn_metadata: T | None) -> None:
|
||||
if self.index_group is not None:
|
||||
self.index_group.prepare_for_batch(self.index_group_index, attn_metadata)
|
||||
|
||||
def _convert_logical_to_physical_topk(
|
||||
self,
|
||||
logical_topk_indices: torch.Tensor,
|
||||
attn_metadata: Any,
|
||||
*,
|
||||
block_stride_rows: int | None,
|
||||
return_valid_counts: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
group = self.index_group
|
||||
assert group is not None
|
||||
assert self.dcp_world_size == 1
|
||||
return group.convert_logical_to_physical_topk(
|
||||
self.index_group_index,
|
||||
logical_topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=block_stride_rows,
|
||||
return_valid_counts=return_valid_counts,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
|
||||
"""Whether this prefill batch's top-k masks fit the workspace."""
|
||||
@@ -853,8 +994,10 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
topk_tokens = attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata is not None
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len
|
||||
topk_tokens = attn_metadata.topk_tokens
|
||||
force_dense = getattr(self, "_sparse_mla_force_dense_mha", False)
|
||||
force_masked = getattr(self, "_sparse_mla_force_masked_mha", False)
|
||||
if force_dense or (prefill_max_seq_len <= topk_tokens and not force_masked):
|
||||
@@ -863,7 +1006,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_c_and_k_pe_cache,
|
||||
cast(MLACommonMetadata, attn_metadata),
|
||||
attn_metadata,
|
||||
k_scale,
|
||||
output,
|
||||
output_scale,
|
||||
@@ -871,13 +1014,11 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
|
||||
assert output_scale is None
|
||||
assert self.masked_mha_available
|
||||
prefill_metadata = attn_metadata.prefill # type: ignore[attr-defined]
|
||||
assert prefill_metadata is not None
|
||||
assert prefill_metadata.query_lens_cpu is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
|
||||
q_lens = prefill_metadata.query_lens_cpu.tolist()
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens # type: ignore[attr-defined]
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
topk_all = self.topk_indices_buffer[
|
||||
num_decode_tokens : num_decode_tokens + q.shape[0]
|
||||
]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,6 +12,11 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLAModules:
|
||||
@@ -30,6 +36,7 @@ class MLAModules:
|
||||
topk_indices_buffer: torch.Tensor | None
|
||||
indexer_rotary_emb: torch.nn.Module | None = None
|
||||
g_proj: torch.nn.Module | None = None
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None
|
||||
|
||||
|
||||
# --8<-- [start:multi_head_latent_attention]
|
||||
@@ -128,6 +135,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
index_group_builder=mla_modules.index_group_builder,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
)
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
@@ -223,6 +231,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
|
||||
if self.indexer and self.is_sparse and not self.skip_topk:
|
||||
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
|
||||
if self.is_sparse:
|
||||
self.mla_attn.impl.record_logical_topk_ready() # type: ignore[attr-defined]
|
||||
|
||||
if llama_4_scaling is not None:
|
||||
q *= llama_4_scaling
|
||||
|
||||
@@ -425,7 +425,6 @@ def sparse_attn_indexer(
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
# The indexer and main MLA may classify the same short extend differently
|
||||
# because they use independent decode thresholds. Only the main MLA route
|
||||
# can determine whether the top-k indices will be consumed.
|
||||
@@ -955,6 +954,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
False,
|
||||
candidate_blocks=self.candidate_blocks,
|
||||
candidate_block_size=self.candidate_block_size,
|
||||
candidate_write=self.candidate_write,
|
||||
|
||||
Reference in New Issue
Block a user