[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -42,6 +42,9 @@ from vllm.v1.attention.ops.pcp import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32Indexer(nn.Module):
|
||||
@@ -126,6 +129,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
prefix: str,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
attn_backend: "type | None" = None,
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
cache_config = vllm_config.cache_config
|
||||
@@ -209,6 +213,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
use_sparse=True,
|
||||
indexer=indexer,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
attn_backend=attn_backend,
|
||||
)
|
||||
|
||||
@@ -304,6 +309,14 @@ class DeepseekV32Attention(MLAAttention):
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
indexer_slot = (
|
||||
slot_mapping.get(self.indexer.k_cache.prefix)
|
||||
if self.indexer is not None
|
||||
else None
|
||||
)
|
||||
hisparse_cache = self.hisparse_cache
|
||||
layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name)
|
||||
self.impl.prepare_for_batch(layer_attn_metadata)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
has_indexer = True
|
||||
@@ -331,12 +344,17 @@ class DeepseekV32Attention(MLAAttention):
|
||||
mla_k_scale = None
|
||||
indexer_k_cache = None
|
||||
mla_slot = None
|
||||
indexer_slot = None
|
||||
else:
|
||||
mla_kv_cache = self.kv_cache
|
||||
mla_kv_cache = None if hisparse_cache is not None else self.kv_cache
|
||||
mla_k_scale = self._k_scale
|
||||
|
||||
kv_c_out = torch.empty_like(kv_c)
|
||||
k_pe_out = torch.empty_like(k_pe)
|
||||
if self.use_pcp or hisparse_cache is None:
|
||||
kv_c_out = torch.empty_like(kv_c)
|
||||
k_pe_out = torch.empty_like(k_pe)
|
||||
else:
|
||||
kv_c_out = torch.empty_like(kv_c)
|
||||
k_pe_out = torch.empty_like(k_pe)
|
||||
q_c = fused_norm_rope(
|
||||
positions,
|
||||
q_c,
|
||||
@@ -354,6 +372,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer_k_rope_cos_sin_cache,
|
||||
self.topk_indices_buffer,
|
||||
slot_mapping=mla_slot,
|
||||
indexer_slot_mapping=indexer_slot,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=self.kv_cache_dtype,
|
||||
@@ -365,6 +384,18 @@ class DeepseekV32Attention(MLAAttention):
|
||||
index_k_out=index_k_out,
|
||||
)
|
||||
|
||||
if hisparse_cache is not None and mla_slot is not None:
|
||||
assert kv_c_out is not None and k_pe_out is not None
|
||||
self.update_kv_cache(
|
||||
kv_c_out,
|
||||
k_pe_out,
|
||||
self.kv_cache,
|
||||
mla_slot,
|
||||
layer_attn_metadata,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
|
||||
@@ -459,6 +490,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
),
|
||||
skip_topk_buffer_clear=True,
|
||||
)
|
||||
self.impl.record_logical_topk_ready() # type: ignore[attr-defined]
|
||||
|
||||
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
self.layer_name
|
||||
@@ -467,6 +499,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
output.zero_()
|
||||
return
|
||||
attn_metadata = cast("MLACommonMetadata", attn_metadata)
|
||||
self.impl.prepare_for_batch(attn_metadata)
|
||||
|
||||
if self.use_pcp:
|
||||
assert kv_c is not None and k_pe is not None
|
||||
|
||||
@@ -68,7 +68,7 @@ def _fp8_quant_and_cache_write(
|
||||
kv_cache_ptr,
|
||||
kv_cache_scale_ptr,
|
||||
cache_block_size,
|
||||
cache_stride,
|
||||
cache_block_stride,
|
||||
offsets,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
):
|
||||
@@ -76,7 +76,7 @@ def _fp8_quant_and_cache_write(
|
||||
|
||||
block_idx = slot_idx // cache_block_size
|
||||
block_offset = slot_idx % cache_block_size
|
||||
block_start = block_idx * cache_block_size * cache_stride
|
||||
block_start = block_idx * cache_block_stride
|
||||
|
||||
tl.store(
|
||||
kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets,
|
||||
@@ -129,8 +129,9 @@ def _fused_norm_rope_kernel(
|
||||
index_k_out_ptr,
|
||||
index_k_out_stride,
|
||||
INDEX_K_HALF_ROT_DIM: tl.constexpr,
|
||||
# Cache params (shared by indexer K and MLA)
|
||||
# Cache params
|
||||
slot_mapping_ptr,
|
||||
indexer_slot_mapping_ptr,
|
||||
# Index K FP8 cache
|
||||
indexer_cache_ptr,
|
||||
indexer_cache_scale_ptr,
|
||||
@@ -400,8 +401,8 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
|
||||
# PCP inserts index K after gathering; other paths write it directly.
|
||||
if indexer_cache_ptr is not None and slot_mapping_ptr is not None:
|
||||
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
|
||||
if indexer_cache_ptr is not None and indexer_slot_mapping_ptr is not None:
|
||||
slot_idx = tl.load(indexer_slot_mapping_ptr + tok_idx)
|
||||
_fp8_quant_and_cache_write(
|
||||
result,
|
||||
index_k_mask,
|
||||
@@ -431,8 +432,9 @@ def fused_norm_rope(
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
# Cache params for fused writes (single slot_mapping for both caches)
|
||||
# Cache params for fused writes
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
indexer_slot_mapping: torch.Tensor | None = None,
|
||||
indexer_k_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache_dtype: str = "auto",
|
||||
@@ -476,9 +478,11 @@ def fused_norm_rope(
|
||||
# --- Indexer K cache setup ---
|
||||
if indexer_k_cache is not None:
|
||||
assert slot_mapping is not None
|
||||
if indexer_slot_mapping is None:
|
||||
indexer_slot_mapping = slot_mapping
|
||||
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
|
||||
idx_cache_block_size = indexer_k_cache.shape[1]
|
||||
idx_cache_stride = indexer_k_cache.shape[2]
|
||||
idx_cache_stride = indexer_k_cache.stride(0)
|
||||
if indexer_k_cache.dtype == torch.uint8:
|
||||
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn)
|
||||
else:
|
||||
@@ -578,6 +582,7 @@ def fused_norm_rope(
|
||||
index_k_rope_cos_sin_cache.shape[-1] // 2,
|
||||
# Cache params
|
||||
slot_mapping,
|
||||
indexer_slot_mapping,
|
||||
indexer_k_cache,
|
||||
idx_cache_scale_view,
|
||||
idx_cache_block_size,
|
||||
|
||||
@@ -46,6 +46,10 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
)
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
|
||||
|
||||
@@ -57,6 +61,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
prefix: str,
|
||||
config=None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -80,6 +85,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
config=config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -189,6 +195,10 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
index_group_builder = SparseMLAIndexGroupBuilder(
|
||||
self.topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = make_input_embedding(
|
||||
@@ -209,6 +219,7 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -48,6 +48,10 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .glm52_low_latency_gemm import (
|
||||
build_glm52_plan,
|
||||
@@ -80,6 +84,10 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
index_group_builder = SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
@@ -88,6 +96,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
prefix,
|
||||
config=config,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -55,6 +55,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
SparseCacheRole,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -148,6 +149,7 @@ class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
)
|
||||
|
||||
def forward(self) -> None: ...
|
||||
|
||||
Reference in New Issue
Block a user