[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+36 -3
View File
@@ -42,6 +42,9 @@ from vllm.v1.attention.ops.pcp import (
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
)
class DeepseekV32Indexer(nn.Module):
@@ -126,6 +129,7 @@ class DeepseekV32Attention(MLAAttention):
prefix: str,
topk_indices_buffer: torch.Tensor | None = None,
attn_backend: "type | None" = None,
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
) -> None:
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
@@ -209,6 +213,7 @@ class DeepseekV32Attention(MLAAttention):
use_sparse=True,
indexer=indexer,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
attn_backend=attn_backend,
)
@@ -304,6 +309,14 @@ class DeepseekV32Attention(MLAAttention):
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
indexer_slot = (
slot_mapping.get(self.indexer.k_cache.prefix)
if self.indexer is not None
else None
)
hisparse_cache = self.hisparse_cache
layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name)
self.impl.prepare_for_batch(layer_attn_metadata)
if self.indexer is not None and not self.skip_topk:
has_indexer = True
@@ -331,12 +344,17 @@ class DeepseekV32Attention(MLAAttention):
mla_k_scale = None
indexer_k_cache = None
mla_slot = None
indexer_slot = None
else:
mla_kv_cache = self.kv_cache
mla_kv_cache = None if hisparse_cache is not None else self.kv_cache
mla_k_scale = self._k_scale
kv_c_out = torch.empty_like(kv_c)
k_pe_out = torch.empty_like(k_pe)
if self.use_pcp or hisparse_cache is None:
kv_c_out = torch.empty_like(kv_c)
k_pe_out = torch.empty_like(k_pe)
else:
kv_c_out = torch.empty_like(kv_c)
k_pe_out = torch.empty_like(k_pe)
q_c = fused_norm_rope(
positions,
q_c,
@@ -354,6 +372,7 @@ class DeepseekV32Attention(MLAAttention):
indexer_k_rope_cos_sin_cache,
self.topk_indices_buffer,
slot_mapping=mla_slot,
indexer_slot_mapping=indexer_slot,
indexer_k_cache=indexer_k_cache,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype,
@@ -365,6 +384,18 @@ class DeepseekV32Attention(MLAAttention):
index_k_out=index_k_out,
)
if hisparse_cache is not None and mla_slot is not None:
assert kv_c_out is not None and k_pe_out is not None
self.update_kv_cache(
kv_c_out,
k_pe_out,
self.kv_cache,
mla_slot,
layer_attn_metadata,
self.kv_cache_dtype,
self._k_scale,
)
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
@@ -459,6 +490,7 @@ class DeepseekV32Attention(MLAAttention):
),
skip_topk_buffer_clear=True,
)
self.impl.record_logical_topk_ready() # type: ignore[attr-defined]
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(
self.layer_name
@@ -467,6 +499,7 @@ class DeepseekV32Attention(MLAAttention):
output.zero_()
return
attn_metadata = cast("MLACommonMetadata", attn_metadata)
self.impl.prepare_for_batch(attn_metadata)
if self.use_pcp:
assert kv_c is not None and k_pe is not None
+12 -7
View File
@@ -68,7 +68,7 @@ def _fp8_quant_and_cache_write(
kv_cache_ptr,
kv_cache_scale_ptr,
cache_block_size,
cache_stride,
cache_block_stride,
offsets,
HEAD_DIM: tl.constexpr,
):
@@ -76,7 +76,7 @@ def _fp8_quant_and_cache_write(
block_idx = slot_idx // cache_block_size
block_offset = slot_idx % cache_block_size
block_start = block_idx * cache_block_size * cache_stride
block_start = block_idx * cache_block_stride
tl.store(
kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets,
@@ -129,8 +129,9 @@ def _fused_norm_rope_kernel(
index_k_out_ptr,
index_k_out_stride,
INDEX_K_HALF_ROT_DIM: tl.constexpr,
# Cache params (shared by indexer K and MLA)
# Cache params
slot_mapping_ptr,
indexer_slot_mapping_ptr,
# Index K FP8 cache
indexer_cache_ptr,
indexer_cache_scale_ptr,
@@ -400,8 +401,8 @@ def _fused_norm_rope_kernel(
)
# PCP inserts index K after gathering; other paths write it directly.
if indexer_cache_ptr is not None and slot_mapping_ptr is not None:
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
if indexer_cache_ptr is not None and indexer_slot_mapping_ptr is not None:
slot_idx = tl.load(indexer_slot_mapping_ptr + tok_idx)
_fp8_quant_and_cache_write(
result,
index_k_mask,
@@ -431,8 +432,9 @@ def fused_norm_rope(
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
# Cache params for fused writes (single slot_mapping for both caches)
# Cache params for fused writes
slot_mapping: torch.Tensor | None = None,
indexer_slot_mapping: torch.Tensor | None = None,
indexer_k_cache: torch.Tensor | None = None,
mla_kv_cache: torch.Tensor | None = None,
mla_kv_cache_dtype: str = "auto",
@@ -476,9 +478,11 @@ def fused_norm_rope(
# --- Indexer K cache setup ---
if indexer_k_cache is not None:
assert slot_mapping is not None
if indexer_slot_mapping is None:
indexer_slot_mapping = slot_mapping
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
idx_cache_block_size = indexer_k_cache.shape[1]
idx_cache_stride = indexer_k_cache.shape[2]
idx_cache_stride = indexer_k_cache.stride(0)
if indexer_k_cache.dtype == torch.uint8:
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn)
else:
@@ -578,6 +582,7 @@ def fused_norm_rope(
index_k_rope_cos_sin_cache.shape[-1] // 2,
# Cache params
slot_mapping,
indexer_slot_mapping,
indexer_k_cache,
idx_cache_scale_view,
idx_cache_block_size,
+11
View File
@@ -46,6 +46,10 @@ from vllm.models.common.ops.sequence_parallel import (
)
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .glm52_low_latency_gemm import enable_glm52_low_latency_gemm
@@ -57,6 +61,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
prefix: str,
config=None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
) -> None:
super().__init__()
@@ -80,6 +85,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
config=config,
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
)
if (
@@ -189,6 +195,10 @@ class DeepseekV32Model(torch.nn.Module):
dtype=torch.int32,
device=self.device,
)
index_group_builder = SparseMLAIndexGroupBuilder(
self.topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
if get_pp_group().is_first_rank:
self.embed_tokens = make_input_embedding(
@@ -209,6 +219,7 @@ class DeepseekV32Model(torch.nn.Module):
vllm_config=vllm_config,
prefix=prefix,
topk_indices_buffer=self.topk_indices_buffer,
index_group_builder=index_group_builder,
),
prefix=f"{prefix}.layers",
)
+9
View File
@@ -48,6 +48,10 @@ from vllm.models.common.ops.sequence_parallel import (
from vllm.models.deepseek_v32.common.kernels import fused_eh_norm
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .glm52_low_latency_gemm import (
build_glm52_plan,
@@ -80,6 +84,10 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
dtype=torch.int32,
device=current_platform.device_type,
)
index_group_builder = SparseMLAIndexGroupBuilder(
topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
@@ -88,6 +96,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
prefix,
config=config,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
)
def forward(
+2
View File
@@ -55,6 +55,7 @@ from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
MLAAttentionSpec,
SparseCacheRole,
)
logger = init_logger(__name__)
@@ -148,6 +149,7 @@ class MiniMaxM3IndexerCache(nn.Module, AttentionLayerBase):
num_kv_heads=1,
head_size=self.head_dim,
dtype=self.dtype,
cache_role=SparseCacheRole.INDEXER,
)
def forward(self) -> None: ...