[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,6 +12,11 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLAModules:
|
||||
@@ -30,6 +36,7 @@ class MLAModules:
|
||||
topk_indices_buffer: torch.Tensor | None
|
||||
indexer_rotary_emb: torch.nn.Module | None = None
|
||||
g_proj: torch.nn.Module | None = None
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None
|
||||
|
||||
|
||||
# --8<-- [start:multi_head_latent_attention]
|
||||
@@ -128,6 +135,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
index_group_builder=mla_modules.index_group_builder,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
)
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
@@ -223,6 +231,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
|
||||
if self.indexer and self.is_sparse and not self.skip_topk:
|
||||
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
|
||||
if self.is_sparse:
|
||||
self.mla_attn.impl.record_logical_topk_ready() # type: ignore[attr-defined]
|
||||
|
||||
if llama_4_scaling is not None:
|
||||
q *= llama_4_scaling
|
||||
|
||||
Reference in New Issue
Block a user