[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -103,10 +103,14 @@ from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionType
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, SparseCacheRole
|
||||
|
||||
from .interfaces import (
|
||||
MixtureOfExperts,
|
||||
@@ -656,6 +660,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
) # Only has one vector instead of K + V
|
||||
|
||||
def forward(self): ...
|
||||
@@ -1012,6 +1017,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
input_size: int | None = None,
|
||||
reduce_results: bool = True,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
@@ -1199,6 +1205,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
indexer_rotary_emb=self.indexer_rope_emb,
|
||||
is_sparse=self.is_v32,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
self.mla_attn = MultiHeadLatentAttentionWrapper(
|
||||
@@ -1242,6 +1249,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
prefix: str,
|
||||
config: DeepseekV2Config | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1290,6 +1298,11 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and is_moe_layer
|
||||
)
|
||||
attn_kwargs = (
|
||||
{"index_group_builder": index_group_builder}
|
||||
if attn_cls is DeepseekV2MLAAttention
|
||||
else {}
|
||||
)
|
||||
self.self_attn = attn_cls(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
@@ -1306,6 +1319,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
reduce_results=not self.use_sequence_parallel_moe,
|
||||
**attn_kwargs,
|
||||
)
|
||||
|
||||
if is_moe_layer:
|
||||
@@ -1429,6 +1443,14 @@ class DeepseekV2Model(nn.Module):
|
||||
)
|
||||
else:
|
||||
topk_indices_buffer = None
|
||||
index_group_builder = (
|
||||
SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
if topk_indices_buffer is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -1445,6 +1467,7 @@ class DeepseekV2Model(nn.Module):
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -67,6 +67,10 @@ from vllm.model_executor.models.glm4_moe import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
@@ -108,6 +112,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
prefix: str,
|
||||
config: "Glm4MoeLiteConfig | None" = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -136,6 +141,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
attn_cls = Glm4MoeLiteMLAAttention
|
||||
else:
|
||||
attn_cls = Glm4MoeLiteAttention
|
||||
attn_kwargs: dict[str, typing.Any] = (
|
||||
{"index_group_builder": index_group_builder}
|
||||
if attn_cls is Glm4MoeLiteMLAAttention
|
||||
else {}
|
||||
)
|
||||
|
||||
self.self_attn = attn_cls(
|
||||
vllm_config=vllm_config,
|
||||
@@ -152,6 +162,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
**attn_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -234,6 +245,14 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
topk_indices_buffer = None
|
||||
index_group_builder = (
|
||||
SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
if topk_indices_buffer is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -252,6 +271,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
config=config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -51,6 +51,10 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .glm4_moe_lite import (
|
||||
Glm4MixtureOfExperts,
|
||||
@@ -108,6 +112,14 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
topk_indices_buffer = None
|
||||
index_group_builder = (
|
||||
SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
if topk_indices_buffer is not None
|
||||
else None
|
||||
)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
@@ -117,6 +129,7 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
|
||||
prefix=prefix,
|
||||
config=self.config,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user