[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
@@ -67,6 +67,10 @@ from vllm.model_executor.models.glm4_moe import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.index_group import (
SparseMLAIndexGroupBuilder,
get_sparse_mla_index_group_max_rows,
)
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
@@ -108,6 +112,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
prefix: str,
config: "Glm4MoeLiteConfig | None" = None,
topk_indices_buffer: torch.Tensor | None = None,
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
) -> None:
super().__init__()
@@ -136,6 +141,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
attn_cls = Glm4MoeLiteMLAAttention
else:
attn_cls = Glm4MoeLiteAttention
attn_kwargs: dict[str, typing.Any] = (
{"index_group_builder": index_group_builder}
if attn_cls is Glm4MoeLiteMLAAttention
else {}
)
self.self_attn = attn_cls(
vllm_config=vllm_config,
@@ -152,6 +162,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
topk_indices_buffer=topk_indices_buffer,
**attn_kwargs,
)
if (
@@ -234,6 +245,14 @@ class Glm4MoeLiteModel(nn.Module):
)
else:
topk_indices_buffer = None
index_group_builder = (
SparseMLAIndexGroupBuilder(
topk_indices_buffer,
get_sparse_mla_index_group_max_rows(vllm_config),
)
if topk_indices_buffer is not None
else None
)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
@@ -252,6 +271,7 @@ class Glm4MoeLiteModel(nn.Module):
config=config,
prefix=prefix,
topk_indices_buffer=topk_indices_buffer,
index_group_builder=index_group_builder,
),
prefix=f"{prefix}.layers",
)