[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+2 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config.attention import AttentionConfig
from vllm.config.attention import AttentionConfig, HiSparseConfig
from vllm.config.cache import CacheConfig
from vllm.config.compilation import (
CompilationConfig,
@@ -68,6 +68,7 @@ from vllm.config.weight_transfer import WeightTransferConfig
__all__ = [
# From vllm.config.attention
"AttentionConfig",
"HiSparseConfig",
# From vllm.config.cache
"CacheConfig",
# From vllm.config.compilation
+24 -1
View File
@@ -4,7 +4,7 @@
from dataclasses import field
from typing import Any, Literal
from pydantic import field_validator
from pydantic import Field, field_validator
from vllm.config.utils import config
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
@@ -14,6 +14,25 @@ IndexerKVDType = Literal["auto", "bf16", "fp8", "mxfp4", "nvfp4"]
MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]
@config
class HiSparseConfig:
"""Configuration for HiSparse sparse-MLA KV offloading."""
device_buffer_size: int | None = Field(default=None, gt=0)
"""Total per-request GPU hot-buffer rows, including the newest-token slot.
Defaults to one top-k per decode query plus one top-k of LRU slack. The
physical allocation is rounded up to the GPU cache block size selected
from the active backends.
"""
eager_host_mirror: bool = True
"""Mirror decode-written KV rows to the host pool during the forward so
page spills complete without moving data. When disabled, decode rows stay
resident-only and evicted pages are copied to host at spill time. Prefill
rows are always mirrored during the forward."""
@config
class AttentionConfig:
"""Configuration for attention mechanisms in vLLM."""
@@ -67,6 +86,10 @@ class AttentionConfig:
indexer). Quantized formats (fp8, mxfp4, nvfp4) require indexer kernel
support in the backend."""
hisparse_config: HiSparseConfig | None = None
"""HiSparse host-resident KV configuration. Setting this enables experimental
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering."""
use_non_causal: bool = False
"""Whether to use non-causal (bidirectional) attention."""
+34
View File
@@ -13,6 +13,40 @@ KVConsumer = Literal["kv_consumer", "kv_both"]
KVRole = Literal[KVProducer, KVConsumer]
def hisparse_host_pool_gib(
kv_transfer_config: "KVTransferConfig | None",
) -> float | None:
if kv_transfer_config is None:
return None
if kv_transfer_config.kv_connector == "MultiConnector":
connectors = kv_transfer_config.kv_connector_extra_config.get("connectors", [])
else:
connectors = [
{
"kv_connector": kv_transfer_config.kv_connector,
"kv_connector_extra_config": (
kv_transfer_config.kv_connector_extra_config
),
}
]
entries = [
connector.get("kv_connector_extra_config", {})
for connector in connectors
if connector.get("kv_connector") == "HiSparseConnector"
]
if len(entries) > 1:
raise ValueError("Only one HiSparseConnector may be configured")
if not entries:
return None
host_pool_gib = entries[0].get("host_pool_gib")
if host_pool_gib is None:
raise ValueError("HiSparseConnector requires host_pool_gib")
host_pool_gib = float(host_pool_gib)
if host_pool_gib <= 0:
raise ValueError("HiSparseConnector host_pool_gib must be positive")
return host_pool_gib
def kv_buffer_device_default_factory() -> str:
from vllm.platforms import current_platform
+7
View File
@@ -1919,6 +1919,13 @@ class SpeculativeConfig:
def uses_draft_model(self) -> bool:
return self.method == "draft_model"
def uses_draft_kv_cache(self) -> bool:
return (
self.use_eagle()
or self.uses_draft_model()
or self.uses_extract_hidden_states()
)
def uses_extract_hidden_states(self) -> bool:
return self.method == "extract_hidden_states"
+51
View File
@@ -666,6 +666,14 @@ class VllmConfig:
@property
def use_v2_model_runner(self) -> bool:
if self.attention_config.hisparse_config is not None:
if envs.VLLM_USE_V2_MODEL_RUNNER is False:
raise ValueError(
"HiSparse requires Model Runner V2; remove "
"VLLM_USE_V2_MODEL_RUNNER=0."
)
return True
if getattr(self, "watermark_config", None) is not None:
if envs.VLLM_USE_V2_MODEL_RUNNER is False:
logger.info_once(
@@ -1559,6 +1567,39 @@ class VllmConfig:
self._maybe_disable_dynamic_sd_for_data_parallel()
self._maybe_override_dynamic_sd_cudagraph_mode()
if self.attention_config.hisparse_config is not None:
if not current_platform.is_cuda():
raise ValueError("HiSparse currently requires NVIDIA CUDA.")
if self.parallel_config.pipeline_parallel_size > 1:
raise ValueError("HiSparse does not support pipeline parallelism.")
if self.parallel_config.decode_context_parallel_size > 1:
raise ValueError(
"HiSparse does not support decode context parallelism."
)
if self.model_config is not None and not hasattr(
self.model_config.hf_config, "index_topk"
):
raise ValueError(
"HiSparse is only supported for DSA models with index_topk."
)
if self.kv_transfer_config is not None and (
self.kv_transfer_config.kv_connector
not in (
None,
"NixlConnector",
"MooncakeStoreConnector",
"MultiConnector",
)
):
logger.warning(
"HiSparse host-resident KV is configured with connector "
"%s. NixlConnector (GPU-staged host imports) and "
"MooncakeStoreConnector (shared-store offload) are the "
"validated paths; other connectors are treated as "
"debug/fallback paths.",
self.kv_transfer_config.kv_connector,
)
self._normalize_piecewise_cudagraph_mode(
breakable_cudagraph_enabled=breakable_cudagraph_enabled
)
@@ -1937,6 +1978,16 @@ class VllmConfig:
# Default to enable HMA if not explicitly disabled by user or logic above.
self.scheduler_config.disable_hybrid_kv_cache_manager = False
if (
self.attention_config.hisparse_config is not None
and self.scheduler_config.disable_hybrid_kv_cache_manager
):
raise ValueError(
"HiSparse requires the hybrid KV cache manager; remove "
"--disable-hybrid-kv-cache-manager or use connectors that "
"support HMA."
)
if self.compilation_config.debug_dump_path:
self.compilation_config.debug_dump_path = (
self.compilation_config.debug_dump_path.absolute().expanduser()