[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.config.attention import AttentionConfig, HiSparseConfig
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.compilation import (
|
||||
CompilationConfig,
|
||||
@@ -68,6 +68,7 @@ from vllm.config.weight_transfer import WeightTransferConfig
|
||||
__all__ = [
|
||||
# From vllm.config.attention
|
||||
"AttentionConfig",
|
||||
"HiSparseConfig",
|
||||
# From vllm.config.cache
|
||||
"CacheConfig",
|
||||
# From vllm.config.compilation
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from dataclasses import field
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import field_validator
|
||||
from pydantic import Field, field_validator
|
||||
|
||||
from vllm.config.utils import config
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
@@ -14,6 +14,25 @@ IndexerKVDType = Literal["auto", "bf16", "fp8", "mxfp4", "nvfp4"]
|
||||
MiniMaxM3MSADecodeBackend = Literal["triton", "cutlass"]
|
||||
|
||||
|
||||
@config
|
||||
class HiSparseConfig:
|
||||
"""Configuration for HiSparse sparse-MLA KV offloading."""
|
||||
|
||||
device_buffer_size: int | None = Field(default=None, gt=0)
|
||||
"""Total per-request GPU hot-buffer rows, including the newest-token slot.
|
||||
|
||||
Defaults to one top-k per decode query plus one top-k of LRU slack. The
|
||||
physical allocation is rounded up to the GPU cache block size selected
|
||||
from the active backends.
|
||||
"""
|
||||
|
||||
eager_host_mirror: bool = True
|
||||
"""Mirror decode-written KV rows to the host pool during the forward so
|
||||
page spills complete without moving data. When disabled, decode rows stay
|
||||
resident-only and evicted pages are copied to host at spill time. Prefill
|
||||
rows are always mirrored during the forward."""
|
||||
|
||||
|
||||
@config
|
||||
class AttentionConfig:
|
||||
"""Configuration for attention mechanisms in vLLM."""
|
||||
@@ -67,6 +86,10 @@ class AttentionConfig:
|
||||
indexer). Quantized formats (fp8, mxfp4, nvfp4) require indexer kernel
|
||||
support in the backend."""
|
||||
|
||||
hisparse_config: HiSparseConfig | None = None
|
||||
"""HiSparse host-resident KV configuration. Setting this enables experimental
|
||||
Model Runner V2-only HiSparse sparse-MLA decode hot-buffering."""
|
||||
|
||||
use_non_causal: bool = False
|
||||
"""Whether to use non-causal (bidirectional) attention."""
|
||||
|
||||
|
||||
@@ -13,6 +13,40 @@ KVConsumer = Literal["kv_consumer", "kv_both"]
|
||||
KVRole = Literal[KVProducer, KVConsumer]
|
||||
|
||||
|
||||
def hisparse_host_pool_gib(
|
||||
kv_transfer_config: "KVTransferConfig | None",
|
||||
) -> float | None:
|
||||
if kv_transfer_config is None:
|
||||
return None
|
||||
if kv_transfer_config.kv_connector == "MultiConnector":
|
||||
connectors = kv_transfer_config.kv_connector_extra_config.get("connectors", [])
|
||||
else:
|
||||
connectors = [
|
||||
{
|
||||
"kv_connector": kv_transfer_config.kv_connector,
|
||||
"kv_connector_extra_config": (
|
||||
kv_transfer_config.kv_connector_extra_config
|
||||
),
|
||||
}
|
||||
]
|
||||
entries = [
|
||||
connector.get("kv_connector_extra_config", {})
|
||||
for connector in connectors
|
||||
if connector.get("kv_connector") == "HiSparseConnector"
|
||||
]
|
||||
if len(entries) > 1:
|
||||
raise ValueError("Only one HiSparseConnector may be configured")
|
||||
if not entries:
|
||||
return None
|
||||
host_pool_gib = entries[0].get("host_pool_gib")
|
||||
if host_pool_gib is None:
|
||||
raise ValueError("HiSparseConnector requires host_pool_gib")
|
||||
host_pool_gib = float(host_pool_gib)
|
||||
if host_pool_gib <= 0:
|
||||
raise ValueError("HiSparseConnector host_pool_gib must be positive")
|
||||
return host_pool_gib
|
||||
|
||||
|
||||
def kv_buffer_device_default_factory() -> str:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -1919,6 +1919,13 @@ class SpeculativeConfig:
|
||||
def uses_draft_model(self) -> bool:
|
||||
return self.method == "draft_model"
|
||||
|
||||
def uses_draft_kv_cache(self) -> bool:
|
||||
return (
|
||||
self.use_eagle()
|
||||
or self.uses_draft_model()
|
||||
or self.uses_extract_hidden_states()
|
||||
)
|
||||
|
||||
def uses_extract_hidden_states(self) -> bool:
|
||||
return self.method == "extract_hidden_states"
|
||||
|
||||
|
||||
@@ -666,6 +666,14 @@ class VllmConfig:
|
||||
|
||||
@property
|
||||
def use_v2_model_runner(self) -> bool:
|
||||
if self.attention_config.hisparse_config is not None:
|
||||
if envs.VLLM_USE_V2_MODEL_RUNNER is False:
|
||||
raise ValueError(
|
||||
"HiSparse requires Model Runner V2; remove "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=0."
|
||||
)
|
||||
return True
|
||||
|
||||
if getattr(self, "watermark_config", None) is not None:
|
||||
if envs.VLLM_USE_V2_MODEL_RUNNER is False:
|
||||
logger.info_once(
|
||||
@@ -1559,6 +1567,39 @@ class VllmConfig:
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if self.attention_config.hisparse_config is not None:
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("HiSparse currently requires NVIDIA CUDA.")
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError("HiSparse does not support pipeline parallelism.")
|
||||
if self.parallel_config.decode_context_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"HiSparse does not support decode context parallelism."
|
||||
)
|
||||
if self.model_config is not None and not hasattr(
|
||||
self.model_config.hf_config, "index_topk"
|
||||
):
|
||||
raise ValueError(
|
||||
"HiSparse is only supported for DSA models with index_topk."
|
||||
)
|
||||
if self.kv_transfer_config is not None and (
|
||||
self.kv_transfer_config.kv_connector
|
||||
not in (
|
||||
None,
|
||||
"NixlConnector",
|
||||
"MooncakeStoreConnector",
|
||||
"MultiConnector",
|
||||
)
|
||||
):
|
||||
logger.warning(
|
||||
"HiSparse host-resident KV is configured with connector "
|
||||
"%s. NixlConnector (GPU-staged host imports) and "
|
||||
"MooncakeStoreConnector (shared-store offload) are the "
|
||||
"validated paths; other connectors are treated as "
|
||||
"debug/fallback paths.",
|
||||
self.kv_transfer_config.kv_connector,
|
||||
)
|
||||
|
||||
self._normalize_piecewise_cudagraph_mode(
|
||||
breakable_cudagraph_enabled=breakable_cudagraph_enabled
|
||||
)
|
||||
@@ -1937,6 +1978,16 @@ class VllmConfig:
|
||||
# Default to enable HMA if not explicitly disabled by user or logic above.
|
||||
self.scheduler_config.disable_hybrid_kv_cache_manager = False
|
||||
|
||||
if (
|
||||
self.attention_config.hisparse_config is not None
|
||||
and self.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
):
|
||||
raise ValueError(
|
||||
"HiSparse requires the hybrid KV cache manager; remove "
|
||||
"--disable-hybrid-kv-cache-manager or use connectors that "
|
||||
"support HMA."
|
||||
)
|
||||
|
||||
if self.compilation_config.debug_dump_path:
|
||||
self.compilation_config.debug_dump_path = (
|
||||
self.compilation_config.debug_dump_path.absolute().expanduser()
|
||||
|
||||
Reference in New Issue
Block a user