[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+34
View File
@@ -13,6 +13,40 @@ KVConsumer = Literal["kv_consumer", "kv_both"]
KVRole = Literal[KVProducer, KVConsumer]
def hisparse_host_pool_gib(
kv_transfer_config: "KVTransferConfig | None",
) -> float | None:
if kv_transfer_config is None:
return None
if kv_transfer_config.kv_connector == "MultiConnector":
connectors = kv_transfer_config.kv_connector_extra_config.get("connectors", [])
else:
connectors = [
{
"kv_connector": kv_transfer_config.kv_connector,
"kv_connector_extra_config": (
kv_transfer_config.kv_connector_extra_config
),
}
]
entries = [
connector.get("kv_connector_extra_config", {})
for connector in connectors
if connector.get("kv_connector") == "HiSparseConnector"
]
if len(entries) > 1:
raise ValueError("Only one HiSparseConnector may be configured")
if not entries:
return None
host_pool_gib = entries[0].get("host_pool_gib")
if host_pool_gib is None:
raise ValueError("HiSparseConnector requires host_pool_gib")
host_pool_gib = float(host_pool_gib)
if host_pool_gib <= 0:
raise ValueError("HiSparseConnector host_pool_gib must be positive")
return host_pool_gib
def kv_buffer_device_default_factory() -> str:
from vllm.platforms import current_platform