[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+154 -3
View File
@@ -147,6 +147,11 @@ class KVCacheSpecKind(str, Enum):
UNKNOWN = "unknown"
class SparseCacheRole(str, Enum):
SPARSE = "sparse"
INDEXER = "indexer"
@dataclass(frozen=True)
class KVCacheSpec:
"""
@@ -192,6 +197,10 @@ class KVCacheSpec:
return kernel_block_size // self.tokens_per_state
return 1
@property
def block_table_token_alignment(self) -> int | None:
return 128
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
"""
The maximum possible memory usage of this KV cache in bytes.
@@ -245,6 +254,25 @@ class KVCacheSpec:
isinstance(spec, uniform_type_base_spec) for spec in kv_cache_specs.values()
)
@property
def has_layer_views(self) -> bool:
"""Whether generic allocation creates per-layer cache views.
Specs without a per-layer shape keep the raw backing tensor and lay it
out themselves. They also have no attention module of their own, so
they take no part in attention-backend or kernel-block selection.
"""
return True
@property
def uses_slot_mapping(self) -> bool:
"""Whether the worker computes a per-token slot mapping for this spec.
Specs that address their pages themselves (raw storage, ring buffers)
take no slot mapping row.
"""
return True
def group_kernel_blocks(cache: torch.Tensor, num_blocks: int) -> torch.Tensor:
"""View a kernel-block-granular layer cache with manager blocks as dim 0.
@@ -383,6 +411,70 @@ def create_kv_cache_views(
return views
@dataclass(frozen=True, kw_only=True)
class HiSparseHotSpec(KVCacheSpec):
"""Ephemeral per-request HiSparse hot-cache allocation."""
page_size: int
blocks_per_request: int
@property
def prefix_cacheable(self) -> bool:
return False
@property
def page_size_bytes(self) -> int:
return self.page_size
@property
def block_table_token_alignment(self) -> int | None:
return None
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
return self.blocks_per_request * self.page_size
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
return self.blocks_per_request
@property
def has_layer_views(self) -> bool:
return False
@property
def uses_slot_mapping(self) -> bool:
return False
@dataclass(frozen=True, kw_only=True)
class HiSparseResidentSpec(KVCacheSpec):
"""Reclaimable GPU-resident pages for host-backed HiSparse KV."""
page_size: int
@property
def prefix_cacheable(self) -> bool:
return False
@property
def page_size_bytes(self) -> int:
return self.page_size
def max_admission_blocks_per_request(
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
num_tokens = min(max_in_flight_tokens, max_model_len)
return cdiv(num_tokens, self.block_size)
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
return cdiv(vllm_config.model_config.max_model_len, self.block_size) * (
self.page_size
)
@property
def has_layer_views(self) -> bool:
return False
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
num_kv_heads: int
@@ -557,6 +649,8 @@ class MLAAttentionSpec(FullAttentionSpec):
# DeepseekV4 only fields. Non-DeepseekV4 MLA models leave these at defaults.
alignment: int | None = None # Default to None for no padding.
model_version: str | None = None
cache_role: SparseCacheRole = SparseCacheRole.SPARSE
is_index_group_leader: bool = False
storage_block_size: int | None = None
"""Token width used to view storage when it differs from the kernel block."""
# Group capability enabled when any member flattens a non-causal query block
@@ -577,16 +671,20 @@ class MLAAttentionSpec(FullAttentionSpec):
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
tokens_per_state_set = set(spec.tokens_per_state for spec in specs)
model_version_set = set(spec.model_version for spec in specs)
cache_role_set = {spec.cache_role for spec in specs}
index_group_leader_set = {spec.is_index_group_leader for spec in specs}
storage_block_size_set = set(spec.storage_block_size for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
and len(model_version_set) == 1
and len(cache_role_set) == 1
and len(index_group_leader_set) == 1
and len(storage_block_size_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, and storage "
"block size."
"quantization method, tokens per state, model version, cache role, "
"index-sharing role, and storage block size."
)
merged_spec = cls(
block_size=specs[0].block_size,
@@ -600,6 +698,8 @@ class MLAAttentionSpec(FullAttentionSpec):
cache_dtype_str=cache_dtype_str_set.pop(),
tokens_per_state=tokens_per_state_set.pop(),
model_version=model_version_set.pop(),
cache_role=cache_role_set.pop(),
is_index_group_leader=index_group_leader_set.pop(),
storage_block_size=storage_block_size_set.pop(),
non_causal_multi_token_decode=any(
spec.non_causal_multi_token_decode for spec in specs
@@ -772,6 +872,10 @@ class CircularBufferSpec(AttentionSpec):
reads the open group's committed keys from the ring.
"""
@property
def block_table_token_alignment(self) -> int | None:
return None
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
# The ring occupies one block per request for its whole lifetime.
del vllm_config
@@ -792,6 +896,10 @@ class CircularBufferSpec(AttentionSpec):
def prefix_cacheable(self) -> bool:
return False
@property
def uses_slot_mapping(self) -> bool:
return False
@dataclass(frozen=True, kw_only=True)
class SlidingWindowMLASpec(SlidingWindowSpec):
@@ -919,6 +1027,10 @@ class MambaSpec(KVCacheSpec):
return self.page_size_padded
return page_size
@property
def block_table_token_alignment(self) -> int | None:
return None
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
if vllm_config.cache_config.mamba_cache_mode == "all":
max_model_len = vllm_config.model_config.max_model_len
@@ -1093,6 +1205,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
def page_size_bytes(self) -> int:
return sum(spec.page_size_bytes for spec in self.kv_cache_specs.values())
@property
def block_table_token_alignment(self) -> int | None:
return self.first_spec.block_table_token_alignment
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
max_num_pages = max(
cdiv(spec.max_memory_usage_bytes(vllm_config), spec.page_size_bytes)
@@ -1154,6 +1270,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
for spec in self.kv_cache_specs.values()
)
@property
def uses_slot_mapping(self) -> bool:
return self.first_spec.uses_slot_mapping
def iter_layer_specs(kv_cache_spec: KVCacheSpec) -> Collection[KVCacheSpec]:
"""The per-layer specs a KV cache group spec covers.
@@ -1259,6 +1379,13 @@ class KVCacheTensor:
layer_stride: int
block_stride: int
offset: int = 0 # byte offset of layers[0]'s block 0
host_resident: bool = False
class KVCacheGroupRole(str, Enum):
DEFAULT = "default"
HISPARSE_SOURCE = "hisparse_source"
HISPARSE_INDEXER = "hisparse_indexer"
@dataclass
@@ -1274,8 +1401,13 @@ class KVCacheGroupSpec:
kv_cache_spec: KVCacheSpec
# Whether this group contains EAGLE/MTP draft attention layers.
is_eagle_group: bool = False
# Host groups use the dedicated HiSparse pool; others share the device pool.
host_resident: bool = False
# Whether this group is part of the externally transferable KV state.
# Ephemeral accelerator-side replicas are rebuilt from their transferable
# source group after a connector load.
enable_kv_transfer: bool = True
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
@dataclass
@@ -1300,6 +1432,8 @@ class KVCacheConfig:
"""Resolved retention policy for local prefix-cache checkpoints."""
kv_cache_layout: str | None = None
"""The KV cache layout resolved by the engine core, adopted by all workers."""
hisparse_host_num_blocks: int | None = None
"""Capacity of the dedicated HiSparse host-block manager, when enabled."""
@cached_property
def transfer_group_ids(self) -> tuple[int, ...]:
@@ -1337,6 +1471,21 @@ class KVCacheConfig:
)
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
@property
def host_group_ids(self) -> tuple[int, ...]:
return tuple(
group_id
for group_id, group in enumerate(self.kv_cache_groups)
if group.host_resident
)
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
"""Number of blocks addressable by the pool backing ``tensor``."""
if not tensor.host_resident:
return self.num_blocks
assert self.hisparse_host_num_blocks is not None
return self.hisparse_host_num_blocks
@property
def has_mamba_layers(self) -> bool:
return any(
@@ -1347,9 +1496,11 @@ class KVCacheConfig:
@property
def has_mixed_precision_kv_cache(self) -> bool:
"""Whether attention groups store their KV cache at more than one precision."""
"""Whether device attention caches use more than one precision."""
kv_cache_precisions: set[tuple[torch.dtype, KVQuantMode]] = set()
for group in self.kv_cache_groups:
if group.host_resident:
continue
kv_cache_precisions.update(
(spec.dtype, spec.kv_quant_mode)
for spec in iter_layer_specs(group.kv_cache_spec)