[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -147,6 +147,11 @@ class KVCacheSpecKind(str, Enum):
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class SparseCacheRole(str, Enum):
|
||||
SPARSE = "sparse"
|
||||
INDEXER = "indexer"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KVCacheSpec:
|
||||
"""
|
||||
@@ -192,6 +197,10 @@ class KVCacheSpec:
|
||||
return kernel_block_size // self.tokens_per_state
|
||||
return 1
|
||||
|
||||
@property
|
||||
def block_table_token_alignment(self) -> int | None:
|
||||
return 128
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
"""
|
||||
The maximum possible memory usage of this KV cache in bytes.
|
||||
@@ -245,6 +254,25 @@ class KVCacheSpec:
|
||||
isinstance(spec, uniform_type_base_spec) for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
@property
|
||||
def has_layer_views(self) -> bool:
|
||||
"""Whether generic allocation creates per-layer cache views.
|
||||
|
||||
Specs without a per-layer shape keep the raw backing tensor and lay it
|
||||
out themselves. They also have no attention module of their own, so
|
||||
they take no part in attention-backend or kernel-block selection.
|
||||
"""
|
||||
return True
|
||||
|
||||
@property
|
||||
def uses_slot_mapping(self) -> bool:
|
||||
"""Whether the worker computes a per-token slot mapping for this spec.
|
||||
|
||||
Specs that address their pages themselves (raw storage, ring buffers)
|
||||
take no slot mapping row.
|
||||
"""
|
||||
return True
|
||||
|
||||
|
||||
def group_kernel_blocks(cache: torch.Tensor, num_blocks: int) -> torch.Tensor:
|
||||
"""View a kernel-block-granular layer cache with manager blocks as dim 0.
|
||||
@@ -383,6 +411,70 @@ def create_kv_cache_views(
|
||||
return views
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class HiSparseHotSpec(KVCacheSpec):
|
||||
"""Ephemeral per-request HiSparse hot-cache allocation."""
|
||||
|
||||
page_size: int
|
||||
blocks_per_request: int
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
return self.page_size
|
||||
|
||||
@property
|
||||
def block_table_token_alignment(self) -> int | None:
|
||||
return None
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
return self.blocks_per_request * self.page_size
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
return self.blocks_per_request
|
||||
|
||||
@property
|
||||
def has_layer_views(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def uses_slot_mapping(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class HiSparseResidentSpec(KVCacheSpec):
|
||||
"""Reclaimable GPU-resident pages for host-backed HiSparse KV."""
|
||||
|
||||
page_size: int
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
return self.page_size
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
num_tokens = min(max_in_flight_tokens, max_model_len)
|
||||
return cdiv(num_tokens, self.block_size)
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
return cdiv(vllm_config.model_config.max_model_len, self.block_size) * (
|
||||
self.page_size
|
||||
)
|
||||
|
||||
@property
|
||||
def has_layer_views(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
num_kv_heads: int
|
||||
@@ -557,6 +649,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
# DeepseekV4 only fields. Non-DeepseekV4 MLA models leave these at defaults.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
model_version: str | None = None
|
||||
cache_role: SparseCacheRole = SparseCacheRole.SPARSE
|
||||
is_index_group_leader: bool = False
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
# Group capability enabled when any member flattens a non-causal query block
|
||||
@@ -577,16 +671,20 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
|
||||
tokens_per_state_set = set(spec.tokens_per_state for spec in specs)
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
cache_role_set = {spec.cache_role for spec in specs}
|
||||
index_group_leader_set = {spec.is_index_group_leader for spec in specs}
|
||||
storage_block_size_set = set(spec.storage_block_size for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(cache_role_set) == 1
|
||||
and len(index_group_leader_set) == 1
|
||||
and len(storage_block_size_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, and storage "
|
||||
"block size."
|
||||
"quantization method, tokens per state, model version, cache role, "
|
||||
"index-sharing role, and storage block size."
|
||||
)
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -600,6 +698,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
tokens_per_state=tokens_per_state_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
cache_role=cache_role_set.pop(),
|
||||
is_index_group_leader=index_group_leader_set.pop(),
|
||||
storage_block_size=storage_block_size_set.pop(),
|
||||
non_causal_multi_token_decode=any(
|
||||
spec.non_causal_multi_token_decode for spec in specs
|
||||
@@ -772,6 +872,10 @@ class CircularBufferSpec(AttentionSpec):
|
||||
reads the open group's committed keys from the ring.
|
||||
"""
|
||||
|
||||
@property
|
||||
def block_table_token_alignment(self) -> int | None:
|
||||
return None
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
# The ring occupies one block per request for its whole lifetime.
|
||||
del vllm_config
|
||||
@@ -792,6 +896,10 @@ class CircularBufferSpec(AttentionSpec):
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def uses_slot_mapping(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
@@ -919,6 +1027,10 @@ class MambaSpec(KVCacheSpec):
|
||||
return self.page_size_padded
|
||||
return page_size
|
||||
|
||||
@property
|
||||
def block_table_token_alignment(self) -> int | None:
|
||||
return None
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
@@ -1093,6 +1205,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
def page_size_bytes(self) -> int:
|
||||
return sum(spec.page_size_bytes for spec in self.kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def block_table_token_alignment(self) -> int | None:
|
||||
return self.first_spec.block_table_token_alignment
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_num_pages = max(
|
||||
cdiv(spec.max_memory_usage_bytes(vllm_config), spec.page_size_bytes)
|
||||
@@ -1154,6 +1270,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
for spec in self.kv_cache_specs.values()
|
||||
)
|
||||
|
||||
@property
|
||||
def uses_slot_mapping(self) -> bool:
|
||||
return self.first_spec.uses_slot_mapping
|
||||
|
||||
|
||||
def iter_layer_specs(kv_cache_spec: KVCacheSpec) -> Collection[KVCacheSpec]:
|
||||
"""The per-layer specs a KV cache group spec covers.
|
||||
@@ -1259,6 +1379,13 @@ class KVCacheTensor:
|
||||
layer_stride: int
|
||||
block_stride: int
|
||||
offset: int = 0 # byte offset of layers[0]'s block 0
|
||||
host_resident: bool = False
|
||||
|
||||
|
||||
class KVCacheGroupRole(str, Enum):
|
||||
DEFAULT = "default"
|
||||
HISPARSE_SOURCE = "hisparse_source"
|
||||
HISPARSE_INDEXER = "hisparse_indexer"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1274,8 +1401,13 @@ class KVCacheGroupSpec:
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
is_eagle_group: bool = False
|
||||
# Host groups use the dedicated HiSparse pool; others share the device pool.
|
||||
host_resident: bool = False
|
||||
# Whether this group is part of the externally transferable KV state.
|
||||
# Ephemeral accelerator-side replicas are rebuilt from their transferable
|
||||
# source group after a connector load.
|
||||
enable_kv_transfer: bool = True
|
||||
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1300,6 +1432,8 @@ class KVCacheConfig:
|
||||
"""Resolved retention policy for local prefix-cache checkpoints."""
|
||||
kv_cache_layout: str | None = None
|
||||
"""The KV cache layout resolved by the engine core, adopted by all workers."""
|
||||
hisparse_host_num_blocks: int | None = None
|
||||
"""Capacity of the dedicated HiSparse host-block manager, when enabled."""
|
||||
|
||||
@cached_property
|
||||
def transfer_group_ids(self) -> tuple[int, ...]:
|
||||
@@ -1337,6 +1471,21 @@ class KVCacheConfig:
|
||||
)
|
||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||
|
||||
@property
|
||||
def host_group_ids(self) -> tuple[int, ...]:
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(self.kv_cache_groups)
|
||||
if group.host_resident
|
||||
)
|
||||
|
||||
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
||||
"""Number of blocks addressable by the pool backing ``tensor``."""
|
||||
if not tensor.host_resident:
|
||||
return self.num_blocks
|
||||
assert self.hisparse_host_num_blocks is not None
|
||||
return self.hisparse_host_num_blocks
|
||||
|
||||
@property
|
||||
def has_mamba_layers(self) -> bool:
|
||||
return any(
|
||||
@@ -1347,9 +1496,11 @@ class KVCacheConfig:
|
||||
|
||||
@property
|
||||
def has_mixed_precision_kv_cache(self) -> bool:
|
||||
"""Whether attention groups store their KV cache at more than one precision."""
|
||||
"""Whether device attention caches use more than one precision."""
|
||||
kv_cache_precisions: set[tuple[torch.dtype, KVQuantMode]] = set()
|
||||
for group in self.kv_cache_groups:
|
||||
if group.host_resident:
|
||||
continue
|
||||
kv_cache_precisions.update(
|
||||
(spec.dtype, spec.kv_quant_mode)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
|
||||
Reference in New Issue
Block a user