[Spec Decode] Fix hidden-state extraction block size for hybrid verifiers (#46301)

Signed-off-by: Igor Margulis <igor.margulis@intel.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Igor Margulis
2026-06-30 08:19:51 -07:00
committed by GitHub
co-authored by Cursor mgoin
parent dc148dc4d7
commit d8f483dc30
3 changed files with 187 additions and 15 deletions
@@ -109,6 +109,49 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
# Must be False so that drafter kv cache isn't merged with verifier's
return False
@classmethod
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
"""Index of the KV cache group holding the extracted hidden states.
Located by spec type so it resolves on both scheduler and worker side.
"""
if kv_cache_config is None:
return 0
from vllm.v1.kv_cache_interface import HiddenStateCacheSpec
groups = kv_cache_config.kv_cache_groups
group_ids = [
gid
for gid, group in enumerate(groups)
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
]
if len(group_ids) == 1:
return group_ids[0]
if not group_ids and len(groups) == 1:
return 0
raise ValueError(
"Could not uniquely identify the extract-hidden-states KV cache "
f"group among {len(groups)} groups; the hidden-states layer must be "
"isolated in its own group (MLA verifiers are unsupported)."
)
@staticmethod
def _get_cache_block_size(
vllm_config: "VllmConfig",
kv_cache_config: "KVCacheConfig | None",
cache_kv_group_id: int,
) -> int:
"""Block size of the hidden-states group, read from its own spec.
cache_config.block_size is bumped to a common multiple for hybrid
verifiers; the page-aligned hidden-states group keeps a smaller one.
"""
if kv_cache_config is None:
return vllm_config.cache_config.block_size
cache_group = kv_cache_config.kv_cache_groups[cache_kv_group_id]
return cache_group.kv_cache_spec.block_size
def __init__(
self,
vllm_config: "VllmConfig",
@@ -120,7 +163,12 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
role=role,
kv_cache_config=kv_cache_config,
)
self._block_size = vllm_config.cache_config.block_size
# Read the hidden-states group and its block size from the group spec;
# cache_config.block_size is bumped (wrong) for hybrid verifiers.
self._cache_kv_group_id = self._find_cache_kv_group_id(kv_cache_config)
self._block_size = self._get_cache_block_size(
vllm_config, kv_cache_config, self._cache_kv_group_id
)
self._storage_path = self._kv_transfer_config.get_from_extra_config(
"shared_storage_path", "/tmp"
)
@@ -151,13 +199,6 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
# Worker-side state (set by register_kv_caches).
self._kv_cache: torch.Tensor | None = None
# Identify which KV cache group holds the hidden-states layer.
self._hs_group_idx: int = 0
if self._kv_cache_config is not None:
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
if any("cache_only_layers" in n for n in group.layer_names):
self._hs_group_idx = i
break
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
# Set in register_kv_caches (after distributed init).
self._is_tp_rank_zero: bool = True
@@ -267,12 +308,14 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
)
self._kv_cache = kv_caches[self.cache_layers[0]]
# Find the KV cache group index for hidden states
if self._kv_cache_config is not None:
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
if self.cache_layers[0] in group.layer_names:
self._hs_group_idx = i
break
# Block size must match the indexed buffer, else reads hit the wrong
# slots. Raise (not assert) so the check survives `python -O`.
if self._block_size != self._kv_cache.shape[1]:
raise ValueError(
f"Hidden-states block-size mismatch: derived {self._block_size} "
f"but buffer block size is {self._kv_cache.shape[1]}; read slots "
"would be wrong (likely a hybrid block-size resolution bug)."
)
@staticmethod
def _write_tensors(
@@ -543,7 +586,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
return self.request_finished(request, block_ids[self._hs_group_idx])
return self.request_finished(request, block_ids[self._cache_kv_group_id])
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None: