[Spec Decode] Fix hidden-state extraction block size for hybrid verifiers (#46301)
Signed-off-by: Igor Margulis <igor.margulis@intel.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Cursor
mgoin
parent
dc148dc4d7
commit
d8f483dc30
@@ -109,6 +109,49 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Must be False so that drafter kv cache isn't merged with verifier's
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
|
||||
"""Index of the KV cache group holding the extracted hidden states.
|
||||
|
||||
Located by spec type so it resolves on both scheduler and worker side.
|
||||
"""
|
||||
if kv_cache_config is None:
|
||||
return 0
|
||||
|
||||
from vllm.v1.kv_cache_interface import HiddenStateCacheSpec
|
||||
|
||||
groups = kv_cache_config.kv_cache_groups
|
||||
group_ids = [
|
||||
gid
|
||||
for gid, group in enumerate(groups)
|
||||
if isinstance(group.kv_cache_spec, HiddenStateCacheSpec)
|
||||
]
|
||||
if len(group_ids) == 1:
|
||||
return group_ids[0]
|
||||
if not group_ids and len(groups) == 1:
|
||||
return 0
|
||||
raise ValueError(
|
||||
"Could not uniquely identify the extract-hidden-states KV cache "
|
||||
f"group among {len(groups)} groups; the hidden-states layer must be "
|
||||
"isolated in its own group (MLA verifiers are unsupported)."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_cache_block_size(
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig | None",
|
||||
cache_kv_group_id: int,
|
||||
) -> int:
|
||||
"""Block size of the hidden-states group, read from its own spec.
|
||||
|
||||
cache_config.block_size is bumped to a common multiple for hybrid
|
||||
verifiers; the page-aligned hidden-states group keeps a smaller one.
|
||||
"""
|
||||
if kv_cache_config is None:
|
||||
return vllm_config.cache_config.block_size
|
||||
cache_group = kv_cache_config.kv_cache_groups[cache_kv_group_id]
|
||||
return cache_group.kv_cache_spec.block_size
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
@@ -120,7 +163,12 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
role=role,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
self._block_size = vllm_config.cache_config.block_size
|
||||
# Read the hidden-states group and its block size from the group spec;
|
||||
# cache_config.block_size is bumped (wrong) for hybrid verifiers.
|
||||
self._cache_kv_group_id = self._find_cache_kv_group_id(kv_cache_config)
|
||||
self._block_size = self._get_cache_block_size(
|
||||
vllm_config, kv_cache_config, self._cache_kv_group_id
|
||||
)
|
||||
self._storage_path = self._kv_transfer_config.get_from_extra_config(
|
||||
"shared_storage_path", "/tmp"
|
||||
)
|
||||
@@ -151,13 +199,6 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Worker-side state (set by register_kv_caches).
|
||||
self._kv_cache: torch.Tensor | None = None
|
||||
|
||||
# Identify which KV cache group holds the hidden-states layer.
|
||||
self._hs_group_idx: int = 0
|
||||
if self._kv_cache_config is not None:
|
||||
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
|
||||
if any("cache_only_layers" in n for n in group.layer_names):
|
||||
self._hs_group_idx = i
|
||||
break
|
||||
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
|
||||
# Set in register_kv_caches (after distributed init).
|
||||
self._is_tp_rank_zero: bool = True
|
||||
@@ -267,12 +308,14 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
self._kv_cache = kv_caches[self.cache_layers[0]]
|
||||
|
||||
# Find the KV cache group index for hidden states
|
||||
if self._kv_cache_config is not None:
|
||||
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
|
||||
if self.cache_layers[0] in group.layer_names:
|
||||
self._hs_group_idx = i
|
||||
break
|
||||
# Block size must match the indexed buffer, else reads hit the wrong
|
||||
# slots. Raise (not assert) so the check survives `python -O`.
|
||||
if self._block_size != self._kv_cache.shape[1]:
|
||||
raise ValueError(
|
||||
f"Hidden-states block-size mismatch: derived {self._block_size} "
|
||||
f"but buffer block size is {self._kv_cache.shape[1]}; read slots "
|
||||
"would be wrong (likely a hybrid block-size resolution bug)."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_tensors(
|
||||
@@ -543,7 +586,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
return self.request_finished(request, block_ids[self._hs_group_idx])
|
||||
return self.request_finished(request, block_ids[self._cache_kv_group_id])
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
|
||||
Reference in New Issue
Block a user