[Spec Decode] Support hybrid attention models in extract_hidden_states (#39949)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-05-13 10:45:53 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 5794c65f8c
commit 2f821faeae
14 changed files with 225 additions and 115 deletions
+9 -5
View File
@@ -42,6 +42,7 @@ class ExtractHiddenStatesProposer:
self.model: nn.Module | None = None
self.attn_layer_names: list[str] = []
self.attn_metadata_builder: AttentionMetadataBuilder | None = None
self.kv_cache_gid: int = -1
# Maximum number of tokens for buffers
max_batch_size = vllm_config.scheduler_config.max_num_seqs
@@ -374,9 +375,12 @@ class ExtractHiddenStatesProposer:
)
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
"""Validate all drafting layers belong to the same KV cache group.
With exactly one attention layer (asserted in load_model), this is
trivially satisfied.
"""
"""Validate all drafting layers belong to the same KV cache group
and record the group index for common_attn_metadata selection."""
assert len(self.attn_layer_names) == 1
layer = self.attn_layer_names[0]
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
if layer in group.layer_names:
self.kv_cache_gid = gid
return
raise ValueError(f"Cache-only layer {layer!r} not in any KV cache group")