[Spec Decode] Support hybrid attention models in extract_hidden_states (#39949)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
5794c65f8c
commit
2f821faeae
@@ -42,6 +42,7 @@ class ExtractHiddenStatesProposer:
|
||||
self.model: nn.Module | None = None
|
||||
self.attn_layer_names: list[str] = []
|
||||
self.attn_metadata_builder: AttentionMetadataBuilder | None = None
|
||||
self.kv_cache_gid: int = -1
|
||||
|
||||
# Maximum number of tokens for buffers
|
||||
max_batch_size = vllm_config.scheduler_config.max_num_seqs
|
||||
@@ -374,9 +375,12 @@ class ExtractHiddenStatesProposer:
|
||||
)
|
||||
|
||||
def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Validate all drafting layers belong to the same KV cache group.
|
||||
|
||||
With exactly one attention layer (asserted in load_model), this is
|
||||
trivially satisfied.
|
||||
"""
|
||||
"""Validate all drafting layers belong to the same KV cache group
|
||||
and record the group index for common_attn_metadata selection."""
|
||||
assert len(self.attn_layer_names) == 1
|
||||
layer = self.attn_layer_names[0]
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if layer in group.layer_names:
|
||||
self.kv_cache_gid = gid
|
||||
return
|
||||
raise ValueError(f"Cache-only layer {layer!r} not in any KV cache group")
|
||||
|
||||
Reference in New Issue
Block a user