From 5fd21eb0b291e16dad66da740b0faf50666a78a5 Mon Sep 17 00:00:00 2001 From: shanjiaz Date: Wed, 17 Jun 2026 18:02:24 -0400 Subject: [PATCH] [BUG] fix hidden states nan for hybrid attention models (#45849) Signed-off-by: shanjiaz Co-authored-by: shanjiaz --- .../kv_connector/v1/example_hidden_states_connector.py | 7 +++++++ vllm/v1/core/single_type_kv_cache_manager.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py index 696d3f7fb4..7e6c95bf8f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py @@ -150,7 +150,14 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): # Worker-side state (set by register_kv_caches). self._kv_cache: torch.Tensor | None = None + + # Identify which KV cache group holds the hidden-states layer. self._hs_group_idx: int = 0 + if self._kv_cache_config is not None: + for i, group in enumerate(self._kv_cache_config.kv_cache_groups): + if any("cache_only_layers" in n for n in group.layer_names): + self._hs_group_idx = i + break # Only TP rank 0 writes hidden states to disk; other TP ranks no-op. # Set in register_kv_caches (after distributed init). self._is_tp_rank_zero: bool = True diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 580b58814b..c98c59017c 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -271,6 +271,7 @@ class SingleTypeKVCacheManager(ABC): FullAttentionSpec, TQFullAttentionSpec, MLAAttentionSpec, + HiddenStateCacheSpec, ): self.new_block_ids.extend(b.block_id for b in allocated_blocks) @@ -303,6 +304,7 @@ class SingleTypeKVCacheManager(ABC): FullAttentionSpec, TQFullAttentionSpec, MLAAttentionSpec, + HiddenStateCacheSpec, ): self.new_block_ids.extend(b.block_id for b in new_blocks) return new_blocks