[BUG] fix hidden states nan for hybrid attention models (#45849)

Signed-off-by: shanjiaz <hezhao@redhat.com>
Co-authored-by: shanjiaz <hezhao@redhat.com>
This commit is contained in:
shanjiaz
2026-06-17 18:02:24 -04:00
committed by GitHub
co-authored by shanjiaz
parent 9d4b87f4f0
commit 5fd21eb0b2
2 changed files with 9 additions and 0 deletions
@@ -150,7 +150,14 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
# Worker-side state (set by register_kv_caches).
self._kv_cache: torch.Tensor | None = None
# Identify which KV cache group holds the hidden-states layer.
self._hs_group_idx: int = 0
if self._kv_cache_config is not None:
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
if any("cache_only_layers" in n for n in group.layer_names):
self._hs_group_idx = i
break
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
# Set in register_kv_caches (after distributed init).
self._is_tp_rank_zero: bool = True
@@ -271,6 +271,7 @@ class SingleTypeKVCacheManager(ABC):
FullAttentionSpec,
TQFullAttentionSpec,
MLAAttentionSpec,
HiddenStateCacheSpec,
):
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
@@ -303,6 +304,7 @@ class SingleTypeKVCacheManager(ABC):
FullAttentionSpec,
TQFullAttentionSpec,
MLAAttentionSpec,
HiddenStateCacheSpec,
):
self.new_block_ids.extend(b.block_id for b in new_blocks)
return new_blocks