[BUG] fix hidden states nan for hybrid attention models (#45849)
Signed-off-by: shanjiaz <hezhao@redhat.com> Co-authored-by: shanjiaz <hezhao@redhat.com>
This commit is contained in:
@@ -150,7 +150,14 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
# Worker-side state (set by register_kv_caches).
|
||||
self._kv_cache: torch.Tensor | None = None
|
||||
|
||||
# Identify which KV cache group holds the hidden-states layer.
|
||||
self._hs_group_idx: int = 0
|
||||
if self._kv_cache_config is not None:
|
||||
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
|
||||
if any("cache_only_layers" in n for n in group.layer_names):
|
||||
self._hs_group_idx = i
|
||||
break
|
||||
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
|
||||
# Set in register_kv_caches (after distributed init).
|
||||
self._is_tp_rank_zero: bool = True
|
||||
|
||||
@@ -271,6 +271,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
FullAttentionSpec,
|
||||
TQFullAttentionSpec,
|
||||
MLAAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
):
|
||||
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
|
||||
|
||||
@@ -303,6 +304,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
FullAttentionSpec,
|
||||
TQFullAttentionSpec,
|
||||
MLAAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
):
|
||||
self.new_block_ids.extend(b.block_id for b in new_blocks)
|
||||
return new_blocks
|
||||
|
||||
Reference in New Issue
Block a user