[Bugfix][DSv4] Keep indexer scoring in breakable graphs (#52492)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Yongye Zhu
parent
6664d397bf
commit
292187dd8c
@@ -906,7 +906,10 @@ class DeepseekV4Indexer(nn.Module):
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix])
|
||||
if indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens:
|
||||
if (
|
||||
indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
# candidates num smaller than topk, every candidate is selected
|
||||
# but we still need to build k cache
|
||||
compressor(compressed_kv_score, positions, rotary_emb)
|
||||
|
||||
Reference in New Issue
Block a user