[BugFix] Scope divergent hybrid cache hits to capable connectors (#50344)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
83ad767eed
commit
d6941300fc
@@ -5,6 +5,7 @@
|
||||
on partial hits, and same-step deferral."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -31,6 +32,41 @@ def _auto_init_hash_fn():
|
||||
init_none_hash(sha256)
|
||||
|
||||
|
||||
def test_connector_without_divergent_hit_support_uses_common_lookup():
|
||||
common_blocks = MagicMock()
|
||||
manager = MagicMock()
|
||||
manager.get_computed_blocks.return_value = (common_blocks, 0, 0)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=False),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
result = Scheduler._get_local_prefix_cache_hit(scheduler, MagicMock())
|
||||
|
||||
assert result == (common_blocks, 0, 0, False)
|
||||
manager.get_computed_blocks_for_connector.assert_not_called()
|
||||
|
||||
|
||||
def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
per_group_blocks = MagicMock()
|
||||
manager = MagicMock()
|
||||
manager.get_computed_blocks_for_connector.return_value = (
|
||||
per_group_blocks,
|
||||
6,
|
||||
0,
|
||||
True,
|
||||
)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=True),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
result = Scheduler._get_local_prefix_cache_hit(scheduler, MagicMock())
|
||||
|
||||
assert result == (per_group_blocks, 6, 0, True)
|
||||
manager.get_computed_blocks.assert_not_called()
|
||||
|
||||
|
||||
def test_mamba_align_split_partial_tail_schedule():
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
|
||||
Reference in New Issue
Block a user