[BugFix] Scope divergent hybrid cache hits to capable connectors (#50344)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
83ad767eed
commit
d6941300fc
@@ -5,6 +5,7 @@
|
||||
on partial hits, and same-step deferral."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -31,6 +32,41 @@ def _auto_init_hash_fn():
|
||||
init_none_hash(sha256)
|
||||
|
||||
|
||||
def test_connector_without_divergent_hit_support_uses_common_lookup():
|
||||
common_blocks = MagicMock()
|
||||
manager = MagicMock()
|
||||
manager.get_computed_blocks.return_value = (common_blocks, 0, 0)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=False),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
result = Scheduler._get_local_prefix_cache_hit(scheduler, MagicMock())
|
||||
|
||||
assert result == (common_blocks, 0, 0, False)
|
||||
manager.get_computed_blocks_for_connector.assert_not_called()
|
||||
|
||||
|
||||
def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
per_group_blocks = MagicMock()
|
||||
manager = MagicMock()
|
||||
manager.get_computed_blocks_for_connector.return_value = (
|
||||
per_group_blocks,
|
||||
6,
|
||||
0,
|
||||
True,
|
||||
)
|
||||
scheduler = SimpleNamespace(
|
||||
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=True),
|
||||
kv_cache_manager=manager,
|
||||
)
|
||||
|
||||
result = Scheduler._get_local_prefix_cache_hit(scheduler, MagicMock())
|
||||
|
||||
assert result == (per_group_blocks, 6, 0, True)
|
||||
manager.get_computed_blocks.assert_not_called()
|
||||
|
||||
|
||||
def test_mamba_align_split_partial_tail_schedule():
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
|
||||
@@ -5547,6 +5547,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
matched_tokens: int,
|
||||
block_size: int = 16,
|
||||
num_blocks: int = 100,
|
||||
supports_divergent_hits: bool = True,
|
||||
) -> Scheduler:
|
||||
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
|
||||
model_config = ModelConfig(
|
||||
@@ -5577,6 +5578,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
kv_connector_extra_config={
|
||||
"matched_tokens": matched_tokens,
|
||||
"is_async": False,
|
||||
"supports_divergent_local_hybrid_hits": supports_divergent_hits,
|
||||
},
|
||||
),
|
||||
)
|
||||
@@ -5686,22 +5688,33 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("matched_tokens", "replay_blocks", "expected_num_computed"),
|
||||
(
|
||||
"supports_divergent_local_hybrid_hits",
|
||||
"matched_tokens",
|
||||
"replay_blocks",
|
||||
"expected_num_computed",
|
||||
),
|
||||
[
|
||||
(0, 5, 16),
|
||||
(16, 6, 80),
|
||||
(True, 0, 5, 16),
|
||||
(True, 16, 6, 80),
|
||||
(False, 16, 6, 32),
|
||||
],
|
||||
)
|
||||
def test_hybrid_fa_deeper_hit_uses_external_mamba_state(
|
||||
def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
supports_divergent_local_hybrid_hits: bool,
|
||||
matched_tokens: int,
|
||||
replay_blocks: int,
|
||||
expected_num_computed: int,
|
||||
):
|
||||
"""A positive external hit supplies the Mamba state at the resume
|
||||
boundary; without one, the scheduler falls back to the common local hit.
|
||||
"""A capable connector may restore missing Mamba state at the deeper FA
|
||||
boundary. An external miss or an incapable connector uses a locally
|
||||
consistent boundary instead.
|
||||
"""
|
||||
block_size = 16
|
||||
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens)
|
||||
scheduler = _create_hybrid_mamba_connector_scheduler(
|
||||
matched_tokens,
|
||||
supports_divergent_hits=supports_divergent_local_hybrid_hits,
|
||||
)
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user