[BugFix] Scope divergent hybrid cache hits to capable connectors (#50344)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Yifan Qiao
2026-08-09 15:38:13 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 83ad767eed
commit d6941300fc
8 changed files with 130 additions and 27 deletions
+20 -7
View File
@@ -5547,6 +5547,7 @@ def _create_hybrid_mamba_connector_scheduler(
matched_tokens: int,
block_size: int = 16,
num_blocks: int = 100,
supports_divergent_hits: bool = True,
) -> Scheduler:
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
model_config = ModelConfig(
@@ -5577,6 +5578,7 @@ def _create_hybrid_mamba_connector_scheduler(
kv_connector_extra_config={
"matched_tokens": matched_tokens,
"is_async": False,
"supports_divergent_local_hybrid_hits": supports_divergent_hits,
},
),
)
@@ -5686,22 +5688,33 @@ def test_hybrid_per_group_hit_divergence_with_connector(
@pytest.mark.parametrize(
("matched_tokens", "replay_blocks", "expected_num_computed"),
(
"supports_divergent_local_hybrid_hits",
"matched_tokens",
"replay_blocks",
"expected_num_computed",
),
[
(0, 5, 16),
(16, 6, 80),
(True, 0, 5, 16),
(True, 16, 6, 80),
(False, 16, 6, 32),
],
)
def test_hybrid_fa_deeper_hit_uses_external_mamba_state(
def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
supports_divergent_local_hybrid_hits: bool,
matched_tokens: int,
replay_blocks: int,
expected_num_computed: int,
):
"""A positive external hit supplies the Mamba state at the resume
boundary; without one, the scheduler falls back to the common local hit.
"""A capable connector may restore missing Mamba state at the deeper FA
boundary. An external miss or an incapable connector uses a locally
consistent boundary instead.
"""
block_size = 16
scheduler = _create_hybrid_mamba_connector_scheduler(matched_tokens)
scheduler = _create_hybrid_mamba_connector_scheduler(
matched_tokens,
supports_divergent_hits=supports_divergent_local_hybrid_hits,
)
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)