[MRV2] Collect KV transfer results before the drafter runs (#62041)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-10-17 10:27:57 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 763e3f7c91
commit 0c5dd18e7a
3 changed files with 52 additions and 10 deletions
+25 -3
View File
@@ -22,7 +22,12 @@ def _make_connector(
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
backend.wait_for_save.side_effect = lambda: events.append("wait")
backend.get_transfer_results.return_value = KVConnectorTransferResults()
def get_transfer_results(_finished_req_ids):
events.append("results")
return KVConnectorTransferResults()
backend.get_transfer_results.side_effect = get_transfer_results
backend.get_block_ids_with_load_errors.return_value = set()
backend.get_kv_connector_stats.return_value = None
backend.get_kv_connector_kv_cache_events.return_value = None
@@ -73,7 +78,7 @@ def test_load_start_phase(
)
connector.post_forward(set())
assert events == ["handle", "bind", "start", "wait", "clear"]
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
assert kwargs["request_state_indices"] is request_indices
@@ -94,4 +99,21 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
connector.no_forward(_scheduler_output(False))
assert events == ["handle", "bind", "start", "wait", "clear"]
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
def test_deferred_finalize_collects_results_before_saving(
monkeypatch: pytest.MonkeyPatch,
):
"""With a drafter, the transfer results are collected right after the
target forward, and the saves and metadata are finalized after the
drafter writes its KV."""
events: list[str] = []
connector = _make_connector(monkeypatch, events)
connector.pre_forward(_scheduler_output(False))
connector.post_forward(set(), defer_finalize=True)
assert events == ["handle", "bind", "start", "results"]
connector.finalize_step()
assert events == ["handle", "bind", "start", "results", "wait", "clear"]