[MRV2] Collect KV transfer results before the drafter runs (#62041)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
763e3f7c91
commit
0c5dd18e7a
@@ -22,7 +22,12 @@ def _make_connector(
|
||||
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
|
||||
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
|
||||
backend.wait_for_save.side_effect = lambda: events.append("wait")
|
||||
backend.get_transfer_results.return_value = KVConnectorTransferResults()
|
||||
|
||||
def get_transfer_results(_finished_req_ids):
|
||||
events.append("results")
|
||||
return KVConnectorTransferResults()
|
||||
|
||||
backend.get_transfer_results.side_effect = get_transfer_results
|
||||
backend.get_block_ids_with_load_errors.return_value = set()
|
||||
backend.get_kv_connector_stats.return_value = None
|
||||
backend.get_kv_connector_kv_cache_events.return_value = None
|
||||
@@ -73,7 +78,7 @@ def test_load_start_phase(
|
||||
)
|
||||
|
||||
connector.post_forward(set())
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
|
||||
|
||||
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
|
||||
assert kwargs["request_state_indices"] is request_indices
|
||||
@@ -94,4 +99,21 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
connector.no_forward(_scheduler_output(False))
|
||||
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
|
||||
|
||||
|
||||
def test_deferred_finalize_collects_results_before_saving(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""With a drafter, the transfer results are collected right after the
|
||||
target forward, and the saves and metadata are finalized after the
|
||||
drafter writes its KV."""
|
||||
events: list[str] = []
|
||||
connector = _make_connector(monkeypatch, events)
|
||||
connector.pre_forward(_scheduler_output(False))
|
||||
|
||||
connector.post_forward(set(), defer_finalize=True)
|
||||
assert events == ["handle", "bind", "start", "results"]
|
||||
|
||||
connector.finalize_step()
|
||||
assert events == ["handle", "bind", "start", "results", "wait", "clear"]
|
||||
|
||||
Reference in New Issue
Block a user