[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Summer Yang
2026-08-28 21:58:00 +00:00
committed by GitHub
co-authored by Nick Hill
parent 96242aa50d
commit 2aac565cae
17 changed files with 293 additions and 118 deletions
+41
View File
@@ -1975,6 +1975,47 @@ def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
return initial_ecos
@pytest.mark.parametrize(
("load_modes", "expected_has_sync_loads"),
[
((True,), False),
((False,), True),
((True, False), True),
],
)
@pytest.mark.skip_global_cleanup
def test_has_sync_kv_loads(
load_modes: tuple[bool, ...],
expected_has_sync_loads: bool,
tmp_path,
):
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
block_size = 16
scheduler = create_scheduler(
model=str(tmp_path),
skip_tokenizer_init=True,
use_kv_connector=mock_kv(matched_tokens=block_size, is_async=False),
block_size=block_size,
)
requests = create_requests(
num_requests=len(load_modes),
num_tokens=block_size * 2,
block_size=block_size,
)
for request in requests:
scheduler.add_request(request)
scheduler.connector.get_num_new_matched_tokens = Mock(
side_effect=[(block_size, is_async) for is_async in load_modes]
)
output = scheduler.schedule()
assert output.has_sync_kv_loads is expected_has_sync_loads
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool):
"""
@@ -2960,7 +2960,7 @@ def test_putting_consumer_queues_decode_save():
with patch.object(torch.cuda, "Event") as event_cls:
event = event_cls.return_value
w.get_finished(set(), meta)
w.wait_for_save(meta)
event.record.assert_called_once_with()
send_thread.add_request.assert_called_once_with(req)
@@ -294,18 +294,23 @@ def test_multi_example_connector_consistency():
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
# First three events are from initialization (register_kv_caches,
# set_host_xfer_buffer_ops, get_handshake_metadata), then generate() events.
assert events["storage1-WORKER"][:8] == [
# First three events are from initialization. During generate(), layer hooks
# run before the deferred load is started after the forward pass.
expected_worker_prefix = [
"register_kv_caches",
"set_host_xfer_buffer_ops",
"get_handshake_metadata",
"handle_preemptions",
"bind_connector_metadata",
"start_load_kv",
"wait_for_layer_load",
"save_kv_layer",
]
for connector_name in ("storage1-WORKER", "storage2-WORKER"):
worker_events = events[connector_name]
assert worker_events[:7] == expected_worker_prefix
assert worker_events.index("start_load_kv") > worker_events.index(
"save_kv_layer"
)
assert storage2_scheduler_events[:7] == [
"bind_gpu_block_pool",
"get_finished_count",
@@ -315,17 +320,6 @@ def test_multi_example_connector_consistency():
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
assert events["storage2-WORKER"][:8] == [
"register_kv_caches",
"set_host_xfer_buffer_ops",
"get_handshake_metadata",
"handle_preemptions",
"bind_connector_metadata",
"start_load_kv",
"wait_for_layer_load",
"save_kv_layer",
]
# Reset prefix cache or else we'll just get the tokens back from there.
llm.reset_prefix_cache()
@@ -213,7 +213,7 @@ def _multi_connector_config(swa_enabled: bool = False):
def test_nixl_wins_load_over_cpu_offload():
"""When NixlConnector (index 0) has matched tokens from a remote prefill, it should
win the load: Nixl metadata tracks the recv while CPU offload metadata has no load
scheduled."""
scheduled."""
vllm_config, kv_cache_config = _multi_connector_config()
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
mc = scheduler.connector
@@ -227,6 +227,9 @@ def test_nixl_wins_load_over_cpu_offload():
)
scheduler.add_request(request)
sched_out = scheduler.schedule()
# The remote-prefill load is async, so the step has no sync KV loads
# and the worker will start the load after the forward.
assert not sched_out.has_sync_kv_loads
assert mc._requests_to_connector[request.request_id] == 0
+4 -3
View File
@@ -152,8 +152,8 @@ class _RecordingBackend:
self.calls.append({"is_store": is_store, "wait_event": wait_event})
def test_get_finished_passes_wait_event_for_store_only():
"""get_finished gates stores on a compute-done event but not loads."""
def test_transfer_hooks_pass_wait_event_for_store_only():
"""wait_for_save gates stores on a compute-done event; start_load_kv does not."""
worker = SimpleCPUOffloadWorker(
vllm_config=None, kv_cache_config=None, cpu_capacity_bytes=0
)
@@ -168,7 +168,8 @@ def test_get_finished_passes_wait_event_for_store_only():
store_cpu_blocks=[1],
)
worker.get_finished(set())
worker.start_load_kv()
worker.wait_for_save()
store_calls = [c for c in recording.calls if c["is_store"]]
load_calls = [c for c in recording.calls if not c["is_store"]]
+79
View File
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import vllm.v1.worker.gpu.kv_connector as kv_connector_module
from vllm.config import KVTransferConfig
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
def _make_connector(
monkeypatch: pytest.MonkeyPatch,
events: list[str],
) -> ActiveKVConnector:
backend = Mock()
backend.handle_preemptions.side_effect = lambda _: events.append("handle")
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
backend.start_load_kv.side_effect = lambda _: events.append("start")
backend.wait_for_save.side_effect = lambda: events.append("wait")
backend.get_finished.side_effect = lambda _: (set(), set())
backend.get_block_ids_with_load_errors.return_value = set()
backend.get_kv_connector_stats.return_value = None
backend.get_kv_connector_kv_cache_events.return_value = None
backend.build_connector_worker_meta.return_value = None
backend.clear_connector_metadata.side_effect = lambda: events.append("clear")
monkeypatch.setattr(kv_connector_module, "get_kv_transfer_group", lambda: backend)
monkeypatch.setattr(
kv_connector_module, "is_forward_context_available", lambda: True
)
monkeypatch.setattr(kv_connector_module, "get_forward_context", object)
kv_config = KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_consumer",
kv_buffer_device="cpu",
)
connector = ActiveKVConnector( # type: ignore[arg-type]
SimpleNamespace(kv_transfer_config=kv_config), {}
)
events.clear()
return connector
def _scheduler_output(has_sync_kv_loads: bool) -> SimpleNamespace:
return SimpleNamespace(
kv_connector_metadata=object(),
finished_req_ids=set(),
has_sync_kv_loads=has_sync_kv_loads,
)
@pytest.mark.parametrize("has_sync_kv_loads", [False, True])
def test_load_start_phase(
monkeypatch: pytest.MonkeyPatch,
has_sync_kv_loads: bool,
):
events: list[str] = []
connector = _make_connector(monkeypatch, events)
output = _scheduler_output(has_sync_kv_loads)
connector.pre_forward(output) # type: ignore[arg-type]
assert events == (
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
)
connector.post_forward(set())
assert events == ["handle", "bind", "start", "wait", "clear"]
def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
events: list[str] = []
connector = _make_connector(monkeypatch, events)
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
assert events == ["handle", "bind", "start", "clear"]