[Core][KV Connector] Start async KV loads after the forward launch when no sync loads are scheduled (#53333)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1975,6 +1975,47 @@ def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
|
||||
return initial_ecos
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("load_modes", "expected_has_sync_loads"),
|
||||
[
|
||||
((True,), False),
|
||||
((False,), True),
|
||||
((True, False), True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_has_sync_kv_loads(
|
||||
load_modes: tuple[bool, ...],
|
||||
expected_has_sync_loads: bool,
|
||||
tmp_path,
|
||||
):
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
skip_tokenizer_init=True,
|
||||
use_kv_connector=mock_kv(matched_tokens=block_size, is_async=False),
|
||||
block_size=block_size,
|
||||
)
|
||||
requests = create_requests(
|
||||
num_requests=len(load_modes),
|
||||
num_tokens=block_size * 2,
|
||||
block_size=block_size,
|
||||
)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.connector.get_num_new_matched_tokens = Mock(
|
||||
side_effect=[(block_size, is_async) for is_async in load_modes]
|
||||
)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.has_sync_kv_loads is expected_has_sync_loads
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""
|
||||
|
||||
@@ -2960,7 +2960,7 @@ def test_putting_consumer_queues_decode_save():
|
||||
|
||||
with patch.object(torch.cuda, "Event") as event_cls:
|
||||
event = event_cls.return_value
|
||||
w.get_finished(set(), meta)
|
||||
w.wait_for_save(meta)
|
||||
|
||||
event.record.assert_called_once_with()
|
||||
send_thread.add_request.assert_called_once_with(req)
|
||||
|
||||
@@ -294,18 +294,23 @@ def test_multi_example_connector_consistency():
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
# First three events are from initialization (register_kv_caches,
|
||||
# set_host_xfer_buffer_ops, get_handshake_metadata), then generate() events.
|
||||
assert events["storage1-WORKER"][:8] == [
|
||||
# First three events are from initialization. During generate(), layer hooks
|
||||
# run before the deferred load is started after the forward pass.
|
||||
expected_worker_prefix = [
|
||||
"register_kv_caches",
|
||||
"set_host_xfer_buffer_ops",
|
||||
"get_handshake_metadata",
|
||||
"handle_preemptions",
|
||||
"bind_connector_metadata",
|
||||
"start_load_kv",
|
||||
"wait_for_layer_load",
|
||||
"save_kv_layer",
|
||||
]
|
||||
for connector_name in ("storage1-WORKER", "storage2-WORKER"):
|
||||
worker_events = events[connector_name]
|
||||
assert worker_events[:7] == expected_worker_prefix
|
||||
assert worker_events.index("start_load_kv") > worker_events.index(
|
||||
"save_kv_layer"
|
||||
)
|
||||
assert storage2_scheduler_events[:7] == [
|
||||
"bind_gpu_block_pool",
|
||||
"get_finished_count",
|
||||
@@ -315,17 +320,6 @@ def test_multi_example_connector_consistency():
|
||||
"update_state_after_alloc num_blocks=[7] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert events["storage2-WORKER"][:8] == [
|
||||
"register_kv_caches",
|
||||
"set_host_xfer_buffer_ops",
|
||||
"get_handshake_metadata",
|
||||
"handle_preemptions",
|
||||
"bind_connector_metadata",
|
||||
"start_load_kv",
|
||||
"wait_for_layer_load",
|
||||
"save_kv_layer",
|
||||
]
|
||||
|
||||
# Reset prefix cache or else we'll just get the tokens back from there.
|
||||
llm.reset_prefix_cache()
|
||||
|
||||
|
||||
@@ -213,7 +213,7 @@ def _multi_connector_config(swa_enabled: bool = False):
|
||||
def test_nixl_wins_load_over_cpu_offload():
|
||||
"""When NixlConnector (index 0) has matched tokens from a remote prefill, it should
|
||||
win the load: Nixl metadata tracks the recv while CPU offload metadata has no load
|
||||
scheduled."""
|
||||
scheduled."""
|
||||
vllm_config, kv_cache_config = _multi_connector_config()
|
||||
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
|
||||
mc = scheduler.connector
|
||||
@@ -227,6 +227,9 @@ def test_nixl_wins_load_over_cpu_offload():
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
sched_out = scheduler.schedule()
|
||||
# The remote-prefill load is async, so the step has no sync KV loads
|
||||
# and the worker will start the load after the forward.
|
||||
assert not sched_out.has_sync_kv_loads
|
||||
|
||||
assert mc._requests_to_connector[request.request_id] == 0
|
||||
|
||||
|
||||
@@ -152,8 +152,8 @@ class _RecordingBackend:
|
||||
self.calls.append({"is_store": is_store, "wait_event": wait_event})
|
||||
|
||||
|
||||
def test_get_finished_passes_wait_event_for_store_only():
|
||||
"""get_finished gates stores on a compute-done event but not loads."""
|
||||
def test_transfer_hooks_pass_wait_event_for_store_only():
|
||||
"""wait_for_save gates stores on a compute-done event; start_load_kv does not."""
|
||||
worker = SimpleCPUOffloadWorker(
|
||||
vllm_config=None, kv_cache_config=None, cpu_capacity_bytes=0
|
||||
)
|
||||
@@ -168,7 +168,8 @@ def test_get_finished_passes_wait_event_for_store_only():
|
||||
store_cpu_blocks=[1],
|
||||
)
|
||||
|
||||
worker.get_finished(set())
|
||||
worker.start_load_kv()
|
||||
worker.wait_for_save()
|
||||
|
||||
store_calls = [c for c in recording.calls if c["is_store"]]
|
||||
load_calls = [c for c in recording.calls if not c["is_store"]]
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.v1.worker.gpu.kv_connector as kv_connector_module
|
||||
from vllm.config import KVTransferConfig
|
||||
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
|
||||
|
||||
|
||||
def _make_connector(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
events: list[str],
|
||||
) -> ActiveKVConnector:
|
||||
backend = Mock()
|
||||
backend.handle_preemptions.side_effect = lambda _: events.append("handle")
|
||||
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
|
||||
backend.start_load_kv.side_effect = lambda _: events.append("start")
|
||||
backend.wait_for_save.side_effect = lambda: events.append("wait")
|
||||
backend.get_finished.side_effect = lambda _: (set(), set())
|
||||
backend.get_block_ids_with_load_errors.return_value = set()
|
||||
backend.get_kv_connector_stats.return_value = None
|
||||
backend.get_kv_connector_kv_cache_events.return_value = None
|
||||
backend.build_connector_worker_meta.return_value = None
|
||||
backend.clear_connector_metadata.side_effect = lambda: events.append("clear")
|
||||
monkeypatch.setattr(kv_connector_module, "get_kv_transfer_group", lambda: backend)
|
||||
monkeypatch.setattr(
|
||||
kv_connector_module, "is_forward_context_available", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(kv_connector_module, "get_forward_context", object)
|
||||
|
||||
kv_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_consumer",
|
||||
kv_buffer_device="cpu",
|
||||
)
|
||||
connector = ActiveKVConnector( # type: ignore[arg-type]
|
||||
SimpleNamespace(kv_transfer_config=kv_config), {}
|
||||
)
|
||||
events.clear()
|
||||
return connector
|
||||
|
||||
|
||||
def _scheduler_output(has_sync_kv_loads: bool) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
kv_connector_metadata=object(),
|
||||
finished_req_ids=set(),
|
||||
has_sync_kv_loads=has_sync_kv_loads,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_sync_kv_loads", [False, True])
|
||||
def test_load_start_phase(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
has_sync_kv_loads: bool,
|
||||
):
|
||||
events: list[str] = []
|
||||
connector = _make_connector(monkeypatch, events)
|
||||
output = _scheduler_output(has_sync_kv_loads)
|
||||
|
||||
connector.pre_forward(output) # type: ignore[arg-type]
|
||||
assert events == (
|
||||
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
|
||||
)
|
||||
|
||||
connector.post_forward(set())
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
|
||||
|
||||
def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
|
||||
events: list[str] = []
|
||||
connector = _make_connector(monkeypatch, events)
|
||||
|
||||
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
|
||||
|
||||
assert events == ["handle", "bind", "start", "clear"]
|
||||
Reference in New Issue
Block a user