[Nixl][PD] Lease renewal TTL KV blocks on P (#41383)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -262,9 +262,11 @@ def test_multi_example_connector_consistency():
|
||||
|
||||
events = get_connector_events()
|
||||
# First event is set_xfer_handshake_metadata from initialization, then
|
||||
# on_new_request when the request is enqueued, then
|
||||
# get_num_new_matched_tokens and update_state_after_alloc from generate().
|
||||
assert events["storage1-SCHEDULER"][:4] == [
|
||||
assert events["storage1-SCHEDULER"][:5] == [
|
||||
"set_xfer_handshake_metadata",
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"build_connector_meta",
|
||||
@@ -281,8 +283,9 @@ def test_multi_example_connector_consistency():
|
||||
"wait_for_layer_load",
|
||||
"save_kv_layer",
|
||||
]
|
||||
assert events["storage2-SCHEDULER"][:4] == [
|
||||
assert events["storage2-SCHEDULER"][:5] == [
|
||||
"set_xfer_handshake_metadata",
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"build_connector_meta",
|
||||
@@ -310,12 +313,14 @@ def test_multi_example_connector_consistency():
|
||||
# connector so update_state_after_alloc will be with allocated blocks
|
||||
# on that one but with zero blocks for others (first nonzero match is
|
||||
# chosen).
|
||||
assert events["storage1-SCHEDULER"][:3] == [
|
||||
assert events["storage1-SCHEDULER"][:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 96",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert events["storage2-SCHEDULER"][:3] == [
|
||||
assert events["storage2-SCHEDULER"][:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"build_connector_meta",
|
||||
@@ -336,12 +341,14 @@ def test_multi_example_connector_consistency():
|
||||
# return 0 from the first connector, but the second connector should have
|
||||
# a hit, so update_state_after_alloc will only be called with allocated
|
||||
# blocks for the second connector.
|
||||
assert events["storage1-SCHEDULER"][:3] == [
|
||||
assert events["storage1-SCHEDULER"][:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert events["storage2-SCHEDULER"][:3] == [
|
||||
assert events["storage2-SCHEDULER"][:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 96",
|
||||
"build_connector_meta",
|
||||
|
||||
@@ -1374,9 +1374,11 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
|
||||
| {eventually free blocks}
|
||||
"""
|
||||
model_name = "Qwen/Qwen3-0.6B"
|
||||
timeout = 6
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"kv_lease_duration": timeout},
|
||||
)
|
||||
llm_kwargs = {
|
||||
"model": model_name,
|
||||
@@ -1386,9 +1388,7 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
|
||||
"distributed_executor_backend": distributed_executor_backend,
|
||||
}
|
||||
|
||||
timeout = 6
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_NIXL_ABORT_REQUEST_TIMEOUT", str(timeout))
|
||||
|
||||
def run_test_and_cleanup():
|
||||
llm = LLM(**llm_kwargs)
|
||||
@@ -1403,8 +1403,6 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
|
||||
runtime_env = {
|
||||
"working_dir": working_dir, # ship fake nixl package
|
||||
"env_vars": {
|
||||
"VLLM_NIXL_ABORT_REQUEST_TIMEOUT": str(timeout),
|
||||
# TODO: for ray to carry over, remove once we set
|
||||
"NIXL_TELEMETRY_ENABLE": "1",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the scheduler-driven heartbeat / lease-renewal system."""
|
||||
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import create_request, make_nixl_scheduler
|
||||
|
||||
_ENGINE_A = "my-engine-id"
|
||||
|
||||
|
||||
def _sched(kv_lease_duration: int = 30):
|
||||
return make_nixl_scheduler(heartbeat=True, kv_lease_duration=kv_lease_duration)
|
||||
|
||||
|
||||
def _req(request_id: int = 1):
|
||||
return create_request(request_id=request_id, do_remote_prefill=True)
|
||||
|
||||
|
||||
def _worker_stub():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
w = object.__new__(NixlConnectorWorker)
|
||||
w._reqs_to_send = {}
|
||||
w._lease_extension = 20
|
||||
return w
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: on_new_request
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_on_new_request_tracks_and_groups():
|
||||
"""Add two reqs to same engine, one to another; verify grouping."""
|
||||
s = _sched()
|
||||
s.on_new_request(_req(1))
|
||||
s.on_new_request(_req(2))
|
||||
|
||||
assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-1", "prefill-2"}
|
||||
info = s._heartbeat_by_engine[_ENGINE_A]
|
||||
assert (info.host, info.port, info.tp_size) == ("my-host", 1234, 1)
|
||||
assert s._heartbeat_req_engine["id-1"] == (_ENGINE_A, "prefill-1")
|
||||
|
||||
# Different engine.
|
||||
r3 = _req(3)
|
||||
r3.kv_transfer_params["remote_engine_id"] = "engine-b"
|
||||
s.on_new_request(r3)
|
||||
assert len(s._heartbeat_by_engine) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"make_req",
|
||||
[
|
||||
lambda: create_request(request_id=2, do_remote_decode=True),
|
||||
lambda: create_request(request_id=3), # no kv_transfer_params
|
||||
],
|
||||
ids=["decode", "plain"],
|
||||
)
|
||||
def test_on_new_request_ignores_non_prefill(make_req):
|
||||
s = _sched()
|
||||
s.on_new_request(make_req())
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: _stop_heartbeat
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_stop_heartbeat_partial_and_full():
|
||||
"""Stop one of two reqs on same engine, then stop the other."""
|
||||
s = _sched()
|
||||
s.on_new_request(_req(1))
|
||||
s.on_new_request(_req(2))
|
||||
|
||||
s._stop_heartbeat("id-1")
|
||||
assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-2"}
|
||||
assert "id-1" not in s._heartbeat_req_engine
|
||||
|
||||
s._stop_heartbeat("id-2")
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
assert len(s._heartbeat_req_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: build_connector_meta throttling
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_build_connector_meta_heartbeat_throttling():
|
||||
# kv_lease_duration=30 => _heartbeat_interval = 30 // 6 = 5
|
||||
s = _sched(kv_lease_duration=30)
|
||||
s.on_new_request(_req(1))
|
||||
|
||||
# Ensure the first call triggers by placing last_heartbeat far in the past.
|
||||
s._last_heartbeat_time = time.perf_counter() - 10
|
||||
meta1 = s.build_connector_meta(MagicMock())
|
||||
assert _ENGINE_A in meta1.heartbeat_by_engine
|
||||
|
||||
# Immediate second call is throttled (< 5s since last).
|
||||
meta2 = s.build_connector_meta(MagicMock())
|
||||
assert len(meta2.heartbeat_by_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: cleanup paths (update_connector_output / request_finished)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_update_connector_output_stops_heartbeat():
|
||||
s = _sched()
|
||||
s.on_new_request(_req(1))
|
||||
|
||||
s.update_connector_output(
|
||||
KVConnectorOutput(
|
||||
finished_sending=None,
|
||||
finished_recving={"id-1"},
|
||||
invalid_block_ids=set(),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
assert len(s._heartbeat_req_engine) == 0
|
||||
|
||||
|
||||
def test_request_finished_stops_heartbeat():
|
||||
s = _sched()
|
||||
r = _req(1)
|
||||
s.on_new_request(r)
|
||||
|
||||
# Simulate update_state_after_alloc having consumed do_remote_prefill.
|
||||
r.kv_transfer_params["do_remote_prefill"] = False
|
||||
s.request_finished(r, block_ids=())
|
||||
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
assert len(s._heartbeat_req_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Worker: _handle_heartbeat
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_handle_heartbeat():
|
||||
w = _worker_stub()
|
||||
far_future = time.perf_counter() + 99999
|
||||
w._reqs_to_send = {"req-a": 100.0, "req-b": far_future}
|
||||
|
||||
before = time.perf_counter()
|
||||
w._handle_heartbeat("req-a,req-b,req-unknown")
|
||||
|
||||
# req-a: pushed forward to ~now+20.
|
||||
assert w._reqs_to_send["req-a"] >= before + 20
|
||||
# req-b: already far out, max() keeps it.
|
||||
assert w._reqs_to_send["req-b"] >= far_future
|
||||
# req-unknown: not added.
|
||||
assert "req-unknown" not in w._reqs_to_send
|
||||
@@ -223,6 +223,7 @@ def create_request(
|
||||
remote_block_ids=list(range(num_remote_blocks)),
|
||||
remote_host="my-host",
|
||||
remote_port=1234,
|
||||
tp_size=1,
|
||||
)
|
||||
|
||||
max_tokens = 1 if do_remote_decode else max_tokens
|
||||
@@ -482,10 +483,16 @@ def make_kv_cache_config(
|
||||
)
|
||||
|
||||
|
||||
def make_nixl_scheduler(has_mamba: bool = False, is_hma_required: bool = False):
|
||||
def make_nixl_scheduler(
|
||||
has_mamba: bool = False,
|
||||
is_hma_required: bool = False,
|
||||
heartbeat: bool = False,
|
||||
kv_lease_duration: int = 30,
|
||||
):
|
||||
"""Create a NixlConnectorScheduler via __new__ (skipping __init__).
|
||||
|
||||
Only sets the two flags needed by the N-1 prefill logic.
|
||||
Only sets the flags needed by the tests. When *heartbeat=True* the
|
||||
scheduler-side heartbeat bookkeeping fields are also initialised.
|
||||
"""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
|
||||
NixlConnectorScheduler,
|
||||
@@ -494,4 +501,23 @@ def make_nixl_scheduler(has_mamba: bool = False, is_hma_required: bool = False):
|
||||
sched = object.__new__(NixlConnectorScheduler)
|
||||
sched._has_mamba = has_mamba
|
||||
sched._is_hma_required = is_hma_required
|
||||
|
||||
if heartbeat:
|
||||
sched._heartbeat_by_engine = {}
|
||||
sched._heartbeat_req_engine = {}
|
||||
sched._last_heartbeat_time = 0.0
|
||||
sched._kv_lease_duration = kv_lease_duration
|
||||
sched._heartbeat_interval = kv_lease_duration // 6
|
||||
# Fields touched by build_connector_meta / request_finished:
|
||||
sched._reqs_need_recv = {}
|
||||
sched._reqs_need_send = {}
|
||||
sched._reqs_in_batch = set()
|
||||
sched._reqs_not_processed = set()
|
||||
sched._reqs_need_save = {}
|
||||
sched.use_host_buffer = False
|
||||
sched.engine_id = "test-engine"
|
||||
sched.side_channel_host = "localhost"
|
||||
sched.side_channel_port = 5555
|
||||
sched.blocks_per_sw = []
|
||||
sched.is_bidirectional_kv_xfer_enabled = False
|
||||
return sched
|
||||
|
||||
Reference in New Issue
Block a user