|
|
|
@@ -24,13 +24,13 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
|
|
|
|
TpKVTopology,
|
|
|
|
|
get_current_attn_backend,
|
|
|
|
|
)
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1 import nixl_connector
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1 import nixl
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
|
|
|
|
|
MultiKVConnectorStats,
|
|
|
|
|
)
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector import (
|
|
|
|
|
KVConnectorRole,
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
|
|
|
|
NixlAgentMetadata,
|
|
|
|
|
NixlConnector,
|
|
|
|
|
NixlConnectorMetadata,
|
|
|
|
@@ -38,6 +38,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector import (
|
|
|
|
|
NixlConnectorWorker,
|
|
|
|
|
NixlHandshakePayload,
|
|
|
|
|
NixlKVConnectorStats,
|
|
|
|
|
)
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
|
|
|
|
compute_nixl_compatibility_hash,
|
|
|
|
|
)
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_transfer_state import (
|
|
|
|
@@ -320,7 +322,7 @@ def test_prompt_less_than_block_size():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_kv_transfer_handshake(dist_init):
|
|
|
|
@@ -534,7 +536,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
|
|
|
|
|
|
|
|
|
class TestNixlHandshake:
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_multi_xfer_one_engine(
|
|
|
|
@@ -621,7 +623,7 @@ class TestNixlHandshake:
|
|
|
|
|
connector.clear_connector_metadata()
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
@@ -691,7 +693,7 @@ class TestNixlHandshake:
|
|
|
|
|
raise TimeoutError("Took too long to complete async handshake.")
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
@pytest.mark.parametrize("local_tp_size", [1, 2])
|
|
|
|
@@ -703,7 +705,7 @@ class TestNixlHandshake:
|
|
|
|
|
remote configurations.
|
|
|
|
|
"""
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.get_tensor_model_parallel_world_size",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size",
|
|
|
|
|
lambda: local_tp_size,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
@@ -760,7 +762,7 @@ class TestNixlHandshake:
|
|
|
|
|
check_handshake(6)
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_prefill_tp_size_greater_than_decode_tp_size_mla(
|
|
|
|
@@ -863,7 +865,7 @@ class TestNixlHandshake:
|
|
|
|
|
assert req_id not in conn_p1.connector_worker._reqs_to_process
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_concurrent_load_kv(
|
|
|
|
@@ -928,7 +930,7 @@ class TestNixlHandshake:
|
|
|
|
|
raise TimeoutError("Took too long to complete async handshake.")
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_handshake_fails_on_kv_cache_layout_mismatch(
|
|
|
|
@@ -943,7 +945,7 @@ class TestNixlHandshake:
|
|
|
|
|
# Mock TP world size to 2 to force heterogeneous TP when
|
|
|
|
|
# remote_tp_size=1
|
|
|
|
|
with patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.get_tensor_model_parallel_world_size", # noqa: E501
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size", # noqa: E501
|
|
|
|
|
return_value=2,
|
|
|
|
|
):
|
|
|
|
|
# Initialize connector and worker (with fake NIXL wrapper)
|
|
|
|
@@ -982,7 +984,7 @@ class TestNixlHandshake:
|
|
|
|
|
worker.add_remote_agent(meta, remote_tp_size=1)
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_handshake_succeed_on_kv_cache_layout_mismatch_with_experimental(
|
|
|
|
@@ -997,7 +999,7 @@ class TestNixlHandshake:
|
|
|
|
|
# Mock TP world size to 2 to force heterogeneous TP when
|
|
|
|
|
# remote_tp_size=1
|
|
|
|
|
with patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.get_tensor_model_parallel_world_size", # noqa: E501
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size", # noqa: E501
|
|
|
|
|
return_value=2,
|
|
|
|
|
):
|
|
|
|
|
# Initialize connector and worker (with fake NIXL wrapper)
|
|
|
|
@@ -1042,7 +1044,7 @@ class TestNixlHandshake:
|
|
|
|
|
# we put here is important. First run ray, it will clean up the resources, then
|
|
|
|
|
# the rest of the tests.
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_kv_connector_stats(default_vllm_config, dist_init):
|
|
|
|
@@ -1227,8 +1229,8 @@ def test_multi_kv_connector_stats_aggregation():
|
|
|
|
|
worker_patterns = [(2, 1), (3, 0), (0, 5)] # (Nixl, Foo)
|
|
|
|
|
|
|
|
|
|
worker_outputs: list[ModelRunnerOutput] = []
|
|
|
|
|
for i, (nixl, foo) in enumerate(worker_patterns):
|
|
|
|
|
stats = make_multi_stats(nixl, foo)
|
|
|
|
|
for i, (nixl_count, foo) in enumerate(worker_patterns):
|
|
|
|
|
stats = make_multi_stats(nixl_count, foo)
|
|
|
|
|
output = ModelRunnerOutput(
|
|
|
|
|
req_ids=[f"req_{i}"],
|
|
|
|
|
req_id_to_index={f"req_{i}": 0},
|
|
|
|
@@ -1256,7 +1258,7 @@ def test_multi_kv_connector_stats_aggregation():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_scheduler_kv_connector_stats_aggregation():
|
|
|
|
@@ -1324,7 +1326,7 @@ def test_scheduler_kv_connector_stats_aggregation():
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("distributed_executor_backend", ["ray", None])
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
|
|
|
|
@@ -1513,13 +1515,14 @@ def test_register_kv_caches(
|
|
|
|
|
|
|
|
|
|
backend_cls = TritonAttentionBackend
|
|
|
|
|
|
|
|
|
|
nixl_module = "vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector"
|
|
|
|
|
nixl_worker = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker"
|
|
|
|
|
nixl_connector = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector"
|
|
|
|
|
with (
|
|
|
|
|
patch(f"{nixl_module}.NixlWrapper") as mock_nixl_wrapper,
|
|
|
|
|
patch(f"{nixl_module}.threading.Event"),
|
|
|
|
|
patch(f"{nixl_module}.threading.Thread") as mock_thread,
|
|
|
|
|
patch(f"{nixl_module}.get_current_attn_backend") as mock_get_attn_backend,
|
|
|
|
|
patch(f"{nixl_module}.get_current_attn_backends") as mock_get_attn_backends,
|
|
|
|
|
patch(f"{nixl_worker}.NixlWrapper") as mock_nixl_wrapper,
|
|
|
|
|
patch(f"{nixl_worker}.threading.Event"),
|
|
|
|
|
patch(f"{nixl_worker}.threading.Thread") as mock_thread,
|
|
|
|
|
patch(f"{nixl_connector}.get_current_attn_backend") as mock_get_attn_backend,
|
|
|
|
|
patch(f"{nixl_worker}.get_current_attn_backends") as mock_get_attn_backends,
|
|
|
|
|
):
|
|
|
|
|
# Ensure get_attn_backend returns the correct value due to
|
|
|
|
|
# _cached_get_attn_backend returning the backend from previous
|
|
|
|
@@ -1754,28 +1757,26 @@ def test_kv_buffer_to_nixl_memory_types(
|
|
|
|
|
vllm_config = create_vllm_config()
|
|
|
|
|
# Override the default memory types in the config
|
|
|
|
|
vllm_config.kv_transfer_config.kv_buffer_device = kv_buffer_device
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector import (
|
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
|
|
|
|
_NIXL_SUPPORTED_DEVICE,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
_NIXL_SUPPORTED_DEVICE.update(FakePlatform.get_nixl_supported_devices())
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch("vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper"),
|
|
|
|
|
patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper"
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.threading.Event"
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.threading.Event"
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.threading.Thread"
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.threading.Thread"
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.current_platform",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.current_platform",
|
|
|
|
|
FakePlatform,
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector._NIXL_SUPPORTED_DEVICE",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils._NIXL_SUPPORTED_DEVICE",
|
|
|
|
|
_NIXL_SUPPORTED_DEVICE,
|
|
|
|
|
),
|
|
|
|
|
): # noqa: E501
|
|
|
|
@@ -1790,7 +1791,7 @@ def test_kv_buffer_to_nixl_memory_types(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
|
|
|
@@ -1855,7 +1856,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_aborted_request_removed_from_worker_in_batch(default_vllm_config, dist_init):
|
|
|
|
@@ -1975,7 +1976,7 @@ class FailingNixlWrapper(FakeNixlWrapper):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FailingNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
@@ -2065,10 +2066,10 @@ def test_transfer_failure_logging(
|
|
|
|
|
slot_mapping={},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Capture logs from the nixl_connector logger specifically
|
|
|
|
|
# Capture logs from the nixl.worker logger specifically
|
|
|
|
|
# vLLM loggers have propagate=False, so we need to capture directly
|
|
|
|
|
nixl_logger = logging.getLogger(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector"
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker"
|
|
|
|
|
)
|
|
|
|
|
captured_logs: list[logging.LogRecord] = []
|
|
|
|
|
|
|
|
|
@@ -2130,7 +2131,7 @@ def test_transfer_failure_logging(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FailingNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
|
|
|
@@ -2181,7 +2182,7 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FailingNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init):
|
|
|
|
@@ -2257,7 +2258,7 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init)
|
|
|
|
|
],
|
|
|
|
|
)
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_compatibility_hash_validation(
|
|
|
|
@@ -2328,7 +2329,7 @@ def test_compatibility_hash_validation(
|
|
|
|
|
elif "connector_version" in version_override:
|
|
|
|
|
stack.enter_context(
|
|
|
|
|
patch.object(
|
|
|
|
|
nixl_connector,
|
|
|
|
|
nixl.metadata,
|
|
|
|
|
"NIXL_CONNECTOR_VERSION",
|
|
|
|
|
version_override["connector_version"],
|
|
|
|
|
)
|
|
|
|
@@ -2365,7 +2366,7 @@ def test_compatibility_hash_validation(
|
|
|
|
|
# Patch zmq_ctx to return our mock socket
|
|
|
|
|
with (
|
|
|
|
|
patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"),
|
|
|
|
|
patch.object(nixl_connector, "zmq_ctx") as mock_zmq_ctx,
|
|
|
|
|
patch.object(nixl.worker, "zmq_ctx") as mock_zmq_ctx,
|
|
|
|
|
):
|
|
|
|
|
mock_zmq_ctx.return_value.__enter__.return_value = mock_socket
|
|
|
|
|
|
|
|
|
@@ -2399,7 +2400,7 @@ def test_compatibility_hash_validation(
|
|
|
|
|
],
|
|
|
|
|
)
|
|
|
|
|
@patch(
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
|
|
|
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
|
|
|
|
FakeNixlWrapper,
|
|
|
|
|
)
|
|
|
|
|
def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario):
|
|
|
|
@@ -2464,7 +2465,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
|
|
|
|
mock_socket.recv.return_value = msg_bytes
|
|
|
|
|
with (
|
|
|
|
|
patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"),
|
|
|
|
|
patch.object(nixl_connector, "zmq_ctx") as mock_zmq_ctx,
|
|
|
|
|
patch.object(nixl.worker, "zmq_ctx") as mock_zmq_ctx,
|
|
|
|
|
):
|
|
|
|
|
mock_zmq_ctx.return_value.__enter__.return_value = mock_socket
|
|
|
|
|
|
|
|
|
|