[BugFix] Make PD work with Ray (#21072)
Signed-off-by: Kourosh Hakhamaneshi <kourosh@anyscale.com>
This commit is contained in:
@@ -1,13 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.config import KVTransferConfig
|
||||
@@ -15,11 +16,32 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector import (
|
||||
KVConnectorRole, NixlAgentMetadata, NixlConnector, NixlConnectorMetadata,
|
||||
NixlConnectorWorker)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.mocks.mock_nixl_connector import FakeNixlWrapper
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from .utils import create_request, create_scheduler, create_vllm_config
|
||||
|
||||
|
||||
def _make_stub_pkg() -> str:
|
||||
"""Return a directory that makes
|
||||
`from nixl._api import nixl_agent` resolve to our FakeNixlWrapper."""
|
||||
td = tempfile.mkdtemp()
|
||||
pkg_root = os.path.join(td, "nixl", "_api")
|
||||
os.makedirs(pkg_root, exist_ok=True)
|
||||
|
||||
stub = textwrap.dedent("""\
|
||||
# Forward the real FakeNixlWrapper that the driver already defined.
|
||||
print("In fake package")
|
||||
from vllm.mocks.mock_nixl_connector import FakeNixlWrapper as nixl_agent
|
||||
""")
|
||||
with open(os.path.join(pkg_root, "__init__.py"), "w") as f:
|
||||
f.write(stub)
|
||||
|
||||
# touch parent package
|
||||
open(os.path.join(td, "nixl", "__init__.py"), "w").close()
|
||||
return td
|
||||
|
||||
|
||||
def test_basic_interface():
|
||||
"""Unit test for basic NixlConnector interface functionality."""
|
||||
|
||||
@@ -87,77 +109,6 @@ def test_prompt_less_than_block_size():
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 1
|
||||
|
||||
|
||||
class FakeNixlWrapper:
|
||||
"""Mock implementation of NixlWrapper for testing.
|
||||
|
||||
We don't inherit from nixl._api.nixl_agent because nixl may not be
|
||||
installed.
|
||||
"""
|
||||
|
||||
AGENT_METADATA = b"fake_agent_metadata"
|
||||
REMOTE_AGENT_NAME = "remote_agent"
|
||||
|
||||
def __init__(self, agent_name: str, *args, **kwargs):
|
||||
self._cycles_before_xfer_done = 0
|
||||
self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(
|
||||
lambda: 0)
|
||||
|
||||
def get_reg_descs(self, caches_data, memory_type: str) -> list:
|
||||
return [str(uuid.uuid4()) for _ in caches_data]
|
||||
|
||||
def register_memory(self, descs) -> None:
|
||||
pass
|
||||
|
||||
def get_xfer_descs(self, blocks_data, memory_type: str) -> list:
|
||||
return [str(uuid.uuid4()) for _ in blocks_data]
|
||||
|
||||
def prep_xfer_dlist(self, agent_name: str, descs: list) -> int:
|
||||
return uuid.uuid4().int
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
return self.AGENT_METADATA
|
||||
|
||||
def add_remote_agent(self, agent_metadata: bytes) -> str:
|
||||
return self.REMOTE_AGENT_NAME
|
||||
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
# Used to collect done_sending, which we don't test yet.
|
||||
return {}
|
||||
|
||||
def check_xfer_state(self, handle: int) -> str:
|
||||
if self._check_xfer_state_cycles[
|
||||
handle] >= self._cycles_before_xfer_done:
|
||||
return "DONE"
|
||||
self._check_xfer_state_cycles[handle] += 1
|
||||
return "PROC"
|
||||
|
||||
def release_xfer_handle(self, handle: int) -> None:
|
||||
pass
|
||||
|
||||
def send_notif(self, agent_name: str, notif_msg: bytes) -> None:
|
||||
pass
|
||||
|
||||
def make_prepped_xfer(self,
|
||||
xfer_type: str,
|
||||
local_xfer_side_handle: int,
|
||||
local_block_descs_ids: list[int],
|
||||
remote_xfer_side_handle: int,
|
||||
remote_block_descs_ids: list[int],
|
||||
notif_msg: Optional[bytes] = None) -> int:
|
||||
return uuid.uuid4().int
|
||||
|
||||
def transfer(self, handle: int) -> str:
|
||||
return "PROC"
|
||||
|
||||
############################################################
|
||||
# Follow are for changing the behavior during testing.
|
||||
############################################################
|
||||
|
||||
def set_cycles_before_xfer_done(self, cycles: int):
|
||||
"""Set the number of cycles before a transfer is considered done."""
|
||||
self._cycles_before_xfer_done = cycles
|
||||
|
||||
|
||||
class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
|
||||
REMOTE_ENGINE_ID = "remote_engine"
|
||||
@@ -378,10 +329,14 @@ class TestNixlHandshake:
|
||||
raise TimeoutError("Took too long to complete async handshake.")
|
||||
|
||||
|
||||
# NOTE: resource cleanup in mp backend is a bit finicky, so the order in which
|
||||
# we put here is important. First run ray, it will clean up the resources, then
|
||||
# the rest of the tests.
|
||||
@pytest.mark.parametrize("distributed_executor_backend", ["ray", None])
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl_connector.NixlWrapper",
|
||||
FakeNixlWrapper)
|
||||
def test_abort_timeout_on_prefiller(monkeypatch):
|
||||
def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
|
||||
"""
|
||||
Test lifecycle of an aborted Remote Prefill request hitting the timeout.
|
||||
-----> P
|
||||
@@ -399,11 +354,23 @@ def test_abort_timeout_on_prefiller(monkeypatch):
|
||||
timeout = 6
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_NIXL_ABORT_REQUEST_TIMEOUT", str(timeout))
|
||||
|
||||
# Build runtime_env only if we’re using Ray
|
||||
if distributed_executor_backend == "ray":
|
||||
runtime_env = {
|
||||
"working_dir": _make_stub_pkg(), # ship stub package
|
||||
"env_vars": {
|
||||
"VLLM_NIXL_ABORT_REQUEST_TIMEOUT": str(timeout),
|
||||
},
|
||||
}
|
||||
ray.init(runtime_env=runtime_env)
|
||||
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
)
|
||||
remote_prefill_opts = {
|
||||
"do_remote_decode": True,
|
||||
|
||||
+9
-28
@@ -1,28 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from typing import Optional
|
||||
|
||||
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
|
||||
|
||||
class DummyMultiprocExecutor(MultiprocExecutor):
|
||||
|
||||
def __init__(self, output_rank, world_size):
|
||||
# Manually initialize minimal required fields
|
||||
self.output_rank = output_rank
|
||||
self.world_size = world_size
|
||||
self._send_remaining_count = defaultdict[str,
|
||||
int](lambda: self.world_size)
|
||||
self._recv_remaining_count = defaultdict[str,
|
||||
int](lambda: self.world_size)
|
||||
self.io_thread_pool = None
|
||||
self.shutdown_event = threading.Event()
|
||||
|
||||
|
||||
class DummyModelRunnerOutput(ModelRunnerOutput):
|
||||
|
||||
def __init__(self,
|
||||
@@ -33,14 +17,14 @@ class DummyModelRunnerOutput(ModelRunnerOutput):
|
||||
|
||||
|
||||
def test_aggregate_workers_output():
|
||||
executor = DummyMultiprocExecutor(output_rank=0, world_size=2)
|
||||
aggregator = KVOutputAggregator(world_size=2)
|
||||
|
||||
output1 = DummyModelRunnerOutput(finished_sending={'req1'},
|
||||
finished_recving={'req2'})
|
||||
output2 = DummyModelRunnerOutput(finished_sending=None,
|
||||
finished_recving=None)
|
||||
|
||||
aggregated = executor._aggregate_workers_output([output1, output2])
|
||||
aggregated = aggregator.aggregate([output1, output2])
|
||||
|
||||
assert aggregated is output1
|
||||
assert aggregated.finished_sending is None
|
||||
@@ -51,7 +35,7 @@ def test_aggregate_workers_output():
|
||||
output2 = DummyModelRunnerOutput(finished_sending={'req1'},
|
||||
finished_recving=None)
|
||||
|
||||
aggregated = executor._aggregate_workers_output([output1, output2])
|
||||
aggregated = aggregator.aggregate([output1, output2])
|
||||
|
||||
assert aggregated is output1
|
||||
assert aggregated.finished_sending == {'req1'}
|
||||
@@ -62,7 +46,7 @@ def test_aggregate_workers_output():
|
||||
output2 = DummyModelRunnerOutput(finished_sending={'req1'},
|
||||
finished_recving={'req2'})
|
||||
|
||||
aggregated = executor._aggregate_workers_output([output1, output2])
|
||||
aggregated = aggregator.aggregate([output1, output2])
|
||||
|
||||
assert aggregated is output1
|
||||
assert aggregated.finished_sending is None
|
||||
@@ -70,12 +54,11 @@ def test_aggregate_workers_output():
|
||||
|
||||
|
||||
def test_async_aggregate_workers_output():
|
||||
executor = DummyMultiprocExecutor(output_rank=0, world_size=2)
|
||||
aggregator = KVOutputAggregator(world_size=2)
|
||||
|
||||
future1: Future[DummyModelRunnerOutput] = Future()
|
||||
future2: Future[DummyModelRunnerOutput] = Future()
|
||||
result_future = executor._async_aggregate_workers_output(
|
||||
[future1, future2])
|
||||
result_future = aggregator.async_aggregate([future1, future2])
|
||||
|
||||
output1 = DummyModelRunnerOutput(finished_sending={'req1'},
|
||||
finished_recving={'req2'})
|
||||
@@ -92,8 +75,7 @@ def test_async_aggregate_workers_output():
|
||||
|
||||
future1 = Future()
|
||||
future2 = Future()
|
||||
result_future = executor._async_aggregate_workers_output(
|
||||
[future1, future2])
|
||||
result_future = aggregator.async_aggregate([future1, future2])
|
||||
|
||||
output1 = DummyModelRunnerOutput(finished_sending=None,
|
||||
finished_recving=None)
|
||||
@@ -110,8 +92,7 @@ def test_async_aggregate_workers_output():
|
||||
|
||||
future1 = Future()
|
||||
future2 = Future()
|
||||
result_future = executor._async_aggregate_workers_output(
|
||||
[future1, future2])
|
||||
result_future = aggregator.async_aggregate([future1, future2])
|
||||
|
||||
output1 = DummyModelRunnerOutput(finished_sending=None,
|
||||
finished_recving=None)
|
||||
Reference in New Issue
Block a user