[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
@@ -51,7 +51,7 @@ def _traffic_loop(
|
||||
stop: threading.Event,
|
||||
finished: threading.Event,
|
||||
is_probe: bool = False,
|
||||
) -> list[tuple[float, float, int | None]]:
|
||||
) -> list[tuple[float, float, int | None, str | None]]:
|
||||
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
|
||||
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
|
||||
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
|
||||
@@ -60,15 +60,18 @@ def _traffic_loop(
|
||||
is_ready = False
|
||||
while not stop.is_set():
|
||||
request_start = time.perf_counter()
|
||||
phase = None
|
||||
try:
|
||||
response = requests.post(
|
||||
url, json=request_payload, headers=headers, timeout=120
|
||||
)
|
||||
status_code = response.status_code
|
||||
if is_probe and status_code == 200:
|
||||
phase = response.json()["phase"]
|
||||
except requests.exceptions.RequestException:
|
||||
status_code = None
|
||||
responses.append((request_start, time.perf_counter(), status_code))
|
||||
if status_code == 200:
|
||||
responses.append((request_start, time.perf_counter(), status_code, phase))
|
||||
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
|
||||
if not is_ready:
|
||||
ready.wait(timeout=120)
|
||||
is_ready = True
|
||||
@@ -78,14 +81,16 @@ def _traffic_loop(
|
||||
return responses
|
||||
|
||||
|
||||
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
|
||||
rejected = [end for _, end, status in responses if status == 503]
|
||||
if not rejected:
|
||||
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
|
||||
committing = [end for _, end, _, phase in responses if phase == "committing"]
|
||||
if not committing:
|
||||
return 0
|
||||
recovered = next(
|
||||
end for _, end, status in responses if status == 200 and end > rejected[-1]
|
||||
end
|
||||
for _, end, status, phase in responses
|
||||
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
|
||||
)
|
||||
return recovered - rejected[0]
|
||||
return recovered - committing[0]
|
||||
|
||||
|
||||
def _scale_with_traffic(
|
||||
@@ -124,16 +129,20 @@ def _scale_with_traffic(
|
||||
bad_statuses = {
|
||||
status
|
||||
for responses in [probe_result, *results]
|
||||
for _, _, status in responses
|
||||
for _, _, status, _ in responses
|
||||
if status not in (200, 503)
|
||||
}
|
||||
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
|
||||
probe_503 = [start for start, _, status in probe_result if status == 503]
|
||||
assert probe_503, "Scaling probe did not observe commit"
|
||||
probe_committing = [
|
||||
start for start, _, _, phase in probe_result if phase == "committing"
|
||||
]
|
||||
assert probe_committing, "Scaling probe did not observe commit"
|
||||
assert not results or any(
|
||||
status == 200 and start_time <= request_start and request_end < probe_503[0]
|
||||
status == 200
|
||||
and start_time <= request_start
|
||||
and request_end < probe_committing[0]
|
||||
for responses in results
|
||||
for request_start, request_end, status in responses
|
||||
for request_start, request_end, status, _ in responses
|
||||
), "No request completed successfully during preparation"
|
||||
|
||||
print(
|
||||
|
||||
@@ -4,17 +4,28 @@ import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AsyncExitStack
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import msgspec
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleCoordinator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine.core_client import DPAsyncMPClient
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
|
||||
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
# Number of data parallel ranks for external LB testing
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
|
||||
class DictStore:
|
||||
def __init__(self, values: dict[str, bytes]):
|
||||
self.values = values
|
||||
|
||||
def check(self, keys: list[str]) -> bool:
|
||||
return all(key in self.values for key in keys)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self.values[key]
|
||||
|
||||
def set(self, key: str, value: bytes) -> None:
|
||||
self.values[key] = value
|
||||
|
||||
|
||||
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_master_port=12345,
|
||||
_data_parallel_master_port_list=[12346, 12347],
|
||||
)
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
control_store, reconfig_store = DictStore({}), DictStore({})
|
||||
coordinator.reconfig_store_ref = reconfig_store
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
|
||||
)
|
||||
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
|
||||
|
||||
for store in (control_store, reconfig_store):
|
||||
# DictStore preserves insertion order: metadata must precede the epoch.
|
||||
assert list(store.values)[-1] == coordinator.key("current_epoch")
|
||||
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
|
||||
bootstrap
|
||||
)
|
||||
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
|
||||
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_get_reconfig_store", lambda store=store: store
|
||||
)
|
||||
assert coordinator.get_status()["requested_data_parallel_size"] == 3
|
||||
|
||||
|
||||
def test_external_elastic_ep_calculates_target_expert_redundancy():
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0)
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
|
||||
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("dp_rank", range(4))
|
||||
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
|
||||
"""Validate expert capacity on every rank before waiting for an epoch."""
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_rank=dp_rank,
|
||||
elastic_ep_max_dp_size=4,
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
_coord_store_port=12345,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
store = DictStore({})
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot retain all model experts"):
|
||||
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
|
||||
|
||||
assert not store.values
|
||||
assert coordinator.prepared_scale is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_elastic_ep_late_rank_observes_epoch_error():
|
||||
coordinator = ExternalElasticEPScaleCoordinator(
|
||||
cast("DPAsyncMPClient", SimpleNamespace())
|
||||
)
|
||||
epoch = "failed-epoch"
|
||||
store = DictStore(
|
||||
{
|
||||
coordinator.key("current_epoch"): epoch.encode(),
|
||||
coordinator.key(epoch, "error"): b"scale failed",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="scale failed"):
|
||||
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
|
||||
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing."""
|
||||
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
|
||||
def start_rank(
|
||||
self,
|
||||
rank: int,
|
||||
dp_size: int | None = None,
|
||||
elastic_ep_scale_up_launch: bool = False,
|
||||
) -> tuple[RemoteOpenAIServer, list[str]]:
|
||||
"""Start one externally managed DP rank."""
|
||||
dp_size = self.dp_size if dp_size is None else dp_size
|
||||
server_args = self.base_server_args.copy()
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank),
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
|
||||
),
|
||||
}
|
||||
if elastic_ep_scale_up_launch:
|
||||
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
|
||||
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
server_args,
|
||||
auto_port=False,
|
||||
env_dict=env_dict,
|
||||
)
|
||||
print(
|
||||
f"Server rank {rank} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
server_and_args = (server, server_args)
|
||||
self.servers.append(server_and_args)
|
||||
return server_and_args
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
"""Start all server instances for external LB mode."""
|
||||
"""Start all initial server instances for external LB mode."""
|
||||
start_errors: list[Exception] = []
|
||||
for rank in range(self.dp_size):
|
||||
# Create server args for this specific rank
|
||||
server_args = self.base_server_args.copy()
|
||||
|
||||
# Add external LB specific arguments
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(self.dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank), # Different port for each rank
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
# Use a thread to start each server to allow parallel initialization
|
||||
def start_server(r: int, sargs: list[str]):
|
||||
def start_server(r: int):
|
||||
try:
|
||||
# Start the server
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
sargs,
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
|
||||
),
|
||||
},
|
||||
)
|
||||
server.__enter__()
|
||||
print(
|
||||
f"Server rank {r} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
self.servers.append((server, sargs))
|
||||
self.start_rank(r)
|
||||
except Exception as e:
|
||||
print(f"Failed to start server rank {r}: {e}")
|
||||
raise
|
||||
start_errors.append(e)
|
||||
|
||||
thread = threading.Thread(target=start_server, args=(rank, server_args))
|
||||
thread = threading.Thread(target=start_server, args=(rank,))
|
||||
thread.start()
|
||||
|
||||
self.server_threads.append(thread)
|
||||
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
|
||||
for thread in self.server_threads:
|
||||
thread.join()
|
||||
|
||||
if start_errors:
|
||||
start_error = start_errors[0]
|
||||
raise RuntimeError("Failed to start external LB servers") from start_error
|
||||
|
||||
# Give servers additional time to fully initialize and coordinate
|
||||
time.sleep(2)
|
||||
|
||||
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
|
||||
assert parallel_config["enable_expert_parallel"] is expected
|
||||
|
||||
|
||||
def _send_scale_command(
|
||||
server: RemoteOpenAIServer, new_dp_size: int
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
server.url_for("scale_elastic_ep"),
|
||||
json={"new_data_parallel_size": new_dp_size},
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_scale_requests_to_start(
|
||||
scale_requests: list[Future[requests.Response]],
|
||||
timeout: float = 30,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for request in scale_requests:
|
||||
if request.done():
|
||||
response = request.result()
|
||||
pytest.fail(
|
||||
"Scale request finished before the new rank was launched: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
if all(request.running() for request in scale_requests):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
pytest.fail("Timed out waiting for all scale requests to start")
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=3)
|
||||
@pytest.mark.skipif(
|
||||
TP_SIZE != 1 or current_platform.device_count() < 3,
|
||||
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
|
||||
reason="Elastic EP scaling is not supported by the Rust frontend",
|
||||
)
|
||||
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
|
||||
server_args = [
|
||||
*default_server_args,
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--attention-backend",
|
||||
"TRITON_MLA",
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
"3",
|
||||
"--enable-eplb",
|
||||
"--eplb-config",
|
||||
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
|
||||
]
|
||||
server_manager = ExternalLBServerManager(
|
||||
ELASTIC_EP_MODEL_NAME,
|
||||
dp_size=2,
|
||||
api_server_count=1,
|
||||
base_server_args=server_args,
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
existing_servers = [server for server, _ in server_manager.servers]
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
scale_requests = [
|
||||
executor.submit(_send_scale_command, server, 3)
|
||||
for server in existing_servers
|
||||
]
|
||||
_wait_for_scale_requests_to_start(scale_requests)
|
||||
server_manager.start_rank(
|
||||
rank=2,
|
||||
dp_size=3,
|
||||
elastic_ep_scale_up_launch=True,
|
||||
)
|
||||
|
||||
for request in scale_requests:
|
||||
response = request.result(timeout=600)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
for server, _ in server_manager.servers:
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["data_parallel_size"] == 3
|
||||
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
|
||||
with server.get_client() as client:
|
||||
completion = client.completions.create(
|
||||
model=ELASTIC_EP_MODEL_NAME,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].finish_reason in ("length", "stop")
|
||||
|
||||
|
||||
def test_external_lb_server_info(server_manager):
|
||||
servers = server_manager.servers
|
||||
api_server_count = server_manager.api_server_count
|
||||
|
||||
_assert_expert_parallel_config(servers)
|
||||
|
||||
for i, (server, _) in enumerate(servers):
|
||||
print(f"Testing {i=}")
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
|
||||
False,
|
||||
),
|
||||
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
|
||||
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
|
||||
],
|
||||
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
|
||||
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
|
||||
) -> None:
|
||||
"""The DP master node holds a coordination store for both engine backends,
|
||||
so engines there pick world-group ports at bind time. Every other
|
||||
deployment keeps the pre-allocated ports and must not hold one."""
|
||||
so engines there pick world-group ports at bind time. Elastic EP also
|
||||
creates its coordination store before startup. The remaining cases
|
||||
below keep the pre-allocated ports and must not hold a store."""
|
||||
|
||||
class FakeManager:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
|
||||
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def getsockopt_string(self, _option):
|
||||
return "inproc://dummy"
|
||||
|
||||
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
|
||||
monkeypatch.setattr(
|
||||
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
|
||||
@@ -448,7 +451,7 @@ def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
@@ -456,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
client.coordinator_input_address = None
|
||||
client.coordinator_output_address = None
|
||||
|
||||
payload = msgspec.msgpack.encode(
|
||||
EngineCoreReadyResponse(
|
||||
@@ -478,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
instance_id="test-instance",
|
||||
supports_lora=False,
|
||||
max_loras=0,
|
||||
coord_store_port=1234,
|
||||
coordinator_input_address="tcp://127.0.0.1:1235",
|
||||
coordinator_output_address="tcp://127.0.0.1:1236",
|
||||
)
|
||||
)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
@@ -498,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
assert client.vllm_config.parallel_config._coord_store_port == 1234
|
||||
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
|
||||
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
@@ -508,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
|
||||
@@ -1578,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
|
||||
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
|
||||
|
||||
@@ -281,6 +281,7 @@ class _FinishedProcess:
|
||||
|
||||
|
||||
def test_wait_for_engine_startup_reports_watched_process_exit():
|
||||
"""Update the startup test argument so it reaches frontend failure handling."""
|
||||
ctx = zmq.Context()
|
||||
handshake_socket = ctx.socket(zmq.ROUTER)
|
||||
recv, send = connection.Pipe(duplex=False)
|
||||
|
||||
Reference in New Issue
Block a user