[Perf] API-server scaleout with many-to-many server-engine comms (#17546)
This commit is contained in:
@@ -44,10 +44,6 @@ class EngineCoreRequest(
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
gc=False): # type: ignore[call-arg]
|
||||
|
||||
# NOTE: prompt and prompt_token_ids should be DecoderOnlyInput,
|
||||
# but this object is currently not playing well with msgspec
|
||||
# due to circular imports and typing we have in data.py
|
||||
|
||||
request_id: str
|
||||
prompt_token_ids: list[int]
|
||||
mm_inputs: Optional[Sequence[Optional[MultiModalKwargs]]]
|
||||
@@ -59,6 +55,10 @@ class EngineCoreRequest(
|
||||
lora_request: Optional[LoRARequest]
|
||||
cache_salt: Optional[str]
|
||||
|
||||
# Index of the client, used to ensure outputs are sent back to the same
|
||||
# client for this request when scaling out the front-end.
|
||||
client_index: int = 0
|
||||
|
||||
# Used in DP case to indicate which wave of requests this is expected to
|
||||
# belong to, to cover a race condition where the request is sent before
|
||||
# a wave finished notification is received.
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.v1.engine.processor import Processor
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.metrics.loggers import (StatLoggerBase, StatLoggerFactory,
|
||||
setup_default_loggers)
|
||||
from vllm.v1.metrics.prometheus import shutdown_prometheus
|
||||
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -54,6 +55,8 @@ class AsyncLLM(EngineClient):
|
||||
log_requests: bool = True,
|
||||
start_engine_loop: bool = True,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_index: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
Create an AsyncLLM.
|
||||
@@ -124,6 +127,8 @@ class AsyncLLM(EngineClient):
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=self.log_stats,
|
||||
client_addresses=client_addresses,
|
||||
client_index=client_index,
|
||||
)
|
||||
if self.stat_loggers:
|
||||
for stat_logger in self.stat_loggers[0]:
|
||||
@@ -145,6 +150,8 @@ class AsyncLLM(EngineClient):
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
disable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_index: int = 0,
|
||||
) -> "AsyncLLM":
|
||||
if not envs.VLLM_USE_V1:
|
||||
raise ValueError(
|
||||
@@ -162,6 +169,8 @@ class AsyncLLM(EngineClient):
|
||||
log_requests=not disable_log_requests,
|
||||
log_stats=not disable_log_stats,
|
||||
usage_context=usage_context,
|
||||
client_addresses=client_addresses,
|
||||
client_index=client_index,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -195,6 +204,8 @@ class AsyncLLM(EngineClient):
|
||||
def shutdown(self):
|
||||
"""Shutdown, cleaning up the background proc and IPC."""
|
||||
|
||||
shutdown_prometheus()
|
||||
|
||||
if engine_core := getattr(self, "engine_core", None):
|
||||
engine_core.shutdown()
|
||||
|
||||
@@ -398,7 +409,6 @@ class AsyncLLM(EngineClient):
|
||||
# TODO(rob): make into a coroutine and launch it in
|
||||
# background thread once Prometheus overhead is non-trivial.
|
||||
if stat_loggers:
|
||||
assert outputs.scheduler_stats is not None
|
||||
AsyncLLM._record_stats(
|
||||
stat_loggers[outputs.engine_index],
|
||||
scheduler_stats=outputs.scheduler_stats,
|
||||
@@ -422,7 +432,7 @@ class AsyncLLM(EngineClient):
|
||||
@staticmethod
|
||||
def _record_stats(
|
||||
stat_loggers: list[StatLoggerBase],
|
||||
scheduler_stats: SchedulerStats,
|
||||
scheduler_stats: Optional[SchedulerStats],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
):
|
||||
"""static so that it can be used from the output_handler task
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import multiprocessing
|
||||
import time
|
||||
import weakref
|
||||
from typing import Optional
|
||||
|
||||
import msgspec.msgpack
|
||||
import zmq
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import get_mp_context, get_open_zmq_ipc_path, make_zmq_socket
|
||||
from vllm.v1.engine import EngineCoreOutputs, EngineCoreRequestType
|
||||
from vllm.v1.serial_utils import MsgpackDecoder
|
||||
from vllm.v1.utils import get_engine_client_zmq_addr, shutdown
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DPCoordinator:
|
||||
"""Coordinator process used for data-parallel deployments (DP>1).
|
||||
|
||||
Intermediates between multiple DP engine rank processes and one or more
|
||||
front-end API server processes.
|
||||
|
||||
* Collects stats from each DP engine (currently just waiting and running
|
||||
queue lengths), and publishes these to all front-ends for use in
|
||||
load-balancing decisions.
|
||||
|
||||
* Keeps track of the current DP "request wave" number and running state
|
||||
of the engines. This is received from the DP rank 0 engine and published
|
||||
to the front-end processes along with the current load stats.
|
||||
|
||||
The engines alternate between a global running/paused state. The global
|
||||
"request wave" number is a count of the number of times that the workers
|
||||
collectively move from a running state to a paused state. This transition
|
||||
is synchronized via the all-reduce operation performed in the
|
||||
DPEngineCoreProc._has_global_unfinished_reqs method.
|
||||
|
||||
* Broadcasts the START_DP_WAVE message to engines to move them from paused
|
||||
to running state when one engine receives a new request. This can happen
|
||||
in two cases:
|
||||
1) A front-end sending a new request while the engines are paused will
|
||||
concurrently notify the coordinator.
|
||||
2) An engine receiving a request for a stale request wave while in paused
|
||||
state will notify the coordinator.
|
||||
|
||||
Engines will move into running state when receiving a new request or
|
||||
START_DP_WAVE message.
|
||||
"""
|
||||
|
||||
def __init__(self, parallel_config: ParallelConfig):
|
||||
|
||||
# Assume coordinator is colocated with front-end procs.
|
||||
front_publish_address = get_open_zmq_ipc_path()
|
||||
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
assert dp_size > 1, "Coordinator only used for data parallel"
|
||||
|
||||
local_only = dp_size == parallel_config.data_parallel_size_local
|
||||
host = parallel_config.data_parallel_master_ip
|
||||
back_publish_address = get_engine_client_zmq_addr(local_only, host)
|
||||
back_output_address = get_engine_client_zmq_addr(local_only, host)
|
||||
|
||||
context = get_mp_context()
|
||||
self.proc: multiprocessing.Process = context.Process(
|
||||
target=CoordinatorProc.run_coordinator,
|
||||
name="VLLM_DP_Coordinator",
|
||||
kwargs={
|
||||
"engine_count": parallel_config.data_parallel_size,
|
||||
"front_publish_address": front_publish_address,
|
||||
"back_output_address": back_output_address,
|
||||
"back_publish_address": back_publish_address,
|
||||
},
|
||||
daemon=True)
|
||||
self.proc.start()
|
||||
|
||||
self.stats_publish_address = front_publish_address
|
||||
self.coord_in_address = back_publish_address
|
||||
self.coord_out_address = back_output_address
|
||||
self._finalizer = weakref.finalize(self, shutdown, [self.proc])
|
||||
|
||||
def get_stats_publish_address(self) -> str:
|
||||
return self.stats_publish_address
|
||||
|
||||
def get_engine_socket_addresses(self) -> tuple[str, str]:
|
||||
"""Returns tuple of ZMQ input address, output address."""
|
||||
return self.coord_in_address, self.coord_out_address
|
||||
|
||||
def close(self):
|
||||
self._finalizer()
|
||||
|
||||
|
||||
class EngineState:
|
||||
|
||||
def __init__(self):
|
||||
self.request_counts = [0, 0] # [waiting, running]
|
||||
|
||||
|
||||
class CoordinatorProc:
|
||||
|
||||
def __init__(self, engine_count: int):
|
||||
|
||||
self.ctx = zmq.Context()
|
||||
|
||||
self.engines = [EngineState() for _ in range(engine_count)]
|
||||
|
||||
self.current_wave = 0
|
||||
self.engines_running = False
|
||||
self.stats_changed = False
|
||||
|
||||
@staticmethod
|
||||
def run_coordinator(
|
||||
engine_count: int,
|
||||
front_publish_address: str,
|
||||
back_output_address: str,
|
||||
back_publish_address: str,
|
||||
):
|
||||
coordinator = CoordinatorProc(engine_count=engine_count)
|
||||
try:
|
||||
coordinator.process_input_socket(
|
||||
front_publish_address,
|
||||
back_output_address,
|
||||
back_publish_address,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("DP Coordinator process exiting")
|
||||
|
||||
def process_input_socket(self, front_publish_address: str,
|
||||
back_output_address: str,
|
||||
back_publish_address: str):
|
||||
|
||||
decoder = MsgpackDecoder(EngineCoreOutputs)
|
||||
|
||||
with make_zmq_socket(
|
||||
path=front_publish_address, # IPC
|
||||
ctx=self.ctx,
|
||||
socket_type=zmq.XPUB,
|
||||
bind=True,
|
||||
) as publish_front, make_zmq_socket(
|
||||
path=back_output_address, # IPC or TCP
|
||||
ctx=self.ctx,
|
||||
socket_type=zmq.PULL,
|
||||
bind=True,
|
||||
) as output_back, make_zmq_socket(
|
||||
path=back_publish_address, # IPC or TCP
|
||||
ctx=self.ctx,
|
||||
socket_type=zmq.XPUB,
|
||||
bind=True,
|
||||
) as publish_back:
|
||||
|
||||
poller = zmq.Poller()
|
||||
poller.register(publish_front, zmq.POLLIN)
|
||||
poller.register(output_back, zmq.POLLIN)
|
||||
last_publish_time = 0
|
||||
while True:
|
||||
elapsed = int(time.time() * 1000) - last_publish_time
|
||||
# Send at 100 ms interval if the stats have changed,
|
||||
# or otherwise every 3 seconds.
|
||||
wait_for = 100 if self.stats_changed else 3000
|
||||
events = poller.poll(timeout=max(0, wait_for - elapsed))
|
||||
if not events:
|
||||
# Poller timeout - publish current stats to front-ends.
|
||||
engine_req_counts_list = self._get_engine_counts()
|
||||
to_publish = (engine_req_counts_list, self.current_wave,
|
||||
self.engines_running)
|
||||
publish_front.send(msgspec.msgpack.encode(to_publish))
|
||||
last_publish_time = int(time.time() * 1000)
|
||||
self.stats_changed = False
|
||||
continue
|
||||
|
||||
events = dict(events)
|
||||
|
||||
if publish_front in events:
|
||||
buffer = publish_front.recv()
|
||||
if buffer == b'\x01':
|
||||
# Ignore subscription messages.
|
||||
continue
|
||||
|
||||
# We received a message on the front-end XPUB socket,
|
||||
# from an API server sending a new request while the
|
||||
# engines are paused, so that we can wake the other
|
||||
# engines.
|
||||
engine_to_exclude, wave = msgspec.msgpack.decode(buffer)
|
||||
if wave < self.current_wave:
|
||||
# If the wave number is stale, ensure the message is
|
||||
# handled by all the engines.
|
||||
engine_to_exclude = None
|
||||
if not self.engines_running:
|
||||
self.engines_running = True
|
||||
self.stats_changed = True
|
||||
self._send_start_wave(publish_back, self.current_wave,
|
||||
engine_to_exclude)
|
||||
|
||||
if output_back in events:
|
||||
# We received a message from one of the engines.
|
||||
|
||||
buffer = output_back.recv()
|
||||
outputs: EngineCoreOutputs = decoder.decode(buffer)
|
||||
|
||||
assert not outputs.outputs
|
||||
assert outputs.utility_output is None
|
||||
|
||||
eng_index = outputs.engine_index
|
||||
if outputs.scheduler_stats:
|
||||
# 1. Updated request load stats - update our local
|
||||
# state with these.
|
||||
stats = self.engines[eng_index].request_counts
|
||||
stats[0] = outputs.scheduler_stats.num_waiting_reqs
|
||||
stats[1] = outputs.scheduler_stats.num_running_reqs
|
||||
self.stats_changed = True
|
||||
|
||||
if (wave := outputs.wave_complete) is not None:
|
||||
# 2. Notification from rank 0 engine that we've
|
||||
# moved into the global paused state
|
||||
# (engines_running==False)
|
||||
if self.current_wave <= wave:
|
||||
logger.debug("Moving DP wave from %d to %d.",
|
||||
self.current_wave, wave)
|
||||
self.current_wave = wave + 1
|
||||
self.engines_running = False
|
||||
self.stats_changed = True
|
||||
elif (wave := outputs.start_wave) is not None and (
|
||||
wave > self.current_wave or
|
||||
(wave == self.current_wave
|
||||
and not self.engines_running)):
|
||||
# 3. The engine received request for a non-current wave
|
||||
# so we must ensure that other engines progress to the
|
||||
# next wave (race condition handling).
|
||||
logger.debug(
|
||||
"Starting wave %d after notification of "
|
||||
"stale wave request from engine.", wave)
|
||||
self.current_wave = wave
|
||||
self.engines_running = True
|
||||
self.stats_changed = True
|
||||
self._send_start_wave(publish_back, wave, eng_index)
|
||||
|
||||
@staticmethod
|
||||
def _send_start_wave(socket: zmq.Socket, wave: int,
|
||||
exclude_engine_index: Optional[int]):
|
||||
"""Broadcast the START_DP_WAVE message to all the engines.
|
||||
It includes the current wave number and index of engine which
|
||||
has already received a request with this wave number and so doesn't
|
||||
require additional notification.
|
||||
"""
|
||||
wave_encoded = msgspec.msgpack.encode((wave, exclude_engine_index))
|
||||
socket.send_multipart(
|
||||
(EngineCoreRequestType.START_DP_WAVE.value, wave_encoded))
|
||||
|
||||
def _get_engine_counts(self) -> list[list[int]]:
|
||||
"""Return list of [waiting, running] count lists for each engine."""
|
||||
return [e.request_counts for e in self.engines]
|
||||
+164
-85
@@ -7,6 +7,7 @@ import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from concurrent.futures import Future
|
||||
from contextlib import ExitStack
|
||||
from inspect import isclass, signature
|
||||
from logging import DEBUG
|
||||
from typing import Any, Callable, Optional, TypeVar, Union
|
||||
@@ -22,7 +23,7 @@ from vllm.logging_utils.dump_input import dump_engine_exception
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.transformers_utils.config import (
|
||||
maybe_register_config_serialize_by_value)
|
||||
from vllm.utils import make_zmq_socket, resolve_obj_by_qualname, zmq_socket_ctx
|
||||
from vllm.utils import make_zmq_socket, resolve_obj_by_qualname
|
||||
from vllm.v1.core.kv_cache_utils import (get_kv_cache_config,
|
||||
unify_kv_cache_configs)
|
||||
from vllm.v1.core.sched.interface import SchedulerInterface
|
||||
@@ -33,10 +34,12 @@ from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
|
||||
from vllm.v1.engine.mm_input_cache import MirroredProcessingCache
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.metrics.stats import SchedulerStats
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.utils import EngineHandshakeMetadata, EngineZmqAddresses
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -211,7 +214,7 @@ class EngineCore:
|
||||
# Re-raise exception
|
||||
raise err
|
||||
|
||||
def step(self) -> tuple[EngineCoreOutputs, bool]:
|
||||
def step(self) -> tuple[dict[int, EngineCoreOutputs], bool]:
|
||||
"""Schedule, execute, and make output.
|
||||
|
||||
Returns tuple of outputs and a flag indicating whether the model
|
||||
@@ -221,10 +224,7 @@ class EngineCore:
|
||||
# Check for any requests remaining in the scheduler - unfinished,
|
||||
# or finished and not yet removed from the batch.
|
||||
if not self.scheduler.has_requests():
|
||||
return EngineCoreOutputs(
|
||||
outputs=[],
|
||||
scheduler_stats=self.scheduler.make_stats(),
|
||||
), False
|
||||
return {}, False
|
||||
scheduler_output = self.scheduler.schedule()
|
||||
model_output = self.execute_model(scheduler_output)
|
||||
engine_core_outputs = self.scheduler.update_from_output(
|
||||
@@ -234,7 +234,7 @@ class EngineCore:
|
||||
scheduler_output.total_num_scheduled_tokens > 0)
|
||||
|
||||
def step_with_batch_queue(
|
||||
self) -> tuple[Optional[EngineCoreOutputs], bool]:
|
||||
self) -> tuple[Optional[dict[int, EngineCoreOutputs]], bool]:
|
||||
"""Schedule and execute batches with the batch queue.
|
||||
Note that if nothing to output in this step, None is returned.
|
||||
|
||||
@@ -276,8 +276,8 @@ class EngineCore:
|
||||
# Blocking until the first result is available.
|
||||
model_output = future.result()
|
||||
self.batch_queue.task_done()
|
||||
engine_core_outputs = self.scheduler.update_from_output(
|
||||
scheduler_output, model_output)
|
||||
engine_core_outputs = (self.scheduler.update_from_output(
|
||||
scheduler_output, model_output))
|
||||
|
||||
return engine_core_outputs, scheduled_batch
|
||||
|
||||
@@ -362,7 +362,7 @@ class EngineCoreProc(EngineCore):
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
on_head_node: bool,
|
||||
input_address: str,
|
||||
handshake_address: str,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
engine_index: int = 0,
|
||||
@@ -375,65 +375,70 @@ class EngineCoreProc(EngineCore):
|
||||
# Create input socket.
|
||||
input_ctx = zmq.Context()
|
||||
identity = engine_index.to_bytes(length=2, byteorder="little")
|
||||
input_socket = make_zmq_socket(input_ctx,
|
||||
input_address,
|
||||
zmq.DEALER,
|
||||
identity=identity,
|
||||
bind=False)
|
||||
try:
|
||||
with make_zmq_socket(input_ctx,
|
||||
handshake_address,
|
||||
zmq.DEALER,
|
||||
identity=identity,
|
||||
linger=5000,
|
||||
bind=False) as handshake_socket:
|
||||
|
||||
# Register engine with front-end.
|
||||
output_address = self.startup_handshake(
|
||||
input_socket, on_head_node, vllm_config.parallel_config)
|
||||
addresses = self.startup_handshake(handshake_socket, on_head_node,
|
||||
vllm_config.parallel_config)
|
||||
self.client_count = len(addresses.outputs)
|
||||
|
||||
# Update config which may have changed from the handshake.
|
||||
vllm_config.__post_init__()
|
||||
|
||||
# Set up data parallel environment.
|
||||
self.has_coordinator = addresses.coordinator_output is not None
|
||||
self._init_data_parallel(vllm_config)
|
||||
|
||||
# Initialize engine core and model.
|
||||
super().__init__(vllm_config, executor_class, log_stats,
|
||||
executor_fail_callback)
|
||||
|
||||
self.engine_index = engine_index
|
||||
self.step_fn = (self.step if self.batch_queue is None else
|
||||
self.step_with_batch_queue)
|
||||
self.engines_running = False
|
||||
self.last_counts = (0, 0)
|
||||
|
||||
# Send ready message.
|
||||
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks
|
||||
input_socket.send(
|
||||
handshake_socket.send(
|
||||
msgspec.msgpack.encode({
|
||||
"status": "READY",
|
||||
"local": on_head_node,
|
||||
"num_gpu_blocks": num_gpu_blocks,
|
||||
}))
|
||||
|
||||
# Background Threads and Queues for IO. These enable us to
|
||||
# overlap ZMQ socket IO with GPU since they release the GIL,
|
||||
# and to overlap some serialization/deserialization with the
|
||||
# model forward pass.
|
||||
# Threads handle Socket <-> Queues and core_busy_loop uses Queue.
|
||||
self.input_queue = input_queue
|
||||
self.output_queue = queue.Queue[Union[EngineCoreOutputs, bytes]]()
|
||||
threading.Thread(target=self.process_input_socket,
|
||||
args=(input_socket, ),
|
||||
daemon=True).start()
|
||||
input_socket = None
|
||||
self.output_thread = threading.Thread(
|
||||
target=self.process_output_socket,
|
||||
args=(output_address, engine_index),
|
||||
daemon=True)
|
||||
self.output_thread.start()
|
||||
finally:
|
||||
if input_socket is not None:
|
||||
input_socket.close(linger=0)
|
||||
# Background Threads and Queues for IO. These enable us to
|
||||
# overlap ZMQ socket IO with GPU since they release the GIL,
|
||||
# and to overlap some serialization/deserialization with the
|
||||
# model forward pass.
|
||||
# Threads handle Socket <-> Queues and core_busy_loop uses Queue.
|
||||
self.input_queue = input_queue
|
||||
self.output_queue = queue.Queue[Union[tuple[int, EngineCoreOutputs],
|
||||
bytes]]()
|
||||
threading.Thread(target=self.process_input_sockets,
|
||||
args=(addresses.inputs, addresses.coordinator_input,
|
||||
identity),
|
||||
daemon=True).start()
|
||||
self.output_thread = threading.Thread(
|
||||
target=self.process_output_sockets,
|
||||
args=(addresses.outputs, addresses.coordinator_output,
|
||||
engine_index),
|
||||
daemon=True)
|
||||
self.output_thread.start()
|
||||
|
||||
@staticmethod
|
||||
def startup_handshake(input_socket: zmq.Socket, on_head_node: bool,
|
||||
parallel_config: ParallelConfig) -> str:
|
||||
def startup_handshake(
|
||||
handshake_socket: zmq.Socket, on_head_node: bool,
|
||||
parallel_config: ParallelConfig) -> EngineZmqAddresses:
|
||||
|
||||
# Send registration message.
|
||||
input_socket.send(
|
||||
handshake_socket.send(
|
||||
msgspec.msgpack.encode({
|
||||
"status": "HELLO",
|
||||
"local": on_head_node,
|
||||
@@ -441,22 +446,20 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
# Receive initialization message.
|
||||
logger.info("Waiting for init message from front-end.")
|
||||
if not input_socket.poll(timeout=HANDSHAKE_TIMEOUT_MINS * 60 * 1000):
|
||||
if not handshake_socket.poll(timeout=HANDSHAKE_TIMEOUT_MINS * 60_000):
|
||||
raise RuntimeError("Did not receive response from front-end "
|
||||
f"process within {HANDSHAKE_TIMEOUT_MINS} "
|
||||
f"minutes")
|
||||
init_bytes = input_socket.recv()
|
||||
init_message = msgspec.msgpack.decode(init_bytes)
|
||||
init_bytes = handshake_socket.recv()
|
||||
init_message: EngineHandshakeMetadata = msgspec.msgpack.decode(
|
||||
init_bytes, type=EngineHandshakeMetadata)
|
||||
logger.debug("Received init message: %s", init_message)
|
||||
|
||||
output_socket_address = init_message["output_socket_address"]
|
||||
#TBD(nick) maybe replace IP with configured head node address
|
||||
|
||||
received_parallel_config = init_message["parallel_config"]
|
||||
received_parallel_config = init_message.parallel_config
|
||||
for key, value in received_parallel_config.items():
|
||||
setattr(parallel_config, key, value)
|
||||
|
||||
return output_socket_address
|
||||
return init_message.addresses
|
||||
|
||||
@staticmethod
|
||||
def run_engine_core(*args,
|
||||
@@ -528,7 +531,7 @@ class EngineCoreProc(EngineCore):
|
||||
"""Exits when an engine step needs to be performed."""
|
||||
|
||||
waited = False
|
||||
while not self.engines_running and not (self.scheduler.has_requests()):
|
||||
while not self.engines_running and not self.scheduler.has_requests():
|
||||
if logger.isEnabledFor(DEBUG) and self.input_queue.empty():
|
||||
logger.debug("EngineCore waiting for work.")
|
||||
waited = True
|
||||
@@ -549,8 +552,8 @@ class EngineCoreProc(EngineCore):
|
||||
# Step the engine core.
|
||||
outputs, model_executed = self.step_fn()
|
||||
# Put EngineCoreOutputs into the output queue.
|
||||
if outputs is not None:
|
||||
self.output_queue.put_nowait(outputs)
|
||||
for output in (outputs.items() if outputs else ()):
|
||||
self.output_queue.put_nowait(output)
|
||||
|
||||
return model_executed
|
||||
|
||||
@@ -563,7 +566,7 @@ class EngineCoreProc(EngineCore):
|
||||
elif request_type == EngineCoreRequestType.ABORT:
|
||||
self.abort_requests(request)
|
||||
elif request_type == EngineCoreRequestType.UTILITY:
|
||||
call_id, method_name, args = request
|
||||
client_idx, call_id, method_name, args = request
|
||||
output = UtilityOutput(call_id)
|
||||
try:
|
||||
method = getattr(self, method_name)
|
||||
@@ -574,7 +577,7 @@ class EngineCoreProc(EngineCore):
|
||||
output.failure_message = (f"Call to {method_name} method"
|
||||
f" failed: {str(e)}")
|
||||
self.output_queue.put_nowait(
|
||||
EngineCoreOutputs(utility_output=output))
|
||||
(client_idx, EngineCoreOutputs(utility_output=output)))
|
||||
elif request_type == EngineCoreRequestType.EXECUTOR_FAILED:
|
||||
raise RuntimeError("Executor failed.")
|
||||
else:
|
||||
@@ -607,27 +610,68 @@ class EngineCoreProc(EngineCore):
|
||||
logger.fatal("vLLM shutdown signal from EngineCore failed "
|
||||
"to send. Please report this issue.")
|
||||
|
||||
def process_input_socket(self, input_socket: zmq.Socket):
|
||||
def process_input_sockets(self, input_addresses: list[str],
|
||||
coord_input_address: Optional[str],
|
||||
identity: bytes):
|
||||
"""Input socket IO thread."""
|
||||
|
||||
# Msgpack serialization decoding.
|
||||
add_request_decoder = MsgpackDecoder(EngineCoreRequest)
|
||||
generic_decoder = MsgpackDecoder()
|
||||
|
||||
while True:
|
||||
# (RequestType, RequestData)
|
||||
type_frame, *data_frames = input_socket.recv_multipart(copy=False)
|
||||
request_type = EngineCoreRequestType(bytes(type_frame.buffer))
|
||||
with ExitStack() as stack, zmq.Context() as ctx:
|
||||
input_sockets = [
|
||||
stack.enter_context(
|
||||
make_zmq_socket(ctx,
|
||||
input_address,
|
||||
zmq.DEALER,
|
||||
identity=identity,
|
||||
bind=False))
|
||||
for input_address in input_addresses
|
||||
]
|
||||
if coord_input_address is None:
|
||||
coord_socket = None
|
||||
else:
|
||||
coord_socket = stack.enter_context(
|
||||
make_zmq_socket(ctx,
|
||||
coord_input_address,
|
||||
zmq.XSUB,
|
||||
identity=identity,
|
||||
bind=False))
|
||||
# Send subscription message to coordinator.
|
||||
coord_socket.send(b'\x01')
|
||||
|
||||
# Deserialize the request data.
|
||||
decoder = add_request_decoder if (
|
||||
request_type == EngineCoreRequestType.ADD) else generic_decoder
|
||||
request = decoder.decode(data_frames)
|
||||
# Register sockets with poller.
|
||||
poller = zmq.Poller()
|
||||
for input_socket in input_sockets:
|
||||
# Send initial message to each input socket - this is required
|
||||
# before the front-end ROUTER socket can send input messages
|
||||
# back to us.
|
||||
input_socket.send(b'')
|
||||
poller.register(input_socket, zmq.POLLIN)
|
||||
if coord_socket is not None:
|
||||
poller.register(coord_socket, zmq.POLLIN)
|
||||
|
||||
# Push to input queue for core busy loop.
|
||||
self.input_queue.put_nowait((request_type, request))
|
||||
while True:
|
||||
for input_socket, _ in poller.poll():
|
||||
# (RequestType, RequestData)
|
||||
type_frame, *data_frames = input_socket.recv_multipart(
|
||||
copy=False)
|
||||
request_type = EngineCoreRequestType(
|
||||
bytes(type_frame.buffer))
|
||||
|
||||
def process_output_socket(self, output_path: str, engine_index: int):
|
||||
# Deserialize the request data.
|
||||
decoder = add_request_decoder if (
|
||||
request_type
|
||||
== EngineCoreRequestType.ADD) else generic_decoder
|
||||
request = decoder.decode(data_frames)
|
||||
|
||||
# Push to input queue for core busy loop.
|
||||
self.input_queue.put_nowait((request_type, request))
|
||||
|
||||
def process_output_sockets(self, output_paths: list[str],
|
||||
coord_output_path: Optional[str],
|
||||
engine_index: int):
|
||||
"""Output socket IO thread."""
|
||||
|
||||
# Msgpack serialization encoding.
|
||||
@@ -641,30 +685,49 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
# We must set linger to ensure the ENGINE_CORE_DEAD
|
||||
# message is sent prior to closing the socket.
|
||||
with zmq_socket_ctx(output_path, zmq.constants.PUSH,
|
||||
linger=4000) as socket:
|
||||
with ExitStack() as stack, zmq.Context() as ctx:
|
||||
sockets = [
|
||||
stack.enter_context(
|
||||
make_zmq_socket(ctx, output_path, zmq.PUSH, linger=4000))
|
||||
for output_path in output_paths
|
||||
]
|
||||
coord_socket = stack.enter_context(
|
||||
make_zmq_socket(
|
||||
ctx, coord_output_path, zmq.PUSH, bind=False,
|
||||
linger=4000)) if coord_output_path is not None else None
|
||||
max_reuse_bufs = len(sockets) + 1
|
||||
|
||||
while True:
|
||||
outputs = self.output_queue.get()
|
||||
if outputs == EngineCoreProc.ENGINE_CORE_DEAD:
|
||||
socket.send(outputs, copy=False)
|
||||
output = self.output_queue.get()
|
||||
if output == EngineCoreProc.ENGINE_CORE_DEAD:
|
||||
for socket in sockets:
|
||||
socket.send(output)
|
||||
break
|
||||
assert not isinstance(outputs, bytes)
|
||||
assert not isinstance(output, bytes)
|
||||
client_index, outputs = output
|
||||
outputs.engine_index = engine_index
|
||||
|
||||
if client_index == -1:
|
||||
# Don't reuse buffer for coordinator message
|
||||
# which will be very small.
|
||||
assert coord_socket is not None
|
||||
coord_socket.send_multipart(encoder.encode(outputs))
|
||||
continue
|
||||
|
||||
# Reclaim buffers that zmq is finished with.
|
||||
while pending and pending[-1][0].done:
|
||||
reuse_buffers.append(pending.pop()[2])
|
||||
|
||||
buffer = reuse_buffers.pop() if reuse_buffers else bytearray()
|
||||
buffers = encoder.encode_into(outputs, buffer)
|
||||
tracker = socket.send_multipart(buffers,
|
||||
copy=False,
|
||||
track=True)
|
||||
tracker = sockets[client_index].send_multipart(buffers,
|
||||
copy=False,
|
||||
track=True)
|
||||
if not tracker.done:
|
||||
ref = outputs if len(buffers) > 1 else None
|
||||
pending.appendleft((tracker, ref, buffer))
|
||||
elif len(reuse_buffers) < 2:
|
||||
# Keep at most 2 buffers to reuse.
|
||||
elif len(reuse_buffers) < max_reuse_bufs:
|
||||
# Limit the number of buffers to reuse.
|
||||
reuse_buffers.append(buffer)
|
||||
|
||||
|
||||
@@ -676,7 +739,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
on_head_node: bool,
|
||||
input_address: str,
|
||||
handshake_address: str,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
):
|
||||
@@ -691,10 +754,11 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
# Counts forward-passes of the model so that we can synchronize
|
||||
# finished with DP peers every N steps.
|
||||
self.counter = 0
|
||||
self.current_wave = 0
|
||||
|
||||
# Initialize the engine.
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
super().__init__(vllm_config, on_head_node, input_address,
|
||||
super().__init__(vllm_config, on_head_node, handshake_address,
|
||||
executor_class, log_stats, dp_rank)
|
||||
|
||||
def _init_data_parallel(self, vllm_config: VllmConfig):
|
||||
@@ -726,7 +790,6 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
self.dp_rank = dp_rank
|
||||
self.dp_group = vllm_config.parallel_config.stateless_init_dp_group()
|
||||
self.current_wave = 0
|
||||
|
||||
def shutdown(self):
|
||||
super().shutdown()
|
||||
@@ -734,22 +797,23 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
stateless_destroy_torch_distributed_process_group(dp_group)
|
||||
|
||||
def add_request(self, request: EngineCoreRequest):
|
||||
if request.current_wave != self.current_wave:
|
||||
if self.has_coordinator and request.current_wave != self.current_wave:
|
||||
if request.current_wave > self.current_wave:
|
||||
self.current_wave = request.current_wave
|
||||
elif not self.engines_running:
|
||||
# Request received for an already-completed wave, notify
|
||||
# front-end that we need to start the next one.
|
||||
self.output_queue.put_nowait(
|
||||
EngineCoreOutputs(start_wave=self.current_wave))
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave)))
|
||||
|
||||
super().add_request(request)
|
||||
|
||||
def _handle_client_request(self, request_type: EngineCoreRequestType,
|
||||
request: Any) -> None:
|
||||
if request_type == EngineCoreRequestType.START_DP_WAVE:
|
||||
new_wave: int = request
|
||||
if new_wave >= self.current_wave:
|
||||
new_wave, exclude_eng_index = request
|
||||
if exclude_eng_index != self.engine_index and (
|
||||
new_wave >= self.current_wave):
|
||||
self.current_wave = new_wave
|
||||
if not self.engines_running:
|
||||
logger.debug("EngineCore starting idle loop for wave %d.",
|
||||
@@ -758,6 +822,18 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
else:
|
||||
super()._handle_client_request(request_type, request)
|
||||
|
||||
def _maybe_publish_request_counts(self):
|
||||
if not self.has_coordinator:
|
||||
return
|
||||
|
||||
# Publish our request counts (if they've changed).
|
||||
counts = self.scheduler.get_request_counts()
|
||||
if counts != self.last_counts:
|
||||
self.last_counts = counts
|
||||
stats = SchedulerStats(*counts)
|
||||
self.output_queue.put_nowait(
|
||||
(-1, EngineCoreOutputs(scheduler_stats=stats)))
|
||||
|
||||
def run_busy_loop(self):
|
||||
"""Core busy loop of the EngineCore for data parallel case."""
|
||||
|
||||
@@ -768,6 +844,8 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
# 2) Step the engine core.
|
||||
executed = self._process_engine_step()
|
||||
self._maybe_publish_request_counts()
|
||||
|
||||
local_unfinished_reqs = self.scheduler.has_unfinished_requests()
|
||||
if not executed:
|
||||
if not local_unfinished_reqs and not self.engines_running:
|
||||
@@ -788,7 +866,8 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
logger.debug("Wave %d finished, pausing engine loop.",
|
||||
self.current_wave)
|
||||
self.output_queue.put_nowait(
|
||||
EngineCoreOutputs(wave_complete=self.current_wave))
|
||||
(-1,
|
||||
EngineCoreOutputs(wave_complete=self.current_wave)))
|
||||
self.current_wave += 1
|
||||
|
||||
def _has_global_unfinished_reqs(self, local_unfinished: bool) -> bool:
|
||||
|
||||
+238
-206
@@ -2,6 +2,7 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
import queue
|
||||
import sys
|
||||
import uuid
|
||||
import weakref
|
||||
from abc import ABC, abstractmethod
|
||||
@@ -9,26 +10,28 @@ from collections import deque
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from threading import Thread
|
||||
from typing import Any, Callable, Optional, TypeVar, Union
|
||||
|
||||
import msgspec
|
||||
import msgspec.msgpack
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
|
||||
from vllm.config import ParallelConfig, VllmConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.utils import (get_open_port, get_open_zmq_inproc_path,
|
||||
get_open_zmq_ipc_path, get_tcp_uri, make_zmq_socket)
|
||||
from vllm.utils import (get_open_zmq_inproc_path, make_zmq_socket,
|
||||
zmq_socket_ctx)
|
||||
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
|
||||
EngineCoreRequestType, UtilityOutput)
|
||||
from vllm.v1.engine.coordinator import DPCoordinator
|
||||
from vllm.v1.engine.core import EngineCore, EngineCoreProc
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
|
||||
from vllm.v1.utils import CoreEngineProcManager
|
||||
from vllm.v1.utils import (CoreEngine, CoreEngineProcManager,
|
||||
EngineZmqAddresses, get_engine_client_zmq_addr,
|
||||
wait_for_engine_startup)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -36,8 +39,6 @@ AnyFuture = Union[asyncio.Future[Any], Future[Any]]
|
||||
|
||||
_R = TypeVar('_R') # Return type for collective_rpc
|
||||
|
||||
STARTUP_POLL_PERIOD_MS = 10000
|
||||
|
||||
|
||||
class EngineCoreClient(ABC):
|
||||
"""
|
||||
@@ -207,7 +208,7 @@ class InprocClient(EngineCoreClient):
|
||||
|
||||
def get_output(self) -> EngineCoreOutputs:
|
||||
outputs, _ = self.engine_core.step()
|
||||
return outputs
|
||||
return outputs.get(0) or EngineCoreOutputs()
|
||||
|
||||
def add_request(self, request: EngineCoreRequest) -> None:
|
||||
self.engine_core.add_request(request)
|
||||
@@ -266,24 +267,6 @@ class InprocClient(EngineCoreClient):
|
||||
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
|
||||
class CoreEngineState(Enum):
|
||||
NEW = auto()
|
||||
CONNECTED = auto()
|
||||
READY = auto()
|
||||
|
||||
|
||||
class CoreEngine:
|
||||
"""One per data parallel rank."""
|
||||
|
||||
def __init__(self, index: int = 0, local: bool = True):
|
||||
self.local = local
|
||||
self.index = index
|
||||
self.identity = index.to_bytes(length=2, byteorder="little")
|
||||
|
||||
self.state = CoreEngineState.NEW
|
||||
self.num_reqs_in_flight = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackgroundResources:
|
||||
"""Used as a finalizer for clean shutdown, avoiding
|
||||
@@ -291,9 +274,12 @@ class BackgroundResources:
|
||||
|
||||
ctx: Union[zmq.Context]
|
||||
local_engine_manager: Optional[CoreEngineProcManager] = None
|
||||
coordinator: Optional[DPCoordinator] = None
|
||||
output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
first_req_send_socket: Optional[zmq.asyncio.Socket] = None
|
||||
output_queue_task: Optional[asyncio.Task] = None
|
||||
stats_update_task: Optional[asyncio.Task] = None
|
||||
shutdown_path: Optional[str] = None
|
||||
|
||||
# Set if any of the engines are dead. Here so that the output
|
||||
@@ -306,16 +292,21 @@ class BackgroundResources:
|
||||
self.engine_dead = True
|
||||
if self.local_engine_manager is not None:
|
||||
self.local_engine_manager.close()
|
||||
if self.coordinator is not None:
|
||||
self.coordinator.close()
|
||||
|
||||
if self.output_queue_task is not None:
|
||||
self.output_queue_task.cancel()
|
||||
if self.stats_update_task is not None:
|
||||
self.stats_update_task.cancel()
|
||||
|
||||
# ZMQ context termination can hang if the sockets
|
||||
# aren't explicitly closed first.
|
||||
if self.output_socket is not None:
|
||||
self.output_socket.close(linger=0)
|
||||
if self.input_socket is not None:
|
||||
self.input_socket.close(linger=0)
|
||||
for socket in (self.output_socket, self.input_socket,
|
||||
self.first_req_send_socket):
|
||||
if socket is not None:
|
||||
socket.close(linger=0)
|
||||
|
||||
if self.shutdown_path is not None:
|
||||
# We must ensure that the sync output socket is
|
||||
# closed cleanly in its own thread.
|
||||
@@ -350,6 +341,7 @@ class MPClient(EngineCoreClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
# Serialization setup.
|
||||
@@ -369,8 +361,8 @@ class MPClient(EngineCoreClient):
|
||||
try:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
local_engine_count = parallel_config.data_parallel_size_local
|
||||
start_index = parallel_config.data_parallel_rank
|
||||
local_start_index = parallel_config.data_parallel_rank_local
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
|
||||
# SPMD mode is where there is an LLM instance per DP rank and
|
||||
# one core engine per LLM, see
|
||||
@@ -382,42 +374,53 @@ class MPClient(EngineCoreClient):
|
||||
CoreEngine(index=local_start_index, local=True)
|
||||
]
|
||||
else:
|
||||
assert start_index == 0
|
||||
assert parallel_config.data_parallel_rank == 0
|
||||
local_start_index = 0
|
||||
self.core_engines = [
|
||||
CoreEngine(index=i, local=(i < local_engine_count))
|
||||
for i in range(parallel_config.data_parallel_size)
|
||||
for i in range(dp_size)
|
||||
]
|
||||
|
||||
input_address, output_address = self._get_zmq_addresses(
|
||||
parallel_config, spmd_mode)
|
||||
local_only = spmd_mode or local_engine_count == dp_size
|
||||
|
||||
self.stats_update_address: Optional[str] = None
|
||||
if client_addresses is not None:
|
||||
input_address = client_addresses["input_address"]
|
||||
output_address = client_addresses["output_address"]
|
||||
self.stats_update_address = client_addresses.get(
|
||||
"stats_update_address")
|
||||
else:
|
||||
host = parallel_config.data_parallel_master_ip
|
||||
input_address = get_engine_client_zmq_addr(local_only, host)
|
||||
output_address = get_engine_client_zmq_addr(local_only, host)
|
||||
|
||||
# Create input and output sockets.
|
||||
self.input_socket = self.resources.input_socket = make_zmq_socket(
|
||||
self.ctx, input_address, zmq.ROUTER, bind=True)
|
||||
|
||||
self.resources.output_socket = make_zmq_socket(
|
||||
self.ctx, output_address, zmq.constants.PULL)
|
||||
# Start local engines.
|
||||
if local_engine_count:
|
||||
# In server mode, start_index and local_start_index will
|
||||
# both be 0.
|
||||
self.resources.local_engine_manager = CoreEngineProcManager(
|
||||
EngineCoreProc.run_engine_core,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
input_address=input_address,
|
||||
on_head_node=True,
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=start_index,
|
||||
local_start_index=local_start_index)
|
||||
self.ctx, output_address, zmq.PULL)
|
||||
|
||||
if client_addresses is None:
|
||||
self._init_engines_direct(vllm_config, local_only,
|
||||
local_start_index, input_address,
|
||||
output_address, executor_class,
|
||||
log_stats)
|
||||
coordinator = self.resources.coordinator
|
||||
if coordinator:
|
||||
self.stats_update_address = (
|
||||
coordinator.get_stats_publish_address())
|
||||
|
||||
# Wait for ready messages from each engine on the input socket.
|
||||
identities = set(e.identity for e in self.core_engines)
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
while identities:
|
||||
if not sync_input_socket.poll(timeout=600_000):
|
||||
raise TimeoutError("Timed out waiting for engines to send"
|
||||
"initial message on input socket.")
|
||||
identity, _ = sync_input_socket.recv_multipart()
|
||||
identities.remove(identity)
|
||||
|
||||
self.core_engine = self.core_engines[0]
|
||||
|
||||
# Wait for engine core process(es) to start.
|
||||
self._wait_for_engine_startup(output_address, parallel_config)
|
||||
|
||||
self.utility_results: dict[int, AnyFuture] = {}
|
||||
|
||||
# Request objects which may contain pytorch-allocated tensors
|
||||
@@ -430,116 +433,67 @@ class MPClient(EngineCoreClient):
|
||||
if not success:
|
||||
self._finalizer()
|
||||
|
||||
@staticmethod
|
||||
def _get_zmq_addresses(parallel_config: ParallelConfig,
|
||||
spmd_mode: bool) -> tuple[str, str]:
|
||||
"""Returns (input_address, output_address)."""
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
def _init_engines_direct(self, vllm_config: VllmConfig, local_only: bool,
|
||||
local_start_index: int, input_address: str,
|
||||
output_address: str,
|
||||
executor_class: type[Executor], log_stats: bool):
|
||||
"""Self-contained client mode, launch engine and coordinator process
|
||||
as needed."""
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
local_engine_count = parallel_config.data_parallel_size_local
|
||||
start_index = parallel_config.data_parallel_rank
|
||||
host = parallel_config.data_parallel_master_ip
|
||||
|
||||
if local_engine_count == dp_size or spmd_mode:
|
||||
input_address = get_open_zmq_ipc_path()
|
||||
output_address = get_open_zmq_ipc_path()
|
||||
else:
|
||||
host = parallel_config.data_parallel_master_ip
|
||||
input_port = parallel_config.data_parallel_rpc_port
|
||||
output_port = get_open_port()
|
||||
input_address = get_tcp_uri(host, input_port)
|
||||
output_address = get_tcp_uri(host, output_port)
|
||||
if len(self.core_engines) > 1:
|
||||
self.resources.coordinator = DPCoordinator(parallel_config)
|
||||
|
||||
return input_address, output_address
|
||||
handshake_address = get_engine_client_zmq_addr(
|
||||
local_only, host, parallel_config.data_parallel_rpc_port)
|
||||
|
||||
def _wait_for_engine_startup(self, output_address: str,
|
||||
parallel_config: ParallelConfig):
|
||||
# Get a sync handle to the socket which can be sync or async.
|
||||
sync_input_socket = zmq.Socket.shadow(self.input_socket)
|
||||
with zmq_socket_ctx(handshake_address, zmq.ROUTER,
|
||||
bind=True) as handshake_socket:
|
||||
|
||||
# Wait for engine core process(es) to send ready messages.
|
||||
local_count = parallel_config.data_parallel_size_local
|
||||
remote_count = len(self.core_engines) - local_count
|
||||
# [local, remote] counts
|
||||
conn_pending, start_pending = [local_count, remote_count], [0, 0]
|
||||
# Start local engines.
|
||||
if local_engine_count:
|
||||
# In server mode, start_index and local_start_index will
|
||||
# both be 0.
|
||||
self.resources.local_engine_manager = CoreEngineProcManager(
|
||||
EngineCoreProc.run_engine_core,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
handshake_address=handshake_address,
|
||||
on_head_node=True,
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=start_index,
|
||||
local_start_index=local_start_index)
|
||||
|
||||
poller = zmq.Poller()
|
||||
poller.register(sync_input_socket, zmq.POLLIN)
|
||||
proc_manager = self.resources.local_engine_manager
|
||||
if proc_manager is not None:
|
||||
for sentinel in proc_manager.sentinels():
|
||||
poller.register(sentinel, zmq.POLLIN)
|
||||
while any(conn_pending) or any(start_pending):
|
||||
events = poller.poll(STARTUP_POLL_PERIOD_MS)
|
||||
if not events:
|
||||
if any(conn_pending):
|
||||
logger.debug(
|
||||
"Waiting for %d local, %d remote core engine proc(s) "
|
||||
"to connect.", *conn_pending)
|
||||
if any(start_pending):
|
||||
logger.debug(
|
||||
"Waiting for %d local, %d remote core engine proc(s) "
|
||||
"to start.", *start_pending)
|
||||
continue
|
||||
if len(events) > 1 or events[0][0] != sync_input_socket:
|
||||
# One of the local core processes exited.
|
||||
finished = proc_manager.finished_procs(
|
||||
) if proc_manager else {}
|
||||
raise RuntimeError("Engine core initialization failed. "
|
||||
"See root cause above. "
|
||||
f"Failed core proc(s): {finished}")
|
||||
# Wait for engine core process(es) to start.
|
||||
self._wait_for_engine_startup(handshake_socket, input_address,
|
||||
output_address)
|
||||
|
||||
# Receive HELLO and READY messages from the input socket.
|
||||
eng_identity, ready_msg_bytes = sync_input_socket.recv_multipart()
|
||||
eng_index = int.from_bytes(eng_identity, byteorder="little")
|
||||
engine = next(
|
||||
(e for e in self.core_engines if e.identity == eng_identity),
|
||||
None)
|
||||
if engine is None:
|
||||
raise RuntimeError(f"Message from engine with unexpected data "
|
||||
f"parallel rank: {eng_index}")
|
||||
msg = msgspec.msgpack.decode(ready_msg_bytes)
|
||||
status, local = msg["status"], msg["local"]
|
||||
if local != engine.local:
|
||||
raise RuntimeError(f"{status} message from "
|
||||
f"{'local' if local else 'remote'} "
|
||||
f"engine {eng_index}, expected it to be "
|
||||
f"{'local' if engine.local else 'remote'}")
|
||||
def _wait_for_engine_startup(self, handshake_socket: zmq.Socket,
|
||||
input_address: str, output_address: str):
|
||||
addresses = EngineZmqAddresses(
|
||||
inputs=[input_address],
|
||||
outputs=[output_address],
|
||||
)
|
||||
|
||||
if status == "HELLO" and engine.state == CoreEngineState.NEW:
|
||||
coordinator = self.resources.coordinator
|
||||
if coordinator is not None:
|
||||
addresses.coordinator_input, addresses.coordinator_output = (
|
||||
coordinator.get_engine_socket_addresses())
|
||||
|
||||
# Send init message with DP config info.
|
||||
init_message = self.encoder.encode({
|
||||
"output_socket_address": output_address,
|
||||
"parallel_config": {
|
||||
"data_parallel_master_ip":
|
||||
parallel_config.data_parallel_master_ip,
|
||||
"data_parallel_master_port":
|
||||
parallel_config.data_parallel_master_port,
|
||||
"data_parallel_size":
|
||||
parallel_config.data_parallel_size,
|
||||
},
|
||||
})
|
||||
sync_input_socket.send_multipart((eng_identity, *init_message),
|
||||
copy=False)
|
||||
conn_pending[0 if local else 1] -= 1
|
||||
start_pending[0 if local else 1] += 1
|
||||
engine.state = CoreEngineState.CONNECTED
|
||||
elif status == "READY" and (engine.state
|
||||
== CoreEngineState.CONNECTED):
|
||||
# Setup KV cache config with initialization state from
|
||||
# engine core process. Sum values from all engines in DP case.
|
||||
cache_config = self.vllm_config.cache_config
|
||||
num_gpu_blocks = cache_config.num_gpu_blocks or 0
|
||||
num_gpu_blocks += msg['num_gpu_blocks']
|
||||
cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
|
||||
start_pending[0 if local else 1] -= 1
|
||||
engine.state = CoreEngineState.READY
|
||||
else:
|
||||
raise RuntimeError(f"Unexpected {status} message for "
|
||||
f"{'local' if local else 'remote'} engine "
|
||||
f"{eng_index} in {engine.state} state.")
|
||||
|
||||
logger.debug("%s from %s core engine process %s.", status,
|
||||
"local" if local else "remote", eng_index)
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
self.core_engines,
|
||||
self.vllm_config.parallel_config,
|
||||
self.vllm_config.cache_config,
|
||||
self.resources.local_engine_manager,
|
||||
coordinator.proc if coordinator else None,
|
||||
)
|
||||
|
||||
def shutdown(self):
|
||||
# Terminate background resources.
|
||||
@@ -605,8 +559,8 @@ class SyncMPClient(MPClient):
|
||||
try:
|
||||
shutdown_socket.bind(shutdown_path)
|
||||
poller = zmq.Poller()
|
||||
poller.register(shutdown_socket)
|
||||
poller.register(out_socket)
|
||||
poller.register(shutdown_socket, zmq.POLLIN)
|
||||
poller.register(out_socket, zmq.POLLIN)
|
||||
while True:
|
||||
socks = poller.poll()
|
||||
if not socks:
|
||||
@@ -668,7 +622,7 @@ class SyncMPClient(MPClient):
|
||||
future: Future[Any] = Future()
|
||||
self.utility_results[call_id] = future
|
||||
self._send_input(EngineCoreRequestType.UTILITY,
|
||||
(call_id, method, args))
|
||||
(0, call_id, method, args))
|
||||
|
||||
return future.result()
|
||||
|
||||
@@ -730,15 +684,21 @@ class SyncMPClient(MPClient):
|
||||
class AsyncMPClient(MPClient):
|
||||
"""Asyncio-compatible client for multi-proc EngineCore."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, executor_class: type[Executor],
|
||||
log_stats: bool):
|
||||
def __init__(self,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_index: int = 0):
|
||||
super().__init__(
|
||||
asyncio_mode=True,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
client_addresses=client_addresses,
|
||||
)
|
||||
|
||||
self.client_index = client_index
|
||||
self.outputs_queue = asyncio.Queue[Union[EngineCoreOutputs,
|
||||
Exception]]()
|
||||
try:
|
||||
@@ -854,12 +814,13 @@ class AsyncMPClient(MPClient):
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self.utility_results[call_id] = future
|
||||
message = (EngineCoreRequestType.UTILITY.value, *self.encoder.encode(
|
||||
(call_id, method, args)))
|
||||
(self.client_index, call_id, method, args)))
|
||||
await self._send_input_message(message, engine, args)
|
||||
self._ensure_output_queue_task()
|
||||
return await future
|
||||
|
||||
async def add_request_async(self, request: EngineCoreRequest) -> None:
|
||||
request.client_index = self.client_index
|
||||
await self._send_input(EngineCoreRequestType.ADD, request)
|
||||
self._ensure_output_queue_task()
|
||||
|
||||
@@ -921,17 +882,120 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
|
||||
EngineCore."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, executor_class: type[Executor],
|
||||
log_stats: bool):
|
||||
def __init__(self,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_index: int = 0):
|
||||
|
||||
self.current_wave = 0
|
||||
self.engines_running = False
|
||||
# To route aborts to the correct engine.
|
||||
self.reqs_in_flight: dict[str, CoreEngine] = {}
|
||||
|
||||
super().__init__(vllm_config, executor_class, log_stats)
|
||||
super().__init__(vllm_config, executor_class, log_stats,
|
||||
client_addresses, client_index)
|
||||
|
||||
assert len(self.core_engines) > 1
|
||||
|
||||
# List of [waiting, running] pair per engine.
|
||||
self.lb_engines: list[list[int]] = []
|
||||
|
||||
self.first_req_sock_addr = get_open_zmq_inproc_path()
|
||||
self.first_req_send_socket = self.resources.first_req_send_socket = (
|
||||
make_zmq_socket(self.ctx,
|
||||
self.first_req_sock_addr,
|
||||
zmq.PAIR,
|
||||
bind=True))
|
||||
try:
|
||||
# If we are running in an asyncio event loop, start the stats task.
|
||||
# Otherwise, it will be started lazily.
|
||||
asyncio.get_running_loop()
|
||||
self._ensure_stats_update_task()
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
def _ensure_stats_update_task(self):
|
||||
resources = self.resources
|
||||
if resources.stats_update_task is not None:
|
||||
return
|
||||
|
||||
assert self.stats_update_address is not None
|
||||
|
||||
async def run_engine_stats_update_task():
|
||||
with make_zmq_socket(self.ctx, self.stats_update_address,
|
||||
zmq.XSUB) as socket, make_zmq_socket(
|
||||
self.ctx,
|
||||
self.first_req_sock_addr,
|
||||
zmq.PAIR,
|
||||
bind=False) as first_req_rcv_socket:
|
||||
# Send subscription message.
|
||||
await socket.send(b'\x01')
|
||||
|
||||
poller = zmq.asyncio.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
poller.register(first_req_rcv_socket, zmq.POLLIN)
|
||||
|
||||
while True:
|
||||
events = await poller.poll()
|
||||
if not self.engines_running and len(events) == 2 or (
|
||||
events[0][0] == first_req_rcv_socket):
|
||||
# Send a message to notify the coordinator that
|
||||
# we're sending a request while the engines are
|
||||
# paused, so that it can wake the others up
|
||||
# (to run dummy EP loop).
|
||||
self.engines_running = True
|
||||
buf = first_req_rcv_socket.recv(
|
||||
flags=zmq.NOBLOCK).result()
|
||||
target_eng_index = int.from_bytes(buf, "little")
|
||||
msg = msgspec.msgpack.encode(
|
||||
(target_eng_index, self.current_wave))
|
||||
await socket.send(msg)
|
||||
|
||||
buf = None
|
||||
while True:
|
||||
# Drain all stats events (we only care about latest).
|
||||
future: asyncio.Future[bytes] = socket.recv(
|
||||
flags=zmq.NOBLOCK)
|
||||
if isinstance(future.exception(), zmq.Again):
|
||||
break
|
||||
buf = future.result()
|
||||
if buf is None:
|
||||
continue
|
||||
|
||||
# Update local load-balancing state.
|
||||
counts, wave, running = msgspec.msgpack.decode(buf)
|
||||
self.current_wave = wave
|
||||
self.engines_running = running
|
||||
self.lb_engines = counts
|
||||
|
||||
resources.stats_update_task = asyncio.create_task(
|
||||
run_engine_stats_update_task())
|
||||
|
||||
def get_core_engine_for_request(self) -> CoreEngine:
|
||||
if not self.lb_engines:
|
||||
return self.core_engines[0]
|
||||
# TODO use P2C alg for larger DP sizes
|
||||
num_engines = len(self.lb_engines)
|
||||
min_counts = [sys.maxsize, sys.maxsize]
|
||||
eng_index = 0
|
||||
for i in range(num_engines):
|
||||
# Start from client_index to help with balancing when engines
|
||||
# are empty.
|
||||
idx = (self.client_index + i) % num_engines
|
||||
counts = self.lb_engines[idx]
|
||||
if counts < min_counts:
|
||||
min_counts = counts
|
||||
eng_index = idx
|
||||
# Adjust local counts for better balancing between stats updates
|
||||
# from the coordinator (which happen every 100ms).
|
||||
if min_counts[0]:
|
||||
min_counts[0] += 1
|
||||
else:
|
||||
min_counts[1] += 1
|
||||
return self.core_engines[eng_index]
|
||||
|
||||
async def call_utility_async(self, method: str, *args) -> Any:
|
||||
# Only the result from the first engine is returned.
|
||||
return (await asyncio.gather(*[
|
||||
@@ -940,62 +1004,30 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
]))[0]
|
||||
|
||||
async def add_request_async(self, request: EngineCoreRequest) -> None:
|
||||
self._ensure_stats_update_task()
|
||||
|
||||
request.current_wave = self.current_wave
|
||||
request.client_index = self.client_index
|
||||
|
||||
chosen_engine = self.get_core_engine_for_request()
|
||||
self.reqs_in_flight[request.request_id] = chosen_engine
|
||||
chosen_engine.num_reqs_in_flight += 1
|
||||
|
||||
to_await = self._send_input(EngineCoreRequestType.ADD, request,
|
||||
chosen_engine)
|
||||
if not self.engines_running:
|
||||
# Send request to chosen engine and dp start loop
|
||||
# control message to all other engines.
|
||||
self.engines_running = True
|
||||
to_await = asyncio.gather(
|
||||
to_await, # type: ignore[assignment]
|
||||
*self._start_wave_coros(exclude_index=chosen_engine.index))
|
||||
# Notify coordinator that we're sending a request
|
||||
await self.first_req_send_socket.send(chosen_engine.identity)
|
||||
|
||||
await to_await
|
||||
|
||||
self._ensure_output_queue_task()
|
||||
|
||||
def get_core_engine_for_request(self) -> CoreEngine:
|
||||
return min(self.core_engines, key=lambda e: e.num_reqs_in_flight)
|
||||
|
||||
@staticmethod
|
||||
async def process_engine_outputs(self: "DPAsyncMPClient",
|
||||
outputs: EngineCoreOutputs):
|
||||
if self.reqs_in_flight:
|
||||
for req_id in outputs.finished_requests or ():
|
||||
if engine := self.reqs_in_flight.pop(req_id, None):
|
||||
engine.num_reqs_in_flight -= 1
|
||||
|
||||
if outputs.wave_complete is not None:
|
||||
# Current wave is complete, move to next wave number
|
||||
# and mark engines as paused.
|
||||
if self.current_wave <= outputs.wave_complete:
|
||||
self.current_wave = outputs.wave_complete + 1
|
||||
self.engines_running = False
|
||||
|
||||
elif outputs.start_wave is not None and (
|
||||
outputs.start_wave > self.current_wave or
|
||||
(outputs.start_wave == self.current_wave
|
||||
and not self.engines_running)):
|
||||
# Engine received request for a non-current wave so we must ensure
|
||||
# that other engines progress to the next wave.
|
||||
self.current_wave = outputs.start_wave
|
||||
self.engines_running = True
|
||||
await asyncio.gather(*self._start_wave_coros(
|
||||
exclude_index=outputs.engine_index))
|
||||
|
||||
def _start_wave_coros(self, exclude_index: int) -> list[Awaitable[None]]:
|
||||
logger.debug("Sending start DP wave %d.", self.current_wave)
|
||||
return [
|
||||
self._send_input(EngineCoreRequestType.START_DP_WAVE,
|
||||
self.current_wave, engine)
|
||||
for engine in self.core_engines if engine.index != exclude_index
|
||||
]
|
||||
if outputs.finished_requests and self.reqs_in_flight:
|
||||
for req_id in outputs.finished_requests:
|
||||
self.reqs_in_flight.pop(req_id, None)
|
||||
|
||||
async def abort_requests_async(self, request_ids: list[str]) -> None:
|
||||
if not request_ids:
|
||||
|
||||
Reference in New Issue
Block a user