Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -45,8 +45,7 @@ class EEPNotificationType(enum.Enum):
|
||||
|
||||
|
||||
class FinishReason(enum.IntEnum):
|
||||
"""
|
||||
Reason a request finished - stop, length, abort, error, or repetition.
|
||||
"""Reason a request finished - stop, length, abort, error, or repetition.
|
||||
|
||||
Int rather than Str for more compact serialization.
|
||||
|
||||
@@ -284,8 +283,7 @@ class EngineCoreOutputs(
|
||||
|
||||
|
||||
class EngineCoreRequestType(enum.Enum):
|
||||
"""
|
||||
Request types defined as hex byte strings, so it can be sent over sockets
|
||||
"""Request types defined as hex byte strings, so it can be sent over sockets
|
||||
without separate encoding step.
|
||||
"""
|
||||
|
||||
@@ -310,9 +308,7 @@ class ReconfigureDistributedRequest(msgspec.Struct):
|
||||
|
||||
|
||||
class ReconfigureRankType(enum.IntEnum):
|
||||
"""
|
||||
Rank type for reconfiguring distributed request.
|
||||
"""
|
||||
"""Rank type for reconfiguring distributed request."""
|
||||
|
||||
KEEP_CURRENT_RANK = -1
|
||||
SHUTDOWN_CURRENT_RANK = -2
|
||||
|
||||
+23
-23
@@ -94,8 +94,7 @@ class AsyncLLM(EngineClient):
|
||||
client_index: int = 0,
|
||||
profiler: TorchProfilerWrapper | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Create an AsyncLLM.
|
||||
"""Create an AsyncLLM.
|
||||
|
||||
Args:
|
||||
vllm_config: global configuration.
|
||||
@@ -105,13 +104,21 @@ class AsyncLLM(EngineClient):
|
||||
mm_registry: Multi-modal registry.
|
||||
log_requests: Whether to log requests.
|
||||
start_engine_loop: Whether to start the engine loop.
|
||||
client_addresses: ZMQ addresses of an externally launched engine,
|
||||
when not launching one in-process.
|
||||
client_count: Number of API-server clients sharing the engine.
|
||||
client_index: Index of this client within ``client_count``.
|
||||
aggregate_engine_logging: Whether to aggregate per-engine logs
|
||||
into a single stat logger.
|
||||
stat_loggers: customized stat loggers for the engine.
|
||||
If not provided, default stat loggers will be used.
|
||||
PLEASE BE AWARE THAT STAT LOGGER IS NOT STABLE
|
||||
IN V1, AND ITS BASE CLASS INTERFACE MIGHT CHANGE.
|
||||
profiler: Torch profiler wrapper used to trace the frontend.
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
"""
|
||||
# Ensure we can serialize custom transformer configs
|
||||
maybe_register_config_serialize_by_value()
|
||||
@@ -255,7 +262,6 @@ class AsyncLLM(EngineClient):
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
) -> "AsyncLLM":
|
||||
"""Create an AsyncLLM from the EngineArgs."""
|
||||
|
||||
# Create the engine configs.
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
@@ -319,6 +325,7 @@ class AsyncLLM(EngineClient):
|
||||
Raises:
|
||||
QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded.
|
||||
MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded.
|
||||
|
||||
"""
|
||||
max_num_reqs = self.scheduler_config.max_num_queued_reqs
|
||||
if max_num_reqs is not None:
|
||||
@@ -378,7 +385,6 @@ class AsyncLLM(EngineClient):
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
"""Add new request to the AsyncLLM."""
|
||||
|
||||
if self.errored:
|
||||
raise EngineDeadError()
|
||||
|
||||
@@ -654,8 +660,7 @@ class AsyncLLM(EngineClient):
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
"""Main function called by the API server to kick off a request
|
||||
* 1) Making an AsyncStream corresponding to the Request.
|
||||
* 2) Processing the Input.
|
||||
* 3) Adding the Request to the Detokenizer.
|
||||
@@ -678,8 +683,8 @@ class AsyncLLM(EngineClient):
|
||||
>>> params = self.renderer.default_cmpl_tok_params
|
||||
>>> (engine_input,) = self.renderer.render_cmpl([parsed], params)
|
||||
>>> gen = self.generate(engine_input, sampling_params, request_id)
|
||||
"""
|
||||
|
||||
"""
|
||||
q: RequestOutputCollector | None = None
|
||||
try:
|
||||
q = await self.add_request(
|
||||
@@ -763,7 +768,6 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
def _run_output_handler(self):
|
||||
"""Background loop: pulls from EngineCore and pushes to AsyncStreams."""
|
||||
|
||||
if self.output_handler is not None:
|
||||
return
|
||||
|
||||
@@ -849,7 +853,6 @@ class AsyncLLM(EngineClient):
|
||||
self, request_id: str | Iterable[str], internal: bool = False
|
||||
) -> None:
|
||||
"""Abort RequestId in OutputProcessor and EngineCore."""
|
||||
|
||||
request_ids = (
|
||||
(request_id,) if isinstance(request_id, str) else as_list(request_id)
|
||||
)
|
||||
@@ -868,7 +871,8 @@ class AsyncLLM(EngineClient):
|
||||
) -> None:
|
||||
"""Submit a pre-aborted request so the connector's request_finished
|
||||
hook runs to free any pre-admission KV-transfer resources (e.g. NIXL
|
||||
prefill blocks pinned on the P node)."""
|
||||
prefill blocks pinned on the P node).
|
||||
"""
|
||||
request = EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=[0],
|
||||
@@ -893,8 +897,7 @@ class AsyncLLM(EngineClient):
|
||||
wait_for_inflight_requests: bool | None = None,
|
||||
clear_cache: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Pause generation to allow model weight updates.
|
||||
"""Pause generation to allow model weight updates.
|
||||
|
||||
All mode handling (abort / wait / keep) and cache clearing is done
|
||||
in the engine. New generation/encoding requests will not be scheduled
|
||||
@@ -910,6 +913,7 @@ class AsyncLLM(EngineClient):
|
||||
wait_for_inflight_requests: DEPRECATED: use mode argument.
|
||||
clear_cache: Whether to clear KV cache and prefix cache after
|
||||
draining. Set to ``False`` to preserve cache for faster resume.
|
||||
|
||||
"""
|
||||
if wait_for_inflight_requests:
|
||||
warnings.warn(
|
||||
@@ -950,8 +954,7 @@ class AsyncLLM(EngineClient):
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
) -> AsyncGenerator[PoolingRequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
"""Main function called by the API server to kick off a request
|
||||
* 1) Making an AsyncStream corresponding to the Request.
|
||||
* 2) Processing the Input.
|
||||
* 3) Adding the Request to the EngineCore (separate process).
|
||||
@@ -963,7 +966,6 @@ class AsyncLLM(EngineClient):
|
||||
The caller of generate() iterates the returned AsyncGenerator,
|
||||
returning the RequestOutput back to the caller.
|
||||
"""
|
||||
|
||||
q: RequestOutputCollector | None = None
|
||||
try:
|
||||
q = await self.add_request(
|
||||
@@ -1116,9 +1118,7 @@ class AsyncLLM(EngineClient):
|
||||
args: tuple = (),
|
||||
kwargs: dict | None = None,
|
||||
):
|
||||
"""
|
||||
Perform a collective RPC call to the given path.
|
||||
"""
|
||||
"""Perform a collective RPC call to the given path."""
|
||||
return await self.engine_core.collective_rpc_async(
|
||||
method, timeout, args, kwargs
|
||||
)
|
||||
@@ -1198,7 +1198,7 @@ class AsyncLLM(EngineClient):
|
||||
async def handle_fault(
|
||||
self, fault_tolerance_request: FaultToleranceRequest
|
||||
) -> FaultToleranceResult:
|
||||
"""send fault tolerance instruction to the engine"""
|
||||
"""Send fault tolerance instruction to the engine."""
|
||||
return await self.engine_core.handle_fault(fault_tolerance_request)
|
||||
|
||||
async def get_status(self):
|
||||
@@ -1224,11 +1224,11 @@ class AsyncLLM(EngineClient):
|
||||
async def init_weight_transfer_engine(
|
||||
self, request: WeightTransferInitRequest
|
||||
) -> None:
|
||||
"""
|
||||
Initialize weight transfer for RL training.
|
||||
"""Initialize weight transfer for RL training.
|
||||
|
||||
Args:
|
||||
request: Weight transfer initialization request with backend-specific info
|
||||
|
||||
"""
|
||||
await self.collective_rpc(
|
||||
"init_weight_transfer_engine", kwargs={"init_info": request.init_info}
|
||||
@@ -1243,11 +1243,11 @@ class AsyncLLM(EngineClient):
|
||||
await self.collective_rpc("start_draft_weight_update")
|
||||
|
||||
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
|
||||
"""
|
||||
Batched weight update for RL training.
|
||||
"""Batched weight update for RL training.
|
||||
|
||||
Args:
|
||||
request: Weight update request with backend-specific update info
|
||||
|
||||
"""
|
||||
await self.collective_rpc(
|
||||
"update_weights", kwargs={"update_info": request.update_info}
|
||||
|
||||
+14
-29
@@ -505,7 +505,6 @@ class EngineCore:
|
||||
|
||||
def abort_requests(self, request_ids: list[str]):
|
||||
"""Abort requests from the scheduler."""
|
||||
|
||||
# TODO: The scheduler doesn't really need to know the
|
||||
# specific finish reason, TBD whether we propagate that
|
||||
# (i.e. client-aborted vs stop criteria met).
|
||||
@@ -599,7 +598,8 @@ class EngineCore:
|
||||
|
||||
def _should_throttle_prefills(self) -> bool:
|
||||
"""Whether to defer new prefills this step (DP prefill balancing).
|
||||
Overridden by the DP engine core; never throttles otherwise."""
|
||||
Overridden by the DP engine core; never throttles otherwise.
|
||||
"""
|
||||
return False
|
||||
|
||||
def step(self) -> tuple[dict[int, EngineCoreOutputs], bool]:
|
||||
@@ -608,7 +608,6 @@ class EngineCore:
|
||||
Returns tuple of outputs and a flag indicating whether the model
|
||||
was executed.
|
||||
"""
|
||||
|
||||
# Check for any requests remaining in the scheduler - unfinished,
|
||||
# or finished and not yet removed from the batch.
|
||||
if not self.scheduler.has_requests():
|
||||
@@ -659,7 +658,6 @@ class EngineCore:
|
||||
batch in the job queue is finished.
|
||||
3. Update the scheduler from the output.
|
||||
"""
|
||||
|
||||
batch_queue = self.batch_queue
|
||||
assert batch_queue is not None
|
||||
|
||||
@@ -903,8 +901,8 @@ class EngineCore:
|
||||
- Level 2: Discard all GPU memory.
|
||||
mode: Pause mode - how to deal with any existing requests, see
|
||||
documentation of pause_scheduler method.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Pause scheduler before sleeping.
|
||||
clear_prefix_cache = level >= 1
|
||||
pause_future = self.pause_scheduler(mode=mode, clear_cache=clear_prefix_cache)
|
||||
@@ -935,6 +933,7 @@ class EngineCore:
|
||||
|
||||
Args:
|
||||
tags: Tags to wake up. Use ["scheduling"] for level 0 wake up.
|
||||
|
||||
"""
|
||||
if tags is not None and "scheduling" in tags:
|
||||
# Remove "scheduling" from tags if there are other tags to process.
|
||||
@@ -1162,8 +1161,7 @@ class EngineCoreProc(EngineCore):
|
||||
vllm_config: VllmConfig,
|
||||
client_handshake_address: str | None,
|
||||
) -> Generator[EngineZmqAddresses, None, None]:
|
||||
"""
|
||||
Perform startup handshakes.
|
||||
"""Perform startup handshakes.
|
||||
|
||||
For DP=1 or offline mode, this is with the colocated front-end process.
|
||||
|
||||
@@ -1298,7 +1296,6 @@ class EngineCoreProc(EngineCore):
|
||||
@staticmethod
|
||||
def run_engine_core(*args, dp_rank: int = 0, local_dp_rank: int = 0, **kwargs):
|
||||
"""Launch EngineCore busy loop in background process."""
|
||||
|
||||
# Ensure we can serialize transformer config after spawning
|
||||
maybe_register_config_serialize_by_value()
|
||||
|
||||
@@ -1444,7 +1441,6 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
def _process_input_queue(self):
|
||||
"""Exits when an engine step needs to be performed."""
|
||||
|
||||
waited = False
|
||||
while not self.has_work() and self.is_running():
|
||||
# Notify callbacks waiting for engine to become idle.
|
||||
@@ -1475,7 +1471,6 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
def _process_engine_step(self) -> bool:
|
||||
"""Called only when there are unfinished local requests."""
|
||||
|
||||
# Step the engine core.
|
||||
outputs, model_executed = self.step_fn()
|
||||
# Put EngineCoreOutputs into the output queue.
|
||||
@@ -1549,7 +1544,6 @@ class EngineCoreProc(EngineCore):
|
||||
self, request_type: EngineCoreRequestType, request: Any
|
||||
) -> None:
|
||||
"""Dispatch request from client."""
|
||||
|
||||
if request_type == EngineCoreRequestType.WAKEUP:
|
||||
return
|
||||
elif request_type == EngineCoreRequestType.ADD:
|
||||
@@ -1629,7 +1623,8 @@ class EngineCoreProc(EngineCore):
|
||||
@staticmethod
|
||||
def _convert_msgspec_args(method, args):
|
||||
"""If a provided arg type doesn't match corresponding target method
|
||||
arg type, try converting to msgspec object."""
|
||||
arg type, try converting to msgspec object.
|
||||
"""
|
||||
if not args:
|
||||
return args
|
||||
arg_types = signature(method).parameters.values()
|
||||
@@ -1645,7 +1640,6 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
def _send_engine_dead(self):
|
||||
"""Send EngineDead status to the EngineCoreClient."""
|
||||
|
||||
# Put ENGINE_CORE_DEAD in the queue.
|
||||
self.output_queue.put_nowait(EngineCoreProc.ENGINE_CORE_DEAD)
|
||||
|
||||
@@ -1710,7 +1704,6 @@ class EngineCoreProc(EngineCore):
|
||||
ready_event: threading.Event,
|
||||
):
|
||||
"""Input socket IO thread."""
|
||||
|
||||
# Msgpack serialization decoding with optional tensor IPC receiver.
|
||||
add_request_decoder = MsgpackDecoder(
|
||||
EngineCoreRequest, oob_tensor_provider=self.tensor_ipc_receiver
|
||||
@@ -1809,7 +1802,6 @@ class EngineCoreProc(EngineCore):
|
||||
self, output_paths: list[str], coord_output_path: str | None, engine_index: int
|
||||
):
|
||||
"""Output socket IO thread."""
|
||||
|
||||
# Msgpack serialization encoding.
|
||||
encoder = MsgpackEncoder()
|
||||
# Send buffers to reuse.
|
||||
@@ -2014,7 +2006,8 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
class DPEngineCoreProc(EngineCoreProc):
|
||||
"""ZMQ-wrapper for running EngineCore in background process
|
||||
in a data parallel context."""
|
||||
in a data parallel context.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -2199,7 +2192,6 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
@fault_tolerant_wrapper
|
||||
def run_busy_loop(self):
|
||||
"""Core busy loop of the EngineCore for data parallel case."""
|
||||
|
||||
# Loop until process is sent a SIGINT or SIGTERM
|
||||
while self._handle_shutdown():
|
||||
# 1) Poll the input queue until there is work to do.
|
||||
@@ -2364,8 +2356,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
def _eep_send_engine_core_notification(
|
||||
self, notification_type: EEPNotificationType
|
||||
):
|
||||
"""
|
||||
Send notifications to EngineCoreClient, which can then forward
|
||||
"""Send notifications to EngineCoreClient, which can then forward
|
||||
the notifications to other engine core processes. It is used for:
|
||||
1) In scale down: removing core engines to notify EngineCoreClient
|
||||
so EngineCoreClient can release their ray placement groups;
|
||||
@@ -2415,9 +2406,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
|
||||
|
||||
class EngineCoreActorMixin:
|
||||
"""
|
||||
Ray actor for running EngineCore in a data parallel context
|
||||
"""
|
||||
"""Ray actor for running EngineCore in a data parallel context."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -2510,16 +2499,14 @@ class EngineCoreActorMixin:
|
||||
vllm_config: VllmConfig,
|
||||
client_handshake_address: str | None,
|
||||
):
|
||||
"""
|
||||
For Ray, we don't need to actually perform handshake.
|
||||
"""For Ray, we don't need to actually perform handshake.
|
||||
All addresses information is known before the actor creation.
|
||||
Therefore, we simply yield these addresses.
|
||||
"""
|
||||
yield self.addresses
|
||||
|
||||
def wait_for_init(self):
|
||||
"""
|
||||
Wait until the engine core is initialized.
|
||||
"""Wait until the engine core is initialized.
|
||||
|
||||
This is just an empty method. When ray.get() on this method
|
||||
(or any other method of the actor) returns, it is guaranteed
|
||||
@@ -2528,9 +2515,7 @@ class EngineCoreActorMixin:
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Run the engine core busy loop.
|
||||
"""
|
||||
"""Run the engine core busy loop."""
|
||||
try:
|
||||
self.run_busy_loop() # type: ignore[attr-defined]
|
||||
except SystemExit:
|
||||
|
||||
@@ -77,8 +77,7 @@ EngineIdentity = bytes
|
||||
|
||||
|
||||
class EngineCoreClient(ABC):
|
||||
"""
|
||||
EngineCoreClient: subclasses handle different methods for pushing
|
||||
"""EngineCoreClient: subclasses handle different methods for pushing
|
||||
and pulling from the EngineCore for asyncio / multiprocessing.
|
||||
|
||||
Subclasses:
|
||||
@@ -249,7 +248,8 @@ class EngineCoreClient(ABC):
|
||||
|
||||
def dp_engines_running(self) -> bool:
|
||||
"""Returns True if data parallel engines are collectively in a
|
||||
running state."""
|
||||
running state.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def commit_elastic_ep(self) -> None:
|
||||
@@ -331,8 +331,7 @@ class EngineCoreClient(ABC):
|
||||
|
||||
|
||||
class InprocClient(EngineCoreClient):
|
||||
"""
|
||||
InprocClient: client for in-process EngineCore. Intended
|
||||
"""InprocClient: client for in-process EngineCore. Intended
|
||||
for use in LLMEngine for V0-style add_request() and step()
|
||||
EngineCore setup in this process (no busy loop).
|
||||
|
||||
@@ -447,7 +446,8 @@ class InprocClient(EngineCoreClient):
|
||||
@dataclass
|
||||
class BackgroundResources:
|
||||
"""Used as a finalizer for clean shutdown, avoiding
|
||||
circular reference back to the client object."""
|
||||
circular reference back to the client object.
|
||||
"""
|
||||
|
||||
ctx: zmq.Context
|
||||
# If CoreEngineProcManager, it manages local engines;
|
||||
@@ -469,7 +469,6 @@ class BackgroundResources:
|
||||
|
||||
def __call__(self):
|
||||
"""Clean up background resources."""
|
||||
|
||||
logger.debug_once("[shutdown] MPClient: background resource cleanup start")
|
||||
self.engine_dead = True
|
||||
if self.engine_manager is not None:
|
||||
@@ -543,16 +542,15 @@ class ElasticScalingCache:
|
||||
|
||||
|
||||
class MPClient(EngineCoreClient):
|
||||
"""
|
||||
MPClient: base client for multi-proc EngineCore.
|
||||
EngineCore runs in a background process busy loop, getting
|
||||
new EngineCoreRequests and returning EngineCoreOutputs
|
||||
"""MPClient: base client for multi-proc EngineCore.
|
||||
EngineCore runs in a background process busy loop, getting
|
||||
new EngineCoreRequests and returning EngineCoreOutputs
|
||||
|
||||
* pushes EngineCoreRequests via input_socket
|
||||
* pulls EngineCoreOutputs via output_socket
|
||||
* pushes EngineCoreRequests via input_socket
|
||||
* pulls EngineCoreOutputs via output_socket
|
||||
|
||||
* AsyncMPClient subclass for AsyncLLM usage
|
||||
* SyncMPClient subclass for LLM usage
|
||||
* AsyncMPClient subclass for AsyncLLM usage
|
||||
* SyncMPClient subclass for LLM usage
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -803,7 +801,8 @@ class MPClient(EngineCoreClient):
|
||||
|
||||
def _apply_ready_response(self, payload: bytes) -> None:
|
||||
"""Decode an EngineCoreReadyResponse and sync any post-initialization
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend.
|
||||
"""
|
||||
vllm_config = self.vllm_config
|
||||
response = (
|
||||
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
@@ -1334,7 +1333,8 @@ class AsyncMPClient(MPClient):
|
||||
|
||||
class DPAsyncMPClient(AsyncMPClient):
|
||||
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
|
||||
EngineCore. Assumes external load-balancing by default."""
|
||||
EngineCore. Assumes external load-balancing by default.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1518,7 +1518,8 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
|
||||
class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
|
||||
EngineCore. Load-balances between multiple engine processes."""
|
||||
EngineCore. Load-balances between multiple engine processes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1754,8 +1755,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts
|
||||
|
||||
def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future:
|
||||
"""
|
||||
Wait for core engines to switch to the new setup.
|
||||
"""Wait for core engines to switch to the new setup.
|
||||
|
||||
In eep_process_engine_core_notification(), a dummy UtilityOutput with
|
||||
EEP_NOTIFICATION_CALL_ID will be set when RECONFIGURE_FINISHED
|
||||
@@ -1827,7 +1827,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
num_redundant_experts: int,
|
||||
) -> None:
|
||||
"""Prepare scale up by creating new engine cores and reconfiguring
|
||||
existing ones."""
|
||||
existing ones.
|
||||
"""
|
||||
self._setup_elastic_ep_reconfig_bootstrap()
|
||||
|
||||
# Phase 1: Send reconfig messages to existing engines
|
||||
@@ -1919,7 +1920,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
|
||||
async def _commit_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
|
||||
"""Scale down the data parallel size by shutting down and
|
||||
reconfiguring existing engine cores."""
|
||||
reconfiguring existing engine cores.
|
||||
"""
|
||||
cur_data_parallel_size = len(self.core_engines)
|
||||
|
||||
self.eep_scaling_cache = ElasticScalingCache(
|
||||
|
||||
@@ -94,8 +94,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
self.output_text = ""
|
||||
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
|
||||
"""
|
||||
Update RequestState for the request_id by:
|
||||
"""Update RequestState for the request_id by:
|
||||
1) Detokenize the new token ids incrementally.
|
||||
2) Evaluate stop criteria.
|
||||
|
||||
@@ -148,8 +147,8 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
|
||||
def get_next_output_text(self, finished: bool, delta: bool) -> str:
|
||||
"""If delta is True, only new text since the last call to
|
||||
this method is returned"""
|
||||
|
||||
this method is returned
|
||||
"""
|
||||
# We return the full output text if the sequence is finished.
|
||||
buffer_length = 0 if finished else self.stop_buffer_length
|
||||
if not delta:
|
||||
|
||||
@@ -206,8 +206,7 @@ class InputProcessor:
|
||||
mm_hash: str,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> str:
|
||||
"""
|
||||
When enable_tower_connector_lora is True, multi-modal embeddings
|
||||
"""When enable_tower_connector_lora is True, multi-modal embeddings
|
||||
vary depending on the LoRA request. Therefore, the mm_hash must be
|
||||
generated based on the LoRA request to prevent incorrect cache hits.
|
||||
"""
|
||||
|
||||
@@ -171,7 +171,6 @@ class LLMEngine:
|
||||
enable_multiprocessing: bool = False,
|
||||
) -> "LLMEngine":
|
||||
"""Creates an LLM engine from the engine arguments."""
|
||||
|
||||
# Create the engine configs.
|
||||
vllm_config = engine_args.create_engine_config(usage_context)
|
||||
executor_class = Executor.get_class(vllm_config)
|
||||
@@ -216,7 +215,6 @@ class LLMEngine:
|
||||
|
||||
def abort_request(self, request_ids: list[str], internal: bool = False) -> None:
|
||||
"""Remove request_ids from EngineCore and Detokenizer."""
|
||||
|
||||
request_ids = self.output_processor.abort_requests(request_ids, internal)
|
||||
self.engine_core.abort_requests(request_ids)
|
||||
|
||||
|
||||
@@ -76,7 +76,6 @@ class LogprobsProcessor:
|
||||
logprobs_lists: the lists of logprob tokens, logprobs, and ranks.
|
||||
|
||||
"""
|
||||
|
||||
assert self.num_logprobs is not None
|
||||
assert self.logprobs is not None
|
||||
assert self.cumulative_logprob is not None
|
||||
@@ -129,7 +128,6 @@ class LogprobsProcessor:
|
||||
tensors.
|
||||
|
||||
"""
|
||||
|
||||
# Prompt logprobs are enabled.
|
||||
assert self.num_prompt_logprobs is not None
|
||||
assert self.prompt_logprobs is not None
|
||||
@@ -187,7 +185,7 @@ class LogprobsProcessor:
|
||||
)
|
||||
|
||||
def pop_prompt_logprobs(self) -> PromptLogprobs | None:
|
||||
"""Pop and return all request prompt logprobs
|
||||
"""Pop and return all request prompt logprobs.
|
||||
|
||||
The logprobs processor aggregates prompt chunk logprobs
|
||||
over one or more prefill chunks. This method returns
|
||||
@@ -199,6 +197,7 @@ class LogprobsProcessor:
|
||||
Returns:
|
||||
None if prompt logprobs are disabled for this request.
|
||||
List of all prompt logprobs, otherwise.
|
||||
|
||||
"""
|
||||
plp = self.prompt_logprobs
|
||||
if plp:
|
||||
@@ -223,6 +222,7 @@ class LogprobsProcessor:
|
||||
|
||||
Returns:
|
||||
List of sampled token IDs, oldest first, most recent last.
|
||||
|
||||
"""
|
||||
if not logprobs_source:
|
||||
return []
|
||||
@@ -265,6 +265,7 @@ class LogprobsProcessor:
|
||||
Returns:
|
||||
The corrected decoded string, or empty string if the byte
|
||||
sequence is genuinely incomplete at this point.
|
||||
|
||||
"""
|
||||
assert self.tokenizer is not None
|
||||
|
||||
@@ -325,6 +326,7 @@ class LogprobsProcessor:
|
||||
context_token_ids: Preceding sampled token IDs providing
|
||||
sequential context. If None, extracted from
|
||||
self.logprobs.
|
||||
|
||||
"""
|
||||
if context_token_ids is None:
|
||||
context_token_ids = self._get_sampled_context_ids(self.logprobs)
|
||||
|
||||
@@ -49,8 +49,7 @@ EMPTY_CPU_TENSOR = torch.empty(0, device="cpu")
|
||||
|
||||
|
||||
class RequestOutputCollector:
|
||||
"""
|
||||
Collects streamed RequestOutputs per individual request,
|
||||
"""Collects streamed RequestOutputs per individual request,
|
||||
for hand-off to the consuming asyncio generate task.
|
||||
|
||||
When streaming deltas, RequestOutputs are merged if the
|
||||
@@ -506,7 +505,6 @@ class OutputProcessor:
|
||||
|
||||
def propagate_error(self, e: Exception):
|
||||
"""Propagate error to all generate() tasks."""
|
||||
|
||||
for _, state in self.request_states.items():
|
||||
assert state.queue is not None
|
||||
state.queue.put(e)
|
||||
@@ -646,8 +644,7 @@ class OutputProcessor:
|
||||
engine_core_timestamp: float | None = None,
|
||||
iteration_stats: IterationStats | None = None,
|
||||
) -> OutputProcessorOutput:
|
||||
"""
|
||||
Process the EngineCoreOutputs:
|
||||
"""Process the EngineCoreOutputs:
|
||||
1) Compute stats for logging
|
||||
2) Detokenize
|
||||
3) Create and handle RequestOutput objects:
|
||||
@@ -667,7 +664,6 @@ class OutputProcessor:
|
||||
If you need to touch every element of the batch, do it from
|
||||
within the loop below.
|
||||
"""
|
||||
|
||||
request_outputs: list[RequestOutput | PoolingRequestOutput] = []
|
||||
reqs_to_abort: list[str] = []
|
||||
for engine_core_output in engine_core_outputs:
|
||||
|
||||
@@ -53,7 +53,7 @@ class ParentRequest:
|
||||
self,
|
||||
index: int,
|
||||
) -> SamplingParams:
|
||||
"""Efficiently obtain child `sampling_params`
|
||||
"""Efficiently obtain child `sampling_params`.
|
||||
|
||||
If `sampling_params.seed` is not `None` then
|
||||
each child request requires a unique clone of
|
||||
@@ -64,6 +64,7 @@ class ParentRequest:
|
||||
|
||||
Returns:
|
||||
Child `sampling_params` instance.
|
||||
|
||||
"""
|
||||
seed = self.sampling_params.seed
|
||||
if self.cached_child_sampling_params:
|
||||
@@ -88,6 +89,7 @@ class ParentRequest:
|
||||
|
||||
Returns:
|
||||
(request ID, sampling_params) tuple
|
||||
|
||||
"""
|
||||
child_req_id = f"{index}_{self.request_id}"
|
||||
self.child_requests.add(child_req_id)
|
||||
|
||||
@@ -29,8 +29,7 @@ TensorIpcQueue = MPQueue
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TensorIpcData:
|
||||
"""
|
||||
Data sent via torch.multiprocessing.Queue for zero-copy IPC.
|
||||
"""Data sent via torch.multiprocessing.Queue for zero-copy IPC.
|
||||
|
||||
Contains the tensor_id and the actual tensor. The tensor is
|
||||
shared in memory (GPU or CPU) for efficient inter-process communication.
|
||||
@@ -130,7 +129,6 @@ class TensorIpcReceiver:
|
||||
the queue, buffering them, until the requested tensor is found.
|
||||
Works for CUDA and CPU.
|
||||
"""
|
||||
|
||||
# Create lookup key from handle
|
||||
sender_id: str = meta["sender_id"]
|
||||
message_id: int = meta["message_id"]
|
||||
|
||||
+8
-18
@@ -142,8 +142,7 @@ def _node_ip_from_resources(node_resources: dict) -> str | None:
|
||||
|
||||
|
||||
class CoreEngineProcManager:
|
||||
"""
|
||||
Utility class to handle creation, readiness, and shutdown
|
||||
"""Utility class to handle creation, readiness, and shutdown
|
||||
of background processes used by the AsyncLLM and LLMEngine.
|
||||
"""
|
||||
|
||||
@@ -255,7 +254,6 @@ class CoreEngineProcManager:
|
||||
|
||||
def monitor_engine_liveness(self) -> None:
|
||||
"""Monitor engine core process liveness."""
|
||||
|
||||
sentinel_to_proc = {proc.sentinel: proc for proc in self.processes}
|
||||
sentinels = set(sentinel_to_proc.keys())
|
||||
|
||||
@@ -316,8 +314,7 @@ def set_assigned_physical_gpu_ids_for_dp_rank(
|
||||
local_dp_rank: int,
|
||||
user_assigned_gpu_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Populate assigned_physical_gpu_ids on the config for the given DP rank.
|
||||
"""Populate assigned_physical_gpu_ids on the config for the given DP rank.
|
||||
|
||||
user_assigned_gpu_ids is the full (un-sharded) --device-ids list, if the
|
||||
user provided one; this DP rank's shard is sliced from it. It is passed
|
||||
@@ -345,8 +342,7 @@ def get_physical_gpu_ids_for_local_dp_rank(
|
||||
local_world_size: int | None = None,
|
||||
user_assigned_gpu_ids: list[int] | None = None,
|
||||
) -> list[int]:
|
||||
"""
|
||||
Returns list of physical GPU IDs for the specified
|
||||
"""Returns list of physical GPU IDs for the specified
|
||||
data parallel rank.
|
||||
|
||||
For example, if world_size=2 and local_dp_rank=1, and there are 4 devices,
|
||||
@@ -399,8 +395,7 @@ def _apply_dp_identity_suffix(dp_vllm_config, dp_rank: int) -> None:
|
||||
|
||||
|
||||
class CoreEngineActorManager:
|
||||
"""
|
||||
Utility class to handle creation, readiness, and shutdown
|
||||
"""Utility class to handle creation, readiness, and shutdown
|
||||
of core engine Ray actors used by the AsyncLLM and LLMEngine.
|
||||
|
||||
Different from CoreEngineProcManager, this class manages
|
||||
@@ -552,10 +547,7 @@ class CoreEngineActorManager:
|
||||
def create_dp_placement_groups(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[list["PlacementGroup"], list[int]]:
|
||||
"""
|
||||
Create placement groups for data parallel.
|
||||
"""
|
||||
|
||||
"""Create placement groups for data parallel."""
|
||||
import ray
|
||||
from ray._private.state import available_resources_per_node
|
||||
|
||||
@@ -769,9 +761,7 @@ class CoreEngineActorManager:
|
||||
def add_dp_placement_groups(
|
||||
old_vllm_config: VllmConfig, new_data_parallel_size: int
|
||||
) -> tuple[list["PlacementGroup"], list[int]]:
|
||||
"""
|
||||
Add placement groups for new data parallel size.
|
||||
"""
|
||||
"""Add placement groups for new data parallel size."""
|
||||
import ray
|
||||
from ray._private.state import (
|
||||
available_resources_per_node,
|
||||
@@ -1051,7 +1041,8 @@ def get_engine_zmq_addresses(
|
||||
|
||||
Set ``defer_api_server_ports=False`` only when the consumer cannot
|
||||
report a bound port back (e.g. the Rust front-end). IPC paths are
|
||||
unaffected."""
|
||||
unaffected.
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
local_engine_count = parallel_config.data_parallel_size_local
|
||||
local_start_index = parallel_config.data_parallel_rank_local
|
||||
@@ -1108,7 +1099,6 @@ def launch_core_engines(
|
||||
addresses: EngineZmqAddresses,
|
||||
) -> Iterator[CoreEngineLaunch]:
|
||||
"""Launch engine and DP coordinator processes as needed."""
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
local_engine_count = parallel_config.data_parallel_size_local
|
||||
|
||||
Reference in New Issue
Block a user