Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+3 -7
View File
@@ -45,8 +45,7 @@ class EEPNotificationType(enum.Enum):
class FinishReason(enum.IntEnum):
"""
Reason a request finished - stop, length, abort, error, or repetition.
"""Reason a request finished - stop, length, abort, error, or repetition.
Int rather than Str for more compact serialization.
@@ -284,8 +283,7 @@ class EngineCoreOutputs(
class EngineCoreRequestType(enum.Enum):
"""
Request types defined as hex byte strings, so it can be sent over sockets
"""Request types defined as hex byte strings, so it can be sent over sockets
without separate encoding step.
"""
@@ -310,9 +308,7 @@ class ReconfigureDistributedRequest(msgspec.Struct):
class ReconfigureRankType(enum.IntEnum):
"""
Rank type for reconfiguring distributed request.
"""
"""Rank type for reconfiguring distributed request."""
KEEP_CURRENT_RANK = -1
SHUTDOWN_CURRENT_RANK = -2
+23 -23
View File
@@ -94,8 +94,7 @@ class AsyncLLM(EngineClient):
client_index: int = 0,
profiler: TorchProfilerWrapper | None = None,
) -> None:
"""
Create an AsyncLLM.
"""Create an AsyncLLM.
Args:
vllm_config: global configuration.
@@ -105,13 +104,21 @@ class AsyncLLM(EngineClient):
mm_registry: Multi-modal registry.
log_requests: Whether to log requests.
start_engine_loop: Whether to start the engine loop.
client_addresses: ZMQ addresses of an externally launched engine,
when not launching one in-process.
client_count: Number of API-server clients sharing the engine.
client_index: Index of this client within ``client_count``.
aggregate_engine_logging: Whether to aggregate per-engine logs
into a single stat logger.
stat_loggers: customized stat loggers for the engine.
If not provided, default stat loggers will be used.
PLEASE BE AWARE THAT STAT LOGGER IS NOT STABLE
IN V1, AND ITS BASE CLASS INTERFACE MIGHT CHANGE.
profiler: Torch profiler wrapper used to trace the frontend.
Returns:
None
"""
# Ensure we can serialize custom transformer configs
maybe_register_config_serialize_by_value()
@@ -255,7 +262,6 @@ class AsyncLLM(EngineClient):
stat_loggers: list[StatLoggerFactory] | None = None,
) -> "AsyncLLM":
"""Create an AsyncLLM from the EngineArgs."""
# Create the engine configs.
vllm_config = engine_args.create_engine_config(usage_context)
executor_class = Executor.get_class(vllm_config)
@@ -319,6 +325,7 @@ class AsyncLLM(EngineClient):
Raises:
QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded.
MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded.
"""
max_num_reqs = self.scheduler_config.max_num_queued_reqs
if max_num_reqs is not None:
@@ -378,7 +385,6 @@ class AsyncLLM(EngineClient):
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> RequestOutputCollector:
"""Add new request to the AsyncLLM."""
if self.errored:
raise EngineDeadError()
@@ -654,8 +660,7 @@ class AsyncLLM(EngineClient):
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
"""
Main function called by the API server to kick off a request
"""Main function called by the API server to kick off a request
* 1) Making an AsyncStream corresponding to the Request.
* 2) Processing the Input.
* 3) Adding the Request to the Detokenizer.
@@ -678,8 +683,8 @@ class AsyncLLM(EngineClient):
>>> params = self.renderer.default_cmpl_tok_params
>>> (engine_input,) = self.renderer.render_cmpl([parsed], params)
>>> gen = self.generate(engine_input, sampling_params, request_id)
"""
"""
q: RequestOutputCollector | None = None
try:
q = await self.add_request(
@@ -763,7 +768,6 @@ class AsyncLLM(EngineClient):
def _run_output_handler(self):
"""Background loop: pulls from EngineCore and pushes to AsyncStreams."""
if self.output_handler is not None:
return
@@ -849,7 +853,6 @@ class AsyncLLM(EngineClient):
self, request_id: str | Iterable[str], internal: bool = False
) -> None:
"""Abort RequestId in OutputProcessor and EngineCore."""
request_ids = (
(request_id,) if isinstance(request_id, str) else as_list(request_id)
)
@@ -868,7 +871,8 @@ class AsyncLLM(EngineClient):
) -> None:
"""Submit a pre-aborted request so the connector's request_finished
hook runs to free any pre-admission KV-transfer resources (e.g. NIXL
prefill blocks pinned on the P node)."""
prefill blocks pinned on the P node).
"""
request = EngineCoreRequest(
request_id=request_id,
prompt_token_ids=[0],
@@ -893,8 +897,7 @@ class AsyncLLM(EngineClient):
wait_for_inflight_requests: bool | None = None,
clear_cache: bool = True,
) -> None:
"""
Pause generation to allow model weight updates.
"""Pause generation to allow model weight updates.
All mode handling (abort / wait / keep) and cache clearing is done
in the engine. New generation/encoding requests will not be scheduled
@@ -910,6 +913,7 @@ class AsyncLLM(EngineClient):
wait_for_inflight_requests: DEPRECATED: use mode argument.
clear_cache: Whether to clear KV cache and prefix cache after
draining. Set to ``False`` to preserve cache for faster resume.
"""
if wait_for_inflight_requests:
warnings.warn(
@@ -950,8 +954,7 @@ class AsyncLLM(EngineClient):
tokenization_kwargs: dict[str, Any] | None = None,
reasoning_ended: bool | None = None,
) -> AsyncGenerator[PoolingRequestOutput, None]:
"""
Main function called by the API server to kick off a request
"""Main function called by the API server to kick off a request
* 1) Making an AsyncStream corresponding to the Request.
* 2) Processing the Input.
* 3) Adding the Request to the EngineCore (separate process).
@@ -963,7 +966,6 @@ class AsyncLLM(EngineClient):
The caller of generate() iterates the returned AsyncGenerator,
returning the RequestOutput back to the caller.
"""
q: RequestOutputCollector | None = None
try:
q = await self.add_request(
@@ -1116,9 +1118,7 @@ class AsyncLLM(EngineClient):
args: tuple = (),
kwargs: dict | None = None,
):
"""
Perform a collective RPC call to the given path.
"""
"""Perform a collective RPC call to the given path."""
return await self.engine_core.collective_rpc_async(
method, timeout, args, kwargs
)
@@ -1198,7 +1198,7 @@ class AsyncLLM(EngineClient):
async def handle_fault(
self, fault_tolerance_request: FaultToleranceRequest
) -> FaultToleranceResult:
"""send fault tolerance instruction to the engine"""
"""Send fault tolerance instruction to the engine."""
return await self.engine_core.handle_fault(fault_tolerance_request)
async def get_status(self):
@@ -1224,11 +1224,11 @@ class AsyncLLM(EngineClient):
async def init_weight_transfer_engine(
self, request: WeightTransferInitRequest
) -> None:
"""
Initialize weight transfer for RL training.
"""Initialize weight transfer for RL training.
Args:
request: Weight transfer initialization request with backend-specific info
"""
await self.collective_rpc(
"init_weight_transfer_engine", kwargs={"init_info": request.init_info}
@@ -1243,11 +1243,11 @@ class AsyncLLM(EngineClient):
await self.collective_rpc("start_draft_weight_update")
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
"""
Batched weight update for RL training.
"""Batched weight update for RL training.
Args:
request: Weight update request with backend-specific update info
"""
await self.collective_rpc(
"update_weights", kwargs={"update_info": request.update_info}
+14 -29
View File
@@ -505,7 +505,6 @@ class EngineCore:
def abort_requests(self, request_ids: list[str]):
"""Abort requests from the scheduler."""
# TODO: The scheduler doesn't really need to know the
# specific finish reason, TBD whether we propagate that
# (i.e. client-aborted vs stop criteria met).
@@ -599,7 +598,8 @@ class EngineCore:
def _should_throttle_prefills(self) -> bool:
"""Whether to defer new prefills this step (DP prefill balancing).
Overridden by the DP engine core; never throttles otherwise."""
Overridden by the DP engine core; never throttles otherwise.
"""
return False
def step(self) -> tuple[dict[int, EngineCoreOutputs], bool]:
@@ -608,7 +608,6 @@ class EngineCore:
Returns tuple of outputs and a flag indicating whether the model
was executed.
"""
# Check for any requests remaining in the scheduler - unfinished,
# or finished and not yet removed from the batch.
if not self.scheduler.has_requests():
@@ -659,7 +658,6 @@ class EngineCore:
batch in the job queue is finished.
3. Update the scheduler from the output.
"""
batch_queue = self.batch_queue
assert batch_queue is not None
@@ -903,8 +901,8 @@ class EngineCore:
- Level 2: Discard all GPU memory.
mode: Pause mode - how to deal with any existing requests, see
documentation of pause_scheduler method.
"""
"""
# Pause scheduler before sleeping.
clear_prefix_cache = level >= 1
pause_future = self.pause_scheduler(mode=mode, clear_cache=clear_prefix_cache)
@@ -935,6 +933,7 @@ class EngineCore:
Args:
tags: Tags to wake up. Use ["scheduling"] for level 0 wake up.
"""
if tags is not None and "scheduling" in tags:
# Remove "scheduling" from tags if there are other tags to process.
@@ -1162,8 +1161,7 @@ class EngineCoreProc(EngineCore):
vllm_config: VllmConfig,
client_handshake_address: str | None,
) -> Generator[EngineZmqAddresses, None, None]:
"""
Perform startup handshakes.
"""Perform startup handshakes.
For DP=1 or offline mode, this is with the colocated front-end process.
@@ -1298,7 +1296,6 @@ class EngineCoreProc(EngineCore):
@staticmethod
def run_engine_core(*args, dp_rank: int = 0, local_dp_rank: int = 0, **kwargs):
"""Launch EngineCore busy loop in background process."""
# Ensure we can serialize transformer config after spawning
maybe_register_config_serialize_by_value()
@@ -1444,7 +1441,6 @@ class EngineCoreProc(EngineCore):
def _process_input_queue(self):
"""Exits when an engine step needs to be performed."""
waited = False
while not self.has_work() and self.is_running():
# Notify callbacks waiting for engine to become idle.
@@ -1475,7 +1471,6 @@ class EngineCoreProc(EngineCore):
def _process_engine_step(self) -> bool:
"""Called only when there are unfinished local requests."""
# Step the engine core.
outputs, model_executed = self.step_fn()
# Put EngineCoreOutputs into the output queue.
@@ -1549,7 +1544,6 @@ class EngineCoreProc(EngineCore):
self, request_type: EngineCoreRequestType, request: Any
) -> None:
"""Dispatch request from client."""
if request_type == EngineCoreRequestType.WAKEUP:
return
elif request_type == EngineCoreRequestType.ADD:
@@ -1629,7 +1623,8 @@ class EngineCoreProc(EngineCore):
@staticmethod
def _convert_msgspec_args(method, args):
"""If a provided arg type doesn't match corresponding target method
arg type, try converting to msgspec object."""
arg type, try converting to msgspec object.
"""
if not args:
return args
arg_types = signature(method).parameters.values()
@@ -1645,7 +1640,6 @@ class EngineCoreProc(EngineCore):
def _send_engine_dead(self):
"""Send EngineDead status to the EngineCoreClient."""
# Put ENGINE_CORE_DEAD in the queue.
self.output_queue.put_nowait(EngineCoreProc.ENGINE_CORE_DEAD)
@@ -1710,7 +1704,6 @@ class EngineCoreProc(EngineCore):
ready_event: threading.Event,
):
"""Input socket IO thread."""
# Msgpack serialization decoding with optional tensor IPC receiver.
add_request_decoder = MsgpackDecoder(
EngineCoreRequest, oob_tensor_provider=self.tensor_ipc_receiver
@@ -1809,7 +1802,6 @@ class EngineCoreProc(EngineCore):
self, output_paths: list[str], coord_output_path: str | None, engine_index: int
):
"""Output socket IO thread."""
# Msgpack serialization encoding.
encoder = MsgpackEncoder()
# Send buffers to reuse.
@@ -2014,7 +2006,8 @@ class EngineCoreProc(EngineCore):
class DPEngineCoreProc(EngineCoreProc):
"""ZMQ-wrapper for running EngineCore in background process
in a data parallel context."""
in a data parallel context.
"""
def __init__(
self,
@@ -2199,7 +2192,6 @@ class DPEngineCoreProc(EngineCoreProc):
@fault_tolerant_wrapper
def run_busy_loop(self):
"""Core busy loop of the EngineCore for data parallel case."""
# Loop until process is sent a SIGINT or SIGTERM
while self._handle_shutdown():
# 1) Poll the input queue until there is work to do.
@@ -2364,8 +2356,7 @@ class DPEngineCoreProc(EngineCoreProc):
def _eep_send_engine_core_notification(
self, notification_type: EEPNotificationType
):
"""
Send notifications to EngineCoreClient, which can then forward
"""Send notifications to EngineCoreClient, which can then forward
the notifications to other engine core processes. It is used for:
1) In scale down: removing core engines to notify EngineCoreClient
so EngineCoreClient can release their ray placement groups;
@@ -2415,9 +2406,7 @@ class DPEngineCoreProc(EngineCoreProc):
class EngineCoreActorMixin:
"""
Ray actor for running EngineCore in a data parallel context
"""
"""Ray actor for running EngineCore in a data parallel context."""
def __init__(
self,
@@ -2510,16 +2499,14 @@ class EngineCoreActorMixin:
vllm_config: VllmConfig,
client_handshake_address: str | None,
):
"""
For Ray, we don't need to actually perform handshake.
"""For Ray, we don't need to actually perform handshake.
All addresses information is known before the actor creation.
Therefore, we simply yield these addresses.
"""
yield self.addresses
def wait_for_init(self):
"""
Wait until the engine core is initialized.
"""Wait until the engine core is initialized.
This is just an empty method. When ray.get() on this method
(or any other method of the actor) returns, it is guaranteed
@@ -2528,9 +2515,7 @@ class EngineCoreActorMixin:
pass
def run(self):
"""
Run the engine core busy loop.
"""
"""Run the engine core busy loop."""
try:
self.run_busy_loop() # type: ignore[attr-defined]
except SystemExit:
+24 -22
View File
@@ -77,8 +77,7 @@ EngineIdentity = bytes
class EngineCoreClient(ABC):
"""
EngineCoreClient: subclasses handle different methods for pushing
"""EngineCoreClient: subclasses handle different methods for pushing
and pulling from the EngineCore for asyncio / multiprocessing.
Subclasses:
@@ -249,7 +248,8 @@ class EngineCoreClient(ABC):
def dp_engines_running(self) -> bool:
"""Returns True if data parallel engines are collectively in a
running state."""
running state.
"""
raise NotImplementedError
async def commit_elastic_ep(self) -> None:
@@ -331,8 +331,7 @@ class EngineCoreClient(ABC):
class InprocClient(EngineCoreClient):
"""
InprocClient: client for in-process EngineCore. Intended
"""InprocClient: client for in-process EngineCore. Intended
for use in LLMEngine for V0-style add_request() and step()
EngineCore setup in this process (no busy loop).
@@ -447,7 +446,8 @@ class InprocClient(EngineCoreClient):
@dataclass
class BackgroundResources:
"""Used as a finalizer for clean shutdown, avoiding
circular reference back to the client object."""
circular reference back to the client object.
"""
ctx: zmq.Context
# If CoreEngineProcManager, it manages local engines;
@@ -469,7 +469,6 @@ class BackgroundResources:
def __call__(self):
"""Clean up background resources."""
logger.debug_once("[shutdown] MPClient: background resource cleanup start")
self.engine_dead = True
if self.engine_manager is not None:
@@ -543,16 +542,15 @@ class ElasticScalingCache:
class MPClient(EngineCoreClient):
"""
MPClient: base client for multi-proc EngineCore.
EngineCore runs in a background process busy loop, getting
new EngineCoreRequests and returning EngineCoreOutputs
"""MPClient: base client for multi-proc EngineCore.
EngineCore runs in a background process busy loop, getting
new EngineCoreRequests and returning EngineCoreOutputs
* pushes EngineCoreRequests via input_socket
* pulls EngineCoreOutputs via output_socket
* pushes EngineCoreRequests via input_socket
* pulls EngineCoreOutputs via output_socket
* AsyncMPClient subclass for AsyncLLM usage
* SyncMPClient subclass for LLM usage
* AsyncMPClient subclass for AsyncLLM usage
* SyncMPClient subclass for LLM usage
"""
def __init__(
@@ -803,7 +801,8 @@ class MPClient(EngineCoreClient):
def _apply_ready_response(self, payload: bytes) -> None:
"""Decode an EngineCoreReadyResponse and sync any post-initialization
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
config changes (e.g. auto-fitted max_model_len) back to the frontend.
"""
vllm_config = self.vllm_config
response = (
msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
@@ -1334,7 +1333,8 @@ class AsyncMPClient(MPClient):
class DPAsyncMPClient(AsyncMPClient):
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
EngineCore. Assumes external load-balancing by default."""
EngineCore. Assumes external load-balancing by default.
"""
def __init__(
self,
@@ -1518,7 +1518,8 @@ class DPAsyncMPClient(AsyncMPClient):
class DPLBAsyncMPClient(DPAsyncMPClient):
"""Asyncio-compatible client for multi-proc, multi-engine (data parallel)
EngineCore. Load-balances between multiple engine processes."""
EngineCore. Load-balances between multiple engine processes.
"""
def __init__(
self,
@@ -1754,8 +1755,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts
def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future:
"""
Wait for core engines to switch to the new setup.
"""Wait for core engines to switch to the new setup.
In eep_process_engine_core_notification(), a dummy UtilityOutput with
EEP_NOTIFICATION_CALL_ID will be set when RECONFIGURE_FINISHED
@@ -1827,7 +1827,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
num_redundant_experts: int,
) -> None:
"""Prepare scale up by creating new engine cores and reconfiguring
existing ones."""
existing ones.
"""
self._setup_elastic_ep_reconfig_bootstrap()
# Phase 1: Send reconfig messages to existing engines
@@ -1919,7 +1920,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
async def _commit_scale_down_elastic_ep(self, new_data_parallel_size: int) -> None:
"""Scale down the data parallel size by shutting down and
reconfiguring existing engine cores."""
reconfiguring existing engine cores.
"""
cur_data_parallel_size = len(self.core_engines)
self.eep_scaling_cache = ElasticScalingCache(
+3 -4
View File
@@ -94,8 +94,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
self.output_text = ""
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
"""
Update RequestState for the request_id by:
"""Update RequestState for the request_id by:
1) Detokenize the new token ids incrementally.
2) Evaluate stop criteria.
@@ -148,8 +147,8 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
def get_next_output_text(self, finished: bool, delta: bool) -> str:
"""If delta is True, only new text since the last call to
this method is returned"""
this method is returned
"""
# We return the full output text if the sequence is finished.
buffer_length = 0 if finished else self.stop_buffer_length
if not delta:
+1 -2
View File
@@ -206,8 +206,7 @@ class InputProcessor:
mm_hash: str,
lora_request: LoRARequest | None,
) -> str:
"""
When enable_tower_connector_lora is True, multi-modal embeddings
"""When enable_tower_connector_lora is True, multi-modal embeddings
vary depending on the LoRA request. Therefore, the mm_hash must be
generated based on the LoRA request to prevent incorrect cache hits.
"""
-2
View File
@@ -171,7 +171,6 @@ class LLMEngine:
enable_multiprocessing: bool = False,
) -> "LLMEngine":
"""Creates an LLM engine from the engine arguments."""
# Create the engine configs.
vllm_config = engine_args.create_engine_config(usage_context)
executor_class = Executor.get_class(vllm_config)
@@ -216,7 +215,6 @@ class LLMEngine:
def abort_request(self, request_ids: list[str], internal: bool = False) -> None:
"""Remove request_ids from EngineCore and Detokenizer."""
request_ids = self.output_processor.abort_requests(request_ids, internal)
self.engine_core.abort_requests(request_ids)
+5 -3
View File
@@ -76,7 +76,6 @@ class LogprobsProcessor:
logprobs_lists: the lists of logprob tokens, logprobs, and ranks.
"""
assert self.num_logprobs is not None
assert self.logprobs is not None
assert self.cumulative_logprob is not None
@@ -129,7 +128,6 @@ class LogprobsProcessor:
tensors.
"""
# Prompt logprobs are enabled.
assert self.num_prompt_logprobs is not None
assert self.prompt_logprobs is not None
@@ -187,7 +185,7 @@ class LogprobsProcessor:
)
def pop_prompt_logprobs(self) -> PromptLogprobs | None:
"""Pop and return all request prompt logprobs
"""Pop and return all request prompt logprobs.
The logprobs processor aggregates prompt chunk logprobs
over one or more prefill chunks. This method returns
@@ -199,6 +197,7 @@ class LogprobsProcessor:
Returns:
None if prompt logprobs are disabled for this request.
List of all prompt logprobs, otherwise.
"""
plp = self.prompt_logprobs
if plp:
@@ -223,6 +222,7 @@ class LogprobsProcessor:
Returns:
List of sampled token IDs, oldest first, most recent last.
"""
if not logprobs_source:
return []
@@ -265,6 +265,7 @@ class LogprobsProcessor:
Returns:
The corrected decoded string, or empty string if the byte
sequence is genuinely incomplete at this point.
"""
assert self.tokenizer is not None
@@ -325,6 +326,7 @@ class LogprobsProcessor:
context_token_ids: Preceding sampled token IDs providing
sequential context. If None, extracted from
self.logprobs.
"""
if context_token_ids is None:
context_token_ids = self._get_sampled_context_ids(self.logprobs)
+2 -6
View File
@@ -49,8 +49,7 @@ EMPTY_CPU_TENSOR = torch.empty(0, device="cpu")
class RequestOutputCollector:
"""
Collects streamed RequestOutputs per individual request,
"""Collects streamed RequestOutputs per individual request,
for hand-off to the consuming asyncio generate task.
When streaming deltas, RequestOutputs are merged if the
@@ -506,7 +505,6 @@ class OutputProcessor:
def propagate_error(self, e: Exception):
"""Propagate error to all generate() tasks."""
for _, state in self.request_states.items():
assert state.queue is not None
state.queue.put(e)
@@ -646,8 +644,7 @@ class OutputProcessor:
engine_core_timestamp: float | None = None,
iteration_stats: IterationStats | None = None,
) -> OutputProcessorOutput:
"""
Process the EngineCoreOutputs:
"""Process the EngineCoreOutputs:
1) Compute stats for logging
2) Detokenize
3) Create and handle RequestOutput objects:
@@ -667,7 +664,6 @@ class OutputProcessor:
If you need to touch every element of the batch, do it from
within the loop below.
"""
request_outputs: list[RequestOutput | PoolingRequestOutput] = []
reqs_to_abort: list[str] = []
for engine_core_output in engine_core_outputs:
+3 -1
View File
@@ -53,7 +53,7 @@ class ParentRequest:
self,
index: int,
) -> SamplingParams:
"""Efficiently obtain child `sampling_params`
"""Efficiently obtain child `sampling_params`.
If `sampling_params.seed` is not `None` then
each child request requires a unique clone of
@@ -64,6 +64,7 @@ class ParentRequest:
Returns:
Child `sampling_params` instance.
"""
seed = self.sampling_params.seed
if self.cached_child_sampling_params:
@@ -88,6 +89,7 @@ class ParentRequest:
Returns:
(request ID, sampling_params) tuple
"""
child_req_id = f"{index}_{self.request_id}"
self.child_requests.add(child_req_id)
+1 -3
View File
@@ -29,8 +29,7 @@ TensorIpcQueue = MPQueue
@dataclasses.dataclass
class TensorIpcData:
"""
Data sent via torch.multiprocessing.Queue for zero-copy IPC.
"""Data sent via torch.multiprocessing.Queue for zero-copy IPC.
Contains the tensor_id and the actual tensor. The tensor is
shared in memory (GPU or CPU) for efficient inter-process communication.
@@ -130,7 +129,6 @@ class TensorIpcReceiver:
the queue, buffering them, until the requested tensor is found.
Works for CUDA and CPU.
"""
# Create lookup key from handle
sender_id: str = meta["sender_id"]
message_id: int = meta["message_id"]
+8 -18
View File
@@ -142,8 +142,7 @@ def _node_ip_from_resources(node_resources: dict) -> str | None:
class CoreEngineProcManager:
"""
Utility class to handle creation, readiness, and shutdown
"""Utility class to handle creation, readiness, and shutdown
of background processes used by the AsyncLLM and LLMEngine.
"""
@@ -255,7 +254,6 @@ class CoreEngineProcManager:
def monitor_engine_liveness(self) -> None:
"""Monitor engine core process liveness."""
sentinel_to_proc = {proc.sentinel: proc for proc in self.processes}
sentinels = set(sentinel_to_proc.keys())
@@ -316,8 +314,7 @@ def set_assigned_physical_gpu_ids_for_dp_rank(
local_dp_rank: int,
user_assigned_gpu_ids: list[int] | None = None,
) -> None:
"""
Populate assigned_physical_gpu_ids on the config for the given DP rank.
"""Populate assigned_physical_gpu_ids on the config for the given DP rank.
user_assigned_gpu_ids is the full (un-sharded) --device-ids list, if the
user provided one; this DP rank's shard is sliced from it. It is passed
@@ -345,8 +342,7 @@ def get_physical_gpu_ids_for_local_dp_rank(
local_world_size: int | None = None,
user_assigned_gpu_ids: list[int] | None = None,
) -> list[int]:
"""
Returns list of physical GPU IDs for the specified
"""Returns list of physical GPU IDs for the specified
data parallel rank.
For example, if world_size=2 and local_dp_rank=1, and there are 4 devices,
@@ -399,8 +395,7 @@ def _apply_dp_identity_suffix(dp_vllm_config, dp_rank: int) -> None:
class CoreEngineActorManager:
"""
Utility class to handle creation, readiness, and shutdown
"""Utility class to handle creation, readiness, and shutdown
of core engine Ray actors used by the AsyncLLM and LLMEngine.
Different from CoreEngineProcManager, this class manages
@@ -552,10 +547,7 @@ class CoreEngineActorManager:
def create_dp_placement_groups(
vllm_config: VllmConfig,
) -> tuple[list["PlacementGroup"], list[int]]:
"""
Create placement groups for data parallel.
"""
"""Create placement groups for data parallel."""
import ray
from ray._private.state import available_resources_per_node
@@ -769,9 +761,7 @@ class CoreEngineActorManager:
def add_dp_placement_groups(
old_vllm_config: VllmConfig, new_data_parallel_size: int
) -> tuple[list["PlacementGroup"], list[int]]:
"""
Add placement groups for new data parallel size.
"""
"""Add placement groups for new data parallel size."""
import ray
from ray._private.state import (
available_resources_per_node,
@@ -1051,7 +1041,8 @@ def get_engine_zmq_addresses(
Set ``defer_api_server_ports=False`` only when the consumer cannot
report a bound port back (e.g. the Rust front-end). IPC paths are
unaffected."""
unaffected.
"""
parallel_config = vllm_config.parallel_config
local_engine_count = parallel_config.data_parallel_size_local
local_start_index = parallel_config.data_parallel_rank_local
@@ -1108,7 +1099,6 @@ def launch_core_engines(
addresses: EngineZmqAddresses,
) -> Iterator[CoreEngineLaunch]:
"""Launch engine and DP coordinator processes as needed."""
parallel_config = vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
local_engine_count = parallel_config.data_parallel_size_local