Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+27 -27
View File
@@ -4,7 +4,7 @@
import enum
import time
from collections.abc import Mapping
from typing import Any, Optional, Union
from typing import Any
import msgspec
import torch
@@ -48,16 +48,16 @@ class EngineCoreRequest(
gc=False,
): # type: ignore[call-arg]
request_id: str
prompt_token_ids: Optional[list[int]]
mm_features: Optional[list[MultiModalFeatureSpec]]
sampling_params: Optional[SamplingParams]
pooling_params: Optional[PoolingParams]
eos_token_id: Optional[int]
prompt_token_ids: list[int] | None
mm_features: list[MultiModalFeatureSpec] | None
sampling_params: SamplingParams | None
pooling_params: PoolingParams | None
eos_token_id: int | None
arrival_time: float
lora_request: Optional[LoRARequest]
cache_salt: Optional[str]
data_parallel_rank: Optional[int]
prompt_embeds: Optional[torch.Tensor] = None
lora_request: LoRARequest | None
cache_salt: str | None
data_parallel_rank: int | None
prompt_embeds: torch.Tensor | None = None
# Index of the client, used to ensure outputs are sent back to the same
# client for this request when scaling out the front-end.
@@ -69,7 +69,7 @@ class EngineCoreRequest(
current_wave: int = 0
priority: int = 0
trace_headers: Optional[Mapping[str, str]] = None
trace_headers: Mapping[str, str] | None = None
class EngineCoreEventType(enum.IntEnum):
@@ -93,7 +93,7 @@ class EngineCoreEvent(msgspec.Struct):
@classmethod
def new_event(
cls, event_type: EngineCoreEventType, timestamp: Optional[float] = None
cls, event_type: EngineCoreEventType, timestamp: float | None = None
) -> "EngineCoreEvent":
timestamp = time.monotonic() if timestamp is None else timestamp
return cls(event_type, timestamp)
@@ -108,17 +108,17 @@ class EngineCoreOutput(
request_id: str
new_token_ids: list[int]
new_logprobs: Optional[LogprobsLists] = None
new_prompt_logprobs_tensors: Optional[LogprobsTensors] = None
new_logprobs: LogprobsLists | None = None
new_prompt_logprobs_tensors: LogprobsTensors | None = None
pooling_output: Optional[torch.Tensor] = None
pooling_output: torch.Tensor | None = None
finish_reason: Optional[FinishReason] = None
stop_reason: Union[int, str, None] = None
events: Optional[list[EngineCoreEvent]] = None
kv_transfer_params: Optional[dict[str, Any]] = None
finish_reason: FinishReason | None = None
stop_reason: int | str | None = None
events: list[EngineCoreEvent] | None = None
kv_transfer_params: dict[str, Any] | None = None
trace_headers: Optional[Mapping[str, str]] = None
trace_headers: Mapping[str, str] | None = None
# The number of tokens with prefix cache hits.
num_cached_tokens: int = 0
@@ -142,8 +142,8 @@ class UtilityOutput(
call_id: int
# Non-None implies the call failed, result should be None.
failure_message: Optional[str] = None
result: Optional[UtilityResult] = None
failure_message: str | None = None
result: UtilityResult | None = None
class EngineCoreOutputs(
@@ -159,18 +159,18 @@ class EngineCoreOutputs(
# [num_reqs]
outputs: list[EngineCoreOutput] = []
scheduler_stats: Optional[SchedulerStats] = None
scheduler_stats: SchedulerStats | None = None
timestamp: float = 0.0
utility_output: Optional[UtilityOutput] = None
finished_requests: Optional[set[str]] = None
utility_output: UtilityOutput | None = None
finished_requests: set[str] | None = None
# In DP case, used to signal that the current wave of requests
# has finished and the engines are paused.
wave_complete: Optional[int] = None
wave_complete: int | None = None
# In DP case, used to signal that a request was received for an
# "old" wave, so the next wave needs to be started in other engines.
start_wave: Optional[int] = None
start_wave: int | None = None
def __post_init__(self):
if self.timestamp == 0.0:
+35 -35
View File
@@ -6,7 +6,7 @@ import socket
import time
from collections.abc import AsyncGenerator, Iterable, Mapping
from copy import copy
from typing import Any, Optional, Union
from typing import Any
import numpy as np
import torch
@@ -56,8 +56,8 @@ class AsyncLLM(EngineClient):
use_cached_outputs: bool = False,
log_requests: bool = True,
start_engine_loop: bool = True,
stat_loggers: Optional[list[StatLoggerFactory]] = None,
client_addresses: Optional[dict[str, str]] = None,
stat_loggers: list[StatLoggerFactory] | None = None,
client_addresses: dict[str, str] | None = None,
client_count: int = 1,
client_index: int = 0,
) -> None:
@@ -136,7 +136,7 @@ class AsyncLLM(EngineClient):
)
# Loggers.
self.logger_manager: Optional[StatLoggerManager] = None
self.logger_manager: StatLoggerManager | None = None
if self.log_stats:
self.logger_manager = StatLoggerManager(
vllm_config=vllm_config,
@@ -147,7 +147,7 @@ class AsyncLLM(EngineClient):
)
self.logger_manager.log_engine_initialized()
self.output_handler: Optional[asyncio.Task] = None
self.output_handler: asyncio.Task | None = None
try:
# Start output handler eagerly if we are in the asyncio eventloop.
asyncio.get_running_loop()
@@ -185,10 +185,10 @@ class AsyncLLM(EngineClient):
vllm_config: VllmConfig,
start_engine_loop: bool = True,
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
stat_loggers: Optional[list[StatLoggerFactory]] = None,
stat_loggers: list[StatLoggerFactory] | None = None,
enable_log_requests: bool = False,
disable_log_stats: bool = False,
client_addresses: Optional[dict[str, str]] = None,
client_addresses: dict[str, str] | None = None,
client_count: int = 1,
client_index: int = 0,
disable_log_requests: bool = True, # Deprecated, will be removed
@@ -221,7 +221,7 @@ class AsyncLLM(EngineClient):
engine_args: AsyncEngineArgs,
start_engine_loop: bool = True,
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
stat_loggers: Optional[list[StatLoggerFactory]] = None,
stat_loggers: list[StatLoggerFactory] | None = None,
) -> "AsyncLLM":
"""Create an AsyncLLM from the EngineArgs."""
@@ -259,15 +259,15 @@ class AsyncLLM(EngineClient):
async def add_request(
self,
request_id: str,
prompt: Union[EngineCoreRequest, PromptType],
params: Union[SamplingParams, PoolingParams],
arrival_time: Optional[float] = None,
lora_request: Optional[LoRARequest] = None,
tokenization_kwargs: Optional[dict[str, Any]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt: EngineCoreRequest | PromptType,
params: SamplingParams | PoolingParams,
arrival_time: float | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
data_parallel_rank: Optional[int] = None,
prompt_text: Optional[str] = None,
data_parallel_rank: int | None = None,
prompt_text: str | None = None,
) -> RequestOutputCollector:
"""Add new request to the AsyncLLM."""
@@ -325,8 +325,8 @@ class AsyncLLM(EngineClient):
async def _add_request(
self,
request: EngineCoreRequest,
prompt: Optional[str],
parent_req: Optional[ParentRequest],
prompt: str | None,
parent_req: ParentRequest | None,
index: int,
queue: RequestOutputCollector,
):
@@ -346,16 +346,16 @@ class AsyncLLM(EngineClient):
# re-multiplexed in the API server anyhow.
async def generate(
self,
prompt: Union[EngineCoreRequest, PromptType],
prompt: EngineCoreRequest | PromptType,
sampling_params: SamplingParams,
request_id: str,
*,
prompt_text: Optional[str] = None,
lora_request: Optional[LoRARequest] = None,
tokenization_kwargs: Optional[dict[str, Any]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt_text: str | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
data_parallel_rank: Optional[int] = None,
data_parallel_rank: int | None = None,
) -> AsyncGenerator[RequestOutput, None]:
"""
Main function called by the API server to kick off a request
@@ -520,7 +520,7 @@ class AsyncLLM(EngineClient):
self.output_handler = asyncio.create_task(output_handler())
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
async def abort(self, request_id: str | Iterable[str]) -> None:
"""Abort RequestId in OutputProcessor and EngineCore."""
request_ids = (
@@ -537,11 +537,11 @@ class AsyncLLM(EngineClient):
prompt: PromptType,
pooling_params: PoolingParams,
request_id: str,
lora_request: Optional[LoRARequest] = None,
trace_headers: Optional[Mapping[str, str]] = None,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
truncate_prompt_tokens: Optional[int] = None,
tokenization_kwargs: Optional[dict[str, Any]] = None,
truncate_prompt_tokens: int | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[PoolingRequestOutput, None]:
"""
Main function called by the API server to kick off a request
@@ -622,11 +622,11 @@ class AsyncLLM(EngineClient):
raise EngineGenerateError() from e
@property
def tokenizer(self) -> Optional[AnyTokenizer]:
def tokenizer(self) -> AnyTokenizer | None:
return self.processor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
self.processor.tokenizer = tokenizer
async def get_tokenizer(self) -> AnyTokenizer:
@@ -665,7 +665,7 @@ class AsyncLLM(EngineClient):
self.processor.clear_mm_cache()
await self.engine_core.reset_mm_cache_async()
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
async def reset_prefix_cache(self, device: Device | None = None) -> None:
if device == Device.CPU:
raise ValueError("Not supported on CPU.")
await self.engine_core.reset_prefix_cache_async()
@@ -674,7 +674,7 @@ class AsyncLLM(EngineClient):
await self.reset_prefix_cache()
await self.engine_core.sleep_async(level)
async def wake_up(self, tags: Optional[list[str]] = None) -> None:
async def wake_up(self, tags: list[str] | None = None) -> None:
await self.engine_core.wake_up_async(tags)
async def is_sleeping(self) -> bool:
@@ -699,9 +699,9 @@ class AsyncLLM(EngineClient):
async def collective_rpc(
self,
method: str,
timeout: Optional[float] = None,
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict] = None,
kwargs: dict | None = None,
):
"""
Perform a collective RPC call to the given path.
+2 -3
View File
@@ -4,7 +4,6 @@ import copy
import multiprocessing
import time
import weakref
from typing import Optional
import msgspec.msgpack
import zmq
@@ -155,7 +154,7 @@ class DPCoordinatorProc:
stats_changed = False
last_stats_step = -1
last_stats_wave = -1
last_step_counts: Optional[list[list[int]]] = None
last_step_counts: list[list[int]] | None = None
with (
make_zmq_socket(
@@ -360,7 +359,7 @@ class DPCoordinatorProc:
@staticmethod
def _send_start_wave(
socket: zmq.Socket, wave: int, exclude_engine_index: Optional[int]
socket: zmq.Socket, wave: int, exclude_engine_index: int | None
):
"""Broadcast the START_DP_WAVE message to all the engines.
It includes the current wave number and index of engine which
+24 -24
View File
@@ -7,12 +7,12 @@ import signal
import threading
import time
from collections import deque
from collections.abc import Generator
from collections.abc import Callable, Generator
from concurrent.futures import Future
from contextlib import ExitStack, contextmanager
from inspect import isclass, signature
from logging import DEBUG
from typing import Any, Callable, Optional, TypeVar, Union
from typing import Any, TypeVar
import msgspec
import zmq
@@ -83,7 +83,7 @@ class EngineCore:
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
executor_fail_callback: Optional[Callable] = None,
executor_fail_callback: Callable | None = None,
):
# plugins need to be loaded at the engine/scheduler level too
from vllm.plugins import load_general_plugins
@@ -171,14 +171,14 @@ class EngineCore:
# schedule and execute batches, and is required by pipeline parallelism
# to eliminate pipeline bubbles.
self.batch_queue_size = self.model_executor.max_concurrent_batches
self.batch_queue: Optional[
deque[tuple[Future[ModelRunnerOutput], SchedulerOutput]]
] = None
self.batch_queue: (
deque[tuple[Future[ModelRunnerOutput], SchedulerOutput]] | None
) = None
if self.batch_queue_size > 1:
logger.info("Batch queue is enabled with size %d", self.batch_queue_size)
self.batch_queue = deque(maxlen=self.batch_queue_size)
self.request_block_hasher: Optional[Callable[[Request], list[BlockHash]]] = None
self.request_block_hasher: Callable[[Request], list[BlockHash]] | None = None
if (
self.vllm_config.cache_config.enable_prefix_caching
or self.scheduler.get_kv_connector() is not None
@@ -337,7 +337,7 @@ class EngineCore:
def step_with_batch_queue(
self,
) -> tuple[Optional[dict[int, EngineCoreOutputs]], bool]:
) -> tuple[dict[int, EngineCoreOutputs] | None, bool]:
"""Schedule and execute batches with the batch queue.
Note that if nothing to output in this step, None is returned.
@@ -424,7 +424,7 @@ class EngineCore:
def sleep(self, level: int = 1):
self.model_executor.sleep(level)
def wake_up(self, tags: Optional[list[str]] = None):
def wake_up(self, tags: list[str] | None = None):
self.model_executor.wake_up(tags)
def is_sleeping(self) -> bool:
@@ -448,8 +448,8 @@ class EngineCore:
def save_sharded_state(
self,
path: str,
pattern: Optional[str] = None,
max_size: Optional[int] = None,
pattern: str | None = None,
max_size: int | None = None,
) -> None:
self.model_executor.save_sharded_state(
path=path, pattern=pattern, max_size=max_size
@@ -457,10 +457,10 @@ class EngineCore:
def collective_rpc(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
return self.model_executor.collective_rpc(method, timeout, args, kwargs)
@@ -509,11 +509,11 @@ class EngineCoreProc(EngineCore):
handshake_address: str,
executor_class: type[Executor],
log_stats: bool,
client_handshake_address: Optional[str] = None,
client_handshake_address: str | None = None,
engine_index: int = 0,
):
self.input_queue = queue.Queue[tuple[EngineCoreRequestType, Any]]()
self.output_queue = queue.Queue[Union[tuple[int, EngineCoreOutputs], bytes]]()
self.output_queue = queue.Queue[tuple[int, EngineCoreOutputs] | bytes]()
executor_fail_callback = lambda: self.input_queue.put_nowait(
(EngineCoreRequestType.EXECUTOR_FAILED, b"")
)
@@ -606,7 +606,7 @@ class EngineCoreProc(EngineCore):
identity: bytes,
local_client: bool,
vllm_config: VllmConfig,
client_handshake_address: Optional[str],
client_handshake_address: str | None,
) -> Generator[EngineZmqAddresses, None, None]:
"""
Perform startup handshakes.
@@ -667,7 +667,7 @@ class EngineCoreProc(EngineCore):
local_client: bool,
headless: bool,
vllm_config: VllmConfig,
parallel_config_to_update: Optional[ParallelConfig] = None,
parallel_config_to_update: ParallelConfig | None = None,
) -> Generator[EngineZmqAddresses, None, None]:
with make_zmq_socket(
ctx,
@@ -710,7 +710,7 @@ class EngineCoreProc(EngineCore):
handshake_socket: zmq.Socket,
local_client: bool,
headless: bool,
parallel_config: Optional[ParallelConfig] = None,
parallel_config: ParallelConfig | None = None,
) -> EngineZmqAddresses:
# Send registration message.
handshake_socket.send(
@@ -765,7 +765,7 @@ class EngineCoreProc(EngineCore):
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
engine_core: Optional[EngineCoreProc] = None
engine_core: EngineCoreProc | None = None
try:
parallel_config: ParallelConfig = kwargs["vllm_config"].parallel_config
if parallel_config.data_parallel_size > 1 or dp_rank > 0:
@@ -911,7 +911,7 @@ class EngineCoreProc(EngineCore):
def process_input_sockets(
self,
input_addresses: list[str],
coord_input_address: Optional[str],
coord_input_address: str | None,
identity: bytes,
ready_event: threading.Event,
):
@@ -980,7 +980,7 @@ class EngineCoreProc(EngineCore):
def process_output_sockets(
self,
output_paths: list[str],
coord_output_path: Optional[str],
coord_output_path: str | None,
engine_index: int,
):
"""Output socket IO thread."""
@@ -1059,7 +1059,7 @@ class DPEngineCoreProc(EngineCoreProc):
handshake_address: str,
executor_class: type[Executor],
log_stats: bool,
client_handshake_address: Optional[str] = None,
client_handshake_address: str | None = None,
):
# Counts forward-passes of the model so that we can synchronize
# finished with DP peers every N steps.
@@ -1332,7 +1332,7 @@ class DPEngineCoreActor(DPEngineCoreProc):
identity: bytes,
local_client: bool,
vllm_config: VllmConfig,
client_handshake_address: Optional[str],
client_handshake_address: str | None,
):
"""
For Ray, we don't need to actually perform handshake.
+50 -52
View File
@@ -9,11 +9,11 @@ import uuid
import weakref
from abc import ABC, abstractmethod
from collections import defaultdict, deque
from collections.abc import Awaitable, Sequence
from collections.abc import Awaitable, Callable, Sequence
from concurrent.futures import Future
from dataclasses import dataclass
from threading import Thread
from typing import Any, Callable, Optional, TypeVar, Union
from typing import Any, TypeAlias, TypeVar
import msgspec.msgpack
import zmq
@@ -51,7 +51,7 @@ from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
logger = init_logger(__name__)
AnyFuture = Union[asyncio.Future[Any], Future[Any]]
AnyFuture: TypeAlias = asyncio.Future[Any] | Future[Any]
_R = TypeVar("_R") # Return type for collective_rpc
@@ -99,7 +99,7 @@ class EngineCoreClient(ABC):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_addresses: dict[str, str] | None = None,
client_count: int = 1,
client_index: int = 0,
) -> "MPClient":
@@ -144,7 +144,7 @@ class EngineCoreClient(ABC):
def sleep(self, level: int = 1) -> None:
raise NotImplementedError
def wake_up(self, tags: Optional[list[str]] = None) -> None:
def wake_up(self, tags: list[str] | None = None) -> None:
raise NotImplementedError
def is_sleeping(self) -> bool:
@@ -172,16 +172,16 @@ class EngineCoreClient(ABC):
raise NotImplementedError
def save_sharded_state(
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
self, path: str, pattern: str | None = None, max_size: int | None = None
) -> None:
raise NotImplementedError
def collective_rpc(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
raise NotImplementedError
@@ -214,7 +214,7 @@ class EngineCoreClient(ABC):
async def sleep_async(self, level: int = 1) -> None:
raise NotImplementedError
async def wake_up_async(self, tags: Optional[list[str]] = None) -> None:
async def wake_up_async(self, tags: list[str] | None = None) -> None:
raise NotImplementedError
async def is_sleeping_async(self) -> bool:
@@ -236,16 +236,16 @@ class EngineCoreClient(ABC):
raise NotImplementedError
async def save_sharded_state_async(
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
self, path: str, pattern: str | None = None, max_size: int | None = None
) -> None:
raise NotImplementedError
async def collective_rpc_async(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
raise NotImplementedError
@@ -293,7 +293,7 @@ class InprocClient(EngineCoreClient):
def sleep(self, level: int = 1) -> None:
self.engine_core.sleep(level)
def wake_up(self, tags: Optional[list[str]] = None) -> None:
def wake_up(self, tags: list[str] | None = None) -> None:
self.engine_core.wake_up(tags)
def is_sleeping(self) -> bool:
@@ -315,16 +315,16 @@ class InprocClient(EngineCoreClient):
return self.engine_core.pin_lora(lora_id)
def save_sharded_state(
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
self, path: str, pattern: str | None = None, max_size: int | None = None
) -> None:
self.engine_core.save_sharded_state(path, pattern, max_size)
def collective_rpc(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
@@ -340,18 +340,16 @@ class BackgroundResources:
ctx: zmq.Context
# If CoreEngineProcManager, it manages local engines;
# if CoreEngineActorManager, it manages all engines.
engine_manager: Optional[Union[CoreEngineProcManager, CoreEngineActorManager]] = (
None
)
coordinator: Optional[DPCoordinator] = None
output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
first_req_send_socket: Optional[zmq.asyncio.Socket] = None
first_req_rcv_socket: Optional[zmq.asyncio.Socket] = None
stats_update_socket: Optional[zmq.asyncio.Socket] = None
output_queue_task: Optional[asyncio.Task] = None
stats_update_task: Optional[asyncio.Task] = None
shutdown_path: Optional[str] = None
engine_manager: CoreEngineProcManager | CoreEngineActorManager | None = None
coordinator: DPCoordinator | None = None
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
first_req_send_socket: zmq.asyncio.Socket | None = None
first_req_rcv_socket: zmq.asyncio.Socket | None = None
stats_update_socket: zmq.asyncio.Socket | None = None
output_queue_task: asyncio.Task | None = None
stats_update_task: asyncio.Task | None = None
shutdown_path: str | None = None
# Set if any of the engines are dead. Here so that the output
# processing threads can access it without holding a ref to the client.
@@ -438,7 +436,7 @@ class MPClient(EngineCoreClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_addresses: dict[str, str] | None = None,
):
self.vllm_config = vllm_config
# Serialization setup.
@@ -459,7 +457,7 @@ class MPClient(EngineCoreClient):
# State used for data parallel.
self.engines_running = False
self.stats_update_address: Optional[str] = None
self.stats_update_address: str | None = None
if client_addresses:
# Engines are managed externally to this client.
input_address = client_addresses["input_address"]
@@ -646,7 +644,7 @@ class SyncMPClient(MPClient):
)
self.is_dp = self.vllm_config.parallel_config.data_parallel_size > 1
self.outputs_queue = queue.Queue[Union[EngineCoreOutputs, Exception]]()
self.outputs_queue = queue.Queue[EngineCoreOutputs | Exception]()
# Ensure that the outputs socket processing thread does not have
# a ref to the client which prevents gc.
@@ -770,7 +768,7 @@ class SyncMPClient(MPClient):
def sleep(self, level: int = 1) -> None:
self.call_utility("sleep", level)
def wake_up(self, tags: Optional[list[str]] = None) -> None:
def wake_up(self, tags: list[str] | None = None) -> None:
self.call_utility("wake_up", tags)
def is_sleeping(self) -> bool:
@@ -781,15 +779,15 @@ class SyncMPClient(MPClient):
def collective_rpc(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
return self.call_utility("collective_rpc", method, timeout, args, kwargs)
def save_sharded_state(
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
self, path: str, pattern: str | None = None, max_size: int | None = None
) -> None:
self.call_utility("save_sharded_state", path, pattern, max_size)
@@ -802,7 +800,7 @@ class AsyncMPClient(MPClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_addresses: dict[str, str] | None = None,
client_count: int = 1,
client_index: int = 0,
):
@@ -816,7 +814,7 @@ class AsyncMPClient(MPClient):
self.client_count = client_count
self.client_index = client_index
self.outputs_queue = asyncio.Queue[Union[EngineCoreOutputs, Exception]]()
self.outputs_queue = asyncio.Queue[EngineCoreOutputs | Exception]()
try:
# If we are running in an asyncio event loop, start the queue task.
# Otherwise, it will be started lazily. If it is not started here,
@@ -837,9 +835,9 @@ class AsyncMPClient(MPClient):
decoder = self.decoder
utility_results = self.utility_results
outputs_queue = self.outputs_queue
output_handler: Optional[
Callable[[AsyncMPClient, EngineCoreOutputs], Awaitable[None]]
] = getattr(self.__class__, "process_engine_outputs", None)
output_handler: (
Callable[[AsyncMPClient, EngineCoreOutputs], Awaitable[None]] | None
) = getattr(self.__class__, "process_engine_outputs", None)
_self_ref = weakref.ref(self) if output_handler else None
output_socket = resources.output_socket
assert output_socket is not None
@@ -888,7 +886,7 @@ class AsyncMPClient(MPClient):
self,
request_type: EngineCoreRequestType,
request: Any,
engine: Optional[EngineIdentity] = None,
engine: EngineIdentity | None = None,
) -> Awaitable[Any]:
if engine is None:
engine = self.core_engine
@@ -962,7 +960,7 @@ class AsyncMPClient(MPClient):
async def sleep_async(self, level: int = 1) -> None:
await self.call_utility_async("sleep", level)
async def wake_up_async(self, tags: Optional[list[str]] = None) -> None:
async def wake_up_async(self, tags: list[str] | None = None) -> None:
await self.call_utility_async("wake_up", tags)
async def is_sleeping_async(self) -> bool:
@@ -984,16 +982,16 @@ class AsyncMPClient(MPClient):
return await self.call_utility_async("pin_lora", lora_id)
async def save_sharded_state_async(
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
self, path: str, pattern: str | None = None, max_size: int | None = None
) -> None:
await self.call_utility_async("save_sharded_state", path, pattern, max_size)
async def collective_rpc_async(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
return await self.call_utility_async(
"collective_rpc", method, timeout, args, kwargs
@@ -1009,7 +1007,7 @@ class DPAsyncMPClient(AsyncMPClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_addresses: dict[str, str] | None = None,
client_count: int = 1,
client_index: int = 0,
):
@@ -1166,7 +1164,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
client_addresses: Optional[dict[str, str]] = None,
client_addresses: dict[str, str] | None = None,
client_count: int = 1,
client_index: int = 0,
):
+5 -6
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Optional
import tokenizers
from packaging import version
@@ -36,7 +35,7 @@ class IncrementalDetokenizer:
def output_token_ids(self) -> list[int]:
return self.token_ids
def update(self, new_token_ids: list[int], stop_terminated: bool) -> Optional[str]:
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
self.token_ids.extend(new_token_ids)
return None
@@ -46,7 +45,7 @@ class IncrementalDetokenizer:
@classmethod
def from_new_request(
cls,
tokenizer: Optional[AnyTokenizer],
tokenizer: AnyTokenizer | None,
request: EngineCoreRequest,
) -> "IncrementalDetokenizer":
assert request.sampling_params is not None
@@ -85,7 +84,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
# Generation data
self.output_text = ""
def update(self, new_token_ids: list[int], stop_terminated: bool) -> Optional[str]:
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
"""
Update RequestState for the request_id by:
1) Detokenize the new token ids incrementally.
@@ -224,7 +223,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
return token or ""
def _protected_step(self, next_token_id: int) -> Optional[str]:
def _protected_step(self, next_token_id: int) -> str | None:
try:
token = self.stream.step(self.tokenizer, next_token_id)
except (OverflowError, TypeError):
@@ -312,7 +311,7 @@ def check_stop_strings(
new_char_count: int,
stop: list[str],
include_in_output: bool,
) -> Optional[tuple[str, int]]:
) -> tuple[str, int] | None:
"""Check if any stop strings are matched and truncate sequence
output text accordingly.
+21 -21
View File
@@ -2,9 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from copy import copy
from typing import Any, Callable, Optional, Union
from typing import Any
import torch.nn as nn
from typing_extensions import TypeVar
@@ -52,7 +52,7 @@ class LLMEngine:
executor_class: type[Executor],
log_stats: bool,
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
stat_loggers: Optional[list[StatLoggerFactory]] = None,
stat_loggers: list[StatLoggerFactory] | None = None,
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
use_cached_outputs: bool = False,
multiprocess_mode: bool = False,
@@ -126,7 +126,7 @@ class LLMEngine:
log_stats=self.log_stats,
)
self.logger_manager: Optional[StatLoggerManager] = None
self.logger_manager: StatLoggerManager | None = None
if self.log_stats:
self.logger_manager = StatLoggerManager(
vllm_config=vllm_config,
@@ -152,7 +152,7 @@ class LLMEngine:
cls,
vllm_config: VllmConfig,
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
stat_loggers: Optional[list[StatLoggerFactory]] = None,
stat_loggers: list[StatLoggerFactory] | None = None,
disable_log_stats: bool = False,
) -> "LLMEngine":
return cls(
@@ -169,7 +169,7 @@ class LLMEngine:
cls,
engine_args: EngineArgs,
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
stat_loggers: Optional[list[StatLoggerFactory]] = None,
stat_loggers: list[StatLoggerFactory] | None = None,
enable_multiprocessing: bool = False,
) -> "LLMEngine":
"""Creates an LLM engine from the engine arguments."""
@@ -225,14 +225,14 @@ class LLMEngine:
def add_request(
self,
request_id: str,
prompt: Union[EngineCoreRequest, PromptType],
params: Union[SamplingParams, PoolingParams],
arrival_time: Optional[float] = None,
lora_request: Optional[LoRARequest] = None,
tokenization_kwargs: Optional[dict[str, Any]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
prompt: EngineCoreRequest | PromptType,
params: SamplingParams | PoolingParams,
arrival_time: float | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
prompt_text: Optional[str] = None,
prompt_text: str | None = None,
) -> None:
# Validate the request_id type.
if not isinstance(request_id, str):
@@ -283,7 +283,7 @@ class LLMEngine:
# Add the request to EngineCore.
self.engine_core.add_request(child_request)
def step(self) -> Union[list[RequestOutput], list[PoolingRequestOutput]]:
def step(self) -> list[RequestOutput] | list[PoolingRequestOutput]:
if self.should_execute_dummy_batch:
self.should_execute_dummy_batch = False
self.engine_core.execute_dummy_batch()
@@ -326,13 +326,13 @@ class LLMEngine:
self.processor.clear_mm_cache()
self.engine_core.reset_mm_cache()
def reset_prefix_cache(self, device: Optional[Device] = None):
def reset_prefix_cache(self, device: Device | None = None):
self.engine_core.reset_prefix_cache()
def sleep(self, level: int = 1):
self.engine_core.sleep(level)
def wake_up(self, tags: Optional[list[str]] = None):
def wake_up(self, tags: list[str] | None = None):
self.engine_core.wake_up(tags)
def is_sleeping(self) -> bool:
@@ -343,11 +343,11 @@ class LLMEngine:
return get_metrics_snapshot()
@property
def tokenizer(self) -> Optional[AnyTokenizer]:
def tokenizer(self) -> AnyTokenizer | None:
return self.processor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
self.processor.tokenizer = tokenizer
def get_tokenizer(self) -> AnyTokenizer:
@@ -390,10 +390,10 @@ class LLMEngine:
def collective_rpc(
self,
method: Union[str, Callable[[WorkerBase], _R]],
timeout: Optional[float] = None,
method: str | Callable[[WorkerBase], _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
+9 -10
View File
@@ -4,7 +4,6 @@
import itertools
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Optional
from vllm.logger import init_logger
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
@@ -24,19 +23,19 @@ NONES = itertools.repeat(None)
class LogprobsProcessor:
# Tokenizer for this request,
# None if detokenization is disabled.
tokenizer: Optional[AnyTokenizer]
tokenizer: AnyTokenizer | None
# Logprobs for this request
logprobs: Optional[SampleLogprobs]
prompt_logprobs: Optional[PromptLogprobs]
cumulative_logprob: Optional[float]
num_logprobs: Optional[int]
num_prompt_logprobs: Optional[int]
logprobs: SampleLogprobs | None
prompt_logprobs: PromptLogprobs | None
cumulative_logprob: float | None
num_logprobs: int | None
num_prompt_logprobs: int | None
@classmethod
def from_new_request(
cls,
tokenizer: Optional[AnyTokenizer],
tokenizer: AnyTokenizer | None,
request: EngineCoreRequest,
) -> "LogprobsProcessor":
assert request.sampling_params is not None
@@ -148,7 +147,7 @@ class LogprobsProcessor:
)
)
def pop_prompt_logprobs(self) -> Optional[PromptLogprobs]:
def pop_prompt_logprobs(self) -> PromptLogprobs | None:
"""Pop and return all request prompt logprobs
The logprobs processor aggregates prompt chunk logprobs
@@ -171,7 +170,7 @@ class LogprobsProcessor:
def _make_logprob_dict(
logprobs: list[float],
logprob_token_ids: list[int],
decoded_tokens: Iterable[Optional[str]],
decoded_tokens: Iterable[str | None],
rank: int,
num_logprobs: int,
) -> dict[int, Logprob]:
+43 -47
View File
@@ -4,7 +4,7 @@
import asyncio
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, Optional, Union, cast
from typing import Any, cast
import torch
@@ -36,14 +36,10 @@ class RequestOutputCollector:
def __init__(self, output_kind: RequestOutputKind):
self.aggregate = output_kind == RequestOutputKind.DELTA
self.output: Optional[Union[RequestOutput, PoolingRequestOutput, Exception]] = (
None
)
self.output: RequestOutput | PoolingRequestOutput | Exception | None = None
self.ready = asyncio.Event()
def put(
self, output: Union[RequestOutput, PoolingRequestOutput, Exception]
) -> None:
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
"""Non-blocking put operation."""
if self.output is None or isinstance(output, Exception):
self.output = output
@@ -53,7 +49,7 @@ class RequestOutputCollector:
# (if n > 1) do not override each other.
self.output.add(output, aggregate=self.aggregate)
async def get(self) -> Union[RequestOutput, PoolingRequestOutput]:
async def get(self) -> RequestOutput | PoolingRequestOutput:
"""Get operation blocks on put event."""
while (output := self.output) is None:
await self.ready.wait()
@@ -63,7 +59,7 @@ class RequestOutputCollector:
raise output
return output
def get_nowait(self) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
"""Non-blocking get operation."""
output = self.output
if output is not None:
@@ -76,7 +72,7 @@ class RequestOutputCollector:
@dataclass
class OutputProcessorOutput:
request_outputs: list[Union[RequestOutput, PoolingRequestOutput]]
request_outputs: list[RequestOutput | PoolingRequestOutput]
reqs_to_abort: list[str]
@@ -84,22 +80,22 @@ class RequestState:
def __init__(
self,
request_id: str,
parent_req: Optional[ParentRequest],
parent_req: ParentRequest | None,
request_index: int,
lora_name: Optional[str],
lora_name: str | None,
output_kind: RequestOutputKind,
prompt: Optional[str],
prompt_token_ids: Optional[list[int]],
prompt_embeds: Optional[torch.Tensor],
logprobs_processor: Optional[LogprobsProcessor],
detokenizer: Optional[IncrementalDetokenizer],
max_tokens_param: Optional[int],
prompt: str | None,
prompt_token_ids: list[int] | None,
prompt_embeds: torch.Tensor | None,
logprobs_processor: LogprobsProcessor | None,
detokenizer: IncrementalDetokenizer | None,
max_tokens_param: int | None,
arrival_time: float,
queue: Optional[RequestOutputCollector],
queue: RequestOutputCollector | None,
log_stats: bool,
top_p: Optional[float] = None,
n: Optional[int] = None,
temperature: Optional[float] = None,
top_p: float | None = None,
n: int | None = None,
temperature: float | None = None,
):
self.request_id = request_id
self.parent_req = parent_req
@@ -129,10 +125,10 @@ class RequestState:
cls,
tokenizer: AnyTokenizer,
request: EngineCoreRequest,
prompt: Optional[str],
parent_req: Optional[ParentRequest],
prompt: str | None,
parent_req: ParentRequest | None,
request_index: int,
queue: Optional[RequestOutputCollector],
queue: RequestOutputCollector | None,
log_stats: bool,
) -> "RequestState":
if sampling_params := request.sampling_params:
@@ -186,11 +182,11 @@ class RequestState:
def make_request_output(
self,
new_token_ids: list[int],
pooling_output: Optional[torch.Tensor],
finish_reason: Optional[FinishReason],
stop_reason: Union[int, str, None],
kv_transfer_params: Optional[dict[str, Any]] = None,
) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
pooling_output: torch.Tensor | None,
finish_reason: FinishReason | None,
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput | None:
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
@@ -222,10 +218,10 @@ class RequestState:
def _new_request_output(
self,
request_id: str,
outputs: Union[list[CompletionOutput], list[PoolingOutput]],
outputs: list[CompletionOutput] | list[PoolingOutput],
finished: bool,
kv_transfer_params: Optional[dict[str, Any]] = None,
) -> Union[RequestOutput, PoolingRequestOutput]:
kv_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput:
first_output = outputs[0]
if isinstance(first_output, PoolingOutput):
assert len(outputs) == 1
@@ -264,8 +260,8 @@ class RequestState:
def _new_completion_output(
self,
token_ids: list[int],
finish_reason: Optional[FinishReason],
stop_reason: Union[int, str, None],
finish_reason: FinishReason | None,
stop_reason: int | str | None,
) -> CompletionOutput:
assert self.detokenizer is not None
assert self.logprobs_processor is not None
@@ -308,7 +304,7 @@ class OutputProcessor:
self.request_states: dict[str, RequestState] = {}
self.parent_requests: dict[str, ParentRequest] = {}
self.lora_states = LoRARequestStates()
self.tracer: Optional[Tracer] = None
self.tracer: Tracer | None = None
def get_num_unfinished_requests(self):
return len(self.request_states)
@@ -360,10 +356,10 @@ class OutputProcessor:
def add_request(
self,
request: EngineCoreRequest,
prompt: Optional[str],
parent_req: Optional[ParentRequest] = None,
prompt: str | None,
parent_req: ParentRequest | None = None,
request_index: int = 0,
queue: Optional[RequestOutputCollector] = None,
queue: RequestOutputCollector | None = None,
) -> None:
request_id = request.request_id
if request_id in self.request_states:
@@ -386,8 +382,8 @@ class OutputProcessor:
def process_outputs(
self,
engine_core_outputs: list[EngineCoreOutput],
engine_core_timestamp: Optional[float] = None,
iteration_stats: Optional[IterationStats] = None,
engine_core_timestamp: float | None = None,
iteration_stats: IterationStats | None = None,
) -> OutputProcessorOutput:
"""
Process the EngineCoreOutputs:
@@ -411,7 +407,7 @@ class OutputProcessor:
within the loop below.
"""
request_outputs: Union[list[RequestOutput], list[PoolingRequestOutput]] = []
request_outputs: list[RequestOutput] | list[PoolingRequestOutput] = []
reqs_to_abort: list[str] = []
for engine_core_output in engine_core_outputs:
req_id = engine_core_output.request_id
@@ -492,7 +488,7 @@ class OutputProcessor:
self,
engine_core_output: EngineCoreOutput,
req_state: RequestState,
iteration_stats: Optional[IterationStats],
iteration_stats: IterationStats | None,
) -> None:
assert req_state.stats is not None
assert iteration_stats is not None
@@ -555,8 +551,8 @@ class OutputProcessor:
self,
req_state: RequestState,
engine_core_output: EngineCoreOutput,
engine_core_timestamp: Optional[float],
iteration_stats: Optional[IterationStats],
engine_core_timestamp: float | None,
iteration_stats: IterationStats | None,
):
if iteration_stats is None:
return
@@ -577,8 +573,8 @@ class OutputProcessor:
def _update_stats_from_finished(
self,
req_state: RequestState,
finish_reason: Optional[FinishReason],
iteration_stats: Optional[IterationStats],
finish_reason: FinishReason | None,
iteration_stats: IterationStats | None,
):
if iteration_stats is None:
return
+1 -1
View File
@@ -29,7 +29,7 @@ class ParentRequest:
max_num_generation_tokens: int
# To efficiently obtain child sampling params
cached_child_sampling_params: Optional[SamplingParams]
cached_child_sampling_params: SamplingParams | None
def __init__(self, request_id: str, sampling_params: SamplingParams) -> None:
self.request_id = request_id
+17 -17
View File
@@ -3,7 +3,7 @@
import time
from collections.abc import Mapping
from typing import Any, Literal, Optional, Union
from typing import Any, Literal
from vllm.config import VllmConfig
from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
@@ -38,7 +38,7 @@ class Processor:
def __init__(
self,
vllm_config: VllmConfig,
tokenizer: Optional[AnyTokenizer],
tokenizer: AnyTokenizer | None,
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
) -> None:
self.vllm_config = vllm_config
@@ -60,11 +60,11 @@ class Processor:
)
@property
def tokenizer(self) -> Optional[AnyTokenizer]:
def tokenizer(self) -> AnyTokenizer | None:
return self.input_preprocessor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
self.input_preprocessor.tokenizer = tokenizer
def _validate_logprobs(
@@ -152,7 +152,7 @@ class Processor:
def _validate_params(
self,
params: Union[SamplingParams, PoolingParams],
params: SamplingParams | PoolingParams,
):
"""
Validate supported SamplingParam.
@@ -174,7 +174,7 @@ class Processor:
auto-hashed downstream.
"""
def _validate_single_prompt(single_prompt: Union[dict, str]) -> None:
def _validate_single_prompt(single_prompt: dict | str) -> None:
if not isinstance(single_prompt, dict):
return
mm_data = single_prompt.get("multi_modal_data")
@@ -214,7 +214,7 @@ class Processor:
else:
_validate_single_prompt(prompt) # type: ignore[arg-type]
def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:
def _validate_lora(self, lora_request: LoRARequest | None) -> None:
if lora_request is None:
return
@@ -309,7 +309,7 @@ class Processor:
self,
request_id: str,
prompt: PromptType,
) -> Optional[MultiModalUUIDDict]:
) -> MultiModalUUIDDict | None:
"""Build per-item multimodal hash overrides when enabled. In this case,
multimodal data items are identified by their request id, modality and
index rather than their content.
@@ -342,13 +342,13 @@ class Processor:
self,
request_id: str,
prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
arrival_time: Optional[float] = None,
lora_request: Optional[LoRARequest] = None,
tokenization_kwargs: Optional[dict[str, Any]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
params: SamplingParams | PoolingParams,
arrival_time: float | None = None,
lora_request: LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
data_parallel_rank: Optional[int] = None,
data_parallel_rank: int | None = None,
) -> EngineCoreRequest:
self._validate_lora(lora_request)
self._validate_params(params)
@@ -445,7 +445,7 @@ class Processor:
pooling_params = params.clone()
# Multimodal related.
mm_features: Optional[list[MultiModalFeatureSpec]] = None
mm_features: list[MultiModalFeatureSpec] | None = None
if decoder_inputs["type"] == "multimodal":
decoder_mm_inputs = decoder_inputs["mm_kwargs"]
@@ -485,7 +485,7 @@ class Processor:
)
def _validate_model_inputs(
self, encoder_inputs: Optional[SingletonInputs], decoder_inputs: SingletonInputs
self, encoder_inputs: SingletonInputs | None, decoder_inputs: SingletonInputs
):
if encoder_inputs is not None:
self._validate_model_input(encoder_inputs, prompt_type="encoder")
@@ -574,7 +574,7 @@ class Processor:
# check that chunked prefill does not truncate them
# max_batch_len = self.scheduler_config.max_num_batched_tokens
def stat_mm_cache(self) -> Optional[MultiModalCacheStats]:
def stat_mm_cache(self) -> MultiModalCacheStats | None:
return self.input_preprocessor.stat_mm_cache()
def clear_mm_cache(self) -> None:
+14 -14
View File
@@ -4,12 +4,12 @@
import contextlib
import os
import weakref
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from enum import Enum, auto
from multiprocessing import Process, connection
from multiprocessing.process import BaseProcess
from typing import TYPE_CHECKING, Callable, Optional, Union
from typing import TYPE_CHECKING
from unittest.mock import patch
import msgspec
@@ -56,13 +56,13 @@ class EngineZmqAddresses:
# ZMQ output socket addresses for each front-end client (responses)
outputs: list[str]
# ZMQ input socket address of DP coordinator if applicable
coordinator_input: Optional[str] = None
coordinator_input: str | None = None
# ZMQ output socket address of DP coordinator if applicable
coordinator_output: Optional[str] = None
coordinator_output: str | None = None
# ZMQ socket for front-end to connect to DP coordinator.
# Not used by engine, just relayed to front-end in handshake response.
# Only required for external DP LB case.
frontend_stats_publish_address: Optional[str] = None
frontend_stats_publish_address: str | None = None
@dataclass
@@ -73,8 +73,8 @@ class EngineHandshakeMetadata:
"""
addresses: EngineZmqAddresses
parallel_config: dict[str, Union[int, str, list[int]]]
parallel_config_hash: Optional[str] = None
parallel_config: dict[str, int | str | list[int]]
parallel_config_hash: str | None = None
class CoreEngineProcManager:
@@ -94,7 +94,7 @@ class CoreEngineProcManager:
handshake_address: str,
executor_class: type[Executor],
log_stats: bool,
client_handshake_address: Optional[str] = None,
client_handshake_address: str | None = None,
):
context = get_mp_context()
common_kwargs = {
@@ -221,8 +221,8 @@ class CoreEngineActorManager:
addresses: EngineZmqAddresses,
executor_class: type[Executor],
log_stats: bool,
placement_groups: Optional[list["PlacementGroup"]] = None,
local_dp_ranks: Optional[list[int]] = None,
placement_groups: list["PlacementGroup"] | None = None,
local_dp_ranks: list[int] | None = None,
):
import copy
@@ -675,8 +675,8 @@ def launch_core_engines(
num_api_servers: int = 1,
) -> Iterator[
tuple[
Optional[Union[CoreEngineProcManager, CoreEngineActorManager]],
Optional[DPCoordinator],
CoreEngineProcManager | CoreEngineActorManager | None,
DPCoordinator | None,
EngineZmqAddresses,
]
]:
@@ -829,8 +829,8 @@ def wait_for_engine_startup(
core_engines: list[CoreEngine],
parallel_config: ParallelConfig,
cache_config: CacheConfig,
proc_manager: Optional[CoreEngineProcManager],
coord_process: Optional[Process],
proc_manager: CoreEngineProcManager | None,
coord_process: Process | None,
):
# Wait for engine core process(es) to send ready messages.
local_count = parallel_config.data_parallel_size_local