Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+27
-27
@@ -4,7 +4,7 @@
|
||||
import enum
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -48,16 +48,16 @@ class EngineCoreRequest(
|
||||
gc=False,
|
||||
): # type: ignore[call-arg]
|
||||
request_id: str
|
||||
prompt_token_ids: Optional[list[int]]
|
||||
mm_features: Optional[list[MultiModalFeatureSpec]]
|
||||
sampling_params: Optional[SamplingParams]
|
||||
pooling_params: Optional[PoolingParams]
|
||||
eos_token_id: Optional[int]
|
||||
prompt_token_ids: list[int] | None
|
||||
mm_features: list[MultiModalFeatureSpec] | None
|
||||
sampling_params: SamplingParams | None
|
||||
pooling_params: PoolingParams | None
|
||||
eos_token_id: int | None
|
||||
arrival_time: float
|
||||
lora_request: Optional[LoRARequest]
|
||||
cache_salt: Optional[str]
|
||||
data_parallel_rank: Optional[int]
|
||||
prompt_embeds: Optional[torch.Tensor] = None
|
||||
lora_request: LoRARequest | None
|
||||
cache_salt: str | None
|
||||
data_parallel_rank: int | None
|
||||
prompt_embeds: torch.Tensor | None = None
|
||||
|
||||
# Index of the client, used to ensure outputs are sent back to the same
|
||||
# client for this request when scaling out the front-end.
|
||||
@@ -69,7 +69,7 @@ class EngineCoreRequest(
|
||||
current_wave: int = 0
|
||||
priority: int = 0
|
||||
|
||||
trace_headers: Optional[Mapping[str, str]] = None
|
||||
trace_headers: Mapping[str, str] | None = None
|
||||
|
||||
|
||||
class EngineCoreEventType(enum.IntEnum):
|
||||
@@ -93,7 +93,7 @@ class EngineCoreEvent(msgspec.Struct):
|
||||
|
||||
@classmethod
|
||||
def new_event(
|
||||
cls, event_type: EngineCoreEventType, timestamp: Optional[float] = None
|
||||
cls, event_type: EngineCoreEventType, timestamp: float | None = None
|
||||
) -> "EngineCoreEvent":
|
||||
timestamp = time.monotonic() if timestamp is None else timestamp
|
||||
return cls(event_type, timestamp)
|
||||
@@ -108,17 +108,17 @@ class EngineCoreOutput(
|
||||
request_id: str
|
||||
new_token_ids: list[int]
|
||||
|
||||
new_logprobs: Optional[LogprobsLists] = None
|
||||
new_prompt_logprobs_tensors: Optional[LogprobsTensors] = None
|
||||
new_logprobs: LogprobsLists | None = None
|
||||
new_prompt_logprobs_tensors: LogprobsTensors | None = None
|
||||
|
||||
pooling_output: Optional[torch.Tensor] = None
|
||||
pooling_output: torch.Tensor | None = None
|
||||
|
||||
finish_reason: Optional[FinishReason] = None
|
||||
stop_reason: Union[int, str, None] = None
|
||||
events: Optional[list[EngineCoreEvent]] = None
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None
|
||||
finish_reason: FinishReason | None = None
|
||||
stop_reason: int | str | None = None
|
||||
events: list[EngineCoreEvent] | None = None
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
trace_headers: Optional[Mapping[str, str]] = None
|
||||
trace_headers: Mapping[str, str] | None = None
|
||||
# The number of tokens with prefix cache hits.
|
||||
num_cached_tokens: int = 0
|
||||
|
||||
@@ -142,8 +142,8 @@ class UtilityOutput(
|
||||
call_id: int
|
||||
|
||||
# Non-None implies the call failed, result should be None.
|
||||
failure_message: Optional[str] = None
|
||||
result: Optional[UtilityResult] = None
|
||||
failure_message: str | None = None
|
||||
result: UtilityResult | None = None
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
@@ -159,18 +159,18 @@ class EngineCoreOutputs(
|
||||
|
||||
# [num_reqs]
|
||||
outputs: list[EngineCoreOutput] = []
|
||||
scheduler_stats: Optional[SchedulerStats] = None
|
||||
scheduler_stats: SchedulerStats | None = None
|
||||
timestamp: float = 0.0
|
||||
|
||||
utility_output: Optional[UtilityOutput] = None
|
||||
finished_requests: Optional[set[str]] = None
|
||||
utility_output: UtilityOutput | None = None
|
||||
finished_requests: set[str] | None = None
|
||||
|
||||
# In DP case, used to signal that the current wave of requests
|
||||
# has finished and the engines are paused.
|
||||
wave_complete: Optional[int] = None
|
||||
wave_complete: int | None = None
|
||||
# In DP case, used to signal that a request was received for an
|
||||
# "old" wave, so the next wave needs to be started in other engines.
|
||||
start_wave: Optional[int] = None
|
||||
start_wave: int | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.timestamp == 0.0:
|
||||
|
||||
+35
-35
@@ -6,7 +6,7 @@ import socket
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping
|
||||
from copy import copy
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -56,8 +56,8 @@ class AsyncLLM(EngineClient):
|
||||
use_cached_outputs: bool = False,
|
||||
log_requests: bool = True,
|
||||
start_engine_loop: bool = True,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> None:
|
||||
@@ -136,7 +136,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
# Loggers.
|
||||
self.logger_manager: Optional[StatLoggerManager] = None
|
||||
self.logger_manager: StatLoggerManager | None = None
|
||||
if self.log_stats:
|
||||
self.logger_manager = StatLoggerManager(
|
||||
vllm_config=vllm_config,
|
||||
@@ -147,7 +147,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
self.logger_manager.log_engine_initialized()
|
||||
|
||||
self.output_handler: Optional[asyncio.Task] = None
|
||||
self.output_handler: asyncio.Task | None = None
|
||||
try:
|
||||
# Start output handler eagerly if we are in the asyncio eventloop.
|
||||
asyncio.get_running_loop()
|
||||
@@ -185,10 +185,10 @@ class AsyncLLM(EngineClient):
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
enable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
disable_log_requests: bool = True, # Deprecated, will be removed
|
||||
@@ -221,7 +221,7 @@ class AsyncLLM(EngineClient):
|
||||
engine_args: AsyncEngineArgs,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
) -> "AsyncLLM":
|
||||
"""Create an AsyncLLM from the EngineArgs."""
|
||||
|
||||
@@ -259,15 +259,15 @@ class AsyncLLM(EngineClient):
|
||||
async def add_request(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: Union[EngineCoreRequest, PromptType],
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
arrival_time: Optional[float] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt: EngineCoreRequest | PromptType,
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
data_parallel_rank: Optional[int] = None,
|
||||
prompt_text: Optional[str] = None,
|
||||
data_parallel_rank: int | None = None,
|
||||
prompt_text: str | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
"""Add new request to the AsyncLLM."""
|
||||
|
||||
@@ -325,8 +325,8 @@ class AsyncLLM(EngineClient):
|
||||
async def _add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest],
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None,
|
||||
index: int,
|
||||
queue: RequestOutputCollector,
|
||||
):
|
||||
@@ -346,16 +346,16 @@ class AsyncLLM(EngineClient):
|
||||
# re-multiplexed in the API server anyhow.
|
||||
async def generate(
|
||||
self,
|
||||
prompt: Union[EngineCoreRequest, PromptType],
|
||||
prompt: EngineCoreRequest | PromptType,
|
||||
sampling_params: SamplingParams,
|
||||
request_id: str,
|
||||
*,
|
||||
prompt_text: Optional[str] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt_text: str | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
data_parallel_rank: Optional[int] = None,
|
||||
data_parallel_rank: int | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
@@ -520,7 +520,7 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
self.output_handler = asyncio.create_task(output_handler())
|
||||
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
async def abort(self, request_id: str | Iterable[str]) -> None:
|
||||
"""Abort RequestId in OutputProcessor and EngineCore."""
|
||||
|
||||
request_ids = (
|
||||
@@ -537,11 +537,11 @@ class AsyncLLM(EngineClient):
|
||||
prompt: PromptType,
|
||||
pooling_params: PoolingParams,
|
||||
request_id: str,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[PoolingRequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
@@ -622,11 +622,11 @@ class AsyncLLM(EngineClient):
|
||||
raise EngineGenerateError() from e
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> Optional[AnyTokenizer]:
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
return self.processor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
self.processor.tokenizer = tokenizer
|
||||
|
||||
async def get_tokenizer(self) -> AnyTokenizer:
|
||||
@@ -665,7 +665,7 @@ class AsyncLLM(EngineClient):
|
||||
self.processor.clear_mm_cache()
|
||||
await self.engine_core.reset_mm_cache_async()
|
||||
|
||||
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
|
||||
async def reset_prefix_cache(self, device: Device | None = None) -> None:
|
||||
if device == Device.CPU:
|
||||
raise ValueError("Not supported on CPU.")
|
||||
await self.engine_core.reset_prefix_cache_async()
|
||||
@@ -674,7 +674,7 @@ class AsyncLLM(EngineClient):
|
||||
await self.reset_prefix_cache()
|
||||
await self.engine_core.sleep_async(level)
|
||||
|
||||
async def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
async def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
await self.engine_core.wake_up_async(tags)
|
||||
|
||||
async def is_sleeping(self) -> bool:
|
||||
@@ -699,9 +699,9 @@ class AsyncLLM(EngineClient):
|
||||
async def collective_rpc(
|
||||
self,
|
||||
method: str,
|
||||
timeout: Optional[float] = None,
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None,
|
||||
kwargs: dict | None = None,
|
||||
):
|
||||
"""
|
||||
Perform a collective RPC call to the given path.
|
||||
|
||||
@@ -4,7 +4,6 @@ import copy
|
||||
import multiprocessing
|
||||
import time
|
||||
import weakref
|
||||
from typing import Optional
|
||||
|
||||
import msgspec.msgpack
|
||||
import zmq
|
||||
@@ -155,7 +154,7 @@ class DPCoordinatorProc:
|
||||
stats_changed = False
|
||||
last_stats_step = -1
|
||||
last_stats_wave = -1
|
||||
last_step_counts: Optional[list[list[int]]] = None
|
||||
last_step_counts: list[list[int]] | None = None
|
||||
|
||||
with (
|
||||
make_zmq_socket(
|
||||
@@ -360,7 +359,7 @@ class DPCoordinatorProc:
|
||||
|
||||
@staticmethod
|
||||
def _send_start_wave(
|
||||
socket: zmq.Socket, wave: int, exclude_engine_index: Optional[int]
|
||||
socket: zmq.Socket, wave: int, exclude_engine_index: int | None
|
||||
):
|
||||
"""Broadcast the START_DP_WAVE message to all the engines.
|
||||
It includes the current wave number and index of engine which
|
||||
|
||||
+24
-24
@@ -7,12 +7,12 @@ import signal
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Callable, Generator
|
||||
from concurrent.futures import Future
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from inspect import isclass, signature
|
||||
from logging import DEBUG
|
||||
from typing import Any, Callable, Optional, TypeVar, Union
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
@@ -83,7 +83,7 @@ class EngineCore:
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
executor_fail_callback: Optional[Callable] = None,
|
||||
executor_fail_callback: Callable | None = None,
|
||||
):
|
||||
# plugins need to be loaded at the engine/scheduler level too
|
||||
from vllm.plugins import load_general_plugins
|
||||
@@ -171,14 +171,14 @@ class EngineCore:
|
||||
# schedule and execute batches, and is required by pipeline parallelism
|
||||
# to eliminate pipeline bubbles.
|
||||
self.batch_queue_size = self.model_executor.max_concurrent_batches
|
||||
self.batch_queue: Optional[
|
||||
deque[tuple[Future[ModelRunnerOutput], SchedulerOutput]]
|
||||
] = None
|
||||
self.batch_queue: (
|
||||
deque[tuple[Future[ModelRunnerOutput], SchedulerOutput]] | None
|
||||
) = None
|
||||
if self.batch_queue_size > 1:
|
||||
logger.info("Batch queue is enabled with size %d", self.batch_queue_size)
|
||||
self.batch_queue = deque(maxlen=self.batch_queue_size)
|
||||
|
||||
self.request_block_hasher: Optional[Callable[[Request], list[BlockHash]]] = None
|
||||
self.request_block_hasher: Callable[[Request], list[BlockHash]] | None = None
|
||||
if (
|
||||
self.vllm_config.cache_config.enable_prefix_caching
|
||||
or self.scheduler.get_kv_connector() is not None
|
||||
@@ -337,7 +337,7 @@ class EngineCore:
|
||||
|
||||
def step_with_batch_queue(
|
||||
self,
|
||||
) -> tuple[Optional[dict[int, EngineCoreOutputs]], bool]:
|
||||
) -> tuple[dict[int, EngineCoreOutputs] | None, bool]:
|
||||
"""Schedule and execute batches with the batch queue.
|
||||
Note that if nothing to output in this step, None is returned.
|
||||
|
||||
@@ -424,7 +424,7 @@ class EngineCore:
|
||||
def sleep(self, level: int = 1):
|
||||
self.model_executor.sleep(level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None):
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
self.model_executor.wake_up(tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -448,8 +448,8 @@ class EngineCore:
|
||||
def save_sharded_state(
|
||||
self,
|
||||
path: str,
|
||||
pattern: Optional[str] = None,
|
||||
max_size: Optional[int] = None,
|
||||
pattern: str | None = None,
|
||||
max_size: int | None = None,
|
||||
) -> None:
|
||||
self.model_executor.save_sharded_state(
|
||||
path=path, pattern=pattern, max_size=max_size
|
||||
@@ -457,10 +457,10 @@ class EngineCore:
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return self.model_executor.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
@@ -509,11 +509,11 @@ class EngineCoreProc(EngineCore):
|
||||
handshake_address: str,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_handshake_address: Optional[str] = None,
|
||||
client_handshake_address: str | None = None,
|
||||
engine_index: int = 0,
|
||||
):
|
||||
self.input_queue = queue.Queue[tuple[EngineCoreRequestType, Any]]()
|
||||
self.output_queue = queue.Queue[Union[tuple[int, EngineCoreOutputs], bytes]]()
|
||||
self.output_queue = queue.Queue[tuple[int, EngineCoreOutputs] | bytes]()
|
||||
executor_fail_callback = lambda: self.input_queue.put_nowait(
|
||||
(EngineCoreRequestType.EXECUTOR_FAILED, b"")
|
||||
)
|
||||
@@ -606,7 +606,7 @@ class EngineCoreProc(EngineCore):
|
||||
identity: bytes,
|
||||
local_client: bool,
|
||||
vllm_config: VllmConfig,
|
||||
client_handshake_address: Optional[str],
|
||||
client_handshake_address: str | None,
|
||||
) -> Generator[EngineZmqAddresses, None, None]:
|
||||
"""
|
||||
Perform startup handshakes.
|
||||
@@ -667,7 +667,7 @@ class EngineCoreProc(EngineCore):
|
||||
local_client: bool,
|
||||
headless: bool,
|
||||
vllm_config: VllmConfig,
|
||||
parallel_config_to_update: Optional[ParallelConfig] = None,
|
||||
parallel_config_to_update: ParallelConfig | None = None,
|
||||
) -> Generator[EngineZmqAddresses, None, None]:
|
||||
with make_zmq_socket(
|
||||
ctx,
|
||||
@@ -710,7 +710,7 @@ class EngineCoreProc(EngineCore):
|
||||
handshake_socket: zmq.Socket,
|
||||
local_client: bool,
|
||||
headless: bool,
|
||||
parallel_config: Optional[ParallelConfig] = None,
|
||||
parallel_config: ParallelConfig | None = None,
|
||||
) -> EngineZmqAddresses:
|
||||
# Send registration message.
|
||||
handshake_socket.send(
|
||||
@@ -765,7 +765,7 @@ class EngineCoreProc(EngineCore):
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
engine_core: Optional[EngineCoreProc] = None
|
||||
engine_core: EngineCoreProc | None = None
|
||||
try:
|
||||
parallel_config: ParallelConfig = kwargs["vllm_config"].parallel_config
|
||||
if parallel_config.data_parallel_size > 1 or dp_rank > 0:
|
||||
@@ -911,7 +911,7 @@ class EngineCoreProc(EngineCore):
|
||||
def process_input_sockets(
|
||||
self,
|
||||
input_addresses: list[str],
|
||||
coord_input_address: Optional[str],
|
||||
coord_input_address: str | None,
|
||||
identity: bytes,
|
||||
ready_event: threading.Event,
|
||||
):
|
||||
@@ -980,7 +980,7 @@ class EngineCoreProc(EngineCore):
|
||||
def process_output_sockets(
|
||||
self,
|
||||
output_paths: list[str],
|
||||
coord_output_path: Optional[str],
|
||||
coord_output_path: str | None,
|
||||
engine_index: int,
|
||||
):
|
||||
"""Output socket IO thread."""
|
||||
@@ -1059,7 +1059,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
handshake_address: str,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_handshake_address: Optional[str] = None,
|
||||
client_handshake_address: str | None = None,
|
||||
):
|
||||
# Counts forward-passes of the model so that we can synchronize
|
||||
# finished with DP peers every N steps.
|
||||
@@ -1332,7 +1332,7 @@ class DPEngineCoreActor(DPEngineCoreProc):
|
||||
identity: bytes,
|
||||
local_client: bool,
|
||||
vllm_config: VllmConfig,
|
||||
client_handshake_address: Optional[str],
|
||||
client_handshake_address: str | None,
|
||||
):
|
||||
"""
|
||||
For Ray, we don't need to actually perform handshake.
|
||||
|
||||
@@ -9,11 +9,11 @@ import uuid
|
||||
import weakref
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from threading import Thread
|
||||
from typing import Any, Callable, Optional, TypeVar, Union
|
||||
from typing import Any, TypeAlias, TypeVar
|
||||
|
||||
import msgspec.msgpack
|
||||
import zmq
|
||||
@@ -51,7 +51,7 @@ from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
AnyFuture = Union[asyncio.Future[Any], Future[Any]]
|
||||
AnyFuture: TypeAlias = asyncio.Future[Any] | Future[Any]
|
||||
|
||||
_R = TypeVar("_R") # Return type for collective_rpc
|
||||
|
||||
@@ -99,7 +99,7 @@ class EngineCoreClient(ABC):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> "MPClient":
|
||||
@@ -144,7 +144,7 @@ class EngineCoreClient(ABC):
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -172,16 +172,16 @@ class EngineCoreClient(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
def save_sharded_state(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -214,7 +214,7 @@ class EngineCoreClient(ABC):
|
||||
async def sleep_async(self, level: int = 1) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def wake_up_async(self, tags: Optional[list[str]] = None) -> None:
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def is_sleeping_async(self) -> bool:
|
||||
@@ -236,16 +236,16 @@ class EngineCoreClient(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
async def save_sharded_state_async(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -293,7 +293,7 @@ class InprocClient(EngineCoreClient):
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
self.engine_core.sleep(level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.engine_core.wake_up(tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -315,16 +315,16 @@ class InprocClient(EngineCoreClient):
|
||||
return self.engine_core.pin_lora(lora_id)
|
||||
|
||||
def save_sharded_state(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
self.engine_core.save_sharded_state(path, pattern, max_size)
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
@@ -340,18 +340,16 @@ class BackgroundResources:
|
||||
ctx: zmq.Context
|
||||
# If CoreEngineProcManager, it manages local engines;
|
||||
# if CoreEngineActorManager, it manages all engines.
|
||||
engine_manager: Optional[Union[CoreEngineProcManager, CoreEngineActorManager]] = (
|
||||
None
|
||||
)
|
||||
coordinator: Optional[DPCoordinator] = None
|
||||
output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
first_req_send_socket: Optional[zmq.asyncio.Socket] = None
|
||||
first_req_rcv_socket: Optional[zmq.asyncio.Socket] = None
|
||||
stats_update_socket: Optional[zmq.asyncio.Socket] = None
|
||||
output_queue_task: Optional[asyncio.Task] = None
|
||||
stats_update_task: Optional[asyncio.Task] = None
|
||||
shutdown_path: Optional[str] = None
|
||||
engine_manager: CoreEngineProcManager | CoreEngineActorManager | None = None
|
||||
coordinator: DPCoordinator | None = None
|
||||
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
first_req_send_socket: zmq.asyncio.Socket | None = None
|
||||
first_req_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
stats_update_socket: zmq.asyncio.Socket | None = None
|
||||
output_queue_task: asyncio.Task | None = None
|
||||
stats_update_task: asyncio.Task | None = None
|
||||
shutdown_path: str | None = None
|
||||
|
||||
# Set if any of the engines are dead. Here so that the output
|
||||
# processing threads can access it without holding a ref to the client.
|
||||
@@ -438,7 +436,7 @@ class MPClient(EngineCoreClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
# Serialization setup.
|
||||
@@ -459,7 +457,7 @@ class MPClient(EngineCoreClient):
|
||||
# State used for data parallel.
|
||||
self.engines_running = False
|
||||
|
||||
self.stats_update_address: Optional[str] = None
|
||||
self.stats_update_address: str | None = None
|
||||
if client_addresses:
|
||||
# Engines are managed externally to this client.
|
||||
input_address = client_addresses["input_address"]
|
||||
@@ -646,7 +644,7 @@ class SyncMPClient(MPClient):
|
||||
)
|
||||
|
||||
self.is_dp = self.vllm_config.parallel_config.data_parallel_size > 1
|
||||
self.outputs_queue = queue.Queue[Union[EngineCoreOutputs, Exception]]()
|
||||
self.outputs_queue = queue.Queue[EngineCoreOutputs | Exception]()
|
||||
|
||||
# Ensure that the outputs socket processing thread does not have
|
||||
# a ref to the client which prevents gc.
|
||||
@@ -770,7 +768,7 @@ class SyncMPClient(MPClient):
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
self.call_utility("sleep", level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.call_utility("wake_up", tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -781,15 +779,15 @@ class SyncMPClient(MPClient):
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return self.call_utility("collective_rpc", method, timeout, args, kwargs)
|
||||
|
||||
def save_sharded_state(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
self.call_utility("save_sharded_state", path, pattern, max_size)
|
||||
|
||||
@@ -802,7 +800,7 @@ class AsyncMPClient(MPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
@@ -816,7 +814,7 @@ class AsyncMPClient(MPClient):
|
||||
|
||||
self.client_count = client_count
|
||||
self.client_index = client_index
|
||||
self.outputs_queue = asyncio.Queue[Union[EngineCoreOutputs, Exception]]()
|
||||
self.outputs_queue = asyncio.Queue[EngineCoreOutputs | Exception]()
|
||||
try:
|
||||
# If we are running in an asyncio event loop, start the queue task.
|
||||
# Otherwise, it will be started lazily. If it is not started here,
|
||||
@@ -837,9 +835,9 @@ class AsyncMPClient(MPClient):
|
||||
decoder = self.decoder
|
||||
utility_results = self.utility_results
|
||||
outputs_queue = self.outputs_queue
|
||||
output_handler: Optional[
|
||||
Callable[[AsyncMPClient, EngineCoreOutputs], Awaitable[None]]
|
||||
] = getattr(self.__class__, "process_engine_outputs", None)
|
||||
output_handler: (
|
||||
Callable[[AsyncMPClient, EngineCoreOutputs], Awaitable[None]] | None
|
||||
) = getattr(self.__class__, "process_engine_outputs", None)
|
||||
_self_ref = weakref.ref(self) if output_handler else None
|
||||
output_socket = resources.output_socket
|
||||
assert output_socket is not None
|
||||
@@ -888,7 +886,7 @@ class AsyncMPClient(MPClient):
|
||||
self,
|
||||
request_type: EngineCoreRequestType,
|
||||
request: Any,
|
||||
engine: Optional[EngineIdentity] = None,
|
||||
engine: EngineIdentity | None = None,
|
||||
) -> Awaitable[Any]:
|
||||
if engine is None:
|
||||
engine = self.core_engine
|
||||
@@ -962,7 +960,7 @@ class AsyncMPClient(MPClient):
|
||||
async def sleep_async(self, level: int = 1) -> None:
|
||||
await self.call_utility_async("sleep", level)
|
||||
|
||||
async def wake_up_async(self, tags: Optional[list[str]] = None) -> None:
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> None:
|
||||
await self.call_utility_async("wake_up", tags)
|
||||
|
||||
async def is_sleeping_async(self) -> bool:
|
||||
@@ -984,16 +982,16 @@ class AsyncMPClient(MPClient):
|
||||
return await self.call_utility_async("pin_lora", lora_id)
|
||||
|
||||
async def save_sharded_state_async(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
await self.call_utility_async("save_sharded_state", path, pattern, max_size)
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return await self.call_utility_async(
|
||||
"collective_rpc", method, timeout, args, kwargs
|
||||
@@ -1009,7 +1007,7 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
@@ -1166,7 +1164,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional
|
||||
|
||||
import tokenizers
|
||||
from packaging import version
|
||||
@@ -36,7 +35,7 @@ class IncrementalDetokenizer:
|
||||
def output_token_ids(self) -> list[int]:
|
||||
return self.token_ids
|
||||
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> Optional[str]:
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
|
||||
self.token_ids.extend(new_token_ids)
|
||||
return None
|
||||
|
||||
@@ -46,7 +45,7 @@ class IncrementalDetokenizer:
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: Optional[AnyTokenizer],
|
||||
tokenizer: AnyTokenizer | None,
|
||||
request: EngineCoreRequest,
|
||||
) -> "IncrementalDetokenizer":
|
||||
assert request.sampling_params is not None
|
||||
@@ -85,7 +84,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
|
||||
# Generation data
|
||||
self.output_text = ""
|
||||
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> Optional[str]:
|
||||
def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None:
|
||||
"""
|
||||
Update RequestState for the request_id by:
|
||||
1) Detokenize the new token ids incrementally.
|
||||
@@ -224,7 +223,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
return token or ""
|
||||
|
||||
def _protected_step(self, next_token_id: int) -> Optional[str]:
|
||||
def _protected_step(self, next_token_id: int) -> str | None:
|
||||
try:
|
||||
token = self.stream.step(self.tokenizer, next_token_id)
|
||||
except (OverflowError, TypeError):
|
||||
@@ -312,7 +311,7 @@ def check_stop_strings(
|
||||
new_char_count: int,
|
||||
stop: list[str],
|
||||
include_in_output: bool,
|
||||
) -> Optional[tuple[str, int]]:
|
||||
) -> tuple[str, int] | None:
|
||||
"""Check if any stop strings are matched and truncate sequence
|
||||
output text accordingly.
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from copy import copy
|
||||
from typing import Any, Callable, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import torch.nn as nn
|
||||
from typing_extensions import TypeVar
|
||||
@@ -52,7 +52,7 @@ class LLMEngine:
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
use_cached_outputs: bool = False,
|
||||
multiprocess_mode: bool = False,
|
||||
@@ -126,7 +126,7 @@ class LLMEngine:
|
||||
log_stats=self.log_stats,
|
||||
)
|
||||
|
||||
self.logger_manager: Optional[StatLoggerManager] = None
|
||||
self.logger_manager: StatLoggerManager | None = None
|
||||
if self.log_stats:
|
||||
self.logger_manager = StatLoggerManager(
|
||||
vllm_config=vllm_config,
|
||||
@@ -152,7 +152,7 @@ class LLMEngine:
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
disable_log_stats: bool = False,
|
||||
) -> "LLMEngine":
|
||||
return cls(
|
||||
@@ -169,7 +169,7 @@ class LLMEngine:
|
||||
cls,
|
||||
engine_args: EngineArgs,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
enable_multiprocessing: bool = False,
|
||||
) -> "LLMEngine":
|
||||
"""Creates an LLM engine from the engine arguments."""
|
||||
@@ -225,14 +225,14 @@ class LLMEngine:
|
||||
def add_request(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: Union[EngineCoreRequest, PromptType],
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
arrival_time: Optional[float] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt: EngineCoreRequest | PromptType,
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
prompt_text: Optional[str] = None,
|
||||
prompt_text: str | None = None,
|
||||
) -> None:
|
||||
# Validate the request_id type.
|
||||
if not isinstance(request_id, str):
|
||||
@@ -283,7 +283,7 @@ class LLMEngine:
|
||||
# Add the request to EngineCore.
|
||||
self.engine_core.add_request(child_request)
|
||||
|
||||
def step(self) -> Union[list[RequestOutput], list[PoolingRequestOutput]]:
|
||||
def step(self) -> list[RequestOutput] | list[PoolingRequestOutput]:
|
||||
if self.should_execute_dummy_batch:
|
||||
self.should_execute_dummy_batch = False
|
||||
self.engine_core.execute_dummy_batch()
|
||||
@@ -326,13 +326,13 @@ class LLMEngine:
|
||||
self.processor.clear_mm_cache()
|
||||
self.engine_core.reset_mm_cache()
|
||||
|
||||
def reset_prefix_cache(self, device: Optional[Device] = None):
|
||||
def reset_prefix_cache(self, device: Device | None = None):
|
||||
self.engine_core.reset_prefix_cache()
|
||||
|
||||
def sleep(self, level: int = 1):
|
||||
self.engine_core.sleep(level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None):
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
self.engine_core.wake_up(tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -343,11 +343,11 @@ class LLMEngine:
|
||||
return get_metrics_snapshot()
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> Optional[AnyTokenizer]:
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
return self.processor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
self.processor.tokenizer = tokenizer
|
||||
|
||||
def get_tokenizer(self) -> AnyTokenizer:
|
||||
@@ -390,10 +390,10 @@ class LLMEngine:
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[[WorkerBase], _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
import itertools
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
|
||||
@@ -24,19 +23,19 @@ NONES = itertools.repeat(None)
|
||||
class LogprobsProcessor:
|
||||
# Tokenizer for this request,
|
||||
# None if detokenization is disabled.
|
||||
tokenizer: Optional[AnyTokenizer]
|
||||
tokenizer: AnyTokenizer | None
|
||||
|
||||
# Logprobs for this request
|
||||
logprobs: Optional[SampleLogprobs]
|
||||
prompt_logprobs: Optional[PromptLogprobs]
|
||||
cumulative_logprob: Optional[float]
|
||||
num_logprobs: Optional[int]
|
||||
num_prompt_logprobs: Optional[int]
|
||||
logprobs: SampleLogprobs | None
|
||||
prompt_logprobs: PromptLogprobs | None
|
||||
cumulative_logprob: float | None
|
||||
num_logprobs: int | None
|
||||
num_prompt_logprobs: int | None
|
||||
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: Optional[AnyTokenizer],
|
||||
tokenizer: AnyTokenizer | None,
|
||||
request: EngineCoreRequest,
|
||||
) -> "LogprobsProcessor":
|
||||
assert request.sampling_params is not None
|
||||
@@ -148,7 +147,7 @@ class LogprobsProcessor:
|
||||
)
|
||||
)
|
||||
|
||||
def pop_prompt_logprobs(self) -> Optional[PromptLogprobs]:
|
||||
def pop_prompt_logprobs(self) -> PromptLogprobs | None:
|
||||
"""Pop and return all request prompt logprobs
|
||||
|
||||
The logprobs processor aggregates prompt chunk logprobs
|
||||
@@ -171,7 +170,7 @@ class LogprobsProcessor:
|
||||
def _make_logprob_dict(
|
||||
logprobs: list[float],
|
||||
logprob_token_ids: list[int],
|
||||
decoded_tokens: Iterable[Optional[str]],
|
||||
decoded_tokens: Iterable[str | None],
|
||||
rank: int,
|
||||
num_logprobs: int,
|
||||
) -> dict[int, Logprob]:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import asyncio
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional, Union, cast
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -36,14 +36,10 @@ class RequestOutputCollector:
|
||||
|
||||
def __init__(self, output_kind: RequestOutputKind):
|
||||
self.aggregate = output_kind == RequestOutputKind.DELTA
|
||||
self.output: Optional[Union[RequestOutput, PoolingRequestOutput, Exception]] = (
|
||||
None
|
||||
)
|
||||
self.output: RequestOutput | PoolingRequestOutput | Exception | None = None
|
||||
self.ready = asyncio.Event()
|
||||
|
||||
def put(
|
||||
self, output: Union[RequestOutput, PoolingRequestOutput, Exception]
|
||||
) -> None:
|
||||
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
|
||||
"""Non-blocking put operation."""
|
||||
if self.output is None or isinstance(output, Exception):
|
||||
self.output = output
|
||||
@@ -53,7 +49,7 @@ class RequestOutputCollector:
|
||||
# (if n > 1) do not override each other.
|
||||
self.output.add(output, aggregate=self.aggregate)
|
||||
|
||||
async def get(self) -> Union[RequestOutput, PoolingRequestOutput]:
|
||||
async def get(self) -> RequestOutput | PoolingRequestOutput:
|
||||
"""Get operation blocks on put event."""
|
||||
while (output := self.output) is None:
|
||||
await self.ready.wait()
|
||||
@@ -63,7 +59,7 @@ class RequestOutputCollector:
|
||||
raise output
|
||||
return output
|
||||
|
||||
def get_nowait(self) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
|
||||
"""Non-blocking get operation."""
|
||||
output = self.output
|
||||
if output is not None:
|
||||
@@ -76,7 +72,7 @@ class RequestOutputCollector:
|
||||
|
||||
@dataclass
|
||||
class OutputProcessorOutput:
|
||||
request_outputs: list[Union[RequestOutput, PoolingRequestOutput]]
|
||||
request_outputs: list[RequestOutput | PoolingRequestOutput]
|
||||
reqs_to_abort: list[str]
|
||||
|
||||
|
||||
@@ -84,22 +80,22 @@ class RequestState:
|
||||
def __init__(
|
||||
self,
|
||||
request_id: str,
|
||||
parent_req: Optional[ParentRequest],
|
||||
parent_req: ParentRequest | None,
|
||||
request_index: int,
|
||||
lora_name: Optional[str],
|
||||
lora_name: str | None,
|
||||
output_kind: RequestOutputKind,
|
||||
prompt: Optional[str],
|
||||
prompt_token_ids: Optional[list[int]],
|
||||
prompt_embeds: Optional[torch.Tensor],
|
||||
logprobs_processor: Optional[LogprobsProcessor],
|
||||
detokenizer: Optional[IncrementalDetokenizer],
|
||||
max_tokens_param: Optional[int],
|
||||
prompt: str | None,
|
||||
prompt_token_ids: list[int] | None,
|
||||
prompt_embeds: torch.Tensor | None,
|
||||
logprobs_processor: LogprobsProcessor | None,
|
||||
detokenizer: IncrementalDetokenizer | None,
|
||||
max_tokens_param: int | None,
|
||||
arrival_time: float,
|
||||
queue: Optional[RequestOutputCollector],
|
||||
queue: RequestOutputCollector | None,
|
||||
log_stats: bool,
|
||||
top_p: Optional[float] = None,
|
||||
n: Optional[int] = None,
|
||||
temperature: Optional[float] = None,
|
||||
top_p: float | None = None,
|
||||
n: int | None = None,
|
||||
temperature: float | None = None,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.parent_req = parent_req
|
||||
@@ -129,10 +125,10 @@ class RequestState:
|
||||
cls,
|
||||
tokenizer: AnyTokenizer,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest],
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None,
|
||||
request_index: int,
|
||||
queue: Optional[RequestOutputCollector],
|
||||
queue: RequestOutputCollector | None,
|
||||
log_stats: bool,
|
||||
) -> "RequestState":
|
||||
if sampling_params := request.sampling_params:
|
||||
@@ -186,11 +182,11 @@ class RequestState:
|
||||
def make_request_output(
|
||||
self,
|
||||
new_token_ids: list[int],
|
||||
pooling_output: Optional[torch.Tensor],
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Union[int, str, None],
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
pooling_output: torch.Tensor | None,
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
|
||||
@@ -222,10 +218,10 @@ class RequestState:
|
||||
def _new_request_output(
|
||||
self,
|
||||
request_id: str,
|
||||
outputs: Union[list[CompletionOutput], list[PoolingOutput]],
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None,
|
||||
) -> Union[RequestOutput, PoolingRequestOutput]:
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
first_output = outputs[0]
|
||||
if isinstance(first_output, PoolingOutput):
|
||||
assert len(outputs) == 1
|
||||
@@ -264,8 +260,8 @@ class RequestState:
|
||||
def _new_completion_output(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Union[int, str, None],
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
) -> CompletionOutput:
|
||||
assert self.detokenizer is not None
|
||||
assert self.logprobs_processor is not None
|
||||
@@ -308,7 +304,7 @@ class OutputProcessor:
|
||||
self.request_states: dict[str, RequestState] = {}
|
||||
self.parent_requests: dict[str, ParentRequest] = {}
|
||||
self.lora_states = LoRARequestStates()
|
||||
self.tracer: Optional[Tracer] = None
|
||||
self.tracer: Tracer | None = None
|
||||
|
||||
def get_num_unfinished_requests(self):
|
||||
return len(self.request_states)
|
||||
@@ -360,10 +356,10 @@ class OutputProcessor:
|
||||
def add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest] = None,
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None = None,
|
||||
request_index: int = 0,
|
||||
queue: Optional[RequestOutputCollector] = None,
|
||||
queue: RequestOutputCollector | None = None,
|
||||
) -> None:
|
||||
request_id = request.request_id
|
||||
if request_id in self.request_states:
|
||||
@@ -386,8 +382,8 @@ class OutputProcessor:
|
||||
def process_outputs(
|
||||
self,
|
||||
engine_core_outputs: list[EngineCoreOutput],
|
||||
engine_core_timestamp: Optional[float] = None,
|
||||
iteration_stats: Optional[IterationStats] = None,
|
||||
engine_core_timestamp: float | None = None,
|
||||
iteration_stats: IterationStats | None = None,
|
||||
) -> OutputProcessorOutput:
|
||||
"""
|
||||
Process the EngineCoreOutputs:
|
||||
@@ -411,7 +407,7 @@ class OutputProcessor:
|
||||
within the loop below.
|
||||
"""
|
||||
|
||||
request_outputs: Union[list[RequestOutput], list[PoolingRequestOutput]] = []
|
||||
request_outputs: list[RequestOutput] | list[PoolingRequestOutput] = []
|
||||
reqs_to_abort: list[str] = []
|
||||
for engine_core_output in engine_core_outputs:
|
||||
req_id = engine_core_output.request_id
|
||||
@@ -492,7 +488,7 @@ class OutputProcessor:
|
||||
self,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
req_state: RequestState,
|
||||
iteration_stats: Optional[IterationStats],
|
||||
iteration_stats: IterationStats | None,
|
||||
) -> None:
|
||||
assert req_state.stats is not None
|
||||
assert iteration_stats is not None
|
||||
@@ -555,8 +551,8 @@ class OutputProcessor:
|
||||
self,
|
||||
req_state: RequestState,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
engine_core_timestamp: Optional[float],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
engine_core_timestamp: float | None,
|
||||
iteration_stats: IterationStats | None,
|
||||
):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
@@ -577,8 +573,8 @@ class OutputProcessor:
|
||||
def _update_stats_from_finished(
|
||||
self,
|
||||
req_state: RequestState,
|
||||
finish_reason: Optional[FinishReason],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
finish_reason: FinishReason | None,
|
||||
iteration_stats: IterationStats | None,
|
||||
):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
@@ -29,7 +29,7 @@ class ParentRequest:
|
||||
max_num_generation_tokens: int
|
||||
|
||||
# To efficiently obtain child sampling params
|
||||
cached_child_sampling_params: Optional[SamplingParams]
|
||||
cached_child_sampling_params: SamplingParams | None
|
||||
|
||||
def __init__(self, request_id: str, sampling_params: SamplingParams) -> None:
|
||||
self.request_id = request_id
|
||||
|
||||
+17
-17
@@ -3,7 +3,7 @@
|
||||
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Literal, Optional, Union
|
||||
from typing import Any, Literal
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
|
||||
@@ -38,7 +38,7 @@ class Processor:
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
tokenizer: Optional[AnyTokenizer],
|
||||
tokenizer: AnyTokenizer | None,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
@@ -60,11 +60,11 @@ class Processor:
|
||||
)
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> Optional[AnyTokenizer]:
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
return self.input_preprocessor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
self.input_preprocessor.tokenizer = tokenizer
|
||||
|
||||
def _validate_logprobs(
|
||||
@@ -152,7 +152,7 @@ class Processor:
|
||||
|
||||
def _validate_params(
|
||||
self,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
params: SamplingParams | PoolingParams,
|
||||
):
|
||||
"""
|
||||
Validate supported SamplingParam.
|
||||
@@ -174,7 +174,7 @@ class Processor:
|
||||
auto-hashed downstream.
|
||||
"""
|
||||
|
||||
def _validate_single_prompt(single_prompt: Union[dict, str]) -> None:
|
||||
def _validate_single_prompt(single_prompt: dict | str) -> None:
|
||||
if not isinstance(single_prompt, dict):
|
||||
return
|
||||
mm_data = single_prompt.get("multi_modal_data")
|
||||
@@ -214,7 +214,7 @@ class Processor:
|
||||
else:
|
||||
_validate_single_prompt(prompt) # type: ignore[arg-type]
|
||||
|
||||
def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:
|
||||
def _validate_lora(self, lora_request: LoRARequest | None) -> None:
|
||||
if lora_request is None:
|
||||
return
|
||||
|
||||
@@ -309,7 +309,7 @@ class Processor:
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
) -> Optional[MultiModalUUIDDict]:
|
||||
) -> MultiModalUUIDDict | None:
|
||||
"""Build per-item multimodal hash overrides when enabled. In this case,
|
||||
multimodal data items are identified by their request id, modality and
|
||||
index rather than their content.
|
||||
@@ -342,13 +342,13 @@ class Processor:
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
arrival_time: Optional[float] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
data_parallel_rank: Optional[int] = None,
|
||||
data_parallel_rank: int | None = None,
|
||||
) -> EngineCoreRequest:
|
||||
self._validate_lora(lora_request)
|
||||
self._validate_params(params)
|
||||
@@ -445,7 +445,7 @@ class Processor:
|
||||
pooling_params = params.clone()
|
||||
|
||||
# Multimodal related.
|
||||
mm_features: Optional[list[MultiModalFeatureSpec]] = None
|
||||
mm_features: list[MultiModalFeatureSpec] | None = None
|
||||
|
||||
if decoder_inputs["type"] == "multimodal":
|
||||
decoder_mm_inputs = decoder_inputs["mm_kwargs"]
|
||||
@@ -485,7 +485,7 @@ class Processor:
|
||||
)
|
||||
|
||||
def _validate_model_inputs(
|
||||
self, encoder_inputs: Optional[SingletonInputs], decoder_inputs: SingletonInputs
|
||||
self, encoder_inputs: SingletonInputs | None, decoder_inputs: SingletonInputs
|
||||
):
|
||||
if encoder_inputs is not None:
|
||||
self._validate_model_input(encoder_inputs, prompt_type="encoder")
|
||||
@@ -574,7 +574,7 @@ class Processor:
|
||||
# check that chunked prefill does not truncate them
|
||||
# max_batch_len = self.scheduler_config.max_num_batched_tokens
|
||||
|
||||
def stat_mm_cache(self) -> Optional[MultiModalCacheStats]:
|
||||
def stat_mm_cache(self) -> MultiModalCacheStats | None:
|
||||
return self.input_preprocessor.stat_mm_cache()
|
||||
|
||||
def clear_mm_cache(self) -> None:
|
||||
|
||||
+14
-14
@@ -4,12 +4,12 @@
|
||||
import contextlib
|
||||
import os
|
||||
import weakref
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from multiprocessing import Process, connection
|
||||
from multiprocessing.process import BaseProcess
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Union
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import patch
|
||||
|
||||
import msgspec
|
||||
@@ -56,13 +56,13 @@ class EngineZmqAddresses:
|
||||
# ZMQ output socket addresses for each front-end client (responses)
|
||||
outputs: list[str]
|
||||
# ZMQ input socket address of DP coordinator if applicable
|
||||
coordinator_input: Optional[str] = None
|
||||
coordinator_input: str | None = None
|
||||
# ZMQ output socket address of DP coordinator if applicable
|
||||
coordinator_output: Optional[str] = None
|
||||
coordinator_output: str | None = None
|
||||
# ZMQ socket for front-end to connect to DP coordinator.
|
||||
# Not used by engine, just relayed to front-end in handshake response.
|
||||
# Only required for external DP LB case.
|
||||
frontend_stats_publish_address: Optional[str] = None
|
||||
frontend_stats_publish_address: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -73,8 +73,8 @@ class EngineHandshakeMetadata:
|
||||
"""
|
||||
|
||||
addresses: EngineZmqAddresses
|
||||
parallel_config: dict[str, Union[int, str, list[int]]]
|
||||
parallel_config_hash: Optional[str] = None
|
||||
parallel_config: dict[str, int | str | list[int]]
|
||||
parallel_config_hash: str | None = None
|
||||
|
||||
|
||||
class CoreEngineProcManager:
|
||||
@@ -94,7 +94,7 @@ class CoreEngineProcManager:
|
||||
handshake_address: str,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_handshake_address: Optional[str] = None,
|
||||
client_handshake_address: str | None = None,
|
||||
):
|
||||
context = get_mp_context()
|
||||
common_kwargs = {
|
||||
@@ -221,8 +221,8 @@ class CoreEngineActorManager:
|
||||
addresses: EngineZmqAddresses,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
placement_groups: Optional[list["PlacementGroup"]] = None,
|
||||
local_dp_ranks: Optional[list[int]] = None,
|
||||
placement_groups: list["PlacementGroup"] | None = None,
|
||||
local_dp_ranks: list[int] | None = None,
|
||||
):
|
||||
import copy
|
||||
|
||||
@@ -675,8 +675,8 @@ def launch_core_engines(
|
||||
num_api_servers: int = 1,
|
||||
) -> Iterator[
|
||||
tuple[
|
||||
Optional[Union[CoreEngineProcManager, CoreEngineActorManager]],
|
||||
Optional[DPCoordinator],
|
||||
CoreEngineProcManager | CoreEngineActorManager | None,
|
||||
DPCoordinator | None,
|
||||
EngineZmqAddresses,
|
||||
]
|
||||
]:
|
||||
@@ -829,8 +829,8 @@ def wait_for_engine_startup(
|
||||
core_engines: list[CoreEngine],
|
||||
parallel_config: ParallelConfig,
|
||||
cache_config: CacheConfig,
|
||||
proc_manager: Optional[CoreEngineProcManager],
|
||||
coord_process: Optional[Process],
|
||||
proc_manager: CoreEngineProcManager | None,
|
||||
coord_process: Process | None,
|
||||
):
|
||||
# Wait for engine core process(es) to send ready messages.
|
||||
local_count = parallel_config.data_parallel_size_local
|
||||
|
||||
Reference in New Issue
Block a user