Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+35
-35
@@ -6,7 +6,7 @@ import socket
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping
|
||||
from copy import copy
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -56,8 +56,8 @@ class AsyncLLM(EngineClient):
|
||||
use_cached_outputs: bool = False,
|
||||
log_requests: bool = True,
|
||||
start_engine_loop: bool = True,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> None:
|
||||
@@ -136,7 +136,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
# Loggers.
|
||||
self.logger_manager: Optional[StatLoggerManager] = None
|
||||
self.logger_manager: StatLoggerManager | None = None
|
||||
if self.log_stats:
|
||||
self.logger_manager = StatLoggerManager(
|
||||
vllm_config=vllm_config,
|
||||
@@ -147,7 +147,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
self.logger_manager.log_engine_initialized()
|
||||
|
||||
self.output_handler: Optional[asyncio.Task] = None
|
||||
self.output_handler: asyncio.Task | None = None
|
||||
try:
|
||||
# Start output handler eagerly if we are in the asyncio eventloop.
|
||||
asyncio.get_running_loop()
|
||||
@@ -185,10 +185,10 @@ class AsyncLLM(EngineClient):
|
||||
vllm_config: VllmConfig,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
enable_log_requests: bool = False,
|
||||
disable_log_stats: bool = False,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
disable_log_requests: bool = True, # Deprecated, will be removed
|
||||
@@ -221,7 +221,7 @@ class AsyncLLM(EngineClient):
|
||||
engine_args: AsyncEngineArgs,
|
||||
start_engine_loop: bool = True,
|
||||
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
|
||||
stat_loggers: Optional[list[StatLoggerFactory]] = None,
|
||||
stat_loggers: list[StatLoggerFactory] | None = None,
|
||||
) -> "AsyncLLM":
|
||||
"""Create an AsyncLLM from the EngineArgs."""
|
||||
|
||||
@@ -259,15 +259,15 @@ class AsyncLLM(EngineClient):
|
||||
async def add_request(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: Union[EngineCoreRequest, PromptType],
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
arrival_time: Optional[float] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt: EngineCoreRequest | PromptType,
|
||||
params: SamplingParams | PoolingParams,
|
||||
arrival_time: float | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
data_parallel_rank: Optional[int] = None,
|
||||
prompt_text: Optional[str] = None,
|
||||
data_parallel_rank: int | None = None,
|
||||
prompt_text: str | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
"""Add new request to the AsyncLLM."""
|
||||
|
||||
@@ -325,8 +325,8 @@ class AsyncLLM(EngineClient):
|
||||
async def _add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest],
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None,
|
||||
index: int,
|
||||
queue: RequestOutputCollector,
|
||||
):
|
||||
@@ -346,16 +346,16 @@ class AsyncLLM(EngineClient):
|
||||
# re-multiplexed in the API server anyhow.
|
||||
async def generate(
|
||||
self,
|
||||
prompt: Union[EngineCoreRequest, PromptType],
|
||||
prompt: EngineCoreRequest | PromptType,
|
||||
sampling_params: SamplingParams,
|
||||
request_id: str,
|
||||
*,
|
||||
prompt_text: Optional[str] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt_text: str | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
data_parallel_rank: Optional[int] = None,
|
||||
data_parallel_rank: int | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
@@ -520,7 +520,7 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
self.output_handler = asyncio.create_task(output_handler())
|
||||
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
async def abort(self, request_id: str | Iterable[str]) -> None:
|
||||
"""Abort RequestId in OutputProcessor and EngineCore."""
|
||||
|
||||
request_ids = (
|
||||
@@ -537,11 +537,11 @@ class AsyncLLM(EngineClient):
|
||||
prompt: PromptType,
|
||||
pooling_params: PoolingParams,
|
||||
request_id: str,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[PoolingRequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
@@ -622,11 +622,11 @@ class AsyncLLM(EngineClient):
|
||||
raise EngineGenerateError() from e
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> Optional[AnyTokenizer]:
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
return self.processor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
self.processor.tokenizer = tokenizer
|
||||
|
||||
async def get_tokenizer(self) -> AnyTokenizer:
|
||||
@@ -665,7 +665,7 @@ class AsyncLLM(EngineClient):
|
||||
self.processor.clear_mm_cache()
|
||||
await self.engine_core.reset_mm_cache_async()
|
||||
|
||||
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
|
||||
async def reset_prefix_cache(self, device: Device | None = None) -> None:
|
||||
if device == Device.CPU:
|
||||
raise ValueError("Not supported on CPU.")
|
||||
await self.engine_core.reset_prefix_cache_async()
|
||||
@@ -674,7 +674,7 @@ class AsyncLLM(EngineClient):
|
||||
await self.reset_prefix_cache()
|
||||
await self.engine_core.sleep_async(level)
|
||||
|
||||
async def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
async def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
await self.engine_core.wake_up_async(tags)
|
||||
|
||||
async def is_sleeping(self) -> bool:
|
||||
@@ -699,9 +699,9 @@ class AsyncLLM(EngineClient):
|
||||
async def collective_rpc(
|
||||
self,
|
||||
method: str,
|
||||
timeout: Optional[float] = None,
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None,
|
||||
kwargs: dict | None = None,
|
||||
):
|
||||
"""
|
||||
Perform a collective RPC call to the given path.
|
||||
|
||||
Reference in New Issue
Block a user