Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+113 -121
View File
@@ -2,8 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
from collections.abc import Callable, Sequence
from typing import TYPE_CHECKING, Any, cast
import cloudpickle
import torch.nn as nn
@@ -191,36 +191,34 @@ class LLM:
*,
runner: RunnerOption = "auto",
convert: ConvertOption = "auto",
tokenizer: Optional[str] = None,
tokenizer: str | None = None,
tokenizer_mode: TokenizerMode = "auto",
skip_tokenizer_init: bool = False,
trust_remote_code: bool = False,
allowed_local_media_path: str = "",
allowed_media_domains: Optional[list[str]] = None,
allowed_media_domains: list[str] | None = None,
tensor_parallel_size: int = 1,
dtype: ModelDType = "auto",
quantization: Optional[QuantizationMethods] = None,
revision: Optional[str] = None,
tokenizer_revision: Optional[str] = None,
seed: Optional[int] = None,
quantization: QuantizationMethods | None = None,
revision: str | None = None,
tokenizer_revision: str | None = None,
seed: int | None = None,
gpu_memory_utilization: float = 0.9,
swap_space: float = 4,
cpu_offload_gb: float = 0,
enforce_eager: bool = False,
disable_custom_all_reduce: bool = False,
hf_token: Optional[Union[bool, str]] = None,
hf_overrides: Optional[HfOverrides] = None,
mm_processor_kwargs: Optional[dict[str, Any]] = None,
pooler_config: Optional[PoolerConfig] = None,
override_pooler_config: Optional[PoolerConfig] = None,
structured_outputs_config: Optional[
Union[dict[str, Any], StructuredOutputsConfig]
] = None,
kv_cache_memory_bytes: Optional[int] = None,
compilation_config: Optional[
Union[int, dict[str, Any], CompilationConfig]
] = None,
logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None,
hf_token: bool | str | None = None,
hf_overrides: HfOverrides | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
pooler_config: PoolerConfig | None = None,
override_pooler_config: PoolerConfig | None = None,
structured_outputs_config: dict[str, Any]
| StructuredOutputsConfig
| None = None,
kv_cache_memory_bytes: int | None = None,
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
logits_processors: list[str | type[LogitsProcessor]] | None = None,
**kwargs: Any,
) -> None:
"""LLM constructor."""
@@ -331,7 +329,7 @@ class LLM:
self.engine_class = type(self.llm_engine)
self.request_counter = Counter()
self.default_sampling_params: Union[dict[str, Any], None] = None
self.default_sampling_params: dict[str, Any] | None = None
supported_tasks = self.llm_engine.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
@@ -367,14 +365,12 @@ class LLM:
def generate(
self,
prompts: Union[PromptType, Sequence[PromptType]],
sampling_params: Optional[
Union[SamplingParams, Sequence[SamplingParams]]
] = None,
prompts: PromptType | Sequence[PromptType],
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
*,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
priority: Optional[list[int]] = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
) -> list[RequestOutput]:
"""Generates the completions for the input prompts.
@@ -437,8 +433,8 @@ class LLM:
def _get_modality_specific_lora_reqs(
self,
prompts: Union[PromptType, Sequence[PromptType]],
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
prompts: PromptType | Sequence[PromptType],
lora_request: list[LoRARequest] | LoRARequest | None,
):
# Grab the lora config off the vllm config on the engine,
# since this is the same for both v0 & v1.
@@ -474,8 +470,8 @@ class LLM:
def _resolve_single_prompt_mm_lora(
self,
prompt: PromptType,
lora_request: Optional[LoRARequest],
default_mm_loras: Optional[dict[str, str]],
lora_request: LoRARequest | None,
default_mm_loras: dict[str, str] | None,
):
if (
not default_mm_loras
@@ -526,10 +522,10 @@ class LLM:
def collective_rpc(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
"""
Execute an RPC call on all workers.
@@ -571,9 +567,9 @@ class LLM:
def _get_beam_search_lora_requests(
self,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
prompts: list[Union[TokensPrompt, TextPrompt]],
) -> list[Optional[LoRARequest]]:
lora_request: list[LoRARequest] | LoRARequest | None,
prompts: list[TokensPrompt | TextPrompt],
) -> list[LoRARequest | None]:
"""Get the optional lora request corresponding to each prompt."""
if isinstance(lora_request, Sequence) and len(lora_request) != len(prompts):
raise ValueError(
@@ -587,11 +583,11 @@ class LLM:
def beam_search(
self,
prompts: list[Union[TokensPrompt, TextPrompt]],
prompts: list[TokensPrompt | TextPrompt],
params: BeamSearchParams,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
use_tqdm: bool = False,
concurrency_limit: Optional[int] = None,
concurrency_limit: int | None = None,
) -> list[BeamSearchOutput]:
"""
Generate sequences using beam search.
@@ -770,16 +766,15 @@ class LLM:
def preprocess_chat(
self,
messages: Union[
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
],
chat_template: Optional[str] = None,
messages: list[ChatCompletionMessageParam]
| list[list[ChatCompletionMessageParam]],
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
continue_final_message: bool = False,
tools: Optional[list[dict[str, Any]]] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None,
mm_processor_kwargs: Optional[dict[str, Any]] = None,
tools: list[dict[str, Any]] | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[TokensPrompt]:
"""
Generate prompt for a chat conversation. The pre-processed
@@ -868,19 +863,18 @@ class LLM:
def chat(
self,
messages: Union[
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
],
sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[LoRARequest] = None,
chat_template: Optional[str] = None,
messages: list[ChatCompletionMessageParam]
| list[list[ChatCompletionMessageParam]],
sampling_params: SamplingParams | list[SamplingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: LoRARequest | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
continue_final_message: bool = False,
tools: Optional[list[dict[str, Any]]] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None,
mm_processor_kwargs: Optional[dict[str, Any]] = None,
tools: list[dict[str, Any]] | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[RequestOutput]:
"""
Generate responses for a chat conversation.
@@ -953,14 +947,14 @@ class LLM:
def encode(
self,
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
prompts: PromptType | Sequence[PromptType] | DataPrompt,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
pooling_task: PoolingTask = "encode",
tokenization_kwargs: Optional[dict[str, Any]] = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[PoolingRequestOutput]:
"""Apply pooling to the hidden states corresponding to the input
prompts.
@@ -1086,12 +1080,12 @@ class LLM:
def embed(
self,
prompts: Union[PromptType, Sequence[PromptType]],
prompts: PromptType | Sequence[PromptType],
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[EmbeddingRequestOutput]:
"""
Generate an embedding vector for each prompt.
@@ -1135,11 +1129,11 @@ class LLM:
def classify(
self,
prompts: Union[PromptType, Sequence[PromptType]],
prompts: PromptType | Sequence[PromptType],
*,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ClassificationRequestOutput]:
"""
Generate class logits for each prompt.
@@ -1181,13 +1175,13 @@ class LLM:
def reward(
self,
prompts: Union[PromptType, Sequence[PromptType]],
prompts: PromptType | Sequence[PromptType],
/,
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[PoolingRequestOutput]:
"""
Generate rewards for each prompt.
@@ -1220,12 +1214,12 @@ class LLM:
def _embedding_score(
self,
tokenizer: AnyTokenizer,
text_1: list[Union[str, TextPrompt, TokensPrompt]],
text_2: list[Union[str, TextPrompt, TokensPrompt]],
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[PoolingParams] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
text_1: list[str | TextPrompt | TokensPrompt],
text_2: list[str | TextPrompt | TokensPrompt],
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ScoringRequestOutput]:
encoded_output: list[PoolingRequestOutput] = self.encode(
text_1 + text_2,
@@ -1252,12 +1246,12 @@ class LLM:
def _cross_encoding_score(
self,
tokenizer: AnyTokenizer,
data_1: Union[list[str], list[ScoreContentPartParam]],
data_2: Union[list[str], list[ScoreContentPartParam]],
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[PoolingParams] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
data_1: list[str] | list[ScoreContentPartParam],
data_2: list[str] | list[ScoreContentPartParam],
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ScoringRequestOutput]:
model_config = self.model_config
@@ -1316,14 +1310,14 @@ class LLM:
def score(
self,
data_1: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
data_2: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
data_1: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
data_2: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
/,
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[PoolingParams] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ScoringRequestOutput]:
"""Generate similarity scores for all pairs `<text,text_pair>` or
`<multi-modal data, multi-modal data pair>`.
@@ -1391,9 +1385,9 @@ class LLM:
if not model_config.is_multimodal_model:
def check_data_type(
data: Union[
SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam
],
data: SingletonPrompt
| Sequence[SingletonPrompt]
| ScoreMultiModalParam,
):
if isinstance(data, dict) and "content" in data:
raise ValueError(
@@ -1470,7 +1464,7 @@ class LLM:
def stop_profile(self) -> None:
self.llm_engine.stop_profile()
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
def reset_prefix_cache(self, device: Device | None = None) -> bool:
return self.llm_engine.reset_prefix_cache(device)
def sleep(self, level: int = 1):
@@ -1495,7 +1489,7 @@ class LLM:
self.reset_prefix_cache()
self.llm_engine.sleep(level=level)
def wake_up(self, tags: Optional[list[str]] = None):
def wake_up(self, tags: list[str] | None = None):
"""
Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
method for more details.
@@ -1523,17 +1517,15 @@ class LLM:
def _validate_and_add_requests(
self,
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
params: Union[
SamplingParams,
Sequence[SamplingParams],
PoolingParams,
Sequence[PoolingParams],
],
prompts: PromptType | Sequence[PromptType] | DataPrompt,
params: SamplingParams
| Sequence[SamplingParams]
| PoolingParams
| Sequence[PoolingParams],
*,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
priority: Optional[list[int]] = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None,
priority: list[int] | None = None,
) -> None:
if isinstance(prompts, (str, dict)):
# Convert a single prompt to a list.
@@ -1575,8 +1567,8 @@ class LLM:
def _validate_mm_data_and_uuids(
self,
multi_modal_data: Optional[Any], # MultiModalDataDict
multi_modal_uuids: Optional[Any], # MultiModalUUIDDict
multi_modal_data: Any | None, # MultiModalDataDict
multi_modal_uuids: Any | None, # MultiModalUUIDDict
):
"""
Validate that if any multi-modal data is skipped (i.e. None),
@@ -1625,9 +1617,9 @@ class LLM:
self,
request_id: str,
engine_prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
params: SamplingParams | PoolingParams,
*,
lora_request: Optional[LoRARequest],
lora_request: LoRARequest | None,
priority: int,
) -> tuple[EngineCoreRequest, dict[str, Any]]:
"""Use the Processor to process inputs for LLMEngine."""
@@ -1651,8 +1643,8 @@ class LLM:
def _add_request(
self,
prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
lora_request: Optional[LoRARequest] = None,
params: SamplingParams | PoolingParams,
lora_request: LoRARequest | None = None,
priority: int = 0,
) -> None:
prompt_text, _, _ = get_prompt_components(prompt)
@@ -1677,8 +1669,8 @@ class LLM:
)
def _run_engine(
self, *, use_tqdm: Union[bool, Callable[..., tqdm]] = True
) -> list[Union[RequestOutput, PoolingRequestOutput]]:
self, *, use_tqdm: bool | Callable[..., tqdm] = True
) -> list[RequestOutput | PoolingRequestOutput]:
# Initialize tqdm.
if use_tqdm:
num_requests = self.llm_engine.get_num_unfinished_requests()
@@ -1691,7 +1683,7 @@ class LLM:
)
# Run the engine.
outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
outputs: list[RequestOutput | PoolingRequestOutput] = []
total_in_toks = 0
total_out_toks = 0
while self.llm_engine.has_unfinished_requests():