Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+113
-121
@@ -2,8 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import cloudpickle
|
||||
import torch.nn as nn
|
||||
@@ -191,36 +191,34 @@ class LLM:
|
||||
*,
|
||||
runner: RunnerOption = "auto",
|
||||
convert: ConvertOption = "auto",
|
||||
tokenizer: Optional[str] = None,
|
||||
tokenizer: str | None = None,
|
||||
tokenizer_mode: TokenizerMode = "auto",
|
||||
skip_tokenizer_init: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
allowed_local_media_path: str = "",
|
||||
allowed_media_domains: Optional[list[str]] = None,
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
dtype: ModelDType = "auto",
|
||||
quantization: Optional[QuantizationMethods] = None,
|
||||
revision: Optional[str] = None,
|
||||
tokenizer_revision: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
quantization: QuantizationMethods | None = None,
|
||||
revision: str | None = None,
|
||||
tokenizer_revision: str | None = None,
|
||||
seed: int | None = None,
|
||||
gpu_memory_utilization: float = 0.9,
|
||||
swap_space: float = 4,
|
||||
cpu_offload_gb: float = 0,
|
||||
enforce_eager: bool = False,
|
||||
disable_custom_all_reduce: bool = False,
|
||||
hf_token: Optional[Union[bool, str]] = None,
|
||||
hf_overrides: Optional[HfOverrides] = None,
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
||||
pooler_config: Optional[PoolerConfig] = None,
|
||||
override_pooler_config: Optional[PoolerConfig] = None,
|
||||
structured_outputs_config: Optional[
|
||||
Union[dict[str, Any], StructuredOutputsConfig]
|
||||
] = None,
|
||||
kv_cache_memory_bytes: Optional[int] = None,
|
||||
compilation_config: Optional[
|
||||
Union[int, dict[str, Any], CompilationConfig]
|
||||
] = None,
|
||||
logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None,
|
||||
hf_token: bool | str | None = None,
|
||||
hf_overrides: HfOverrides | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
pooler_config: PoolerConfig | None = None,
|
||||
override_pooler_config: PoolerConfig | None = None,
|
||||
structured_outputs_config: dict[str, Any]
|
||||
| StructuredOutputsConfig
|
||||
| None = None,
|
||||
kv_cache_memory_bytes: int | None = None,
|
||||
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
|
||||
logits_processors: list[str | type[LogitsProcessor]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""LLM constructor."""
|
||||
@@ -331,7 +329,7 @@ class LLM:
|
||||
self.engine_class = type(self.llm_engine)
|
||||
|
||||
self.request_counter = Counter()
|
||||
self.default_sampling_params: Union[dict[str, Any], None] = None
|
||||
self.default_sampling_params: dict[str, Any] | None = None
|
||||
|
||||
supported_tasks = self.llm_engine.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
@@ -367,14 +365,12 @@ class LLM:
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
sampling_params: Optional[
|
||||
Union[SamplingParams, Sequence[SamplingParams]]
|
||||
] = None,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
priority: Optional[list[int]] = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""Generates the completions for the input prompts.
|
||||
|
||||
@@ -437,8 +433,8 @@ class LLM:
|
||||
|
||||
def _get_modality_specific_lora_reqs(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
lora_request: list[LoRARequest] | LoRARequest | None,
|
||||
):
|
||||
# Grab the lora config off the vllm config on the engine,
|
||||
# since this is the same for both v0 & v1.
|
||||
@@ -474,8 +470,8 @@ class LLM:
|
||||
def _resolve_single_prompt_mm_lora(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
lora_request: Optional[LoRARequest],
|
||||
default_mm_loras: Optional[dict[str, str]],
|
||||
lora_request: LoRARequest | None,
|
||||
default_mm_loras: dict[str, str] | None,
|
||||
):
|
||||
if (
|
||||
not default_mm_loras
|
||||
@@ -526,10 +522,10 @@ class LLM:
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
"""
|
||||
Execute an RPC call on all workers.
|
||||
@@ -571,9 +567,9 @@ class LLM:
|
||||
|
||||
def _get_beam_search_lora_requests(
|
||||
self,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
|
||||
prompts: list[Union[TokensPrompt, TextPrompt]],
|
||||
) -> list[Optional[LoRARequest]]:
|
||||
lora_request: list[LoRARequest] | LoRARequest | None,
|
||||
prompts: list[TokensPrompt | TextPrompt],
|
||||
) -> list[LoRARequest | None]:
|
||||
"""Get the optional lora request corresponding to each prompt."""
|
||||
if isinstance(lora_request, Sequence) and len(lora_request) != len(prompts):
|
||||
raise ValueError(
|
||||
@@ -587,11 +583,11 @@ class LLM:
|
||||
|
||||
def beam_search(
|
||||
self,
|
||||
prompts: list[Union[TokensPrompt, TextPrompt]],
|
||||
prompts: list[TokensPrompt | TextPrompt],
|
||||
params: BeamSearchParams,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
use_tqdm: bool = False,
|
||||
concurrency_limit: Optional[int] = None,
|
||||
concurrency_limit: int | None = None,
|
||||
) -> list[BeamSearchOutput]:
|
||||
"""
|
||||
Generate sequences using beam search.
|
||||
@@ -770,16 +766,15 @@ class LLM:
|
||||
|
||||
def preprocess_chat(
|
||||
self,
|
||||
messages: Union[
|
||||
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
|
||||
],
|
||||
chat_template: Optional[str] = None,
|
||||
messages: list[ChatCompletionMessageParam]
|
||||
| list[list[ChatCompletionMessageParam]],
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
continue_final_message: bool = False,
|
||||
tools: Optional[list[dict[str, Any]]] = None,
|
||||
chat_template_kwargs: Optional[dict[str, Any]] = None,
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[TokensPrompt]:
|
||||
"""
|
||||
Generate prompt for a chat conversation. The pre-processed
|
||||
@@ -868,19 +863,18 @@ class LLM:
|
||||
|
||||
def chat(
|
||||
self,
|
||||
messages: Union[
|
||||
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
|
||||
],
|
||||
sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
chat_template: Optional[str] = None,
|
||||
messages: list[ChatCompletionMessageParam]
|
||||
| list[list[ChatCompletionMessageParam]],
|
||||
sampling_params: SamplingParams | list[SamplingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: LoRARequest | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
continue_final_message: bool = False,
|
||||
tools: Optional[list[dict[str, Any]]] = None,
|
||||
chat_template_kwargs: Optional[dict[str, Any]] = None,
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""
|
||||
Generate responses for a chat conversation.
|
||||
@@ -953,14 +947,14 @@ class LLM:
|
||||
|
||||
def encode(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
pooling_task: PoolingTask = "encode",
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""Apply pooling to the hidden states corresponding to the input
|
||||
prompts.
|
||||
@@ -1086,12 +1080,12 @@ class LLM:
|
||||
|
||||
def embed(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[EmbeddingRequestOutput]:
|
||||
"""
|
||||
Generate an embedding vector for each prompt.
|
||||
@@ -1135,11 +1129,11 @@ class LLM:
|
||||
|
||||
def classify(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ClassificationRequestOutput]:
|
||||
"""
|
||||
Generate class logits for each prompt.
|
||||
@@ -1181,13 +1175,13 @@ class LLM:
|
||||
|
||||
def reward(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
/,
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""
|
||||
Generate rewards for each prompt.
|
||||
@@ -1220,12 +1214,12 @@ class LLM:
|
||||
def _embedding_score(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
text_1: list[Union[str, TextPrompt, TokensPrompt]],
|
||||
text_2: list[Union[str, TextPrompt, TokensPrompt]],
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
text_1: list[str | TextPrompt | TokensPrompt],
|
||||
text_2: list[str | TextPrompt | TokensPrompt],
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
encoded_output: list[PoolingRequestOutput] = self.encode(
|
||||
text_1 + text_2,
|
||||
@@ -1252,12 +1246,12 @@ class LLM:
|
||||
def _cross_encoding_score(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
data_1: Union[list[str], list[ScoreContentPartParam]],
|
||||
data_2: Union[list[str], list[ScoreContentPartParam]],
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
data_1: list[str] | list[ScoreContentPartParam],
|
||||
data_2: list[str] | list[ScoreContentPartParam],
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
model_config = self.model_config
|
||||
|
||||
@@ -1316,14 +1310,14 @@ class LLM:
|
||||
|
||||
def score(
|
||||
self,
|
||||
data_1: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
|
||||
data_2: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
|
||||
data_1: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
|
||||
data_2: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
|
||||
/,
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
"""Generate similarity scores for all pairs `<text,text_pair>` or
|
||||
`<multi-modal data, multi-modal data pair>`.
|
||||
@@ -1391,9 +1385,9 @@ class LLM:
|
||||
if not model_config.is_multimodal_model:
|
||||
|
||||
def check_data_type(
|
||||
data: Union[
|
||||
SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam
|
||||
],
|
||||
data: SingletonPrompt
|
||||
| Sequence[SingletonPrompt]
|
||||
| ScoreMultiModalParam,
|
||||
):
|
||||
if isinstance(data, dict) and "content" in data:
|
||||
raise ValueError(
|
||||
@@ -1470,7 +1464,7 @@ class LLM:
|
||||
def stop_profile(self) -> None:
|
||||
self.llm_engine.stop_profile()
|
||||
|
||||
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
|
||||
def reset_prefix_cache(self, device: Device | None = None) -> bool:
|
||||
return self.llm_engine.reset_prefix_cache(device)
|
||||
|
||||
def sleep(self, level: int = 1):
|
||||
@@ -1495,7 +1489,7 @@ class LLM:
|
||||
self.reset_prefix_cache()
|
||||
self.llm_engine.sleep(level=level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None):
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
"""
|
||||
Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
|
||||
method for more details.
|
||||
@@ -1523,17 +1517,15 @@ class LLM:
|
||||
|
||||
def _validate_and_add_requests(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
|
||||
params: Union[
|
||||
SamplingParams,
|
||||
Sequence[SamplingParams],
|
||||
PoolingParams,
|
||||
Sequence[PoolingParams],
|
||||
],
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
params: SamplingParams
|
||||
| Sequence[SamplingParams]
|
||||
| PoolingParams
|
||||
| Sequence[PoolingParams],
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
|
||||
priority: Optional[list[int]] = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None,
|
||||
priority: list[int] | None = None,
|
||||
) -> None:
|
||||
if isinstance(prompts, (str, dict)):
|
||||
# Convert a single prompt to a list.
|
||||
@@ -1575,8 +1567,8 @@ class LLM:
|
||||
|
||||
def _validate_mm_data_and_uuids(
|
||||
self,
|
||||
multi_modal_data: Optional[Any], # MultiModalDataDict
|
||||
multi_modal_uuids: Optional[Any], # MultiModalUUIDDict
|
||||
multi_modal_data: Any | None, # MultiModalDataDict
|
||||
multi_modal_uuids: Any | None, # MultiModalUUIDDict
|
||||
):
|
||||
"""
|
||||
Validate that if any multi-modal data is skipped (i.e. None),
|
||||
@@ -1625,9 +1617,9 @@ class LLM:
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: PromptType,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
params: SamplingParams | PoolingParams,
|
||||
*,
|
||||
lora_request: Optional[LoRARequest],
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
) -> tuple[EngineCoreRequest, dict[str, Any]]:
|
||||
"""Use the Processor to process inputs for LLMEngine."""
|
||||
@@ -1651,8 +1643,8 @@ class LLM:
|
||||
def _add_request(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
params: SamplingParams | PoolingParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
) -> None:
|
||||
prompt_text, _, _ = get_prompt_components(prompt)
|
||||
@@ -1677,8 +1669,8 @@ class LLM:
|
||||
)
|
||||
|
||||
def _run_engine(
|
||||
self, *, use_tqdm: Union[bool, Callable[..., tqdm]] = True
|
||||
) -> list[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
self, *, use_tqdm: bool | Callable[..., tqdm] = True
|
||||
) -> list[RequestOutput | PoolingRequestOutput]:
|
||||
# Initialize tqdm.
|
||||
if use_tqdm:
|
||||
num_requests = self.llm_engine.get_num_unfinished_requests()
|
||||
@@ -1691,7 +1683,7 @@ class LLM:
|
||||
)
|
||||
|
||||
# Run the engine.
|
||||
outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
|
||||
outputs: list[RequestOutput | PoolingRequestOutput] = []
|
||||
total_in_toks = 0
|
||||
total_out_toks = 0
|
||||
while self.llm_engine.has_unfinished_requests():
|
||||
|
||||
Reference in New Issue
Block a user