Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+20 -20
View File
@@ -16,10 +16,10 @@ import socket
import tempfile
import uuid
from argparse import Namespace
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
from contextlib import asynccontextmanager
from http import HTTPStatus
from typing import Annotated, Any, Callable, Literal, Optional
from typing import Annotated, Any, Literal
import prometheus_client
import pydantic
@@ -166,8 +166,8 @@ async def build_async_engine_client(
args: Namespace,
*,
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
disable_frontend_multiprocessing: Optional[bool] = None,
client_config: Optional[dict[str, Any]] = None,
disable_frontend_multiprocessing: bool | None = None,
client_config: dict[str, Any] | None = None,
) -> AsyncIterator[EngineClient]:
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
# The executor is expected to be mp.
@@ -203,7 +203,7 @@ async def build_async_engine_client_from_engine_args(
*,
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
disable_frontend_multiprocessing: bool = False,
client_config: Optional[dict[str, Any]] = None,
client_config: dict[str, Any] | None = None,
) -> AsyncIterator[EngineClient]:
"""
Create EngineClient, either:
@@ -227,7 +227,7 @@ async def build_async_engine_client_from_engine_args(
from vllm.v1.engine.async_llm import AsyncLLM
async_llm: Optional[AsyncLLM] = None
async_llm: AsyncLLM | None = None
# Don't mutate the input client_config
client_config = dict(client_config) if client_config else {}
@@ -308,35 +308,35 @@ def models(request: Request) -> OpenAIServingModels:
return request.app.state.openai_serving_models
def responses(request: Request) -> Optional[OpenAIServingResponses]:
def responses(request: Request) -> OpenAIServingResponses | None:
return request.app.state.openai_serving_responses
def chat(request: Request) -> Optional[OpenAIServingChat]:
def chat(request: Request) -> OpenAIServingChat | None:
return request.app.state.openai_serving_chat
def completion(request: Request) -> Optional[OpenAIServingCompletion]:
def completion(request: Request) -> OpenAIServingCompletion | None:
return request.app.state.openai_serving_completion
def pooling(request: Request) -> Optional[OpenAIServingPooling]:
def pooling(request: Request) -> OpenAIServingPooling | None:
return request.app.state.openai_serving_pooling
def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
def embedding(request: Request) -> OpenAIServingEmbedding | None:
return request.app.state.openai_serving_embedding
def score(request: Request) -> Optional[ServingScores]:
def score(request: Request) -> ServingScores | None:
return request.app.state.openai_serving_scores
def classify(request: Request) -> Optional[ServingClassification]:
def classify(request: Request) -> ServingClassification | None:
return request.app.state.openai_serving_classification
def rerank(request: Request) -> Optional[ServingScores]:
def rerank(request: Request) -> ServingScores | None:
return request.app.state.openai_serving_scores
@@ -542,8 +542,8 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
async def retrieve_responses(
response_id: str,
raw_request: Request,
starting_after: Optional[int] = None,
stream: Optional[bool] = False,
starting_after: int | None = None,
stream: bool | None = False,
):
handler = responses(raw_request)
if handler is None:
@@ -1039,7 +1039,7 @@ if envs.VLLM_SERVER_DEV_MODE:
# User-defined `method` is responsible for deserialization if needed.
args: list[str] = body.get("args", [])
kwargs: dict[str, str] = body.get("kwargs", {})
timeout: Optional[float] = body.get("timeout")
timeout: float | None = body.get("timeout")
results = await engine_client(raw_request).collective_rpc(
method=method, timeout=timeout, args=tuple(args), kwargs=kwargs
)
@@ -1120,7 +1120,7 @@ async def is_scaling_elastic_ep(raw_request: Request):
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
# (requires typing_extensions >= 4.13)
RequestType = Any
GetHandlerFn = Callable[[Request], Optional[OpenAIServing]]
GetHandlerFn = Callable[[Request], OpenAIServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
# NOTE: Items defined earlier take higher priority
@@ -1236,7 +1236,7 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
return Response(status_code=200, content=response)
def load_log_config(log_config_file: Optional[str]) -> Optional[dict]:
def load_log_config(log_config_file: str | None) -> dict | None:
if not log_config_file:
return None
try:
@@ -1655,7 +1655,7 @@ async def init_app_state(
)
if args.tool_server == "demo":
tool_server: Optional[ToolServer] = DemoToolServer()
tool_server: ToolServer | None = DemoToolServer()
assert isinstance(tool_server, DemoToolServer)
await tool_server.init_and_validate()
elif args.tool_server:
+16 -16
View File
@@ -11,7 +11,7 @@ import json
import ssl
from collections.abc import Sequence
from dataclasses import field
from typing import Literal, Optional, Union
from typing import Literal
from pydantic.dataclasses import dataclass
@@ -39,8 +39,8 @@ class LoRAParserAction(argparse.Action):
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Optional[Union[str, Sequence[str]]],
option_string: Optional[str] = None,
values: str | Sequence[str] | None,
option_string: str | None = None,
):
if values is None:
values = []
@@ -73,11 +73,11 @@ class LoRAParserAction(argparse.Action):
class FrontendArgs:
"""Arguments for the OpenAI-compatible frontend server."""
host: Optional[str] = None
host: str | None = None
"""Host name."""
port: int = 8000
"""Port number."""
uds: Optional[str] = None
uds: str | None = None
"""Unix domain socket path. If set, host and port arguments are ignored."""
uvicorn_log_level: Literal[
"debug", "info", "warning", "error", "critical", "trace"
@@ -93,15 +93,15 @@ class FrontendArgs:
"""Allowed methods."""
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
"""Allowed headers."""
api_key: Optional[list[str]] = None
api_key: list[str] | None = None
"""If provided, the server will require one of these keys to be presented in
the header."""
lora_modules: Optional[list[LoRAModulePath]] = None
lora_modules: list[LoRAModulePath] | None = None
"""LoRA modules configurations in either 'name=path' format or JSON format
or JSON list format. Example (old format): `'name=path'` Example (new
format): `{\"name\": \"name\", \"path\": \"lora_path\",
\"base_model_name\": \"id\"}`"""
chat_template: Optional[str] = None
chat_template: str | None = None
"""The file path to the chat template, or the template in single-line form
for the specified model."""
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
@@ -116,17 +116,17 @@ class FrontendArgs:
or the ones from tokenizer."""
response_role: str = "assistant"
"""The role name to return if `request.add_generation_prompt=true`."""
ssl_keyfile: Optional[str] = None
ssl_keyfile: str | None = None
"""The file path to the SSL key file."""
ssl_certfile: Optional[str] = None
ssl_certfile: str | None = None
"""The file path to the SSL cert file."""
ssl_ca_certs: Optional[str] = None
ssl_ca_certs: str | None = None
"""The CA certificates file."""
enable_ssl_refresh: bool = False
"""Refresh SSL Context when SSL certificate files change"""
ssl_cert_reqs: int = int(ssl.CERT_NONE)
"""Whether client certificate is required (see stdlib ssl module's)."""
root_path: Optional[str] = None
root_path: str | None = None
"""FastAPI root_path when app is behind a path based routing proxy."""
middleware: list[str] = field(default_factory=lambda: [])
"""Additional ASGI middleware to apply to the app. We accept multiple
@@ -149,7 +149,7 @@ class FrontendArgs:
exclude_tools_when_tool_choice_none: bool = False
"""If specified, exclude tool definitions in prompts when
tool_choice='none'."""
tool_call_parser: Optional[str] = None
tool_call_parser: str | None = None
"""Select the tool call parser depending on the model that you're using.
This is used to parse the model-generated tool call into OpenAI API format.
Required for `--enable-auto-tool-choice`. You can choose any option from
@@ -158,13 +158,13 @@ class FrontendArgs:
"""Special the tool parser plugin write to parse the model-generated tool
into OpenAI API format, the name register in this plugin can be used in
`--tool-call-parser`."""
tool_server: Optional[str] = None
tool_server: str | None = None
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for demo
purpose."""
log_config_file: Optional[str] = envs.VLLM_LOGGING_CONFIG_PATH
log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH
"""Path to logging config JSON file for both vllm and uvicorn"""
max_log_len: Optional[int] = None
max_log_len: int | None = None
"""Max number of prompt characters or prompt ID numbers being printed in
log. The default of None means unlimited."""
disable_fastapi_docs: bool = False
+4 -5
View File
@@ -3,7 +3,6 @@
from collections.abc import Iterable
from functools import lru_cache, partial
from typing import Optional, Union
import torch
@@ -16,8 +15,8 @@ class AllowedTokenIdsLogitsProcessor:
specific set of token ids."""
def __init__(self, allowed_ids: Iterable[int]):
self.allowed_ids: Optional[list[int]] = list(allowed_ids)
self.mask: Optional[torch.Tensor] = None
self.allowed_ids: list[int] | None = list(allowed_ids)
self.mask: torch.Tensor | None = None
def __call__(self, token_ids: list[int], logits: torch.Tensor) -> torch.Tensor:
if self.mask is None:
@@ -53,8 +52,8 @@ def logit_bias_logits_processor(
def get_logits_processors(
logit_bias: Optional[Union[dict[int, float], dict[str, float]]],
allowed_token_ids: Optional[list[int]],
logit_bias: dict[int, float] | dict[str, float] | None,
allowed_token_ids: list[int] | None,
tokenizer: AnyTokenizer,
) -> list[LogitsProcessor]:
logits_processors: list[LogitsProcessor] = []
File diff suppressed because it is too large Load Diff
+2 -3
View File
@@ -4,10 +4,9 @@
import asyncio
import tempfile
from argparse import Namespace
from collections.abc import Awaitable
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from io import StringIO
from typing import Callable, Optional
import aiohttp
import torch
@@ -124,7 +123,7 @@ _BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elap
class BatchProgressTracker:
def __init__(self):
self._total = 0
self._pbar: Optional[tqdm] = None
self._pbar: tqdm | None = None
def submitted(self):
self._total += 1
+22 -22
View File
@@ -6,7 +6,7 @@ import json
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Final, Optional, Union
from typing import Final
import jinja2
import partial_json_parser
@@ -82,15 +82,15 @@ class OpenAIServingChat(OpenAIServing):
models: OpenAIServingModels,
response_role: str,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
return_tokens_as_token_ids: bool = False,
reasoning_parser: str = "",
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: Optional[str] = None,
tool_parser: str | None = None,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_log_outputs: bool = False,
@@ -159,8 +159,8 @@ class OpenAIServingChat(OpenAIServing):
async def create_chat_completion(
self,
request: ChatCompletionRequest,
raw_request: Optional[Request] = None,
) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
"""
Chat Completion API similar to OpenAI's API.
@@ -281,7 +281,7 @@ class OpenAIServingChat(OpenAIServing):
default_sampling_params=self.default_sampling_params,
)
sampling_params: Union[SamplingParams, BeamSearchParams]
sampling_params: SamplingParams | BeamSearchParams
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
@@ -416,11 +416,11 @@ class OpenAIServingChat(OpenAIServing):
def extract_tool_call_required_streaming(
self,
previous_text: str,
current_text: Optional[str],
current_text: str | None,
delta_text: str,
function_name_returned: bool,
tool_call_idx: Optional[int] = None,
) -> tuple[Optional[DeltaMessage], bool]:
tool_call_idx: int | None = None,
) -> tuple[DeltaMessage | None, bool]:
if current_text is None or current_text == "":
# if the current text is empty, we cannot parse it
return None, function_name_returned
@@ -548,7 +548,7 @@ class OpenAIServingChat(OpenAIServing):
and self._should_stream_with_auto_tool_parsing(request)
)
all_previous_token_ids: Optional[list[list[int]]]
all_previous_token_ids: list[list[int]] | None
function_name_returned = [False] * num_choices
if self.tool_call_id_type == "kimi_k2":
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
@@ -583,7 +583,7 @@ class OpenAIServingChat(OpenAIServing):
# Prepare the tool parser if it's needed
try:
if tool_choice_auto and self.tool_parser:
tool_parsers: list[Optional[ToolParser]] = [
tool_parsers: list[ToolParser | None] = [
self.tool_parser(tokenizer)
] * num_choices
else:
@@ -661,7 +661,7 @@ class OpenAIServingChat(OpenAIServing):
# Send response to echo the input portion of the
# last message
if request.echo:
last_msg_content: Union[str, list[dict[str, str]]] = ""
last_msg_content: str | list[dict[str, str]] = ""
if (
conversation
and "content" in conversation[-1]
@@ -734,7 +734,7 @@ class OpenAIServingChat(OpenAIServing):
# Chunked prefill case, don't return empty chunks
continue
delta_message: Optional[DeltaMessage]
delta_message: DeltaMessage | None
# just update previous_texts and previous_token_ids
if tool_choice_auto or self.reasoning_parser:
@@ -1260,9 +1260,9 @@ class OpenAIServingChat(OpenAIServing):
conversation: list[ConversationMessage],
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
) -> Union[ErrorResponse, ChatCompletionResponse]:
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: Optional[RequestOutput] = None
final_res: RequestOutput | None = None
try:
async for res in result_generator:
@@ -1512,7 +1512,7 @@ class OpenAIServingChat(OpenAIServing):
choices.append(choice_data)
if request.echo:
last_msg_content: Union[str, list[dict[str, str]]] = ""
last_msg_content: str | list[dict[str, str]] = ""
if (
conversation
and "content" in conversation[-1]
@@ -1597,7 +1597,7 @@ class OpenAIServingChat(OpenAIServing):
def _get_top_logprobs(
self,
logprobs: dict[int, Logprob],
top_logprobs: Optional[int],
top_logprobs: int | None,
tokenizer: AnyTokenizer,
should_return_as_token_id: bool,
) -> list[ChatCompletionLogProb]:
@@ -1621,10 +1621,10 @@ class OpenAIServingChat(OpenAIServing):
def _create_chat_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
tokenizer: AnyTokenizer,
num_output_top_logprobs: Optional[int] = None,
return_as_token_id: Optional[bool] = None,
num_output_top_logprobs: int | None = None,
return_as_token_id: bool | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs."""
logprobs_content: list[ChatCompletionLogProbsContent] = []
@@ -1693,7 +1693,7 @@ class OpenAIServingChat(OpenAIServing):
def _should_check_for_unstreamed_tool_arg_tokens(
self,
delta_message: Optional[DeltaMessage],
delta_message: DeltaMessage | None,
output: CompletionOutput,
) -> bool:
"""
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import Optional, Union, cast
from typing import cast
import numpy as np
from fastapi import Request
@@ -36,7 +36,7 @@ class ClassificationMixin(OpenAIServing):
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""
Process classification inputs: tokenize text, resolve adapters,
and prepare model-specific inputs.
@@ -70,7 +70,7 @@ class ClassificationMixin(OpenAIServing):
def _build_response(
self,
ctx: ServeContext,
) -> Union[ClassificationResponse, ErrorResponse]:
) -> ClassificationResponse | ErrorResponse:
"""
Convert model outputs to a formatted classification response
with probabilities and labels.
@@ -129,7 +129,7 @@ class ServingClassification(ClassificationMixin):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
log_error_stack: bool = False,
) -> None:
super().__init__(
@@ -143,7 +143,7 @@ class ServingClassification(ClassificationMixin):
self,
request: ClassificationRequest,
raw_request: Request,
) -> Union[ClassificationResponse, ErrorResponse]:
) -> ClassificationResponse | ErrorResponse:
model_name = self.models.model_name()
request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
@@ -160,7 +160,7 @@ class ServingClassification(ClassificationMixin):
def _create_pooling_params(
self,
ctx: ClassificationServeContext,
) -> Union[PoolingParams, ErrorResponse]:
) -> PoolingParams | ErrorResponse:
pooling_params = super()._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
+16 -16
View File
@@ -5,7 +5,7 @@ import asyncio
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Optional, Union, cast
from typing import cast
import jinja2
from fastapi import Request
@@ -45,7 +45,7 @@ class OpenAIServingCompletion(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
@@ -73,8 +73,8 @@ class OpenAIServingCompletion(OpenAIServing):
async def create_completion(
self,
request: CompletionRequest,
raw_request: Optional[Request] = None,
) -> Union[AsyncGenerator[str, None], CompletionResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
"""Completion API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/completions/create
@@ -166,7 +166,7 @@ class OpenAIServingCompletion(OpenAIServing):
default_sampling_params=self.default_sampling_params,
)
sampling_params: Union[SamplingParams, BeamSearchParams]
sampling_params: SamplingParams | BeamSearchParams
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
@@ -196,7 +196,7 @@ class OpenAIServingCompletion(OpenAIServing):
# Mypy inconsistently requires this second cast in different
# environments. It shouldn't be necessary (redundant from above)
# but pre-commit in CI fails without it.
engine_prompt = cast(Union[EmbedsPrompt, TokensPrompt], engine_prompt)
engine_prompt = cast(EmbedsPrompt | TokensPrompt, engine_prompt)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_prompt,
@@ -260,7 +260,7 @@ class OpenAIServingCompletion(OpenAIServing):
)
# Non-streaming response
final_res_batch: list[Optional[RequestOutput]] = [None] * num_prompts
final_res_batch: list[RequestOutput | None] = [None] * num_prompts
try:
async for i, res in result_generator:
final_res_batch[i] = res
@@ -312,7 +312,7 @@ class OpenAIServingCompletion(OpenAIServing):
async def completion_stream_generator(
self,
request: CompletionRequest,
engine_prompts: list[Union[TokensPrompt, EmbedsPrompt]],
engine_prompts: list[TokensPrompt | EmbedsPrompt],
result_generator: AsyncIterator[tuple[int, RequestOutput]],
request_id: str,
created_time: int,
@@ -362,7 +362,7 @@ class OpenAIServingCompletion(OpenAIServing):
num_prompt_tokens[prompt_idx] = len(prompt_token_ids)
delta_token_ids: GenericSequence[int]
out_logprobs: Optional[GenericSequence[Optional[dict[int, Logprob]]]]
out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
for output in res.outputs:
i = output.index + prompt_idx * num_choices
@@ -370,7 +370,7 @@ class OpenAIServingCompletion(OpenAIServing):
# Useful when request.return_token_ids is True
# Returning prompt token IDs shares the same logic
# with the echo implementation.
prompt_token_ids_to_return: Optional[list[int]] = None
prompt_token_ids_to_return: list[int] | None = None
assert request.max_tokens is not None
if request.echo and not has_echoed[i]:
@@ -524,7 +524,7 @@ class OpenAIServingCompletion(OpenAIServing):
prompt_text = final_res.prompt
token_ids: GenericSequence[int]
out_logprobs: Optional[GenericSequence[Optional[dict[int, Logprob]]]]
out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
for output in final_res.outputs:
assert request.max_tokens is not None
@@ -617,17 +617,17 @@ class OpenAIServingCompletion(OpenAIServing):
def _create_completion_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int,
tokenizer: AnyTokenizer,
initial_text_offset: int = 0,
return_as_token_id: Optional[bool] = None,
return_as_token_id: bool | None = None,
) -> CompletionLogProbs:
"""Create logprobs for OpenAI Completion API."""
out_text_offset: list[int] = []
out_token_logprobs: list[Optional[float]] = []
out_token_logprobs: list[float | None] = []
out_tokens: list[str] = []
out_top_logprobs: list[Optional[dict[str, float]]] = []
out_top_logprobs: list[dict[str, float] | None] = []
last_token_len = 0
@@ -695,7 +695,7 @@ class OpenAIServingCompletion(OpenAIServing):
def _build_render_config(
self,
request: CompletionRequest,
max_input_length: Optional[int] = None,
max_input_length: int | None = None,
) -> RenderConfig:
max_input_tokens_len = self.max_model_len - (request.max_tokens or 0)
return RenderConfig(
+17 -19
View File
@@ -3,7 +3,7 @@
import base64
from collections.abc import AsyncGenerator, Mapping
from typing import Any, Final, Literal, Optional, Union, cast
from typing import Any, Final, Literal, cast
import numpy as np
import torch
@@ -48,7 +48,7 @@ logger = init_logger(__name__)
def _get_embedding(
output: EmbeddingOutput,
encoding_format: Literal["float", "base64"],
) -> Union[list[float], str]:
) -> list[float] | str:
if encoding_format == "float":
return output.embedding
elif encoding_format == "base64":
@@ -80,7 +80,7 @@ class EmbeddingMixin(OpenAIServing):
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
ctx = cast(EmbeddingServeContext, ctx)
try:
ctx.lora_request = self._maybe_get_adapters(ctx.request)
@@ -130,7 +130,7 @@ class EmbeddingMixin(OpenAIServing):
def _build_response(
self,
ctx: ServeContext,
) -> Union[EmbeddingResponse, ErrorResponse]:
) -> EmbeddingResponse | ErrorResponse:
items: list[EmbeddingResponseData] = []
num_prompt_tokens = 0
@@ -314,9 +314,9 @@ class EmbeddingMixin(OpenAIServing):
ctx: EmbeddingServeContext,
engine_prompt: EngineTokensPrompt,
pooling_params: PoolingParams,
trace_headers: Optional[Mapping[str, str]],
trace_headers: Mapping[str, str] | None,
prompt_index: int,
) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
) -> AsyncGenerator[RequestOutput | PoolingRequestOutput, None]:
"""Create a generator for a single prompt using standard processing."""
request_id_item = f"{ctx.request_id}-{prompt_index}"
@@ -341,7 +341,7 @@ class EmbeddingMixin(OpenAIServing):
async def _prepare_generators(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Override to support chunked processing."""
ctx = cast(EmbeddingServeContext, ctx)
@@ -354,7 +354,7 @@ class EmbeddingMixin(OpenAIServing):
# Custom logic for chunked processing
generators: list[
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
] = []
try:
@@ -413,7 +413,7 @@ class EmbeddingMixin(OpenAIServing):
async def _collect_batch(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Collect and aggregate batch results
with support for chunked processing.
@@ -522,9 +522,7 @@ class EmbeddingMixin(OpenAIServing):
)
# Finalize aggregated results
final_res_batch: list[
Union[PoolingRequestOutput, EmbeddingRequestOutput]
] = []
final_res_batch: list[PoolingRequestOutput | EmbeddingRequestOutput] = []
num_prompts = len(ctx.engine_prompts)
for prompt_idx in range(num_prompts):
@@ -581,7 +579,7 @@ class EmbeddingMixin(OpenAIServing):
)
ctx.final_res_batch = cast(
list[Union[RequestOutput, PoolingRequestOutput]], final_res_batch
list[RequestOutput | PoolingRequestOutput], final_res_batch
)
return None
@@ -598,8 +596,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
@@ -618,8 +616,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
async def create_embedding(
self,
request: EmbeddingRequest,
raw_request: Optional[Request] = None,
) -> Union[EmbeddingResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> EmbeddingResponse | ErrorResponse:
"""
Embedding API similar to OpenAI's API.
@@ -647,7 +645,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
def _create_pooling_params(
self,
ctx: ServeContext[EmbeddingRequest],
) -> Union[PoolingParams, ErrorResponse]:
) -> PoolingParams | ErrorResponse:
pooling_params = super()._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
@@ -662,7 +660,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
if isinstance(ctx.request, EmbeddingChatRequest):
error_check_ret = self._validate_chat_template(
request_chat_template=ctx.request.chat_template,
+93 -95
View File
@@ -5,10 +5,10 @@ import json
import sys
import time
import traceback
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from http import HTTPStatus
from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
import torch
from fastapi import Request
@@ -102,38 +102,38 @@ from vllm.v1.engine import EngineCoreRequest
logger = init_logger(__name__)
CompletionLikeRequest = Union[
CompletionRequest,
DetokenizeRequest,
EmbeddingCompletionRequest,
RerankRequest,
ClassificationRequest,
ScoreRequest,
TokenizeCompletionRequest,
]
CompletionLikeRequest: TypeAlias = (
CompletionRequest
| DetokenizeRequest
| EmbeddingCompletionRequest
| RerankRequest
| ClassificationRequest
| ScoreRequest
| TokenizeCompletionRequest
)
ChatLikeRequest = Union[
ChatCompletionRequest, EmbeddingChatRequest, TokenizeChatRequest
]
SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]
AnyRequest = Union[
CompletionLikeRequest,
ChatLikeRequest,
SpeechToTextRequest,
ResponsesRequest,
IOProcessorRequest,
]
ChatLikeRequest: TypeAlias = (
ChatCompletionRequest | EmbeddingChatRequest | TokenizeChatRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
AnyRequest: TypeAlias = (
CompletionLikeRequest
| ChatLikeRequest
| SpeechToTextRequest
| ResponsesRequest
| IOProcessorRequest
)
AnyResponse = Union[
CompletionResponse,
ChatCompletionResponse,
EmbeddingResponse,
TranscriptionResponse,
TokenizeResponse,
PoolingResponse,
ClassificationResponse,
ScoreResponse,
]
AnyResponse: TypeAlias = (
CompletionResponse
| ChatCompletionResponse
| EmbeddingResponse
| TranscriptionResponse
| TokenizeResponse
| PoolingResponse
| ClassificationResponse
| ScoreResponse
)
class TextTokensPrompt(TypedDict):
@@ -145,7 +145,7 @@ class EmbedsPrompt(TypedDict):
prompt_embeds: torch.Tensor
RequestPrompt = Union[list[int], str, TextTokensPrompt, EmbedsPrompt]
RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt
def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]:
@@ -173,8 +173,8 @@ class RequestProcessingMixin(BaseModel):
handling prompt preparation and engine input.
"""
request_prompts: Optional[Sequence[RequestPrompt]] = []
engine_prompts: Optional[list[EngineTokensPrompt]] = []
request_prompts: Sequence[RequestPrompt] | None = []
engine_prompts: list[EngineTokensPrompt] | None = []
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -185,10 +185,10 @@ class ResponseGenerationMixin(BaseModel):
managing result generators and final batch results.
"""
result_generator: Optional[
AsyncGenerator[tuple[int, Union[RequestOutput, PoolingRequestOutput]], None]
] = None
final_res_batch: list[Union[RequestOutput, PoolingRequestOutput]] = Field(
result_generator: (
AsyncGenerator[tuple[int, RequestOutput | PoolingRequestOutput], None] | None
) = None
final_res_batch: list[RequestOutput | PoolingRequestOutput] = Field(
default_factory=list
)
@@ -203,14 +203,14 @@ class ServeContext(
):
# Shared across all requests
request: RequestT
raw_request: Optional[Request] = None
raw_request: Request | None = None
model_name: str
request_id: str
created_time: int = Field(default_factory=lambda: int(time.time()))
lora_request: Optional[LoRARequest] = None
lora_request: LoRARequest | None = None
# Shared across most requests
tokenizer: Optional[AnyTokenizer] = None
tokenizer: AnyTokenizer | None = None
# `protected_namespaces` resolves Pydantic v2's warning
# on conflict with protected namespace "model_"
@@ -224,7 +224,7 @@ ClassificationServeContext = ServeContext[ClassificationRequest]
class EmbeddingServeContext(ServeContext[EmbeddingRequest]):
chat_template: Optional[str] = None
chat_template: str | None = None
chat_template_content_format: ChatTemplateContentFormatOption
@@ -247,7 +247,7 @@ class OpenAIServing:
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
log_error_stack: bool = False,
@@ -276,8 +276,8 @@ class OpenAIServing:
self.max_model_len = self.model_config.max_model_len
def _get_tool_parser(
self, tool_parser_name: Optional[str] = None, enable_auto_tools: bool = False
) -> Optional[Callable[[AnyTokenizer], ToolParser]]:
self, tool_parser_name: str | None = None, enable_auto_tools: bool = False
) -> Callable[[AnyTokenizer], ToolParser] | None:
"""Get the tool parser based on the name."""
parser = None
if not enable_auto_tools or tool_parser_name is None:
@@ -307,7 +307,7 @@ class OpenAIServing:
def _get_reasoning_parser(
self,
reasoning_parser_name: str,
) -> Optional[Callable[[AnyTokenizer], ReasoningParser]]:
) -> Callable[[AnyTokenizer], ReasoningParser] | None:
"""Get the reasoning parser based on the name."""
parser = None
if not reasoning_parser_name:
@@ -328,7 +328,7 @@ class OpenAIServing:
prompt: PromptType,
request_id: str,
params: BeamSearchParams,
lora_request: Optional[LoRARequest] = None,
lora_request: LoRARequest | None = None,
) -> AsyncGenerator[RequestOutput, None]:
beam_width = params.beam_width
max_tokens = params.max_tokens
@@ -364,9 +364,9 @@ class OpenAIServing:
# this happens again in generation, so the double expansion causes
# a mismatch.
# TODO - would be ideal to handle this more gracefully.
prompt_text: Optional[str]
prompt_text: str | None
prompt_token_ids: list[int]
multi_modal_data: Optional[MultiModalDataDict]
multi_modal_data: MultiModalDataDict | None
if isinstance(prompt, str):
prompt_text = prompt
prompt_token_ids = []
@@ -376,7 +376,7 @@ class OpenAIServing:
prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore
multi_modal_data = prompt.get("multi_modal_data") # type: ignore
mm_processor_kwargs: Optional[dict[str, Any]] = processed_inputs.get(
mm_processor_kwargs: dict[str, Any] | None = processed_inputs.get(
"mm_processor_kwargs"
) # type: ignore
@@ -507,7 +507,7 @@ class OpenAIServing:
prompt_logprobs=None,
)
def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:
def _get_renderer(self, tokenizer: AnyTokenizer | None) -> BaseRenderer:
"""
Get a Renderer instance with the provided tokenizer.
Uses shared async tokenizer pool for efficiency.
@@ -545,7 +545,7 @@ class OpenAIServing:
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""
Default preprocessing hook. Subclasses may override
to prepare `ctx` (classification, embedding, etc.).
@@ -555,7 +555,7 @@ class OpenAIServing:
def _build_response(
self,
ctx: ServeContext,
) -> Union[AnyResponse, ErrorResponse]:
) -> AnyResponse | ErrorResponse:
"""
Default response builder. Subclass may override this method
to return the appropriate response object.
@@ -565,8 +565,8 @@ class OpenAIServing:
async def handle(
self,
ctx: ServeContext,
) -> Union[AnyResponse, ErrorResponse]:
generation: AsyncGenerator[Union[AnyResponse, ErrorResponse], None]
) -> AnyResponse | ErrorResponse:
generation: AsyncGenerator[AnyResponse | ErrorResponse, None]
generation = self._pipeline(ctx)
async for response in generation:
@@ -577,7 +577,7 @@ class OpenAIServing:
async def _pipeline(
self,
ctx: ServeContext,
) -> AsyncGenerator[Union[AnyResponse, ErrorResponse], None]:
) -> AsyncGenerator[AnyResponse | ErrorResponse, None]:
"""Execute the request processing pipeline yielding responses."""
if error := await self._check_model(ctx.request):
yield error
@@ -598,7 +598,7 @@ class OpenAIServing:
yield self._build_response(ctx)
def _validate_request(self, ctx: ServeContext) -> Optional[ErrorResponse]:
def _validate_request(self, ctx: ServeContext) -> ErrorResponse | None:
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
if (
@@ -615,7 +615,7 @@ class OpenAIServing:
def _create_pooling_params(
self,
ctx: ServeContext,
) -> Union[PoolingParams, ErrorResponse]:
) -> PoolingParams | ErrorResponse:
if not hasattr(ctx.request, "to_pooling_params"):
return self.create_error_response(
"Request type does not support pooling parameters"
@@ -626,10 +626,10 @@ class OpenAIServing:
async def _prepare_generators(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Schedule the request and get the result generator."""
generators: list[
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
] = []
try:
@@ -678,14 +678,14 @@ class OpenAIServing:
async def _collect_batch(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Collect batch results from the result generator."""
try:
if ctx.engine_prompts is None:
return self.create_error_response("Engine prompts not available")
num_prompts = len(ctx.engine_prompts)
final_res_batch: list[Optional[Union[RequestOutput, PoolingRequestOutput]]]
final_res_batch: list[RequestOutput | PoolingRequestOutput | None]
final_res_batch = [None] * num_prompts
if ctx.result_generator is None:
@@ -738,7 +738,7 @@ class OpenAIServing:
async def _check_model(
self,
request: AnyRequest,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
error_response = None
if self._is_model_supported(request.model):
@@ -764,9 +764,7 @@ class OpenAIServing:
status_code=HTTPStatus.NOT_FOUND,
)
def _get_active_default_mm_loras(
self, request: AnyRequest
) -> Optional[LoRARequest]:
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
@@ -793,7 +791,7 @@ class OpenAIServing:
self,
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> Optional[LoRARequest]:
) -> LoRARequest | None:
if request.model in self.models.lora_requests:
return self.models.lora_requests[request.model]
@@ -877,7 +875,7 @@ class OpenAIServing:
self,
request: AnyRequest,
prompt_ids: list[int],
tokenizer: Optional[AnyTokenizer],
tokenizer: AnyTokenizer | None,
) -> TextTokensPrompt:
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
@@ -972,7 +970,7 @@ class OpenAIServing:
self,
request: AnyRequest,
tokenizer: AnyTokenizer,
prompt_input: Union[str, list[int]],
prompt_input: str | list[int],
add_special_tokens: bool = True,
) -> TextTokensPrompt:
"""
@@ -991,7 +989,7 @@ class OpenAIServing:
self,
request: AnyRequest,
tokenizer: AnyTokenizer,
prompt_inputs: Iterable[Union[str, list[int]]],
prompt_inputs: Iterable[str | list[int]],
add_special_tokens: bool = True,
) -> AsyncGenerator[TextTokensPrompt, None]:
"""
@@ -1014,10 +1012,10 @@ class OpenAIServing:
def _validate_chat_template(
self,
request_chat_template: Optional[str],
chat_template_kwargs: Optional[dict[str, Any]],
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
if not trust_request_chat_template and (
request_chat_template is not None
or (
@@ -1034,17 +1032,17 @@ class OpenAIServing:
async def _preprocess_chat(
self,
request: Union[ChatLikeRequest, ResponsesRequest],
request: ChatLikeRequest | ResponsesRequest,
tokenizer: AnyTokenizer,
messages: list[ChatCompletionMessageParam],
chat_template: Optional[str],
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
add_generation_prompt: bool = True,
continue_final_message: bool = False,
tool_dicts: Optional[list[dict[str, Any]]] = None,
documents: Optional[list[dict[str, str]]] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None,
tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
tool_dicts: list[dict[str, Any]] | None = None,
documents: list[dict[str, str]] | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
tool_parser: Callable[[AnyTokenizer], ToolParser] | None = None,
add_special_tokens: bool = False,
) -> tuple[
list[ConversationMessage],
@@ -1076,7 +1074,7 @@ class OpenAIServing:
)
_chat_template_kwargs.update(chat_template_kwargs or {})
request_prompt: Union[str, list[int]]
request_prompt: str | list[int]
if tokenizer is None:
request_prompt = "placeholder"
@@ -1158,10 +1156,10 @@ class OpenAIServing:
self,
request_id: str,
engine_prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
params: SamplingParams | PoolingParams,
*,
lora_request: Optional[LoRARequest],
trace_headers: Optional[Mapping[str, str]],
lora_request: LoRARequest | None,
trace_headers: Mapping[str, str] | None,
priority: int,
) -> tuple[EngineCoreRequest, dict[str, Any]]:
"""Use the Processor to process inputs for AsyncLLM."""
@@ -1188,7 +1186,7 @@ class OpenAIServing:
engine_prompt: EngineTokensPrompt,
sampling_params: SamplingParams,
context: ConversationContext,
lora_request: Optional[LoRARequest] = None,
lora_request: LoRARequest | None = None,
priority: int = 0,
**kwargs,
):
@@ -1250,7 +1248,7 @@ class OpenAIServing:
def _get_prompt_components(
self,
prompt: Union[RequestPrompt, PromptType],
prompt: RequestPrompt | PromptType,
) -> PromptComponents:
if isinstance(prompt, list):
return PromptComponents(token_ids=prompt)
@@ -1260,9 +1258,9 @@ class OpenAIServing:
def _log_inputs(
self,
request_id: str,
inputs: Union[RequestPrompt, PromptType],
params: Optional[Union[SamplingParams, PoolingParams, BeamSearchParams]],
lora_request: Optional[LoRARequest],
inputs: RequestPrompt | PromptType,
params: SamplingParams | PoolingParams | BeamSearchParams | None,
lora_request: LoRARequest | None,
) -> None:
if self.request_logger is None:
return
@@ -1281,7 +1279,7 @@ class OpenAIServing:
async def _get_trace_headers(
self,
headers: Headers,
) -> Optional[Mapping[str, str]]:
) -> Mapping[str, str] | None:
is_tracing_enabled = await self.engine_client.is_tracing_enabled()
if is_tracing_enabled:
@@ -1294,8 +1292,8 @@ class OpenAIServing:
@staticmethod
def _base_request_id(
raw_request: Optional[Request], default: Optional[str] = None
) -> Optional[str]:
raw_request: Request | None, default: str | None = None
) -> str | None:
"""Pulls the request id to use from a header, if provided"""
default = default or random_uuid()
if raw_request is None:
@@ -1317,15 +1315,15 @@ class OpenAIServing:
return logprob.decoded_token
return tokenizer.decode(token_id)
def _is_model_supported(self, model_name: Optional[str]) -> bool:
def _is_model_supported(self, model_name: str | None) -> bool:
if not model_name:
return True
return self.models.is_base_model(model_name)
def clamp_prompt_logprobs(
prompt_logprobs: Union[PromptLogprobs, None],
) -> Union[PromptLogprobs, None]:
prompt_logprobs: PromptLogprobs | None,
) -> PromptLogprobs | None:
if prompt_logprobs is None:
return prompt_logprobs
+9 -10
View File
@@ -5,7 +5,6 @@ from asyncio import Lock
from collections import defaultdict
from dataclasses import dataclass
from http import HTTPStatus
from typing import Optional, Union
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.protocol import (
@@ -35,7 +34,7 @@ class BaseModelPath:
class LoRAModulePath:
name: str
path: str
base_model_name: Optional[str] = None
base_model_name: str | None = None
class OpenAIServingModels:
@@ -52,7 +51,7 @@ class OpenAIServingModels:
engine_client: EngineClient,
base_model_paths: list[BaseModelPath],
*,
lora_modules: Optional[list[LoRAModulePath]] = None,
lora_modules: list[LoRAModulePath] | None = None,
):
super().__init__()
@@ -93,7 +92,7 @@ class OpenAIServingModels:
def is_base_model(self, model_name) -> bool:
return any(model.name == model_name for model in self.base_model_paths)
def model_name(self, lora_request: Optional[LoRARequest] = None) -> str:
def model_name(self, lora_request: LoRARequest | None = None) -> str:
"""Returns the appropriate model name depending on the availability
and support of the LoRA or base model.
Parameters:
@@ -132,8 +131,8 @@ class OpenAIServingModels:
return ModelList(data=model_cards)
async def load_lora_adapter(
self, request: LoadLoRAAdapterRequest, base_model_name: Optional[str] = None
) -> Union[ErrorResponse, str]:
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
) -> ErrorResponse | str:
lora_name = request.lora_name
# Ensure atomicity based on the lora name
@@ -173,7 +172,7 @@ class OpenAIServingModels:
async def unload_lora_adapter(
self, request: UnloadLoRAAdapterRequest
) -> Union[ErrorResponse, str]:
) -> ErrorResponse | str:
lora_name = request.lora_name
# Ensure atomicity based on the lora name
@@ -189,7 +188,7 @@ class OpenAIServingModels:
async def _check_load_lora_adapter_request(
self, request: LoadLoRAAdapterRequest
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
# Check if both 'lora_name' and 'lora_path' are provided
if not request.lora_name or not request.lora_path:
return create_error_response(
@@ -211,7 +210,7 @@ class OpenAIServingModels:
async def _check_unload_lora_adapter_request(
self, request: UnloadLoRAAdapterRequest
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
# Check if 'lora_name' is not provided return an error
if not request.lora_name:
return create_error_response(
@@ -230,7 +229,7 @@ class OpenAIServingModels:
return None
async def resolve_lora(self, lora_name: str) -> Union[LoRARequest, ErrorResponse]:
async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse:
"""Attempt to resolve a LoRA adapter using available resolvers.
Args:
+7 -7
View File
@@ -5,7 +5,7 @@ import asyncio
import base64
import time
from collections.abc import AsyncGenerator
from typing import Final, Literal, Optional, Union, cast
from typing import Final, Literal, cast
import jinja2
import numpy as np
@@ -41,7 +41,7 @@ logger = init_logger(__name__)
def _get_data(
output: PoolingOutput,
encoding_format: Literal["float", "base64"],
) -> Union[list[float], str]:
) -> list[float] | str:
if encoding_format == "float":
return output.data.tolist()
elif encoding_format == "base64":
@@ -60,8 +60,8 @@ class OpenAIServingPooling(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
@@ -80,8 +80,8 @@ class OpenAIServingPooling(OpenAIServing):
async def create_pooling(
self,
request: PoolingRequest,
raw_request: Optional[Request] = None,
) -> Union[PoolingResponse, IOProcessorResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> PoolingResponse | IOProcessorResponse | ErrorResponse:
"""
See https://platform.openai.com/docs/api-reference/embeddings/create
for the API specification. This API mimics the OpenAI Embedding API.
@@ -219,7 +219,7 @@ class OpenAIServingPooling(OpenAIServing):
num_prompts = len(engine_prompts)
# Non-streaming response
final_res_batch: list[Optional[PoolingRequestOutput]]
final_res_batch: list[PoolingRequestOutput | None]
final_res_batch = [None] * num_prompts
try:
async for i, res in result_generator:
+36 -36
View File
@@ -6,11 +6,11 @@ import json
import time
import uuid
from collections import deque
from collections.abc import AsyncGenerator, AsyncIterator, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
from contextlib import AsyncExitStack
from copy import copy
from http import HTTPStatus
from typing import Callable, Final, Optional, Union
from typing import Final
import jinja2
from fastapi import Request
@@ -109,14 +109,14 @@ class OpenAIServingResponses(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
return_tokens_as_token_ids: bool = False,
reasoning_parser: str = "",
enable_auto_tools: bool = False,
tool_parser: Optional[str] = None,
tool_server: Optional[ToolServer] = None,
tool_parser: str | None = None,
tool_server: ToolServer | None = None,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_log_outputs: bool = False,
@@ -210,7 +210,7 @@ class OpenAIServingResponses(OpenAIServing):
def _validate_generator_input(
self, engine_prompt: EngineTokensPrompt
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Add validations to the input to the generator here."""
if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
error_message = (
@@ -229,12 +229,12 @@ class OpenAIServingResponses(OpenAIServing):
async def create_responses(
self,
request: ResponsesRequest,
raw_request: Optional[Request] = None,
) -> Union[
AsyncGenerator[StreamingResponsesResponse, None],
ResponsesResponse,
ErrorResponse,
]:
raw_request: Request | None = None,
) -> (
AsyncGenerator[StreamingResponsesResponse, None]
| ResponsesResponse
| ErrorResponse
):
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
@@ -460,7 +460,7 @@ class OpenAIServingResponses(OpenAIServing):
async def _make_request(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse],
prev_response: ResponsesResponse | None,
tokenizer: AnyTokenizer,
):
if len(request.tools) > 0:
@@ -481,7 +481,7 @@ class OpenAIServingResponses(OpenAIServing):
def _make_request_with_harmony(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse],
prev_response: ResponsesResponse | None,
):
if request.tool_choice != "auto":
raise NotImplementedError(
@@ -522,8 +522,8 @@ class OpenAIServingResponses(OpenAIServing):
model_name: str,
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
created_time: Optional[int] = None,
) -> Union[ErrorResponse, ResponsesResponse]:
created_time: int | None = None,
) -> ErrorResponse | ResponsesResponse:
if created_time is None:
created_time = int(time.time())
@@ -642,9 +642,9 @@ class OpenAIServingResponses(OpenAIServing):
def _create_response_logprobs(
self,
token_ids: Sequence[int],
logprobs: Optional[SampleLogprobs],
logprobs: SampleLogprobs | None,
tokenizer: AnyTokenizer,
top_logprobs: Optional[int] = None,
top_logprobs: int | None = None,
) -> list[Logprob]:
assert logprobs is not None, "logprobs must be provided"
assert len(token_ids) == len(logprobs), (
@@ -676,9 +676,9 @@ class OpenAIServingResponses(OpenAIServing):
def _create_stream_response_logprobs(
self,
token_ids: Sequence[int],
logprobs: Optional[SampleLogprobs],
logprobs: SampleLogprobs | None,
tokenizer: AnyTokenizer,
top_logprobs: Optional[int] = None,
top_logprobs: int | None = None,
) -> list[response_text_delta_event.Logprob]:
lgs = self._create_response_logprobs(
token_ids=token_ids,
@@ -793,7 +793,7 @@ class OpenAIServingResponses(OpenAIServing):
def _construct_input_messages(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse] = None,
prev_response: ResponsesResponse | None = None,
) -> list[ChatCompletionMessageParam]:
messages: list[ChatCompletionMessageParam] = []
if request.instructions:
@@ -833,7 +833,7 @@ class OpenAIServingResponses(OpenAIServing):
def _construct_input_messages_with_harmony(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse],
prev_response: ResponsesResponse | None,
) -> list[OpenAIHarmonyMessage]:
messages: list[OpenAIHarmonyMessage] = []
if prev_response is None:
@@ -986,7 +986,7 @@ class OpenAIServingResponses(OpenAIServing):
async def responses_background_stream_generator(
self,
response_id: str,
starting_after: Optional[int] = None,
starting_after: int | None = None,
) -> AsyncGenerator[StreamingResponsesResponse, None]:
if response_id not in self.event_store:
raise ValueError(f"Unknown response_id: {response_id}")
@@ -1011,13 +1011,13 @@ class OpenAIServingResponses(OpenAIServing):
async def retrieve_responses(
self,
response_id: str,
starting_after: Optional[int],
stream: Optional[bool],
) -> Union[
ErrorResponse,
ResponsesResponse,
AsyncGenerator[StreamingResponsesResponse, None],
]:
starting_after: int | None,
stream: bool | None,
) -> (
ErrorResponse
| ResponsesResponse
| AsyncGenerator[StreamingResponsesResponse, None]
):
async with self.response_store_lock:
response = self.response_store.get(response_id)
@@ -1034,7 +1034,7 @@ class OpenAIServingResponses(OpenAIServing):
async def cancel_responses(
self,
response_id: str,
) -> Union[ErrorResponse, ResponsesResponse]:
) -> ErrorResponse | ResponsesResponse:
async with self.response_store_lock:
response = self.response_store.get(response_id)
if response is None:
@@ -1082,7 +1082,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
sampling_params: SamplingParams,
result_generator: AsyncIterator[Optional[ConversationContext]],
result_generator: AsyncIterator[ConversationContext | None],
context: ConversationContext,
model_name: str,
tokenizer: AnyTokenizer,
@@ -1385,7 +1385,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
sampling_params: SamplingParams,
result_generator: AsyncIterator[Optional[ConversationContext]],
result_generator: AsyncIterator[ConversationContext | None],
context: ConversationContext,
model_name: str,
tokenizer: AnyTokenizer,
@@ -1772,12 +1772,12 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
sampling_params: SamplingParams,
result_generator: AsyncIterator[Optional[ConversationContext]],
result_generator: AsyncIterator[ConversationContext | None],
context: ConversationContext,
model_name: str,
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
created_time: Optional[int] = None,
created_time: int | None = None,
) -> AsyncGenerator[StreamingResponsesResponse, None]:
# TODO:
# 1. Handle disconnect
+29 -29
View File
@@ -3,7 +3,7 @@
import asyncio
import time
from collections.abc import AsyncGenerator, Mapping
from typing import Any, Optional, Union
from typing import Any
from fastapi import Request
@@ -48,7 +48,7 @@ class ServingScores(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
log_error_stack: bool = False,
) -> None:
super().__init__(
@@ -63,12 +63,12 @@ class ServingScores(OpenAIServing):
tokenizer: AnyTokenizer,
texts_1: list[str],
texts_2: list[str],
request: Union[RerankRequest, ScoreRequest],
request: RerankRequest | ScoreRequest,
request_id: str,
tokenization_kwargs: Optional[dict[str, Any]] = None,
lora_request: Optional[Union[LoRARequest, None]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
tokenization_kwargs: dict[str, Any] | None = None,
lora_request: LoRARequest | None | None = None,
trace_headers: Mapping[str, str] | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
input_texts = texts_1 + texts_2
engine_prompts: list[TokensPrompt] = []
@@ -125,7 +125,7 @@ class ServingScores(OpenAIServing):
# Non-streaming response
final_res_batch: list[PoolingRequestOutput] = []
embeddings: list[Optional[PoolingRequestOutput]] = [None] * len(engine_prompts)
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)
async for i, res in result_generator:
embeddings[i] = res
@@ -152,11 +152,11 @@ class ServingScores(OpenAIServing):
def _preprocess_score(
self,
request: Union[RerankRequest, ScoreRequest],
request: RerankRequest | ScoreRequest,
tokenizer: AnyTokenizer,
tokenization_kwargs: dict[str, Any],
data_1: Union[str, ScoreContentPartParam],
data_2: Union[str, ScoreContentPartParam],
data_1: str | ScoreContentPartParam,
data_2: str | ScoreContentPartParam,
) -> tuple[str, TokensPrompt]:
model_config = self.model_config
@@ -176,14 +176,14 @@ class ServingScores(OpenAIServing):
async def _cross_encoding_score(
self,
tokenizer: AnyTokenizer,
data_1: Union[list[str], list[ScoreContentPartParam]],
data_2: Union[list[str], list[ScoreContentPartParam]],
request: Union[RerankRequest, ScoreRequest],
data_1: list[str] | list[ScoreContentPartParam],
data_2: list[str] | list[ScoreContentPartParam],
request: RerankRequest | ScoreRequest,
request_id: str,
tokenization_kwargs: Optional[dict[str, Any]] = None,
lora_request: Optional[Union[LoRARequest, None]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
tokenization_kwargs: dict[str, Any] | None = None,
lora_request: LoRARequest | None | None = None,
trace_headers: Mapping[str, str] | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
request_prompts: list[str] = []
engine_prompts: list[TokensPrompt] = []
@@ -259,7 +259,7 @@ class ServingScores(OpenAIServing):
result_generator = merge_async_iterators(*generators)
# Non-streaming response
final_res_batch: list[Optional[PoolingRequestOutput]] = [None] * len(
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
engine_prompts
)
@@ -270,12 +270,12 @@ class ServingScores(OpenAIServing):
async def _run_scoring(
self,
data_1: Union[list[str], str, ScoreMultiModalParam],
data_2: Union[list[str], str, ScoreMultiModalParam],
request: Union[ScoreRequest, RerankRequest],
data_1: list[str] | str | ScoreMultiModalParam,
data_2: list[str] | str | ScoreMultiModalParam,
request: ScoreRequest | RerankRequest,
request_id: str,
raw_request: Optional[Request] = None,
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
raw_request: Request | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
lora_request = self._maybe_get_adapters(request)
tokenizer = await self.engine_client.get_tokenizer()
@@ -339,8 +339,8 @@ class ServingScores(OpenAIServing):
async def create_score(
self,
request: ScoreRequest,
raw_request: Optional[Request] = None,
) -> Union[ScoreResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> ScoreResponse | ErrorResponse:
"""
Score API similar to Sentence Transformers cross encoder
@@ -377,8 +377,8 @@ class ServingScores(OpenAIServing):
return self.create_error_response(str(e))
async def do_rerank(
self, request: RerankRequest, raw_request: Optional[Request] = None
) -> Union[RerankResponse, ErrorResponse]:
self, request: RerankRequest, raw_request: Request | None = None
) -> RerankResponse | ErrorResponse:
"""
Rerank API based on JinaAI's rerank API; implements the same
API interface. Designed for compatibility with off-the-shelf
@@ -468,7 +468,7 @@ class ServingScores(OpenAIServing):
final_res_batch: list[PoolingRequestOutput],
request_id: str,
model_name: str,
documents: Union[list[str], ScoreMultiModalParam],
documents: list[str] | ScoreMultiModalParam,
top_n: int,
) -> RerankResponse:
"""
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any, Final, Optional, Union
from typing import Any, Final
import jinja2
from fastapi import Request
@@ -33,8 +33,8 @@ class OpenAIServingTokenization(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
@@ -54,7 +54,7 @@ class OpenAIServingTokenization(OpenAIServing):
self,
request: TokenizeRequest,
raw_request: Request,
) -> Union[TokenizeResponse, ErrorResponse]:
) -> TokenizeResponse | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
@@ -129,7 +129,7 @@ class OpenAIServingTokenization(OpenAIServing):
self,
request: DetokenizeRequest,
raw_request: Request,
) -> Union[DetokenizeResponse, ErrorResponse]:
) -> DetokenizeResponse | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
@@ -155,7 +155,7 @@ class OpenAIServingTokenization(OpenAIServing):
async def get_tokenizer_info(
self,
) -> Union[TokenizerInfoResponse, ErrorResponse]:
) -> TokenizerInfoResponse | ErrorResponse:
"""Get comprehensive tokenizer information."""
try:
tokenizer = await self.engine_client.get_tokenizer()
@@ -171,7 +171,7 @@ class OpenAIServingTokenization(OpenAIServing):
@dataclass
class TokenizerInfo:
tokenizer: AnyTokenizer
chat_template: Optional[str]
chat_template: str | None
def to_dict(self) -> dict[str, Any]:
"""Return the tokenizer configuration."""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator
from typing import Optional, Union
from fastapi import Request
@@ -35,7 +34,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
@@ -50,7 +49,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
async def create_transcription(
self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request
) -> Union[TranscriptionResponse, AsyncGenerator[str, None], ErrorResponse]:
) -> TranscriptionResponse | AsyncGenerator[str, None] | ErrorResponse:
"""Transcription API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranscription
@@ -94,7 +93,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
@@ -109,7 +108,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
async def create_translation(
self, audio_data: bytes, request: TranslationRequest, raw_request: Request
) -> Union[TranslationResponse, AsyncGenerator[str, None], ErrorResponse]:
) -> TranslationResponse | AsyncGenerator[str, None] | ErrorResponse:
"""Translation API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranslation
+10 -15
View File
@@ -4,9 +4,9 @@ import asyncio
import io
import math
import time
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Callable
from functools import cached_property
from typing import Callable, Literal, Optional, TypeVar, Union, cast
from typing import Literal, TypeAlias, TypeVar, cast
import numpy as np
from fastapi import Request
@@ -39,7 +39,7 @@ try:
except ImportError:
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
SpeechToTextResponse = Union[TranscriptionResponse, TranslationResponse]
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
T = TypeVar("T", bound=SpeechToTextResponse)
logger = init_logger(__name__)
@@ -54,7 +54,7 @@ class OpenAISpeechToText(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
task_type: Literal["transcribe", "translate"] = "transcribe",
log_error_stack: bool = False,
@@ -140,7 +140,7 @@ class OpenAISpeechToText(OpenAIServing):
raw_request: Request,
response_class: type[T],
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
) -> Union[T, AsyncGenerator[str, None], ErrorResponse]:
) -> T | AsyncGenerator[str, None] | ErrorResponse:
"""Base method for speech-to-text operations like transcription and
translation."""
error_check_ret = await self._check_model(request)
@@ -181,9 +181,7 @@ class OpenAISpeechToText(OpenAIServing):
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(str(e))
list_result_generator: Optional[list[AsyncGenerator[RequestOutput, None]]] = (
None
)
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
try:
# Unlike most decoder-only models, whisper generation length is not
# constrained by the size of the input audio, which is mapped to a
@@ -252,13 +250,10 @@ class OpenAISpeechToText(OpenAIServing):
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
response_stream_choice_class: Union[
type[TranscriptionResponseStreamChoice],
type[TranslationResponseStreamChoice],
],
stream_response_class: Union[
type[TranscriptionStreamResponse], type[TranslationStreamResponse]
],
response_stream_choice_class: type[TranscriptionResponseStreamChoice]
| type[TranslationResponseStreamChoice],
stream_response_class: type[TranscriptionStreamResponse]
| type[TranslationStreamResponse],
) -> AsyncGenerator[str, None]:
created_time = int(time.time())
model_name = request.model
@@ -2,9 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Sequence
from collections.abc import Callable, Sequence
from functools import cached_property
from typing import Callable, Optional, Union
from vllm.entrypoints.openai.protocol import (
ChatCompletionRequest,
@@ -69,7 +68,7 @@ class ToolParser:
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Instance method that should be implemented for extracting tool calls
from an incomplete response; for use when handling tool calls and
@@ -101,7 +100,7 @@ class ToolParserManager:
def _register_module(
cls,
module: type,
module_name: Optional[Union[str, list[str]]] = None,
module_name: str | list[str] | None = None,
force: bool = True,
) -> None:
if not issubclass(module, ToolParser):
@@ -123,10 +122,10 @@ class ToolParserManager:
@classmethod
def register_module(
cls,
name: Optional[Union[str, list[str]]] = None,
name: str | list[str] | None = None,
force: bool = True,
module: Union[type, None] = None,
) -> Union[type, Callable]:
module: type | None = None,
) -> type | Callable:
"""
Register module with the given name or name list. it can be used as a
decoder(with module as None) or normal function(with module as not
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Union
import regex as re
@@ -129,7 +128,7 @@ class DeepSeekV31ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
logger.debug("delta_text: %s", delta_text)
logger.debug("delta_token_ids: %s", delta_token_ids)
# check to see if we should be streaming a tool call - is there a
@@ -272,7 +271,7 @@ class DeepSeekV31ToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
if function_name:
self.current_tool_name_sent = True
return DeltaMessage(
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Union
import regex as re
@@ -129,7 +128,7 @@ class DeepSeekV3ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
logger.debug("delta_text: %s", delta_text)
logger.debug("delta_token_ids: %s", delta_token_ids)
# check to see if we should be streaming a tool call - is there a
@@ -272,7 +271,7 @@ class DeepSeekV3ToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
if function_name:
self.current_tool_name_sent = True
return DeltaMessage(
@@ -4,7 +4,7 @@
import ast
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -66,7 +66,7 @@ class Glm4MoeModelToolParser(ToolParser):
def _is_string_type(
tool_name: str,
arg_name: str,
tools: Optional[list[ChatCompletionToolsParam]],
tools: list[ChatCompletionToolsParam] | None,
) -> bool:
if tools is None:
return False
@@ -144,7 +144,7 @@ class Glm4MoeModelToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
self._buffer += delta_text
cur_text = self._buffer
start_idx = cur_text.find(self.tool_call_start_token)
@@ -4,7 +4,6 @@
import json
from collections.abc import Sequence
from json import JSONDecoder
from typing import Union
import partial_json_parser
import regex as re
@@ -121,7 +120,7 @@ class Granite20bFCToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if len(current_text) < len(self.bot_token) and self.bot_token.startswith(
current_text
):
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
from partial_json_parser.core.options import Allow
@@ -108,7 +107,7 @@ class GraniteToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
start_idx = consume_space(0, current_text)
if current_text[start_idx:].startswith(self.bot_token):
start_idx = consume_space(start_idx + len(self.bot_token), current_text)
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
import regex as re
@@ -181,7 +180,7 @@ class Hermes2ProToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# 1. All tokens are parsed based on _text, not token_ids.
# 2. All incoming text data is processed by the tool_call_delta_buffer
# function for buffering before being used for parsing.
@@ -333,7 +332,7 @@ class Hermes2ProToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
if function_name:
self.current_tool_name_sent = True
return DeltaMessage(
@@ -4,7 +4,7 @@
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -73,7 +73,7 @@ class HunyuanA13BToolParser(ToolParser):
def preprocess_model_output(
self, model_output: str
) -> tuple[Optional[str], Optional[str]]:
) -> tuple[str | None, str | None]:
# find the location tool call
for match in self.answer_tool_calls_pattern.finditer(model_output):
start, end = match.span()
@@ -176,7 +176,7 @@ class HunyuanA13BToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Extract tool calls for streaming mode.
"""
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
from partial_json_parser.core.options import Allow
@@ -59,7 +58,7 @@ class Internlm2ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if "<|action_start|>" not in current_text:
self.position = len(current_text)
return DeltaMessage(content=delta_text)
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
import regex as re
@@ -129,7 +128,7 @@ class JambaToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# if the tool call token is not in the tokens generated so far, append
# output to contents since it's not a tool
if self.tool_calls_start_token not in current_text:
@@ -190,7 +189,7 @@ class JambaToolParser(ToolParser):
# auto-generated due to JSON completions, but wasn't
# streamed to the client yet.
if self.current_tool_id >= 0:
diff: Union[str, None] = current_tool_call.get("arguments")
diff: str | None = current_tool_call.get("arguments")
if diff:
diff = json.dumps(diff, ensure_ascii=False).replace(
@@ -3,7 +3,6 @@
# code modified from deepseekv3_tool_parser.py
from collections.abc import Sequence
from typing import Union
import regex as re
@@ -131,7 +130,7 @@ class KimiK2ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
logger.debug("delta_text: %s", delta_text)
logger.debug("delta_token_ids: %s", delta_token_ids)
# check to see if we should be streaming a tool call - is there a
@@ -278,7 +277,7 @@ class KimiK2ToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
tool_id = current_tool_call.get("id")
if function_name:
self.current_tool_name_sent = True
@@ -3,7 +3,7 @@
import ast
import json
from collections.abc import Sequence
from typing import Any, Union
from typing import Any
import regex as re
from transformers import PreTrainedTokenizerBase
@@ -128,7 +128,7 @@ class Llama4PythonicToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not current_text.startswith("[") and not current_text.startswith(
"<|python_start|>"
):
@@ -245,7 +245,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
)
def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _make_valid_python(text: str) -> tuple[str, str] | None:
bracket_stack = []
for index, char in enumerate(text):
if char in {"[", "(", "{"}:
@@ -317,7 +317,7 @@ def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _compute_tool_delta(
previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
) -> Union[DeltaToolCall, None]:
) -> DeltaToolCall | None:
new_call_args = new_call.function.arguments
if withheld_suffix:
assert new_call_args.endswith(withheld_suffix)
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
import regex as re
@@ -134,7 +133,7 @@ class Llama3JsonToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not (
current_text.startswith(self.bot_token) or current_text.startswith("{")
):
@@ -3,7 +3,7 @@
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -509,7 +509,7 @@ class MinimaxToolParser(ToolParser):
def _get_current_tool_content(
self, text: str, tool_index: int
) -> tuple[Optional[str], Optional[str]]:
) -> tuple[str | None, str | None]:
"""
Get the content of a specific tool by index.
@@ -545,7 +545,7 @@ class MinimaxToolParser(ToolParser):
def _handle_tool_name_streaming(
self, tool_content: str, tool_count: int
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Handle streaming of tool names.
@@ -595,7 +595,7 @@ class MinimaxToolParser(ToolParser):
def _handle_tool_args_streaming(
self, tool_content: str, tool_count: int
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Handle streaming of tool arguments.
@@ -702,7 +702,7 @@ class MinimaxToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
self._update_thinking_state(current_text)
if self.in_thinking_tag:
@@ -776,7 +776,7 @@ class MinimaxToolParser(ToolParser):
)
return None
def _find_tool_start_outside_thinking(self, current_text: str) -> Optional[int]:
def _find_tool_start_outside_thinking(self, current_text: str) -> int | None:
"""
Find the start position of tool calls outside of thinking tags.
@@ -809,7 +809,7 @@ class MinimaxToolParser(ToolParser):
def _extract_content_before_tools(
self, current_text: str, delta_text: str, tool_start: int
) -> Optional[str]:
) -> str | None:
"""
Extract content that appears before tool calls.
@@ -5,7 +5,6 @@ import json
from collections.abc import Sequence
from random import choices
from string import ascii_letters, digits
from typing import Union
import partial_json_parser
import regex as re
@@ -194,7 +193,7 @@ class MistralToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# if the tool call token is not in the tokens generated so far, append
# output to contents since it's not a tool
if self.bot_token not in current_text:
@@ -252,7 +251,7 @@ class MistralToolParser(ToolParser):
# auto-generated due to JSON completions, but wasn't
# streamed to the client yet.
if self.current_tool_id >= 0:
diff: Union[str, None] = current_tool_call.get("arguments")
diff: str | None = current_tool_call.get("arguments")
if diff:
diff = json.dumps(diff, ensure_ascii=False).replace(
@@ -1,7 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import json
from collections.abc import Sequence
from typing import TYPE_CHECKING
@@ -22,13 +20,15 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.transformers_utils.tokenizer import AnyTokenizer
else:
AnyTokenizer = object
logger = init_logger(__name__)
@ToolParserManager.register_module("openai")
class OpenAIToolParser(ToolParser):
def __init__(self, tokenizer: AnyTokenizer):
def __init__(self, tokenizer: "AnyTokenizer"):
super().__init__(tokenizer)
def extract_tool_calls(
@@ -3,7 +3,7 @@
import json
from collections.abc import Sequence
from typing import Any, Optional
from typing import Any
import regex as re
from transformers import PreTrainedTokenizerBase
@@ -118,5 +118,5 @@ class Phi4MiniJsonToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Optional[DeltaMessage]:
) -> DeltaMessage | None:
return None
@@ -4,7 +4,7 @@
import ast
import json
from collections.abc import Sequence
from typing import Any, Union
from typing import Any
import regex as re
from transformers import PreTrainedTokenizerBase
@@ -124,7 +124,7 @@ class PythonicToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not current_text.startswith("["):
return DeltaMessage(content=delta_text)
@@ -236,7 +236,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
)
def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _make_valid_python(text: str) -> tuple[str, str] | None:
bracket_stack = []
for index, char in enumerate(text):
if char in {"[", "(", "{"}:
@@ -308,7 +308,7 @@ def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _compute_tool_delta(
previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
) -> Union[DeltaToolCall, None]:
) -> DeltaToolCall | None:
new_call_args = new_call.function.arguments
if withheld_suffix:
assert new_call_args.endswith(withheld_suffix)
@@ -4,7 +4,7 @@ import ast
import json
import uuid
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -36,7 +36,7 @@ class Qwen3CoderToolParser(ToolParser):
self.current_tool_name_sent: bool = False
self.prev_tool_call_arr: list[dict] = []
# Override base class type - we use string IDs for tool calls
self.current_tool_id: Optional[str] = None # type: ignore
self.current_tool_id: str | None = None # type: ignore
self.streamed_args_for_tool: list[str] = []
# Sentinel tokens for streaming mode
@@ -110,7 +110,7 @@ class Qwen3CoderToolParser(ToolParser):
self.streaming_request = None
def _get_arguments_config(
self, func_name: str, tools: Optional[list[ChatCompletionToolsParam]]
self, func_name: str, tools: list[ChatCompletionToolsParam] | None
) -> dict:
"""Extract argument configuration for a function."""
if tools is None:
@@ -240,8 +240,8 @@ class Qwen3CoderToolParser(ToolParser):
return param_value
def _parse_xml_function_call(
self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
) -> Optional[ToolCall]:
self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None
) -> ToolCall | None:
# Extract function name
end_index = function_call_str.index(">")
function_name = function_call_str[:end_index]
@@ -349,7 +349,7 @@ class Qwen3CoderToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# Store request for type conversion
if not previous_text:
self._reset_streaming_state()
@@ -4,7 +4,7 @@ import ast
import json
import uuid
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
from xml.parsers.expat import ParserCreate
import regex as re
@@ -39,7 +39,7 @@ class StreamingXMLToolCallParser:
self.reset_streaming_state()
# Tool configuration information
self.tools: Union[list[ChatCompletionToolsParam], None] = None
self.tools: list[ChatCompletionToolsParam] | None = None
self.tool_call_start_token: str = "<tool_call>"
self.tool_call_end_token: str = "</tool_call>"
self.function_start_token: str = "<function="
@@ -341,7 +341,7 @@ class StreamingXMLToolCallParser:
# Skip blank content
return not element
def _find_next_complete_element(self, start_pos: int) -> tuple[Optional[str], int]:
def _find_next_complete_element(self, start_pos: int) -> tuple[str | None, int]:
"""
Find next complete XML element from specified position
@@ -584,7 +584,7 @@ class StreamingXMLToolCallParser:
"""Emit Delta response (streaming output)"""
self.deltas.append(delta)
def _auto_close_open_parameter_if_needed(self, incoming_tag: Optional[str] = None):
def _auto_close_open_parameter_if_needed(self, incoming_tag: str | None = None):
"""Before starting to process new elements,
if there are unclosed tags from before,
automatically complete their endings to the parser.
@@ -953,7 +953,7 @@ class StreamingXMLToolCallParser:
self.parser.EndElementHandler = self._end_element
self.parser.CharacterDataHandler = self._char_data
def set_tools(self, tools: Union[list[ChatCompletionToolsParam], None]):
def set_tools(self, tools: list[ChatCompletionToolsParam] | None):
"""Set tool configuration information"""
self.tools = tools
@@ -961,7 +961,7 @@ class StreamingXMLToolCallParser:
"""Generate unique call ID"""
return f"call_{uuid.uuid4().hex[:24]}"
def _extract_function_name(self, name: str, attrs: dict[str, str]) -> Optional[str]:
def _extract_function_name(self, name: str, attrs: dict[str, str]) -> str | None:
"""Extract function name from various formats"""
if attrs and "name" in attrs:
return attrs["name"]
@@ -973,9 +973,7 @@ class StreamingXMLToolCallParser:
return None
def _extract_parameter_name(
self, name: str, attrs: dict[str, str]
) -> Optional[str]:
def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> str | None:
"""Extract parameter name from various formats"""
if attrs and "name" in attrs:
return attrs["name"]
@@ -1218,7 +1216,7 @@ class Qwen3XMLToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not previous_text:
self.parser.reset_streaming_state()
if request:
@@ -7,7 +7,7 @@ import ast
import json
import uuid
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -109,8 +109,8 @@ class SeedOssToolParser(ToolParser):
self.json_closed = False
def _parse_xml_function_call(
self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
) -> Optional[ToolCall]:
self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None
) -> ToolCall | None:
def get_arguments_config(func_name: str) -> dict:
if tools is None:
return {}
@@ -357,7 +357,7 @@ class SeedOssToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# If no delta text, return None unless
# it's an EOS token after tool calls
if not delta_text:
@@ -4,7 +4,7 @@
import contextlib
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -58,7 +58,7 @@ class Step3ToolParser(ToolParser):
@staticmethod
def _parse_steptml_invoke(
action_text: str,
) -> tuple[Optional[str], Optional[dict[str, str]]]:
) -> tuple[str | None, dict[str, str] | None]:
func_name_match = re.search(r'<steptml:invoke name="([^"]+)">', action_text)
if not func_name_match:
return None, None
@@ -117,7 +117,7 @@ class Step3ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# The main loop processes the stream from the last known position.
while True:
if self.position >= len(current_text):