Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+3 -3
View File
@@ -13,7 +13,7 @@ import json
import ssl
from argparse import Namespace
from collections.abc import AsyncGenerator
from typing import Any, Optional
from typing import Any
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -101,7 +101,7 @@ def build_app(args: Namespace) -> FastAPI:
async def init_app(
args: Namespace,
llm_engine: Optional[AsyncLLMEngine] = None,
llm_engine: AsyncLLMEngine | None = None,
) -> FastAPI:
app = build_app(args)
@@ -120,7 +120,7 @@ async def init_app(
async def run_server(
args: Namespace, llm_engine: Optional[AsyncLLMEngine] = None, **uvicorn_kwargs: Any
args: Namespace, llm_engine: AsyncLLMEngine | None = None, **uvicorn_kwargs: Any
) -> None:
logger.info("vLLM API server version %s", VLLM_VERSION)
logger.info("args: %s", args)
+104 -107
View File
@@ -5,10 +5,10 @@ import asyncio
import json
from abc import ABC, abstractmethod
from collections import Counter, defaultdict, deque
from collections.abc import Awaitable, Iterable
from collections.abc import Awaitable, Callable, Iterable
from functools import cached_property, lru_cache, partial
from pathlib import Path
from typing import Any, Callable, Generic, Literal, Optional, TypeVar, Union, cast
from typing import Any, Generic, Literal, TypeAlias, TypeVar, cast
import jinja2
import jinja2.ext
@@ -40,7 +40,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin
# pydantic needs the TypedDict from typing_extensions
from typing_extensions import Required, TypeAlias, TypedDict
from typing_extensions import Required, TypedDict
from vllm.config import ModelConfig
from vllm.logger import init_logger
@@ -76,7 +76,7 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
image_embeds: Optional[Union[str, dict[str, str]]]
image_embeds: str | dict[str, str] | None
"""
The image embeddings. It can be either:
- A single base64 string.
@@ -84,7 +84,7 @@ class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
"""
type: Required[Literal["image_embeds"]]
"""The type of the content part."""
uuid: Optional[str]
uuid: str | None
"""
User-provided UUID of a media. User must guarantee that it is properly
generated and unique for different medias.
@@ -123,8 +123,8 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
}
"""
image_pil: Optional[PILImage]
uuid: Optional[str]
image_pil: PILImage | None
uuid: str | None
"""
User-provided UUID of a media. User must guarantee that it is properly
generated and unique for different medias.
@@ -141,8 +141,8 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
}
"""
image_url: Optional[str]
uuid: Optional[str]
image_url: str | None
uuid: str | None
"""
User-provided UUID of a media. User must guarantee that it is properly
generated and unique for different medias.
@@ -158,7 +158,7 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
}
"""
audio_url: Optional[str]
audio_url: str | None
class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
@@ -170,8 +170,8 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
}
"""
video_url: Optional[str]
uuid: Optional[str]
video_url: str | None
uuid: str | None
"""
User-provided UUID of a media. User must guarantee that it is properly
generated and unique for different medias.
@@ -199,20 +199,20 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
"""The thinking type."""
ChatCompletionContentPartParam: TypeAlias = Union[
OpenAIChatCompletionContentPartParam,
ChatCompletionContentPartAudioParam,
ChatCompletionContentPartInputAudioParam,
ChatCompletionContentPartVideoParam,
ChatCompletionContentPartRefusalParam,
CustomChatCompletionContentPILImageParam,
CustomChatCompletionContentSimpleImageParam,
ChatCompletionContentPartImageEmbedsParam,
CustomChatCompletionContentSimpleAudioParam,
CustomChatCompletionContentSimpleVideoParam,
str,
CustomThinkCompletionContentParam,
]
ChatCompletionContentPartParam: TypeAlias = (
OpenAIChatCompletionContentPartParam
| ChatCompletionContentPartAudioParam
| ChatCompletionContentPartInputAudioParam
| ChatCompletionContentPartVideoParam
| ChatCompletionContentPartRefusalParam
| CustomChatCompletionContentPILImageParam
| CustomChatCompletionContentSimpleImageParam
| ChatCompletionContentPartImageEmbedsParam
| CustomChatCompletionContentSimpleAudioParam
| CustomChatCompletionContentSimpleVideoParam
| str
| CustomThinkCompletionContentParam
)
class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -221,7 +221,7 @@ class CustomChatCompletionMessageParam(TypedDict, total=False):
role: Required[str]
"""The role of the message's author."""
content: Union[str, list[ChatCompletionContentPartParam]]
content: str | list[ChatCompletionContentPartParam]
"""The contents of the message."""
name: str
@@ -231,18 +231,18 @@ class CustomChatCompletionMessageParam(TypedDict, total=False):
same role.
"""
tool_call_id: Optional[str]
tool_call_id: str | None
"""Tool call that this message is responding to."""
tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None
"""The tool calls generated by the model, such as function calls."""
ChatCompletionMessageParam = Union[
OpenAIChatCompletionMessageParam,
CustomChatCompletionMessageParam,
OpenAIHarmonyMessage,
]
ChatCompletionMessageParam: TypeAlias = (
OpenAIChatCompletionMessageParam
| CustomChatCompletionMessageParam
| OpenAIHarmonyMessage
)
# TODO: Make fields ReadOnly once mypy supports it
@@ -250,16 +250,16 @@ class ConversationMessage(TypedDict, total=False):
role: Required[str]
"""The role of the message's author."""
content: Union[Optional[str], list[dict[str, str]]]
content: str | None | list[dict[str, str]]
"""The contents of the message"""
tool_call_id: Optional[str]
tool_call_id: str | None
"""Tool call that this message is responding to."""
name: Optional[str]
name: str | None
"""The name of the function to call"""
tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None
"""The tool calls generated by the model, such as function calls."""
@@ -294,7 +294,7 @@ def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
def _is_var_or_elems_access(
node: jinja2.nodes.Node,
varname: str,
key: Optional[str] = None,
key: str | None = None,
) -> bool:
if isinstance(node, jinja2.nodes.Filter):
return node.node is not None and _is_var_or_elems_access(
@@ -369,7 +369,7 @@ def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
break
def _try_extract_ast(chat_template: str) -> Optional[jinja2.nodes.Template]:
def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None:
try:
jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
return jinja_compiled.environment.parse(chat_template)
@@ -400,9 +400,9 @@ def _detect_content_format(
def resolve_mistral_chat_template(
chat_template: Optional[str],
chat_template: str | None,
**kwargs: Any,
) -> Optional[str]:
) -> str | None:
if chat_template is not None or kwargs.get("chat_template_kwargs") is not None:
raise ValueError(
"'chat_template' or 'chat_template_kwargs' cannot be overridden "
@@ -412,7 +412,7 @@ def resolve_mistral_chat_template(
return None
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], Optional[str]]()
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
"""
Used in `_try_get_processor_chat_template` to avoid calling
`cached_get_processor` again if the processor fails to be loaded.
@@ -422,9 +422,9 @@ This is needed because `lru_cache` does not cache when an exception happens.
def _try_get_processor_chat_template(
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
model_config: ModelConfig,
) -> Optional[str]:
) -> str | None:
cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)
if cache_key in _PROCESSOR_CHAT_TEMPLATES:
return _PROCESSOR_CHAT_TEMPLATES[cache_key]
@@ -458,12 +458,12 @@ def _try_get_processor_chat_template(
def resolve_hf_chat_template(
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
chat_template: Optional[str],
tools: Optional[list[dict[str, Any]]],
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
chat_template: str | None,
tools: list[dict[str, Any]] | None,
*,
model_config: ModelConfig,
) -> Optional[str]:
) -> str | None:
# 1st priority: The given chat template
if chat_template is not None:
return chat_template
@@ -505,8 +505,8 @@ def resolve_hf_chat_template(
def _resolve_chat_template_content_format(
chat_template: Optional[str],
tools: Optional[list[dict[str, Any]]],
chat_template: str | None,
tools: list[dict[str, Any]] | None,
tokenizer: AnyTokenizer,
*,
model_config: ModelConfig,
@@ -538,7 +538,7 @@ def _resolve_chat_template_content_format(
@lru_cache
def _log_chat_template_content_format(
chat_template: Optional[str],
chat_template: str | None,
given_format: ChatTemplateContentFormatOption,
detected_format: ChatTemplateContentFormatOption,
):
@@ -561,8 +561,8 @@ def _log_chat_template_content_format(
def resolve_chat_template_content_format(
chat_template: Optional[str],
tools: Optional[list[dict[str, Any]]],
chat_template: str | None,
tools: list[dict[str, Any]] | None,
given_format: ChatTemplateContentFormatOption,
tokenizer: AnyTokenizer,
*,
@@ -604,8 +604,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
self._model_config = model_config
self._tokenizer = tokenizer
self._items_by_modality = defaultdict[str, list[Optional[_T]]](list)
self._uuids_by_modality = defaultdict[str, list[Optional[str]]](list)
self._items_by_modality = defaultdict[str, list[_T | None]](list)
self._uuids_by_modality = defaultdict[str, list[str | None]](list)
@property
def model_config(self) -> ModelConfig:
@@ -637,9 +637,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
def add(
self,
modality: ModalityStr,
item: Optional[_T],
uuid: Optional[str] = None,
) -> Optional[str]:
item: _T | None,
uuid: str | None = None,
) -> str | None:
"""
Add a multi-modal item to the current prompt and returns the
placeholder string to use, if any.
@@ -657,7 +657,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
return self.model_cls.get_placeholder_str(modality, num_items)
def all_mm_uuids(self) -> Optional[MultiModalUUIDDict]:
def all_mm_uuids(self) -> MultiModalUUIDDict | None:
if not self._items_by_modality:
return None
mm_uuids = {}
@@ -684,7 +684,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
def all_mm_data(self) -> Optional[MultiModalDataDict]:
def all_mm_data(self) -> MultiModalDataDict | None:
if not self._items_by_modality:
return None
mm_inputs = {}
@@ -710,7 +710,7 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
async def all_mm_data(self) -> Optional[MultiModalDataDict]:
async def all_mm_data(self) -> MultiModalDataDict | None:
if not self._items_by_modality:
return None
mm_inputs = {}
@@ -756,7 +756,7 @@ class BaseMultiModalContentParser(ABC):
# }
self._placeholder_storage: dict[str, list] = defaultdict(list)
def _add_placeholder(self, modality: ModalityStr, placeholder: Optional[str]):
def _add_placeholder(self, modality: ModalityStr, placeholder: str | None):
mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
if placeholder:
self._placeholder_storage[mod_placeholder].append(placeholder)
@@ -765,35 +765,35 @@ class BaseMultiModalContentParser(ABC):
return dict(self._placeholder_storage)
@abstractmethod
def parse_image(self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
raise NotImplementedError
@abstractmethod
def parse_image_embeds(
self,
image_embeds: Union[str, dict[str, str], None],
uuid: Optional[str] = None,
image_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
raise NotImplementedError
@abstractmethod
def parse_image_pil(
self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
self, image_pil: Image.Image | None, uuid: str | None = None
) -> None:
raise NotImplementedError
@abstractmethod
def parse_audio(self, audio_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
raise NotImplementedError
@abstractmethod
def parse_input_audio(
self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
self, input_audio: InputAudio | None, uuid: str | None = None
) -> None:
raise NotImplementedError
@abstractmethod
def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
raise NotImplementedError
@@ -810,7 +810,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
allowed_media_domains=tracker.allowed_media_domains,
)
def parse_image(self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
image = self._connector.fetch_image(image_url) if image_url else None
placeholder = self._tracker.add("image", image, uuid)
@@ -818,8 +818,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def parse_image_embeds(
self,
image_embeds: Union[str, dict[str, str], None],
uuid: Optional[str] = None,
image_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
if isinstance(image_embeds, dict):
embeds = {
@@ -838,19 +838,19 @@ class MultiModalContentParser(BaseMultiModalContentParser):
self._add_placeholder("image", placeholder)
def parse_image_pil(
self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
self, image_pil: Image.Image | None, uuid: str | None = None
) -> None:
placeholder = self._tracker.add("image", image_pil, uuid)
self._add_placeholder("image", placeholder)
def parse_audio(self, audio_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
audio = self._connector.fetch_audio(audio_url) if audio_url else None
placeholder = self._tracker.add("audio", audio, uuid)
self._add_placeholder("audio", placeholder)
def parse_input_audio(
self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
self, input_audio: InputAudio | None, uuid: str | None = None
) -> None:
if input_audio:
audio_data = input_audio.get("data", "")
@@ -865,7 +865,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
return self.parse_audio(audio_url, uuid)
def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
video = self._connector.fetch_video(video_url=video_url) if video_url else None
placeholder = self._tracker.add("video", video, uuid)
@@ -885,7 +885,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
allowed_media_domains=tracker.allowed_media_domains,
)
def parse_image(self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
image_coro = self._connector.fetch_image_async(image_url) if image_url else None
placeholder = self._tracker.add("image", image_coro, uuid)
@@ -893,10 +893,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def parse_image_embeds(
self,
image_embeds: Union[str, dict[str, str], None],
uuid: Optional[str] = None,
image_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
future: asyncio.Future[Union[str, dict[str, str], None]] = asyncio.Future()
future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()
if isinstance(image_embeds, dict):
embeds = {
@@ -916,9 +916,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
self._add_placeholder("image", placeholder)
def parse_image_pil(
self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
self, image_pil: Image.Image | None, uuid: str | None = None
) -> None:
future: asyncio.Future[Optional[Image.Image]] = asyncio.Future()
future: asyncio.Future[Image.Image | None] = asyncio.Future()
if image_pil:
future.set_result(image_pil)
else:
@@ -927,14 +927,14 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
placeholder = self._tracker.add("image", future, uuid)
self._add_placeholder("image", placeholder)
def parse_audio(self, audio_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
audio_coro = self._connector.fetch_audio_async(audio_url) if audio_url else None
placeholder = self._tracker.add("audio", audio_coro, uuid)
self._add_placeholder("audio", placeholder)
def parse_input_audio(
self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
self, input_audio: InputAudio | None, uuid: str | None = None
) -> None:
if input_audio:
audio_data = input_audio.get("data", "")
@@ -949,7 +949,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
return self.parse_audio(audio_url, uuid)
def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
video = (
self._connector.fetch_video_async(video_url=video_url)
if video_url
@@ -960,7 +960,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
self._add_placeholder("video", placeholder)
def validate_chat_template(chat_template: Optional[Union[Path, str]]):
def validate_chat_template(chat_template: Path | str | None):
"""Raises if the provided chat template appears invalid."""
if chat_template is None:
return
@@ -984,10 +984,10 @@ def validate_chat_template(chat_template: Optional[Union[Path, str]]):
def _load_chat_template(
chat_template: Optional[Union[Path, str]],
chat_template: Path | str | None,
*,
is_literal: bool = False,
) -> Optional[str]:
) -> str | None:
if chat_template is None:
return None
@@ -1024,10 +1024,10 @@ _cached_load_chat_template = lru_cache(_load_chat_template)
def load_chat_template(
chat_template: Optional[Union[Path, str]],
chat_template: Path | str | None,
*,
is_literal: bool = False,
) -> Optional[str]:
) -> str | None:
return _cached_load_chat_template(chat_template, is_literal=is_literal)
@@ -1107,7 +1107,7 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
_ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python
_ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio, PILImage]
_ContentPart: TypeAlias = str | dict[str, str] | InputAudio | PILImage
# Define a mapping from part types to their corresponding parsing functions.
MM_PARSER_MAP: dict[
@@ -1264,7 +1264,7 @@ def _parse_chat_message_content_part(
*,
wrap_dicts: bool,
interleave_strings: bool,
) -> Optional[_ContentPart]:
) -> _ContentPart | None:
"""Parses a single part of a conversation. If wrap_dicts is True,
structured dictionary pieces for texts and images will be
wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
@@ -1310,10 +1310,7 @@ def _parse_chat_message_content_part(
mm_parser.parse_image(str_content, uuid)
modality = "image"
elif part_type == "image_embeds":
if content is not None:
content = cast(Union[str, dict[str, str]], content)
else:
content = None
content = cast(str | dict[str, str], content) if content is not None else None
mm_parser.parse_image_embeds(content, uuid)
modality = "image"
elif part_type == "audio_url":
@@ -1411,8 +1408,8 @@ def parse_chat_messages(
content_format: _ChatTemplateContentFormat,
) -> tuple[
list[ConversationMessage],
Optional[MultiModalDataDict],
Optional[MultiModalUUIDDict],
MultiModalDataDict | None,
MultiModalUUIDDict | None,
]:
conversation: list[ConversationMessage] = []
mm_tracker = MultiModalItemTracker(model_config, tokenizer)
@@ -1443,8 +1440,8 @@ def parse_chat_messages_futures(
content_format: _ChatTemplateContentFormat,
) -> tuple[
list[ConversationMessage],
Awaitable[Optional[MultiModalDataDict]],
Optional[MultiModalUUIDDict],
Awaitable[MultiModalDataDict | None],
MultiModalUUIDDict | None,
]:
conversation: list[ConversationMessage] = []
mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
@@ -1498,7 +1495,7 @@ _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
def resolve_chat_template_kwargs(
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
chat_template: str,
chat_template_kwargs: dict[str, Any],
) -> dict[str, Any]:
@@ -1518,10 +1515,10 @@ def resolve_chat_template_kwargs(
def apply_hf_chat_template(
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
conversation: list[ConversationMessage],
chat_template: Optional[str],
tools: Optional[list[dict[str, Any]]],
chat_template: str | None,
tools: list[dict[str, Any]] | None,
*,
model_config: ModelConfig,
tokenize: bool = False, # Different from HF's default
@@ -1569,8 +1566,8 @@ def apply_hf_chat_template(
def apply_mistral_chat_template(
tokenizer: MistralTokenizer,
messages: list[ChatCompletionMessageParam],
chat_template: Optional[str],
tools: Optional[list[dict[str, Any]]],
chat_template: str | None,
tools: list[dict[str, Any]] | None,
**kwargs: Any,
) -> list[int]:
from mistral_common.exceptions import MistralCommonException
+2 -2
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import typing
@@ -12,6 +10,8 @@ from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
if typing.TYPE_CHECKING:
from vllm.utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
class BenchmarkSubcommand(CLISubcommand):
+2 -2
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import typing
@@ -11,6 +9,8 @@ from vllm.entrypoints.cli.types import CLISubcommand
if typing.TYPE_CHECKING:
from vllm.utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
class CollectEnvSubcommand(CLISubcommand):
-2
View File
@@ -5,8 +5,6 @@
Note that all future modules must be lazily loaded within main
to avoid certain eager import breakage."""
from __future__ import annotations
import importlib.metadata
import sys
+2 -2
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import os
import signal
@@ -16,6 +14,8 @@ from vllm.entrypoints.cli.types import CLISubcommand
if TYPE_CHECKING:
from vllm.utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
def _register_signal_handlers():
+2 -2
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import asyncio
import importlib.metadata
@@ -14,6 +12,8 @@ from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
+1 -2
View File
@@ -3,7 +3,6 @@
import argparse
import signal
from typing import Optional
import uvloop
@@ -179,7 +178,7 @@ def run_multi_api_server(args: argparse.Namespace):
hybrid_dp_lb = parallel_config.data_parallel_hybrid_lb
assert external_dp_lb or hybrid_dp_lb or dp_rank == 0
api_server_manager: Optional[APIServerProcessManager] = None
api_server_manager: APIServerProcessManager | None = None
with launch_core_engines(
vllm_config, executor_class, log_stats, num_api_servers
+2 -2
View File
@@ -1,13 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import typing
if typing.TYPE_CHECKING:
from vllm.utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
class CLISubcommand:
+8 -8
View File
@@ -6,7 +6,7 @@ import json
import logging
from abc import ABC, abstractmethod
from contextlib import AsyncExitStack
from typing import TYPE_CHECKING, Optional, Union
from typing import TYPE_CHECKING, Union
from openai.types.responses.tool import Mcp
from openai_harmony import Author, Message, Role, StreamState, TextContent
@@ -82,7 +82,7 @@ class ConversationContext(ABC):
@abstractmethod
async def init_tool_sessions(
self,
tool_server: Optional[ToolServer],
tool_server: ToolServer | None,
exit_stack: AsyncExitStack,
request_id: str,
mcp_tools: dict[str, Mcp],
@@ -122,7 +122,7 @@ class SimpleContext(ConversationContext):
async def init_tool_sessions(
self,
tool_server: Optional[ToolServer],
tool_server: ToolServer | None,
exit_stack: AsyncExitStack,
request_id: str,
mcp_tools: dict[str, Mcp],
@@ -140,9 +140,9 @@ class HarmonyContext(ConversationContext):
available_tools: list[str],
):
self._messages = messages
self.finish_reason: Optional[str] = None
self.finish_reason: str | None = None
self.available_tools = available_tools
self._tool_sessions: dict[str, Union[ClientSession, Tool]] = {}
self._tool_sessions: dict[str, ClientSession | Tool] = {}
self.called_tools: set[str] = set()
self.parser = get_streamable_parser_for_assistant()
@@ -164,7 +164,7 @@ class HarmonyContext(ConversationContext):
if self.parser.current_channel in {"analysis", "commentary"}:
self.num_reasoning_tokens += 1
def append_output(self, output: Union[RequestOutput, list[Message]]) -> None:
def append_output(self, output: RequestOutput | list[Message]) -> None:
if isinstance(output, RequestOutput):
output_token_ids = output.outputs[0].token_ids
self.parser = get_streamable_parser_for_assistant()
@@ -358,7 +358,7 @@ class HarmonyContext(ConversationContext):
async def init_tool_sessions(
self,
tool_server: Optional[ToolServer],
tool_server: ToolServer | None,
exit_stack: AsyncExitStack,
request_id: str,
mcp_tools: dict[str, Mcp],
@@ -446,7 +446,7 @@ class StreamingHarmonyContext(HarmonyContext):
def messages(self) -> list:
return self._messages
def append_output(self, output: Union[RequestOutput, list[Message]]) -> None:
def append_output(self, output: RequestOutput | list[Message]) -> None:
if isinstance(output, RequestOutput):
# append_output is called for each output token in streaming case,
# so we only want to add the prompt tokens once for each message.
+5 -7
View File
@@ -1,12 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import datetime
import json
from collections.abc import Iterable, Sequence
from typing import Literal, Union
from typing import Literal
from openai.types.responses import (
ResponseFunctionToolCall,
@@ -122,7 +120,7 @@ def get_system_message(
return sys_msg
def create_tool_definition(tool: Union[ChatCompletionToolsParam, Tool]):
def create_tool_definition(tool: ChatCompletionToolsParam | Tool):
if isinstance(tool, ChatCompletionToolsParam):
return ToolDescription.new(
name=tool.function.name,
@@ -138,13 +136,13 @@ def create_tool_definition(tool: Union[ChatCompletionToolsParam, Tool]):
def get_developer_message(
instructions: str | None = None,
tools: list[Union[Tool, ChatCompletionToolsParam]] | None = None,
tools: list[Tool | ChatCompletionToolsParam] | None = None,
) -> Message:
dev_msg_content = DeveloperContent.new()
if instructions is not None and not envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
dev_msg_content = dev_msg_content.with_instructions(instructions)
if tools is not None:
function_tools: list[Union[Tool, ChatCompletionToolsParam]] = []
function_tools: list[Tool | ChatCompletionToolsParam] = []
for tool in tools:
if tool.type in (
"web_search_preview",
@@ -178,7 +176,7 @@ def get_user_message(content: str) -> Message:
def parse_response_input(
response_msg: ResponseInputOutputItem,
prev_responses: list[Union[ResponseOutputItem, ResponseReasoningItem]],
prev_responses: list[ResponseOutputItem | ResponseReasoningItem],
) -> Message:
if not isinstance(response_msg, dict):
response_msg = response_msg.model_dump()
+2 -2
View File
@@ -5,7 +5,7 @@ import asyncio
import signal
import socket
from http import HTTPStatus
from typing import Any, Optional
from typing import Any
import uvicorn
from fastapi import FastAPI, Request, Response
@@ -26,7 +26,7 @@ logger = init_logger(__name__)
async def serve_http(
app: FastAPI,
sock: Optional[socket.socket],
sock: socket.socket | None,
enable_ssl_refresh: bool = False,
**uvicorn_kwargs: Any,
):
+113 -121
View File
@@ -2,8 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
from collections.abc import Callable, Sequence
from typing import TYPE_CHECKING, Any, cast
import cloudpickle
import torch.nn as nn
@@ -191,36 +191,34 @@ class LLM:
*,
runner: RunnerOption = "auto",
convert: ConvertOption = "auto",
tokenizer: Optional[str] = None,
tokenizer: str | None = None,
tokenizer_mode: TokenizerMode = "auto",
skip_tokenizer_init: bool = False,
trust_remote_code: bool = False,
allowed_local_media_path: str = "",
allowed_media_domains: Optional[list[str]] = None,
allowed_media_domains: list[str] | None = None,
tensor_parallel_size: int = 1,
dtype: ModelDType = "auto",
quantization: Optional[QuantizationMethods] = None,
revision: Optional[str] = None,
tokenizer_revision: Optional[str] = None,
seed: Optional[int] = None,
quantization: QuantizationMethods | None = None,
revision: str | None = None,
tokenizer_revision: str | None = None,
seed: int | None = None,
gpu_memory_utilization: float = 0.9,
swap_space: float = 4,
cpu_offload_gb: float = 0,
enforce_eager: bool = False,
disable_custom_all_reduce: bool = False,
hf_token: Optional[Union[bool, str]] = None,
hf_overrides: Optional[HfOverrides] = None,
mm_processor_kwargs: Optional[dict[str, Any]] = None,
pooler_config: Optional[PoolerConfig] = None,
override_pooler_config: Optional[PoolerConfig] = None,
structured_outputs_config: Optional[
Union[dict[str, Any], StructuredOutputsConfig]
] = None,
kv_cache_memory_bytes: Optional[int] = None,
compilation_config: Optional[
Union[int, dict[str, Any], CompilationConfig]
] = None,
logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None,
hf_token: bool | str | None = None,
hf_overrides: HfOverrides | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
pooler_config: PoolerConfig | None = None,
override_pooler_config: PoolerConfig | None = None,
structured_outputs_config: dict[str, Any]
| StructuredOutputsConfig
| None = None,
kv_cache_memory_bytes: int | None = None,
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
logits_processors: list[str | type[LogitsProcessor]] | None = None,
**kwargs: Any,
) -> None:
"""LLM constructor."""
@@ -331,7 +329,7 @@ class LLM:
self.engine_class = type(self.llm_engine)
self.request_counter = Counter()
self.default_sampling_params: Union[dict[str, Any], None] = None
self.default_sampling_params: dict[str, Any] | None = None
supported_tasks = self.llm_engine.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
@@ -367,14 +365,12 @@ class LLM:
def generate(
self,
prompts: Union[PromptType, Sequence[PromptType]],
sampling_params: Optional[
Union[SamplingParams, Sequence[SamplingParams]]
] = None,
prompts: PromptType | Sequence[PromptType],
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
*,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
priority: Optional[list[int]] = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
priority: list[int] | None = None,
) -> list[RequestOutput]:
"""Generates the completions for the input prompts.
@@ -437,8 +433,8 @@ class LLM:
def _get_modality_specific_lora_reqs(
self,
prompts: Union[PromptType, Sequence[PromptType]],
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
prompts: PromptType | Sequence[PromptType],
lora_request: list[LoRARequest] | LoRARequest | None,
):
# Grab the lora config off the vllm config on the engine,
# since this is the same for both v0 & v1.
@@ -474,8 +470,8 @@ class LLM:
def _resolve_single_prompt_mm_lora(
self,
prompt: PromptType,
lora_request: Optional[LoRARequest],
default_mm_loras: Optional[dict[str, str]],
lora_request: LoRARequest | None,
default_mm_loras: dict[str, str] | None,
):
if (
not default_mm_loras
@@ -526,10 +522,10 @@ class LLM:
def collective_rpc(
self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
method: str | Callable[..., _R],
timeout: float | None = None,
args: tuple = (),
kwargs: Optional[dict[str, Any]] = None,
kwargs: dict[str, Any] | None = None,
) -> list[_R]:
"""
Execute an RPC call on all workers.
@@ -571,9 +567,9 @@ class LLM:
def _get_beam_search_lora_requests(
self,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
prompts: list[Union[TokensPrompt, TextPrompt]],
) -> list[Optional[LoRARequest]]:
lora_request: list[LoRARequest] | LoRARequest | None,
prompts: list[TokensPrompt | TextPrompt],
) -> list[LoRARequest | None]:
"""Get the optional lora request corresponding to each prompt."""
if isinstance(lora_request, Sequence) and len(lora_request) != len(prompts):
raise ValueError(
@@ -587,11 +583,11 @@ class LLM:
def beam_search(
self,
prompts: list[Union[TokensPrompt, TextPrompt]],
prompts: list[TokensPrompt | TextPrompt],
params: BeamSearchParams,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
use_tqdm: bool = False,
concurrency_limit: Optional[int] = None,
concurrency_limit: int | None = None,
) -> list[BeamSearchOutput]:
"""
Generate sequences using beam search.
@@ -770,16 +766,15 @@ class LLM:
def preprocess_chat(
self,
messages: Union[
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
],
chat_template: Optional[str] = None,
messages: list[ChatCompletionMessageParam]
| list[list[ChatCompletionMessageParam]],
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
continue_final_message: bool = False,
tools: Optional[list[dict[str, Any]]] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None,
mm_processor_kwargs: Optional[dict[str, Any]] = None,
tools: list[dict[str, Any]] | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[TokensPrompt]:
"""
Generate prompt for a chat conversation. The pre-processed
@@ -868,19 +863,18 @@ class LLM:
def chat(
self,
messages: Union[
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
],
sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[LoRARequest] = None,
chat_template: Optional[str] = None,
messages: list[ChatCompletionMessageParam]
| list[list[ChatCompletionMessageParam]],
sampling_params: SamplingParams | list[SamplingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: LoRARequest | None = None,
chat_template: str | None = None,
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
add_generation_prompt: bool = True,
continue_final_message: bool = False,
tools: Optional[list[dict[str, Any]]] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None,
mm_processor_kwargs: Optional[dict[str, Any]] = None,
tools: list[dict[str, Any]] | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[RequestOutput]:
"""
Generate responses for a chat conversation.
@@ -953,14 +947,14 @@ class LLM:
def encode(
self,
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
prompts: PromptType | Sequence[PromptType] | DataPrompt,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
pooling_task: PoolingTask = "encode",
tokenization_kwargs: Optional[dict[str, Any]] = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[PoolingRequestOutput]:
"""Apply pooling to the hidden states corresponding to the input
prompts.
@@ -1086,12 +1080,12 @@ class LLM:
def embed(
self,
prompts: Union[PromptType, Sequence[PromptType]],
prompts: PromptType | Sequence[PromptType],
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[EmbeddingRequestOutput]:
"""
Generate an embedding vector for each prompt.
@@ -1135,11 +1129,11 @@ class LLM:
def classify(
self,
prompts: Union[PromptType, Sequence[PromptType]],
prompts: PromptType | Sequence[PromptType],
*,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ClassificationRequestOutput]:
"""
Generate class logits for each prompt.
@@ -1181,13 +1175,13 @@ class LLM:
def reward(
self,
prompts: Union[PromptType, Sequence[PromptType]],
prompts: PromptType | Sequence[PromptType],
/,
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[PoolingRequestOutput]:
"""
Generate rewards for each prompt.
@@ -1220,12 +1214,12 @@ class LLM:
def _embedding_score(
self,
tokenizer: AnyTokenizer,
text_1: list[Union[str, TextPrompt, TokensPrompt]],
text_2: list[Union[str, TextPrompt, TokensPrompt]],
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[PoolingParams] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
text_1: list[str | TextPrompt | TokensPrompt],
text_2: list[str | TextPrompt | TokensPrompt],
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ScoringRequestOutput]:
encoded_output: list[PoolingRequestOutput] = self.encode(
text_1 + text_2,
@@ -1252,12 +1246,12 @@ class LLM:
def _cross_encoding_score(
self,
tokenizer: AnyTokenizer,
data_1: Union[list[str], list[ScoreContentPartParam]],
data_2: Union[list[str], list[ScoreContentPartParam]],
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[PoolingParams] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
data_1: list[str] | list[ScoreContentPartParam],
data_2: list[str] | list[ScoreContentPartParam],
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ScoringRequestOutput]:
model_config = self.model_config
@@ -1316,14 +1310,14 @@ class LLM:
def score(
self,
data_1: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
data_2: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
data_1: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
data_2: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
/,
*,
truncate_prompt_tokens: Optional[int] = None,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
pooling_params: Optional[PoolingParams] = None,
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
truncate_prompt_tokens: int | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
) -> list[ScoringRequestOutput]:
"""Generate similarity scores for all pairs `<text,text_pair>` or
`<multi-modal data, multi-modal data pair>`.
@@ -1391,9 +1385,9 @@ class LLM:
if not model_config.is_multimodal_model:
def check_data_type(
data: Union[
SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam
],
data: SingletonPrompt
| Sequence[SingletonPrompt]
| ScoreMultiModalParam,
):
if isinstance(data, dict) and "content" in data:
raise ValueError(
@@ -1470,7 +1464,7 @@ class LLM:
def stop_profile(self) -> None:
self.llm_engine.stop_profile()
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
def reset_prefix_cache(self, device: Device | None = None) -> bool:
return self.llm_engine.reset_prefix_cache(device)
def sleep(self, level: int = 1):
@@ -1495,7 +1489,7 @@ class LLM:
self.reset_prefix_cache()
self.llm_engine.sleep(level=level)
def wake_up(self, tags: Optional[list[str]] = None):
def wake_up(self, tags: list[str] | None = None):
"""
Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
method for more details.
@@ -1523,17 +1517,15 @@ class LLM:
def _validate_and_add_requests(
self,
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
params: Union[
SamplingParams,
Sequence[SamplingParams],
PoolingParams,
Sequence[PoolingParams],
],
prompts: PromptType | Sequence[PromptType] | DataPrompt,
params: SamplingParams
| Sequence[SamplingParams]
| PoolingParams
| Sequence[PoolingParams],
*,
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
priority: Optional[list[int]] = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: Sequence[LoRARequest] | LoRARequest | None,
priority: list[int] | None = None,
) -> None:
if isinstance(prompts, (str, dict)):
# Convert a single prompt to a list.
@@ -1575,8 +1567,8 @@ class LLM:
def _validate_mm_data_and_uuids(
self,
multi_modal_data: Optional[Any], # MultiModalDataDict
multi_modal_uuids: Optional[Any], # MultiModalUUIDDict
multi_modal_data: Any | None, # MultiModalDataDict
multi_modal_uuids: Any | None, # MultiModalUUIDDict
):
"""
Validate that if any multi-modal data is skipped (i.e. None),
@@ -1625,9 +1617,9 @@ class LLM:
self,
request_id: str,
engine_prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
params: SamplingParams | PoolingParams,
*,
lora_request: Optional[LoRARequest],
lora_request: LoRARequest | None,
priority: int,
) -> tuple[EngineCoreRequest, dict[str, Any]]:
"""Use the Processor to process inputs for LLMEngine."""
@@ -1651,8 +1643,8 @@ class LLM:
def _add_request(
self,
prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
lora_request: Optional[LoRARequest] = None,
params: SamplingParams | PoolingParams,
lora_request: LoRARequest | None = None,
priority: int = 0,
) -> None:
prompt_text, _, _ = get_prompt_components(prompt)
@@ -1677,8 +1669,8 @@ class LLM:
)
def _run_engine(
self, *, use_tqdm: Union[bool, Callable[..., tqdm]] = True
) -> list[Union[RequestOutput, PoolingRequestOutput]]:
self, *, use_tqdm: bool | Callable[..., tqdm] = True
) -> list[RequestOutput | PoolingRequestOutput]:
# Initialize tqdm.
if use_tqdm:
num_requests = self.llm_engine.get_num_unfinished_requests()
@@ -1691,7 +1683,7 @@ class LLM:
)
# Run the engine.
outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
outputs: list[RequestOutput | PoolingRequestOutput] = []
total_in_toks = 0
total_out_toks = 0
while self.llm_engine.has_unfinished_requests():
+8 -9
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Optional, Union
import torch
@@ -15,17 +14,17 @@ logger = init_logger(__name__)
class RequestLogger:
def __init__(self, *, max_log_len: Optional[int]) -> None:
def __init__(self, *, max_log_len: int | None) -> None:
self.max_log_len = max_log_len
def log_inputs(
self,
request_id: str,
prompt: Optional[str],
prompt_token_ids: Optional[list[int]],
prompt_embeds: Optional[torch.Tensor],
params: Optional[Union[SamplingParams, PoolingParams, BeamSearchParams]],
lora_request: Optional[LoRARequest],
prompt: str | None,
prompt_token_ids: list[int] | None,
prompt_embeds: torch.Tensor | None,
params: SamplingParams | PoolingParams | BeamSearchParams | None,
lora_request: LoRARequest | None,
) -> None:
max_log_len = self.max_log_len
if max_log_len is not None:
@@ -52,8 +51,8 @@ class RequestLogger:
self,
request_id: str,
outputs: str,
output_token_ids: Optional[Sequence[int]],
finish_reason: Optional[str] = None,
output_token_ids: Sequence[int] | None,
finish_reason: str | None = None,
is_streaming: bool = False,
delta: bool = False,
) -> None:
+20 -20
View File
@@ -16,10 +16,10 @@ import socket
import tempfile
import uuid
from argparse import Namespace
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
from contextlib import asynccontextmanager
from http import HTTPStatus
from typing import Annotated, Any, Callable, Literal, Optional
from typing import Annotated, Any, Literal
import prometheus_client
import pydantic
@@ -166,8 +166,8 @@ async def build_async_engine_client(
args: Namespace,
*,
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
disable_frontend_multiprocessing: Optional[bool] = None,
client_config: Optional[dict[str, Any]] = None,
disable_frontend_multiprocessing: bool | None = None,
client_config: dict[str, Any] | None = None,
) -> AsyncIterator[EngineClient]:
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
# The executor is expected to be mp.
@@ -203,7 +203,7 @@ async def build_async_engine_client_from_engine_args(
*,
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
disable_frontend_multiprocessing: bool = False,
client_config: Optional[dict[str, Any]] = None,
client_config: dict[str, Any] | None = None,
) -> AsyncIterator[EngineClient]:
"""
Create EngineClient, either:
@@ -227,7 +227,7 @@ async def build_async_engine_client_from_engine_args(
from vllm.v1.engine.async_llm import AsyncLLM
async_llm: Optional[AsyncLLM] = None
async_llm: AsyncLLM | None = None
# Don't mutate the input client_config
client_config = dict(client_config) if client_config else {}
@@ -308,35 +308,35 @@ def models(request: Request) -> OpenAIServingModels:
return request.app.state.openai_serving_models
def responses(request: Request) -> Optional[OpenAIServingResponses]:
def responses(request: Request) -> OpenAIServingResponses | None:
return request.app.state.openai_serving_responses
def chat(request: Request) -> Optional[OpenAIServingChat]:
def chat(request: Request) -> OpenAIServingChat | None:
return request.app.state.openai_serving_chat
def completion(request: Request) -> Optional[OpenAIServingCompletion]:
def completion(request: Request) -> OpenAIServingCompletion | None:
return request.app.state.openai_serving_completion
def pooling(request: Request) -> Optional[OpenAIServingPooling]:
def pooling(request: Request) -> OpenAIServingPooling | None:
return request.app.state.openai_serving_pooling
def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
def embedding(request: Request) -> OpenAIServingEmbedding | None:
return request.app.state.openai_serving_embedding
def score(request: Request) -> Optional[ServingScores]:
def score(request: Request) -> ServingScores | None:
return request.app.state.openai_serving_scores
def classify(request: Request) -> Optional[ServingClassification]:
def classify(request: Request) -> ServingClassification | None:
return request.app.state.openai_serving_classification
def rerank(request: Request) -> Optional[ServingScores]:
def rerank(request: Request) -> ServingScores | None:
return request.app.state.openai_serving_scores
@@ -542,8 +542,8 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
async def retrieve_responses(
response_id: str,
raw_request: Request,
starting_after: Optional[int] = None,
stream: Optional[bool] = False,
starting_after: int | None = None,
stream: bool | None = False,
):
handler = responses(raw_request)
if handler is None:
@@ -1039,7 +1039,7 @@ if envs.VLLM_SERVER_DEV_MODE:
# User-defined `method` is responsible for deserialization if needed.
args: list[str] = body.get("args", [])
kwargs: dict[str, str] = body.get("kwargs", {})
timeout: Optional[float] = body.get("timeout")
timeout: float | None = body.get("timeout")
results = await engine_client(raw_request).collective_rpc(
method=method, timeout=timeout, args=tuple(args), kwargs=kwargs
)
@@ -1120,7 +1120,7 @@ async def is_scaling_elastic_ep(raw_request: Request):
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
# (requires typing_extensions >= 4.13)
RequestType = Any
GetHandlerFn = Callable[[Request], Optional[OpenAIServing]]
GetHandlerFn = Callable[[Request], OpenAIServing | None]
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
# NOTE: Items defined earlier take higher priority
@@ -1236,7 +1236,7 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
return Response(status_code=200, content=response)
def load_log_config(log_config_file: Optional[str]) -> Optional[dict]:
def load_log_config(log_config_file: str | None) -> dict | None:
if not log_config_file:
return None
try:
@@ -1655,7 +1655,7 @@ async def init_app_state(
)
if args.tool_server == "demo":
tool_server: Optional[ToolServer] = DemoToolServer()
tool_server: ToolServer | None = DemoToolServer()
assert isinstance(tool_server, DemoToolServer)
await tool_server.init_and_validate()
elif args.tool_server:
+16 -16
View File
@@ -11,7 +11,7 @@ import json
import ssl
from collections.abc import Sequence
from dataclasses import field
from typing import Literal, Optional, Union
from typing import Literal
from pydantic.dataclasses import dataclass
@@ -39,8 +39,8 @@ class LoRAParserAction(argparse.Action):
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Optional[Union[str, Sequence[str]]],
option_string: Optional[str] = None,
values: str | Sequence[str] | None,
option_string: str | None = None,
):
if values is None:
values = []
@@ -73,11 +73,11 @@ class LoRAParserAction(argparse.Action):
class FrontendArgs:
"""Arguments for the OpenAI-compatible frontend server."""
host: Optional[str] = None
host: str | None = None
"""Host name."""
port: int = 8000
"""Port number."""
uds: Optional[str] = None
uds: str | None = None
"""Unix domain socket path. If set, host and port arguments are ignored."""
uvicorn_log_level: Literal[
"debug", "info", "warning", "error", "critical", "trace"
@@ -93,15 +93,15 @@ class FrontendArgs:
"""Allowed methods."""
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
"""Allowed headers."""
api_key: Optional[list[str]] = None
api_key: list[str] | None = None
"""If provided, the server will require one of these keys to be presented in
the header."""
lora_modules: Optional[list[LoRAModulePath]] = None
lora_modules: list[LoRAModulePath] | None = None
"""LoRA modules configurations in either 'name=path' format or JSON format
or JSON list format. Example (old format): `'name=path'` Example (new
format): `{\"name\": \"name\", \"path\": \"lora_path\",
\"base_model_name\": \"id\"}`"""
chat_template: Optional[str] = None
chat_template: str | None = None
"""The file path to the chat template, or the template in single-line form
for the specified model."""
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
@@ -116,17 +116,17 @@ class FrontendArgs:
or the ones from tokenizer."""
response_role: str = "assistant"
"""The role name to return if `request.add_generation_prompt=true`."""
ssl_keyfile: Optional[str] = None
ssl_keyfile: str | None = None
"""The file path to the SSL key file."""
ssl_certfile: Optional[str] = None
ssl_certfile: str | None = None
"""The file path to the SSL cert file."""
ssl_ca_certs: Optional[str] = None
ssl_ca_certs: str | None = None
"""The CA certificates file."""
enable_ssl_refresh: bool = False
"""Refresh SSL Context when SSL certificate files change"""
ssl_cert_reqs: int = int(ssl.CERT_NONE)
"""Whether client certificate is required (see stdlib ssl module's)."""
root_path: Optional[str] = None
root_path: str | None = None
"""FastAPI root_path when app is behind a path based routing proxy."""
middleware: list[str] = field(default_factory=lambda: [])
"""Additional ASGI middleware to apply to the app. We accept multiple
@@ -149,7 +149,7 @@ class FrontendArgs:
exclude_tools_when_tool_choice_none: bool = False
"""If specified, exclude tool definitions in prompts when
tool_choice='none'."""
tool_call_parser: Optional[str] = None
tool_call_parser: str | None = None
"""Select the tool call parser depending on the model that you're using.
This is used to parse the model-generated tool call into OpenAI API format.
Required for `--enable-auto-tool-choice`. You can choose any option from
@@ -158,13 +158,13 @@ class FrontendArgs:
"""Special the tool parser plugin write to parse the model-generated tool
into OpenAI API format, the name register in this plugin can be used in
`--tool-call-parser`."""
tool_server: Optional[str] = None
tool_server: str | None = None
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for demo
purpose."""
log_config_file: Optional[str] = envs.VLLM_LOGGING_CONFIG_PATH
log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH
"""Path to logging config JSON file for both vllm and uvicorn"""
max_log_len: Optional[int] = None
max_log_len: int | None = None
"""Max number of prompt characters or prompt ID numbers being printed in
log. The default of None means unlimited."""
disable_fastapi_docs: bool = False
+4 -5
View File
@@ -3,7 +3,6 @@
from collections.abc import Iterable
from functools import lru_cache, partial
from typing import Optional, Union
import torch
@@ -16,8 +15,8 @@ class AllowedTokenIdsLogitsProcessor:
specific set of token ids."""
def __init__(self, allowed_ids: Iterable[int]):
self.allowed_ids: Optional[list[int]] = list(allowed_ids)
self.mask: Optional[torch.Tensor] = None
self.allowed_ids: list[int] | None = list(allowed_ids)
self.mask: torch.Tensor | None = None
def __call__(self, token_ids: list[int], logits: torch.Tensor) -> torch.Tensor:
if self.mask is None:
@@ -53,8 +52,8 @@ def logit_bias_logits_processor(
def get_logits_processors(
logit_bias: Optional[Union[dict[int, float], dict[str, float]]],
allowed_token_ids: Optional[list[int]],
logit_bias: dict[int, float] | dict[str, float] | None,
allowed_token_ids: list[int] | None,
tokenizer: AnyTokenizer,
) -> list[LogitsProcessor]:
logits_processors: list[LogitsProcessor] = []
File diff suppressed because it is too large Load Diff
+2 -3
View File
@@ -4,10 +4,9 @@
import asyncio
import tempfile
from argparse import Namespace
from collections.abc import Awaitable
from collections.abc import Awaitable, Callable
from http import HTTPStatus
from io import StringIO
from typing import Callable, Optional
import aiohttp
import torch
@@ -124,7 +123,7 @@ _BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elap
class BatchProgressTracker:
def __init__(self):
self._total = 0
self._pbar: Optional[tqdm] = None
self._pbar: tqdm | None = None
def submitted(self):
self._total += 1
+22 -22
View File
@@ -6,7 +6,7 @@ import json
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Final, Optional, Union
from typing import Final
import jinja2
import partial_json_parser
@@ -82,15 +82,15 @@ class OpenAIServingChat(OpenAIServing):
models: OpenAIServingModels,
response_role: str,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
return_tokens_as_token_ids: bool = False,
reasoning_parser: str = "",
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: Optional[str] = None,
tool_parser: str | None = None,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_log_outputs: bool = False,
@@ -159,8 +159,8 @@ class OpenAIServingChat(OpenAIServing):
async def create_chat_completion(
self,
request: ChatCompletionRequest,
raw_request: Optional[Request] = None,
) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
"""
Chat Completion API similar to OpenAI's API.
@@ -281,7 +281,7 @@ class OpenAIServingChat(OpenAIServing):
default_sampling_params=self.default_sampling_params,
)
sampling_params: Union[SamplingParams, BeamSearchParams]
sampling_params: SamplingParams | BeamSearchParams
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
@@ -416,11 +416,11 @@ class OpenAIServingChat(OpenAIServing):
def extract_tool_call_required_streaming(
self,
previous_text: str,
current_text: Optional[str],
current_text: str | None,
delta_text: str,
function_name_returned: bool,
tool_call_idx: Optional[int] = None,
) -> tuple[Optional[DeltaMessage], bool]:
tool_call_idx: int | None = None,
) -> tuple[DeltaMessage | None, bool]:
if current_text is None or current_text == "":
# if the current text is empty, we cannot parse it
return None, function_name_returned
@@ -548,7 +548,7 @@ class OpenAIServingChat(OpenAIServing):
and self._should_stream_with_auto_tool_parsing(request)
)
all_previous_token_ids: Optional[list[list[int]]]
all_previous_token_ids: list[list[int]] | None
function_name_returned = [False] * num_choices
if self.tool_call_id_type == "kimi_k2":
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
@@ -583,7 +583,7 @@ class OpenAIServingChat(OpenAIServing):
# Prepare the tool parser if it's needed
try:
if tool_choice_auto and self.tool_parser:
tool_parsers: list[Optional[ToolParser]] = [
tool_parsers: list[ToolParser | None] = [
self.tool_parser(tokenizer)
] * num_choices
else:
@@ -661,7 +661,7 @@ class OpenAIServingChat(OpenAIServing):
# Send response to echo the input portion of the
# last message
if request.echo:
last_msg_content: Union[str, list[dict[str, str]]] = ""
last_msg_content: str | list[dict[str, str]] = ""
if (
conversation
and "content" in conversation[-1]
@@ -734,7 +734,7 @@ class OpenAIServingChat(OpenAIServing):
# Chunked prefill case, don't return empty chunks
continue
delta_message: Optional[DeltaMessage]
delta_message: DeltaMessage | None
# just update previous_texts and previous_token_ids
if tool_choice_auto or self.reasoning_parser:
@@ -1260,9 +1260,9 @@ class OpenAIServingChat(OpenAIServing):
conversation: list[ConversationMessage],
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
) -> Union[ErrorResponse, ChatCompletionResponse]:
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: Optional[RequestOutput] = None
final_res: RequestOutput | None = None
try:
async for res in result_generator:
@@ -1512,7 +1512,7 @@ class OpenAIServingChat(OpenAIServing):
choices.append(choice_data)
if request.echo:
last_msg_content: Union[str, list[dict[str, str]]] = ""
last_msg_content: str | list[dict[str, str]] = ""
if (
conversation
and "content" in conversation[-1]
@@ -1597,7 +1597,7 @@ class OpenAIServingChat(OpenAIServing):
def _get_top_logprobs(
self,
logprobs: dict[int, Logprob],
top_logprobs: Optional[int],
top_logprobs: int | None,
tokenizer: AnyTokenizer,
should_return_as_token_id: bool,
) -> list[ChatCompletionLogProb]:
@@ -1621,10 +1621,10 @@ class OpenAIServingChat(OpenAIServing):
def _create_chat_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
tokenizer: AnyTokenizer,
num_output_top_logprobs: Optional[int] = None,
return_as_token_id: Optional[bool] = None,
num_output_top_logprobs: int | None = None,
return_as_token_id: bool | None = None,
) -> ChatCompletionLogProbs:
"""Create OpenAI-style logprobs."""
logprobs_content: list[ChatCompletionLogProbsContent] = []
@@ -1693,7 +1693,7 @@ class OpenAIServingChat(OpenAIServing):
def _should_check_for_unstreamed_tool_arg_tokens(
self,
delta_message: Optional[DeltaMessage],
delta_message: DeltaMessage | None,
output: CompletionOutput,
) -> bool:
"""
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import Optional, Union, cast
from typing import cast
import numpy as np
from fastapi import Request
@@ -36,7 +36,7 @@ class ClassificationMixin(OpenAIServing):
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""
Process classification inputs: tokenize text, resolve adapters,
and prepare model-specific inputs.
@@ -70,7 +70,7 @@ class ClassificationMixin(OpenAIServing):
def _build_response(
self,
ctx: ServeContext,
) -> Union[ClassificationResponse, ErrorResponse]:
) -> ClassificationResponse | ErrorResponse:
"""
Convert model outputs to a formatted classification response
with probabilities and labels.
@@ -129,7 +129,7 @@ class ServingClassification(ClassificationMixin):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
log_error_stack: bool = False,
) -> None:
super().__init__(
@@ -143,7 +143,7 @@ class ServingClassification(ClassificationMixin):
self,
request: ClassificationRequest,
raw_request: Request,
) -> Union[ClassificationResponse, ErrorResponse]:
) -> ClassificationResponse | ErrorResponse:
model_name = self.models.model_name()
request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
@@ -160,7 +160,7 @@ class ServingClassification(ClassificationMixin):
def _create_pooling_params(
self,
ctx: ClassificationServeContext,
) -> Union[PoolingParams, ErrorResponse]:
) -> PoolingParams | ErrorResponse:
pooling_params = super()._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
+16 -16
View File
@@ -5,7 +5,7 @@ import asyncio
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Optional, Union, cast
from typing import cast
import jinja2
from fastapi import Request
@@ -45,7 +45,7 @@ class OpenAIServingCompletion(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
@@ -73,8 +73,8 @@ class OpenAIServingCompletion(OpenAIServing):
async def create_completion(
self,
request: CompletionRequest,
raw_request: Optional[Request] = None,
) -> Union[AsyncGenerator[str, None], CompletionResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
"""Completion API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/completions/create
@@ -166,7 +166,7 @@ class OpenAIServingCompletion(OpenAIServing):
default_sampling_params=self.default_sampling_params,
)
sampling_params: Union[SamplingParams, BeamSearchParams]
sampling_params: SamplingParams | BeamSearchParams
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
@@ -196,7 +196,7 @@ class OpenAIServingCompletion(OpenAIServing):
# Mypy inconsistently requires this second cast in different
# environments. It shouldn't be necessary (redundant from above)
# but pre-commit in CI fails without it.
engine_prompt = cast(Union[EmbedsPrompt, TokensPrompt], engine_prompt)
engine_prompt = cast(EmbedsPrompt | TokensPrompt, engine_prompt)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_prompt,
@@ -260,7 +260,7 @@ class OpenAIServingCompletion(OpenAIServing):
)
# Non-streaming response
final_res_batch: list[Optional[RequestOutput]] = [None] * num_prompts
final_res_batch: list[RequestOutput | None] = [None] * num_prompts
try:
async for i, res in result_generator:
final_res_batch[i] = res
@@ -312,7 +312,7 @@ class OpenAIServingCompletion(OpenAIServing):
async def completion_stream_generator(
self,
request: CompletionRequest,
engine_prompts: list[Union[TokensPrompt, EmbedsPrompt]],
engine_prompts: list[TokensPrompt | EmbedsPrompt],
result_generator: AsyncIterator[tuple[int, RequestOutput]],
request_id: str,
created_time: int,
@@ -362,7 +362,7 @@ class OpenAIServingCompletion(OpenAIServing):
num_prompt_tokens[prompt_idx] = len(prompt_token_ids)
delta_token_ids: GenericSequence[int]
out_logprobs: Optional[GenericSequence[Optional[dict[int, Logprob]]]]
out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
for output in res.outputs:
i = output.index + prompt_idx * num_choices
@@ -370,7 +370,7 @@ class OpenAIServingCompletion(OpenAIServing):
# Useful when request.return_token_ids is True
# Returning prompt token IDs shares the same logic
# with the echo implementation.
prompt_token_ids_to_return: Optional[list[int]] = None
prompt_token_ids_to_return: list[int] | None = None
assert request.max_tokens is not None
if request.echo and not has_echoed[i]:
@@ -524,7 +524,7 @@ class OpenAIServingCompletion(OpenAIServing):
prompt_text = final_res.prompt
token_ids: GenericSequence[int]
out_logprobs: Optional[GenericSequence[Optional[dict[int, Logprob]]]]
out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
for output in final_res.outputs:
assert request.max_tokens is not None
@@ -617,17 +617,17 @@ class OpenAIServingCompletion(OpenAIServing):
def _create_completion_logprobs(
self,
token_ids: GenericSequence[int],
top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
top_logprobs: GenericSequence[dict[int, Logprob] | None],
num_output_top_logprobs: int,
tokenizer: AnyTokenizer,
initial_text_offset: int = 0,
return_as_token_id: Optional[bool] = None,
return_as_token_id: bool | None = None,
) -> CompletionLogProbs:
"""Create logprobs for OpenAI Completion API."""
out_text_offset: list[int] = []
out_token_logprobs: list[Optional[float]] = []
out_token_logprobs: list[float | None] = []
out_tokens: list[str] = []
out_top_logprobs: list[Optional[dict[str, float]]] = []
out_top_logprobs: list[dict[str, float] | None] = []
last_token_len = 0
@@ -695,7 +695,7 @@ class OpenAIServingCompletion(OpenAIServing):
def _build_render_config(
self,
request: CompletionRequest,
max_input_length: Optional[int] = None,
max_input_length: int | None = None,
) -> RenderConfig:
max_input_tokens_len = self.max_model_len - (request.max_tokens or 0)
return RenderConfig(
+17 -19
View File
@@ -3,7 +3,7 @@
import base64
from collections.abc import AsyncGenerator, Mapping
from typing import Any, Final, Literal, Optional, Union, cast
from typing import Any, Final, Literal, cast
import numpy as np
import torch
@@ -48,7 +48,7 @@ logger = init_logger(__name__)
def _get_embedding(
output: EmbeddingOutput,
encoding_format: Literal["float", "base64"],
) -> Union[list[float], str]:
) -> list[float] | str:
if encoding_format == "float":
return output.embedding
elif encoding_format == "base64":
@@ -80,7 +80,7 @@ class EmbeddingMixin(OpenAIServing):
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
ctx = cast(EmbeddingServeContext, ctx)
try:
ctx.lora_request = self._maybe_get_adapters(ctx.request)
@@ -130,7 +130,7 @@ class EmbeddingMixin(OpenAIServing):
def _build_response(
self,
ctx: ServeContext,
) -> Union[EmbeddingResponse, ErrorResponse]:
) -> EmbeddingResponse | ErrorResponse:
items: list[EmbeddingResponseData] = []
num_prompt_tokens = 0
@@ -314,9 +314,9 @@ class EmbeddingMixin(OpenAIServing):
ctx: EmbeddingServeContext,
engine_prompt: EngineTokensPrompt,
pooling_params: PoolingParams,
trace_headers: Optional[Mapping[str, str]],
trace_headers: Mapping[str, str] | None,
prompt_index: int,
) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
) -> AsyncGenerator[RequestOutput | PoolingRequestOutput, None]:
"""Create a generator for a single prompt using standard processing."""
request_id_item = f"{ctx.request_id}-{prompt_index}"
@@ -341,7 +341,7 @@ class EmbeddingMixin(OpenAIServing):
async def _prepare_generators(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Override to support chunked processing."""
ctx = cast(EmbeddingServeContext, ctx)
@@ -354,7 +354,7 @@ class EmbeddingMixin(OpenAIServing):
# Custom logic for chunked processing
generators: list[
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
] = []
try:
@@ -413,7 +413,7 @@ class EmbeddingMixin(OpenAIServing):
async def _collect_batch(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Collect and aggregate batch results
with support for chunked processing.
@@ -522,9 +522,7 @@ class EmbeddingMixin(OpenAIServing):
)
# Finalize aggregated results
final_res_batch: list[
Union[PoolingRequestOutput, EmbeddingRequestOutput]
] = []
final_res_batch: list[PoolingRequestOutput | EmbeddingRequestOutput] = []
num_prompts = len(ctx.engine_prompts)
for prompt_idx in range(num_prompts):
@@ -581,7 +579,7 @@ class EmbeddingMixin(OpenAIServing):
)
ctx.final_res_batch = cast(
list[Union[RequestOutput, PoolingRequestOutput]], final_res_batch
list[RequestOutput | PoolingRequestOutput], final_res_batch
)
return None
@@ -598,8 +596,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
@@ -618,8 +616,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
async def create_embedding(
self,
request: EmbeddingRequest,
raw_request: Optional[Request] = None,
) -> Union[EmbeddingResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> EmbeddingResponse | ErrorResponse:
"""
Embedding API similar to OpenAI's API.
@@ -647,7 +645,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
def _create_pooling_params(
self,
ctx: ServeContext[EmbeddingRequest],
) -> Union[PoolingParams, ErrorResponse]:
) -> PoolingParams | ErrorResponse:
pooling_params = super()._create_pooling_params(ctx)
if isinstance(pooling_params, ErrorResponse):
return pooling_params
@@ -662,7 +660,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
if isinstance(ctx.request, EmbeddingChatRequest):
error_check_ret = self._validate_chat_template(
request_chat_template=ctx.request.chat_template,
+93 -95
View File
@@ -5,10 +5,10 @@ import json
import sys
import time
import traceback
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from http import HTTPStatus
from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
import torch
from fastapi import Request
@@ -102,38 +102,38 @@ from vllm.v1.engine import EngineCoreRequest
logger = init_logger(__name__)
CompletionLikeRequest = Union[
CompletionRequest,
DetokenizeRequest,
EmbeddingCompletionRequest,
RerankRequest,
ClassificationRequest,
ScoreRequest,
TokenizeCompletionRequest,
]
CompletionLikeRequest: TypeAlias = (
CompletionRequest
| DetokenizeRequest
| EmbeddingCompletionRequest
| RerankRequest
| ClassificationRequest
| ScoreRequest
| TokenizeCompletionRequest
)
ChatLikeRequest = Union[
ChatCompletionRequest, EmbeddingChatRequest, TokenizeChatRequest
]
SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]
AnyRequest = Union[
CompletionLikeRequest,
ChatLikeRequest,
SpeechToTextRequest,
ResponsesRequest,
IOProcessorRequest,
]
ChatLikeRequest: TypeAlias = (
ChatCompletionRequest | EmbeddingChatRequest | TokenizeChatRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
AnyRequest: TypeAlias = (
CompletionLikeRequest
| ChatLikeRequest
| SpeechToTextRequest
| ResponsesRequest
| IOProcessorRequest
)
AnyResponse = Union[
CompletionResponse,
ChatCompletionResponse,
EmbeddingResponse,
TranscriptionResponse,
TokenizeResponse,
PoolingResponse,
ClassificationResponse,
ScoreResponse,
]
AnyResponse: TypeAlias = (
CompletionResponse
| ChatCompletionResponse
| EmbeddingResponse
| TranscriptionResponse
| TokenizeResponse
| PoolingResponse
| ClassificationResponse
| ScoreResponse
)
class TextTokensPrompt(TypedDict):
@@ -145,7 +145,7 @@ class EmbedsPrompt(TypedDict):
prompt_embeds: torch.Tensor
RequestPrompt = Union[list[int], str, TextTokensPrompt, EmbedsPrompt]
RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt
def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]:
@@ -173,8 +173,8 @@ class RequestProcessingMixin(BaseModel):
handling prompt preparation and engine input.
"""
request_prompts: Optional[Sequence[RequestPrompt]] = []
engine_prompts: Optional[list[EngineTokensPrompt]] = []
request_prompts: Sequence[RequestPrompt] | None = []
engine_prompts: list[EngineTokensPrompt] | None = []
model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -185,10 +185,10 @@ class ResponseGenerationMixin(BaseModel):
managing result generators and final batch results.
"""
result_generator: Optional[
AsyncGenerator[tuple[int, Union[RequestOutput, PoolingRequestOutput]], None]
] = None
final_res_batch: list[Union[RequestOutput, PoolingRequestOutput]] = Field(
result_generator: (
AsyncGenerator[tuple[int, RequestOutput | PoolingRequestOutput], None] | None
) = None
final_res_batch: list[RequestOutput | PoolingRequestOutput] = Field(
default_factory=list
)
@@ -203,14 +203,14 @@ class ServeContext(
):
# Shared across all requests
request: RequestT
raw_request: Optional[Request] = None
raw_request: Request | None = None
model_name: str
request_id: str
created_time: int = Field(default_factory=lambda: int(time.time()))
lora_request: Optional[LoRARequest] = None
lora_request: LoRARequest | None = None
# Shared across most requests
tokenizer: Optional[AnyTokenizer] = None
tokenizer: AnyTokenizer | None = None
# `protected_namespaces` resolves Pydantic v2's warning
# on conflict with protected namespace "model_"
@@ -224,7 +224,7 @@ ClassificationServeContext = ServeContext[ClassificationRequest]
class EmbeddingServeContext(ServeContext[EmbeddingRequest]):
chat_template: Optional[str] = None
chat_template: str | None = None
chat_template_content_format: ChatTemplateContentFormatOption
@@ -247,7 +247,7 @@ class OpenAIServing:
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
log_error_stack: bool = False,
@@ -276,8 +276,8 @@ class OpenAIServing:
self.max_model_len = self.model_config.max_model_len
def _get_tool_parser(
self, tool_parser_name: Optional[str] = None, enable_auto_tools: bool = False
) -> Optional[Callable[[AnyTokenizer], ToolParser]]:
self, tool_parser_name: str | None = None, enable_auto_tools: bool = False
) -> Callable[[AnyTokenizer], ToolParser] | None:
"""Get the tool parser based on the name."""
parser = None
if not enable_auto_tools or tool_parser_name is None:
@@ -307,7 +307,7 @@ class OpenAIServing:
def _get_reasoning_parser(
self,
reasoning_parser_name: str,
) -> Optional[Callable[[AnyTokenizer], ReasoningParser]]:
) -> Callable[[AnyTokenizer], ReasoningParser] | None:
"""Get the reasoning parser based on the name."""
parser = None
if not reasoning_parser_name:
@@ -328,7 +328,7 @@ class OpenAIServing:
prompt: PromptType,
request_id: str,
params: BeamSearchParams,
lora_request: Optional[LoRARequest] = None,
lora_request: LoRARequest | None = None,
) -> AsyncGenerator[RequestOutput, None]:
beam_width = params.beam_width
max_tokens = params.max_tokens
@@ -364,9 +364,9 @@ class OpenAIServing:
# this happens again in generation, so the double expansion causes
# a mismatch.
# TODO - would be ideal to handle this more gracefully.
prompt_text: Optional[str]
prompt_text: str | None
prompt_token_ids: list[int]
multi_modal_data: Optional[MultiModalDataDict]
multi_modal_data: MultiModalDataDict | None
if isinstance(prompt, str):
prompt_text = prompt
prompt_token_ids = []
@@ -376,7 +376,7 @@ class OpenAIServing:
prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore
multi_modal_data = prompt.get("multi_modal_data") # type: ignore
mm_processor_kwargs: Optional[dict[str, Any]] = processed_inputs.get(
mm_processor_kwargs: dict[str, Any] | None = processed_inputs.get(
"mm_processor_kwargs"
) # type: ignore
@@ -507,7 +507,7 @@ class OpenAIServing:
prompt_logprobs=None,
)
def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:
def _get_renderer(self, tokenizer: AnyTokenizer | None) -> BaseRenderer:
"""
Get a Renderer instance with the provided tokenizer.
Uses shared async tokenizer pool for efficiency.
@@ -545,7 +545,7 @@ class OpenAIServing:
async def _preprocess(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""
Default preprocessing hook. Subclasses may override
to prepare `ctx` (classification, embedding, etc.).
@@ -555,7 +555,7 @@ class OpenAIServing:
def _build_response(
self,
ctx: ServeContext,
) -> Union[AnyResponse, ErrorResponse]:
) -> AnyResponse | ErrorResponse:
"""
Default response builder. Subclass may override this method
to return the appropriate response object.
@@ -565,8 +565,8 @@ class OpenAIServing:
async def handle(
self,
ctx: ServeContext,
) -> Union[AnyResponse, ErrorResponse]:
generation: AsyncGenerator[Union[AnyResponse, ErrorResponse], None]
) -> AnyResponse | ErrorResponse:
generation: AsyncGenerator[AnyResponse | ErrorResponse, None]
generation = self._pipeline(ctx)
async for response in generation:
@@ -577,7 +577,7 @@ class OpenAIServing:
async def _pipeline(
self,
ctx: ServeContext,
) -> AsyncGenerator[Union[AnyResponse, ErrorResponse], None]:
) -> AsyncGenerator[AnyResponse | ErrorResponse, None]:
"""Execute the request processing pipeline yielding responses."""
if error := await self._check_model(ctx.request):
yield error
@@ -598,7 +598,7 @@ class OpenAIServing:
yield self._build_response(ctx)
def _validate_request(self, ctx: ServeContext) -> Optional[ErrorResponse]:
def _validate_request(self, ctx: ServeContext) -> ErrorResponse | None:
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
if (
@@ -615,7 +615,7 @@ class OpenAIServing:
def _create_pooling_params(
self,
ctx: ServeContext,
) -> Union[PoolingParams, ErrorResponse]:
) -> PoolingParams | ErrorResponse:
if not hasattr(ctx.request, "to_pooling_params"):
return self.create_error_response(
"Request type does not support pooling parameters"
@@ -626,10 +626,10 @@ class OpenAIServing:
async def _prepare_generators(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Schedule the request and get the result generator."""
generators: list[
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
] = []
try:
@@ -678,14 +678,14 @@ class OpenAIServing:
async def _collect_batch(
self,
ctx: ServeContext,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Collect batch results from the result generator."""
try:
if ctx.engine_prompts is None:
return self.create_error_response("Engine prompts not available")
num_prompts = len(ctx.engine_prompts)
final_res_batch: list[Optional[Union[RequestOutput, PoolingRequestOutput]]]
final_res_batch: list[RequestOutput | PoolingRequestOutput | None]
final_res_batch = [None] * num_prompts
if ctx.result_generator is None:
@@ -738,7 +738,7 @@ class OpenAIServing:
async def _check_model(
self,
request: AnyRequest,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
error_response = None
if self._is_model_supported(request.model):
@@ -764,9 +764,7 @@ class OpenAIServing:
status_code=HTTPStatus.NOT_FOUND,
)
def _get_active_default_mm_loras(
self, request: AnyRequest
) -> Optional[LoRARequest]:
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
@@ -793,7 +791,7 @@ class OpenAIServing:
self,
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> Optional[LoRARequest]:
) -> LoRARequest | None:
if request.model in self.models.lora_requests:
return self.models.lora_requests[request.model]
@@ -877,7 +875,7 @@ class OpenAIServing:
self,
request: AnyRequest,
prompt_ids: list[int],
tokenizer: Optional[AnyTokenizer],
tokenizer: AnyTokenizer | None,
) -> TextTokensPrompt:
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
@@ -972,7 +970,7 @@ class OpenAIServing:
self,
request: AnyRequest,
tokenizer: AnyTokenizer,
prompt_input: Union[str, list[int]],
prompt_input: str | list[int],
add_special_tokens: bool = True,
) -> TextTokensPrompt:
"""
@@ -991,7 +989,7 @@ class OpenAIServing:
self,
request: AnyRequest,
tokenizer: AnyTokenizer,
prompt_inputs: Iterable[Union[str, list[int]]],
prompt_inputs: Iterable[str | list[int]],
add_special_tokens: bool = True,
) -> AsyncGenerator[TextTokensPrompt, None]:
"""
@@ -1014,10 +1012,10 @@ class OpenAIServing:
def _validate_chat_template(
self,
request_chat_template: Optional[str],
chat_template_kwargs: Optional[dict[str, Any]],
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
if not trust_request_chat_template and (
request_chat_template is not None
or (
@@ -1034,17 +1032,17 @@ class OpenAIServing:
async def _preprocess_chat(
self,
request: Union[ChatLikeRequest, ResponsesRequest],
request: ChatLikeRequest | ResponsesRequest,
tokenizer: AnyTokenizer,
messages: list[ChatCompletionMessageParam],
chat_template: Optional[str],
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
add_generation_prompt: bool = True,
continue_final_message: bool = False,
tool_dicts: Optional[list[dict[str, Any]]] = None,
documents: Optional[list[dict[str, str]]] = None,
chat_template_kwargs: Optional[dict[str, Any]] = None,
tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
tool_dicts: list[dict[str, Any]] | None = None,
documents: list[dict[str, str]] | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
tool_parser: Callable[[AnyTokenizer], ToolParser] | None = None,
add_special_tokens: bool = False,
) -> tuple[
list[ConversationMessage],
@@ -1076,7 +1074,7 @@ class OpenAIServing:
)
_chat_template_kwargs.update(chat_template_kwargs or {})
request_prompt: Union[str, list[int]]
request_prompt: str | list[int]
if tokenizer is None:
request_prompt = "placeholder"
@@ -1158,10 +1156,10 @@ class OpenAIServing:
self,
request_id: str,
engine_prompt: PromptType,
params: Union[SamplingParams, PoolingParams],
params: SamplingParams | PoolingParams,
*,
lora_request: Optional[LoRARequest],
trace_headers: Optional[Mapping[str, str]],
lora_request: LoRARequest | None,
trace_headers: Mapping[str, str] | None,
priority: int,
) -> tuple[EngineCoreRequest, dict[str, Any]]:
"""Use the Processor to process inputs for AsyncLLM."""
@@ -1188,7 +1186,7 @@ class OpenAIServing:
engine_prompt: EngineTokensPrompt,
sampling_params: SamplingParams,
context: ConversationContext,
lora_request: Optional[LoRARequest] = None,
lora_request: LoRARequest | None = None,
priority: int = 0,
**kwargs,
):
@@ -1250,7 +1248,7 @@ class OpenAIServing:
def _get_prompt_components(
self,
prompt: Union[RequestPrompt, PromptType],
prompt: RequestPrompt | PromptType,
) -> PromptComponents:
if isinstance(prompt, list):
return PromptComponents(token_ids=prompt)
@@ -1260,9 +1258,9 @@ class OpenAIServing:
def _log_inputs(
self,
request_id: str,
inputs: Union[RequestPrompt, PromptType],
params: Optional[Union[SamplingParams, PoolingParams, BeamSearchParams]],
lora_request: Optional[LoRARequest],
inputs: RequestPrompt | PromptType,
params: SamplingParams | PoolingParams | BeamSearchParams | None,
lora_request: LoRARequest | None,
) -> None:
if self.request_logger is None:
return
@@ -1281,7 +1279,7 @@ class OpenAIServing:
async def _get_trace_headers(
self,
headers: Headers,
) -> Optional[Mapping[str, str]]:
) -> Mapping[str, str] | None:
is_tracing_enabled = await self.engine_client.is_tracing_enabled()
if is_tracing_enabled:
@@ -1294,8 +1292,8 @@ class OpenAIServing:
@staticmethod
def _base_request_id(
raw_request: Optional[Request], default: Optional[str] = None
) -> Optional[str]:
raw_request: Request | None, default: str | None = None
) -> str | None:
"""Pulls the request id to use from a header, if provided"""
default = default or random_uuid()
if raw_request is None:
@@ -1317,15 +1315,15 @@ class OpenAIServing:
return logprob.decoded_token
return tokenizer.decode(token_id)
def _is_model_supported(self, model_name: Optional[str]) -> bool:
def _is_model_supported(self, model_name: str | None) -> bool:
if not model_name:
return True
return self.models.is_base_model(model_name)
def clamp_prompt_logprobs(
prompt_logprobs: Union[PromptLogprobs, None],
) -> Union[PromptLogprobs, None]:
prompt_logprobs: PromptLogprobs | None,
) -> PromptLogprobs | None:
if prompt_logprobs is None:
return prompt_logprobs
+9 -10
View File
@@ -5,7 +5,6 @@ from asyncio import Lock
from collections import defaultdict
from dataclasses import dataclass
from http import HTTPStatus
from typing import Optional, Union
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.protocol import (
@@ -35,7 +34,7 @@ class BaseModelPath:
class LoRAModulePath:
name: str
path: str
base_model_name: Optional[str] = None
base_model_name: str | None = None
class OpenAIServingModels:
@@ -52,7 +51,7 @@ class OpenAIServingModels:
engine_client: EngineClient,
base_model_paths: list[BaseModelPath],
*,
lora_modules: Optional[list[LoRAModulePath]] = None,
lora_modules: list[LoRAModulePath] | None = None,
):
super().__init__()
@@ -93,7 +92,7 @@ class OpenAIServingModels:
def is_base_model(self, model_name) -> bool:
return any(model.name == model_name for model in self.base_model_paths)
def model_name(self, lora_request: Optional[LoRARequest] = None) -> str:
def model_name(self, lora_request: LoRARequest | None = None) -> str:
"""Returns the appropriate model name depending on the availability
and support of the LoRA or base model.
Parameters:
@@ -132,8 +131,8 @@ class OpenAIServingModels:
return ModelList(data=model_cards)
async def load_lora_adapter(
self, request: LoadLoRAAdapterRequest, base_model_name: Optional[str] = None
) -> Union[ErrorResponse, str]:
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
) -> ErrorResponse | str:
lora_name = request.lora_name
# Ensure atomicity based on the lora name
@@ -173,7 +172,7 @@ class OpenAIServingModels:
async def unload_lora_adapter(
self, request: UnloadLoRAAdapterRequest
) -> Union[ErrorResponse, str]:
) -> ErrorResponse | str:
lora_name = request.lora_name
# Ensure atomicity based on the lora name
@@ -189,7 +188,7 @@ class OpenAIServingModels:
async def _check_load_lora_adapter_request(
self, request: LoadLoRAAdapterRequest
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
# Check if both 'lora_name' and 'lora_path' are provided
if not request.lora_name or not request.lora_path:
return create_error_response(
@@ -211,7 +210,7 @@ class OpenAIServingModels:
async def _check_unload_lora_adapter_request(
self, request: UnloadLoRAAdapterRequest
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
# Check if 'lora_name' is not provided return an error
if not request.lora_name:
return create_error_response(
@@ -230,7 +229,7 @@ class OpenAIServingModels:
return None
async def resolve_lora(self, lora_name: str) -> Union[LoRARequest, ErrorResponse]:
async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse:
"""Attempt to resolve a LoRA adapter using available resolvers.
Args:
+7 -7
View File
@@ -5,7 +5,7 @@ import asyncio
import base64
import time
from collections.abc import AsyncGenerator
from typing import Final, Literal, Optional, Union, cast
from typing import Final, Literal, cast
import jinja2
import numpy as np
@@ -41,7 +41,7 @@ logger = init_logger(__name__)
def _get_data(
output: PoolingOutput,
encoding_format: Literal["float", "base64"],
) -> Union[list[float], str]:
) -> list[float] | str:
if encoding_format == "float":
return output.data.tolist()
elif encoding_format == "base64":
@@ -60,8 +60,8 @@ class OpenAIServingPooling(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
@@ -80,8 +80,8 @@ class OpenAIServingPooling(OpenAIServing):
async def create_pooling(
self,
request: PoolingRequest,
raw_request: Optional[Request] = None,
) -> Union[PoolingResponse, IOProcessorResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> PoolingResponse | IOProcessorResponse | ErrorResponse:
"""
See https://platform.openai.com/docs/api-reference/embeddings/create
for the API specification. This API mimics the OpenAI Embedding API.
@@ -219,7 +219,7 @@ class OpenAIServingPooling(OpenAIServing):
num_prompts = len(engine_prompts)
# Non-streaming response
final_res_batch: list[Optional[PoolingRequestOutput]]
final_res_batch: list[PoolingRequestOutput | None]
final_res_batch = [None] * num_prompts
try:
async for i, res in result_generator:
+36 -36
View File
@@ -6,11 +6,11 @@ import json
import time
import uuid
from collections import deque
from collections.abc import AsyncGenerator, AsyncIterator, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
from contextlib import AsyncExitStack
from copy import copy
from http import HTTPStatus
from typing import Callable, Final, Optional, Union
from typing import Final
import jinja2
from fastapi import Request
@@ -109,14 +109,14 @@ class OpenAIServingResponses(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
return_tokens_as_token_ids: bool = False,
reasoning_parser: str = "",
enable_auto_tools: bool = False,
tool_parser: Optional[str] = None,
tool_server: Optional[ToolServer] = None,
tool_parser: str | None = None,
tool_server: ToolServer | None = None,
enable_prompt_tokens_details: bool = False,
enable_force_include_usage: bool = False,
enable_log_outputs: bool = False,
@@ -210,7 +210,7 @@ class OpenAIServingResponses(OpenAIServing):
def _validate_generator_input(
self, engine_prompt: EngineTokensPrompt
) -> Optional[ErrorResponse]:
) -> ErrorResponse | None:
"""Add validations to the input to the generator here."""
if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
error_message = (
@@ -229,12 +229,12 @@ class OpenAIServingResponses(OpenAIServing):
async def create_responses(
self,
request: ResponsesRequest,
raw_request: Optional[Request] = None,
) -> Union[
AsyncGenerator[StreamingResponsesResponse, None],
ResponsesResponse,
ErrorResponse,
]:
raw_request: Request | None = None,
) -> (
AsyncGenerator[StreamingResponsesResponse, None]
| ResponsesResponse
| ErrorResponse
):
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
@@ -460,7 +460,7 @@ class OpenAIServingResponses(OpenAIServing):
async def _make_request(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse],
prev_response: ResponsesResponse | None,
tokenizer: AnyTokenizer,
):
if len(request.tools) > 0:
@@ -481,7 +481,7 @@ class OpenAIServingResponses(OpenAIServing):
def _make_request_with_harmony(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse],
prev_response: ResponsesResponse | None,
):
if request.tool_choice != "auto":
raise NotImplementedError(
@@ -522,8 +522,8 @@ class OpenAIServingResponses(OpenAIServing):
model_name: str,
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
created_time: Optional[int] = None,
) -> Union[ErrorResponse, ResponsesResponse]:
created_time: int | None = None,
) -> ErrorResponse | ResponsesResponse:
if created_time is None:
created_time = int(time.time())
@@ -642,9 +642,9 @@ class OpenAIServingResponses(OpenAIServing):
def _create_response_logprobs(
self,
token_ids: Sequence[int],
logprobs: Optional[SampleLogprobs],
logprobs: SampleLogprobs | None,
tokenizer: AnyTokenizer,
top_logprobs: Optional[int] = None,
top_logprobs: int | None = None,
) -> list[Logprob]:
assert logprobs is not None, "logprobs must be provided"
assert len(token_ids) == len(logprobs), (
@@ -676,9 +676,9 @@ class OpenAIServingResponses(OpenAIServing):
def _create_stream_response_logprobs(
self,
token_ids: Sequence[int],
logprobs: Optional[SampleLogprobs],
logprobs: SampleLogprobs | None,
tokenizer: AnyTokenizer,
top_logprobs: Optional[int] = None,
top_logprobs: int | None = None,
) -> list[response_text_delta_event.Logprob]:
lgs = self._create_response_logprobs(
token_ids=token_ids,
@@ -793,7 +793,7 @@ class OpenAIServingResponses(OpenAIServing):
def _construct_input_messages(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse] = None,
prev_response: ResponsesResponse | None = None,
) -> list[ChatCompletionMessageParam]:
messages: list[ChatCompletionMessageParam] = []
if request.instructions:
@@ -833,7 +833,7 @@ class OpenAIServingResponses(OpenAIServing):
def _construct_input_messages_with_harmony(
self,
request: ResponsesRequest,
prev_response: Optional[ResponsesResponse],
prev_response: ResponsesResponse | None,
) -> list[OpenAIHarmonyMessage]:
messages: list[OpenAIHarmonyMessage] = []
if prev_response is None:
@@ -986,7 +986,7 @@ class OpenAIServingResponses(OpenAIServing):
async def responses_background_stream_generator(
self,
response_id: str,
starting_after: Optional[int] = None,
starting_after: int | None = None,
) -> AsyncGenerator[StreamingResponsesResponse, None]:
if response_id not in self.event_store:
raise ValueError(f"Unknown response_id: {response_id}")
@@ -1011,13 +1011,13 @@ class OpenAIServingResponses(OpenAIServing):
async def retrieve_responses(
self,
response_id: str,
starting_after: Optional[int],
stream: Optional[bool],
) -> Union[
ErrorResponse,
ResponsesResponse,
AsyncGenerator[StreamingResponsesResponse, None],
]:
starting_after: int | None,
stream: bool | None,
) -> (
ErrorResponse
| ResponsesResponse
| AsyncGenerator[StreamingResponsesResponse, None]
):
async with self.response_store_lock:
response = self.response_store.get(response_id)
@@ -1034,7 +1034,7 @@ class OpenAIServingResponses(OpenAIServing):
async def cancel_responses(
self,
response_id: str,
) -> Union[ErrorResponse, ResponsesResponse]:
) -> ErrorResponse | ResponsesResponse:
async with self.response_store_lock:
response = self.response_store.get(response_id)
if response is None:
@@ -1082,7 +1082,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
sampling_params: SamplingParams,
result_generator: AsyncIterator[Optional[ConversationContext]],
result_generator: AsyncIterator[ConversationContext | None],
context: ConversationContext,
model_name: str,
tokenizer: AnyTokenizer,
@@ -1385,7 +1385,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
sampling_params: SamplingParams,
result_generator: AsyncIterator[Optional[ConversationContext]],
result_generator: AsyncIterator[ConversationContext | None],
context: ConversationContext,
model_name: str,
tokenizer: AnyTokenizer,
@@ -1772,12 +1772,12 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
sampling_params: SamplingParams,
result_generator: AsyncIterator[Optional[ConversationContext]],
result_generator: AsyncIterator[ConversationContext | None],
context: ConversationContext,
model_name: str,
tokenizer: AnyTokenizer,
request_metadata: RequestResponseMetadata,
created_time: Optional[int] = None,
created_time: int | None = None,
) -> AsyncGenerator[StreamingResponsesResponse, None]:
# TODO:
# 1. Handle disconnect
+29 -29
View File
@@ -3,7 +3,7 @@
import asyncio
import time
from collections.abc import AsyncGenerator, Mapping
from typing import Any, Optional, Union
from typing import Any
from fastapi import Request
@@ -48,7 +48,7 @@ class ServingScores(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
log_error_stack: bool = False,
) -> None:
super().__init__(
@@ -63,12 +63,12 @@ class ServingScores(OpenAIServing):
tokenizer: AnyTokenizer,
texts_1: list[str],
texts_2: list[str],
request: Union[RerankRequest, ScoreRequest],
request: RerankRequest | ScoreRequest,
request_id: str,
tokenization_kwargs: Optional[dict[str, Any]] = None,
lora_request: Optional[Union[LoRARequest, None]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
tokenization_kwargs: dict[str, Any] | None = None,
lora_request: LoRARequest | None | None = None,
trace_headers: Mapping[str, str] | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
input_texts = texts_1 + texts_2
engine_prompts: list[TokensPrompt] = []
@@ -125,7 +125,7 @@ class ServingScores(OpenAIServing):
# Non-streaming response
final_res_batch: list[PoolingRequestOutput] = []
embeddings: list[Optional[PoolingRequestOutput]] = [None] * len(engine_prompts)
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)
async for i, res in result_generator:
embeddings[i] = res
@@ -152,11 +152,11 @@ class ServingScores(OpenAIServing):
def _preprocess_score(
self,
request: Union[RerankRequest, ScoreRequest],
request: RerankRequest | ScoreRequest,
tokenizer: AnyTokenizer,
tokenization_kwargs: dict[str, Any],
data_1: Union[str, ScoreContentPartParam],
data_2: Union[str, ScoreContentPartParam],
data_1: str | ScoreContentPartParam,
data_2: str | ScoreContentPartParam,
) -> tuple[str, TokensPrompt]:
model_config = self.model_config
@@ -176,14 +176,14 @@ class ServingScores(OpenAIServing):
async def _cross_encoding_score(
self,
tokenizer: AnyTokenizer,
data_1: Union[list[str], list[ScoreContentPartParam]],
data_2: Union[list[str], list[ScoreContentPartParam]],
request: Union[RerankRequest, ScoreRequest],
data_1: list[str] | list[ScoreContentPartParam],
data_2: list[str] | list[ScoreContentPartParam],
request: RerankRequest | ScoreRequest,
request_id: str,
tokenization_kwargs: Optional[dict[str, Any]] = None,
lora_request: Optional[Union[LoRARequest, None]] = None,
trace_headers: Optional[Mapping[str, str]] = None,
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
tokenization_kwargs: dict[str, Any] | None = None,
lora_request: LoRARequest | None | None = None,
trace_headers: Mapping[str, str] | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
request_prompts: list[str] = []
engine_prompts: list[TokensPrompt] = []
@@ -259,7 +259,7 @@ class ServingScores(OpenAIServing):
result_generator = merge_async_iterators(*generators)
# Non-streaming response
final_res_batch: list[Optional[PoolingRequestOutput]] = [None] * len(
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
engine_prompts
)
@@ -270,12 +270,12 @@ class ServingScores(OpenAIServing):
async def _run_scoring(
self,
data_1: Union[list[str], str, ScoreMultiModalParam],
data_2: Union[list[str], str, ScoreMultiModalParam],
request: Union[ScoreRequest, RerankRequest],
data_1: list[str] | str | ScoreMultiModalParam,
data_2: list[str] | str | ScoreMultiModalParam,
request: ScoreRequest | RerankRequest,
request_id: str,
raw_request: Optional[Request] = None,
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
raw_request: Request | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
lora_request = self._maybe_get_adapters(request)
tokenizer = await self.engine_client.get_tokenizer()
@@ -339,8 +339,8 @@ class ServingScores(OpenAIServing):
async def create_score(
self,
request: ScoreRequest,
raw_request: Optional[Request] = None,
) -> Union[ScoreResponse, ErrorResponse]:
raw_request: Request | None = None,
) -> ScoreResponse | ErrorResponse:
"""
Score API similar to Sentence Transformers cross encoder
@@ -377,8 +377,8 @@ class ServingScores(OpenAIServing):
return self.create_error_response(str(e))
async def do_rerank(
self, request: RerankRequest, raw_request: Optional[Request] = None
) -> Union[RerankResponse, ErrorResponse]:
self, request: RerankRequest, raw_request: Request | None = None
) -> RerankResponse | ErrorResponse:
"""
Rerank API based on JinaAI's rerank API; implements the same
API interface. Designed for compatibility with off-the-shelf
@@ -468,7 +468,7 @@ class ServingScores(OpenAIServing):
final_res_batch: list[PoolingRequestOutput],
request_id: str,
model_name: str,
documents: Union[list[str], ScoreMultiModalParam],
documents: list[str] | ScoreMultiModalParam,
top_n: int,
) -> RerankResponse:
"""
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any, Final, Optional, Union
from typing import Any, Final
import jinja2
from fastapi import Request
@@ -33,8 +33,8 @@ class OpenAIServingTokenization(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
chat_template: Optional[str],
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
log_error_stack: bool = False,
@@ -54,7 +54,7 @@ class OpenAIServingTokenization(OpenAIServing):
self,
request: TokenizeRequest,
raw_request: Request,
) -> Union[TokenizeResponse, ErrorResponse]:
) -> TokenizeResponse | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
@@ -129,7 +129,7 @@ class OpenAIServingTokenization(OpenAIServing):
self,
request: DetokenizeRequest,
raw_request: Request,
) -> Union[DetokenizeResponse, ErrorResponse]:
) -> DetokenizeResponse | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
@@ -155,7 +155,7 @@ class OpenAIServingTokenization(OpenAIServing):
async def get_tokenizer_info(
self,
) -> Union[TokenizerInfoResponse, ErrorResponse]:
) -> TokenizerInfoResponse | ErrorResponse:
"""Get comprehensive tokenizer information."""
try:
tokenizer = await self.engine_client.get_tokenizer()
@@ -171,7 +171,7 @@ class OpenAIServingTokenization(OpenAIServing):
@dataclass
class TokenizerInfo:
tokenizer: AnyTokenizer
chat_template: Optional[str]
chat_template: str | None
def to_dict(self) -> dict[str, Any]:
"""Return the tokenizer configuration."""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator
from typing import Optional, Union
from fastapi import Request
@@ -35,7 +34,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
@@ -50,7 +49,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
async def create_transcription(
self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request
) -> Union[TranscriptionResponse, AsyncGenerator[str, None], ErrorResponse]:
) -> TranscriptionResponse | AsyncGenerator[str, None] | ErrorResponse:
"""Transcription API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranscription
@@ -94,7 +93,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
@@ -109,7 +108,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
async def create_translation(
self, audio_data: bytes, request: TranslationRequest, raw_request: Request
) -> Union[TranslationResponse, AsyncGenerator[str, None], ErrorResponse]:
) -> TranslationResponse | AsyncGenerator[str, None] | ErrorResponse:
"""Translation API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranslation
+10 -15
View File
@@ -4,9 +4,9 @@ import asyncio
import io
import math
import time
from collections.abc import AsyncGenerator
from collections.abc import AsyncGenerator, Callable
from functools import cached_property
from typing import Callable, Literal, Optional, TypeVar, Union, cast
from typing import Literal, TypeAlias, TypeVar, cast
import numpy as np
from fastapi import Request
@@ -39,7 +39,7 @@ try:
except ImportError:
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
SpeechToTextResponse = Union[TranscriptionResponse, TranslationResponse]
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
T = TypeVar("T", bound=SpeechToTextResponse)
logger = init_logger(__name__)
@@ -54,7 +54,7 @@ class OpenAISpeechToText(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: Optional[RequestLogger],
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
task_type: Literal["transcribe", "translate"] = "transcribe",
log_error_stack: bool = False,
@@ -140,7 +140,7 @@ class OpenAISpeechToText(OpenAIServing):
raw_request: Request,
response_class: type[T],
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
) -> Union[T, AsyncGenerator[str, None], ErrorResponse]:
) -> T | AsyncGenerator[str, None] | ErrorResponse:
"""Base method for speech-to-text operations like transcription and
translation."""
error_check_ret = await self._check_model(request)
@@ -181,9 +181,7 @@ class OpenAISpeechToText(OpenAIServing):
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(str(e))
list_result_generator: Optional[list[AsyncGenerator[RequestOutput, None]]] = (
None
)
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
try:
# Unlike most decoder-only models, whisper generation length is not
# constrained by the size of the input audio, which is mapped to a
@@ -252,13 +250,10 @@ class OpenAISpeechToText(OpenAIServing):
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
response_stream_choice_class: Union[
type[TranscriptionResponseStreamChoice],
type[TranslationResponseStreamChoice],
],
stream_response_class: Union[
type[TranscriptionStreamResponse], type[TranslationStreamResponse]
],
response_stream_choice_class: type[TranscriptionResponseStreamChoice]
| type[TranslationResponseStreamChoice],
stream_response_class: type[TranscriptionStreamResponse]
| type[TranslationStreamResponse],
) -> AsyncGenerator[str, None]:
created_time = int(time.time())
model_name = request.model
@@ -2,9 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Sequence
from collections.abc import Callable, Sequence
from functools import cached_property
from typing import Callable, Optional, Union
from vllm.entrypoints.openai.protocol import (
ChatCompletionRequest,
@@ -69,7 +68,7 @@ class ToolParser:
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Instance method that should be implemented for extracting tool calls
from an incomplete response; for use when handling tool calls and
@@ -101,7 +100,7 @@ class ToolParserManager:
def _register_module(
cls,
module: type,
module_name: Optional[Union[str, list[str]]] = None,
module_name: str | list[str] | None = None,
force: bool = True,
) -> None:
if not issubclass(module, ToolParser):
@@ -123,10 +122,10 @@ class ToolParserManager:
@classmethod
def register_module(
cls,
name: Optional[Union[str, list[str]]] = None,
name: str | list[str] | None = None,
force: bool = True,
module: Union[type, None] = None,
) -> Union[type, Callable]:
module: type | None = None,
) -> type | Callable:
"""
Register module with the given name or name list. it can be used as a
decoder(with module as None) or normal function(with module as not
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Union
import regex as re
@@ -129,7 +128,7 @@ class DeepSeekV31ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
logger.debug("delta_text: %s", delta_text)
logger.debug("delta_token_ids: %s", delta_token_ids)
# check to see if we should be streaming a tool call - is there a
@@ -272,7 +271,7 @@ class DeepSeekV31ToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
if function_name:
self.current_tool_name_sent = True
return DeltaMessage(
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Union
import regex as re
@@ -129,7 +128,7 @@ class DeepSeekV3ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
logger.debug("delta_text: %s", delta_text)
logger.debug("delta_token_ids: %s", delta_token_ids)
# check to see if we should be streaming a tool call - is there a
@@ -272,7 +271,7 @@ class DeepSeekV3ToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
if function_name:
self.current_tool_name_sent = True
return DeltaMessage(
@@ -4,7 +4,7 @@
import ast
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -66,7 +66,7 @@ class Glm4MoeModelToolParser(ToolParser):
def _is_string_type(
tool_name: str,
arg_name: str,
tools: Optional[list[ChatCompletionToolsParam]],
tools: list[ChatCompletionToolsParam] | None,
) -> bool:
if tools is None:
return False
@@ -144,7 +144,7 @@ class Glm4MoeModelToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
self._buffer += delta_text
cur_text = self._buffer
start_idx = cur_text.find(self.tool_call_start_token)
@@ -4,7 +4,6 @@
import json
from collections.abc import Sequence
from json import JSONDecoder
from typing import Union
import partial_json_parser
import regex as re
@@ -121,7 +120,7 @@ class Granite20bFCToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if len(current_text) < len(self.bot_token) and self.bot_token.startswith(
current_text
):
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
from partial_json_parser.core.options import Allow
@@ -108,7 +107,7 @@ class GraniteToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
start_idx = consume_space(0, current_text)
if current_text[start_idx:].startswith(self.bot_token):
start_idx = consume_space(start_idx + len(self.bot_token), current_text)
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
import regex as re
@@ -181,7 +180,7 @@ class Hermes2ProToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# 1. All tokens are parsed based on _text, not token_ids.
# 2. All incoming text data is processed by the tool_call_delta_buffer
# function for buffering before being used for parsing.
@@ -333,7 +332,7 @@ class Hermes2ProToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
if function_name:
self.current_tool_name_sent = True
return DeltaMessage(
@@ -4,7 +4,7 @@
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -73,7 +73,7 @@ class HunyuanA13BToolParser(ToolParser):
def preprocess_model_output(
self, model_output: str
) -> tuple[Optional[str], Optional[str]]:
) -> tuple[str | None, str | None]:
# find the location tool call
for match in self.answer_tool_calls_pattern.finditer(model_output):
start, end = match.span()
@@ -176,7 +176,7 @@ class HunyuanA13BToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Extract tool calls for streaming mode.
"""
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
from partial_json_parser.core.options import Allow
@@ -59,7 +58,7 @@ class Internlm2ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if "<|action_start|>" not in current_text:
self.position = len(current_text)
return DeltaMessage(content=delta_text)
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
import regex as re
@@ -129,7 +128,7 @@ class JambaToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# if the tool call token is not in the tokens generated so far, append
# output to contents since it's not a tool
if self.tool_calls_start_token not in current_text:
@@ -190,7 +189,7 @@ class JambaToolParser(ToolParser):
# auto-generated due to JSON completions, but wasn't
# streamed to the client yet.
if self.current_tool_id >= 0:
diff: Union[str, None] = current_tool_call.get("arguments")
diff: str | None = current_tool_call.get("arguments")
if diff:
diff = json.dumps(diff, ensure_ascii=False).replace(
@@ -3,7 +3,6 @@
# code modified from deepseekv3_tool_parser.py
from collections.abc import Sequence
from typing import Union
import regex as re
@@ -131,7 +130,7 @@ class KimiK2ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
logger.debug("delta_text: %s", delta_text)
logger.debug("delta_token_ids: %s", delta_token_ids)
# check to see if we should be streaming a tool call - is there a
@@ -278,7 +277,7 @@ class KimiK2ToolParser(ToolParser):
if not self.current_tool_name_sent:
if current_tool_call is None:
return None
function_name: Union[str, None] = current_tool_call.get("name")
function_name: str | None = current_tool_call.get("name")
tool_id = current_tool_call.get("id")
if function_name:
self.current_tool_name_sent = True
@@ -3,7 +3,7 @@
import ast
import json
from collections.abc import Sequence
from typing import Any, Union
from typing import Any
import regex as re
from transformers import PreTrainedTokenizerBase
@@ -128,7 +128,7 @@ class Llama4PythonicToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not current_text.startswith("[") and not current_text.startswith(
"<|python_start|>"
):
@@ -245,7 +245,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
)
def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _make_valid_python(text: str) -> tuple[str, str] | None:
bracket_stack = []
for index, char in enumerate(text):
if char in {"[", "(", "{"}:
@@ -317,7 +317,7 @@ def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _compute_tool_delta(
previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
) -> Union[DeltaToolCall, None]:
) -> DeltaToolCall | None:
new_call_args = new_call.function.arguments
if withheld_suffix:
assert new_call_args.endswith(withheld_suffix)
@@ -3,7 +3,6 @@
import json
from collections.abc import Sequence
from typing import Union
import partial_json_parser
import regex as re
@@ -134,7 +133,7 @@ class Llama3JsonToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not (
current_text.startswith(self.bot_token) or current_text.startswith("{")
):
@@ -3,7 +3,7 @@
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -509,7 +509,7 @@ class MinimaxToolParser(ToolParser):
def _get_current_tool_content(
self, text: str, tool_index: int
) -> tuple[Optional[str], Optional[str]]:
) -> tuple[str | None, str | None]:
"""
Get the content of a specific tool by index.
@@ -545,7 +545,7 @@ class MinimaxToolParser(ToolParser):
def _handle_tool_name_streaming(
self, tool_content: str, tool_count: int
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Handle streaming of tool names.
@@ -595,7 +595,7 @@ class MinimaxToolParser(ToolParser):
def _handle_tool_args_streaming(
self, tool_content: str, tool_count: int
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
"""
Handle streaming of tool arguments.
@@ -702,7 +702,7 @@ class MinimaxToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
self._update_thinking_state(current_text)
if self.in_thinking_tag:
@@ -776,7 +776,7 @@ class MinimaxToolParser(ToolParser):
)
return None
def _find_tool_start_outside_thinking(self, current_text: str) -> Optional[int]:
def _find_tool_start_outside_thinking(self, current_text: str) -> int | None:
"""
Find the start position of tool calls outside of thinking tags.
@@ -809,7 +809,7 @@ class MinimaxToolParser(ToolParser):
def _extract_content_before_tools(
self, current_text: str, delta_text: str, tool_start: int
) -> Optional[str]:
) -> str | None:
"""
Extract content that appears before tool calls.
@@ -5,7 +5,6 @@ import json
from collections.abc import Sequence
from random import choices
from string import ascii_letters, digits
from typing import Union
import partial_json_parser
import regex as re
@@ -194,7 +193,7 @@ class MistralToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# if the tool call token is not in the tokens generated so far, append
# output to contents since it's not a tool
if self.bot_token not in current_text:
@@ -252,7 +251,7 @@ class MistralToolParser(ToolParser):
# auto-generated due to JSON completions, but wasn't
# streamed to the client yet.
if self.current_tool_id >= 0:
diff: Union[str, None] = current_tool_call.get("arguments")
diff: str | None = current_tool_call.get("arguments")
if diff:
diff = json.dumps(diff, ensure_ascii=False).replace(
@@ -1,7 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import json
from collections.abc import Sequence
from typing import TYPE_CHECKING
@@ -22,13 +20,15 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.transformers_utils.tokenizer import AnyTokenizer
else:
AnyTokenizer = object
logger = init_logger(__name__)
@ToolParserManager.register_module("openai")
class OpenAIToolParser(ToolParser):
def __init__(self, tokenizer: AnyTokenizer):
def __init__(self, tokenizer: "AnyTokenizer"):
super().__init__(tokenizer)
def extract_tool_calls(
@@ -3,7 +3,7 @@
import json
from collections.abc import Sequence
from typing import Any, Optional
from typing import Any
import regex as re
from transformers import PreTrainedTokenizerBase
@@ -118,5 +118,5 @@ class Phi4MiniJsonToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Optional[DeltaMessage]:
) -> DeltaMessage | None:
return None
@@ -4,7 +4,7 @@
import ast
import json
from collections.abc import Sequence
from typing import Any, Union
from typing import Any
import regex as re
from transformers import PreTrainedTokenizerBase
@@ -124,7 +124,7 @@ class PythonicToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not current_text.startswith("["):
return DeltaMessage(content=delta_text)
@@ -236,7 +236,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
)
def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _make_valid_python(text: str) -> tuple[str, str] | None:
bracket_stack = []
for index, char in enumerate(text):
if char in {"[", "(", "{"}:
@@ -308,7 +308,7 @@ def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
def _compute_tool_delta(
previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
) -> Union[DeltaToolCall, None]:
) -> DeltaToolCall | None:
new_call_args = new_call.function.arguments
if withheld_suffix:
assert new_call_args.endswith(withheld_suffix)
@@ -4,7 +4,7 @@ import ast
import json
import uuid
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -36,7 +36,7 @@ class Qwen3CoderToolParser(ToolParser):
self.current_tool_name_sent: bool = False
self.prev_tool_call_arr: list[dict] = []
# Override base class type - we use string IDs for tool calls
self.current_tool_id: Optional[str] = None # type: ignore
self.current_tool_id: str | None = None # type: ignore
self.streamed_args_for_tool: list[str] = []
# Sentinel tokens for streaming mode
@@ -110,7 +110,7 @@ class Qwen3CoderToolParser(ToolParser):
self.streaming_request = None
def _get_arguments_config(
self, func_name: str, tools: Optional[list[ChatCompletionToolsParam]]
self, func_name: str, tools: list[ChatCompletionToolsParam] | None
) -> dict:
"""Extract argument configuration for a function."""
if tools is None:
@@ -240,8 +240,8 @@ class Qwen3CoderToolParser(ToolParser):
return param_value
def _parse_xml_function_call(
self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
) -> Optional[ToolCall]:
self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None
) -> ToolCall | None:
# Extract function name
end_index = function_call_str.index(">")
function_name = function_call_str[:end_index]
@@ -349,7 +349,7 @@ class Qwen3CoderToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# Store request for type conversion
if not previous_text:
self._reset_streaming_state()
@@ -4,7 +4,7 @@ import ast
import json
import uuid
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
from xml.parsers.expat import ParserCreate
import regex as re
@@ -39,7 +39,7 @@ class StreamingXMLToolCallParser:
self.reset_streaming_state()
# Tool configuration information
self.tools: Union[list[ChatCompletionToolsParam], None] = None
self.tools: list[ChatCompletionToolsParam] | None = None
self.tool_call_start_token: str = "<tool_call>"
self.tool_call_end_token: str = "</tool_call>"
self.function_start_token: str = "<function="
@@ -341,7 +341,7 @@ class StreamingXMLToolCallParser:
# Skip blank content
return not element
def _find_next_complete_element(self, start_pos: int) -> tuple[Optional[str], int]:
def _find_next_complete_element(self, start_pos: int) -> tuple[str | None, int]:
"""
Find next complete XML element from specified position
@@ -584,7 +584,7 @@ class StreamingXMLToolCallParser:
"""Emit Delta response (streaming output)"""
self.deltas.append(delta)
def _auto_close_open_parameter_if_needed(self, incoming_tag: Optional[str] = None):
def _auto_close_open_parameter_if_needed(self, incoming_tag: str | None = None):
"""Before starting to process new elements,
if there are unclosed tags from before,
automatically complete their endings to the parser.
@@ -953,7 +953,7 @@ class StreamingXMLToolCallParser:
self.parser.EndElementHandler = self._end_element
self.parser.CharacterDataHandler = self._char_data
def set_tools(self, tools: Union[list[ChatCompletionToolsParam], None]):
def set_tools(self, tools: list[ChatCompletionToolsParam] | None):
"""Set tool configuration information"""
self.tools = tools
@@ -961,7 +961,7 @@ class StreamingXMLToolCallParser:
"""Generate unique call ID"""
return f"call_{uuid.uuid4().hex[:24]}"
def _extract_function_name(self, name: str, attrs: dict[str, str]) -> Optional[str]:
def _extract_function_name(self, name: str, attrs: dict[str, str]) -> str | None:
"""Extract function name from various formats"""
if attrs and "name" in attrs:
return attrs["name"]
@@ -973,9 +973,7 @@ class StreamingXMLToolCallParser:
return None
def _extract_parameter_name(
self, name: str, attrs: dict[str, str]
) -> Optional[str]:
def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> str | None:
"""Extract parameter name from various formats"""
if attrs and "name" in attrs:
return attrs["name"]
@@ -1218,7 +1216,7 @@ class Qwen3XMLToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
if not previous_text:
self.parser.reset_streaming_state()
if request:
@@ -7,7 +7,7 @@ import ast
import json
import uuid
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -109,8 +109,8 @@ class SeedOssToolParser(ToolParser):
self.json_closed = False
def _parse_xml_function_call(
self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
) -> Optional[ToolCall]:
self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None
) -> ToolCall | None:
def get_arguments_config(func_name: str) -> dict:
if tools is None:
return {}
@@ -357,7 +357,7 @@ class SeedOssToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# If no delta text, return None unless
# it's an EOS token after tool calls
if not delta_text:
@@ -4,7 +4,7 @@
import contextlib
import json
from collections.abc import Sequence
from typing import Any, Optional, Union
from typing import Any
import regex as re
@@ -58,7 +58,7 @@ class Step3ToolParser(ToolParser):
@staticmethod
def _parse_steptml_invoke(
action_text: str,
) -> tuple[Optional[str], Optional[dict[str, str]]]:
) -> tuple[str | None, dict[str, str] | None]:
func_name_match = re.search(r'<steptml:invoke name="([^"]+)">', action_text)
if not func_name_match:
return None, None
@@ -117,7 +117,7 @@ class Step3ToolParser(ToolParser):
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> Union[DeltaMessage, None]:
) -> DeltaMessage | None:
# The main loop processes the stream from the last known position.
while True:
if self.position >= len(current_text):
+38 -43
View File
@@ -5,7 +5,7 @@ import asyncio
import io
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Annotated, Optional, Union
from typing import Annotated
import pybase64
import torch
@@ -24,25 +24,25 @@ from vllm.utils import AsyncMicrobatchTokenizer
class RenderConfig:
"""Configuration to control how prompts are prepared."""
max_length: Optional[int] = None
max_length: int | None = None
"""Maximum allowable total input token length. If provided,
token inputs longer than this raise ``ValueError``."""
truncate_prompt_tokens: Optional[int] = None
truncate_prompt_tokens: int | None = None
"""Number of tokens to keep. ``None`` means no truncation.
``0`` yields an empty list (and skips embeds).
``-1`` maps to ``model_config.max_model_len``."""
add_special_tokens: Optional[bool] = True
add_special_tokens: bool | None = True
"""Whether to add model-specific special tokens during tokenization."""
cache_salt: Optional[str] = None
cache_salt: str | None = None
"""String to disambiguate prefix cache entries."""
needs_detokenization: Optional[bool] = False
needs_detokenization: bool | None = False
"""If True, detokenize IDs back to text for inclusion in outputs."""
def verify_truncate_prompt_tokens(self, model_config: ModelConfig) -> Optional[int]:
def verify_truncate_prompt_tokens(self, model_config: ModelConfig) -> int | None:
"""Validate and normalize `truncate_prompt_tokens` parameter."""
truncate_prompt_tokens = self.truncate_prompt_tokens
if truncate_prompt_tokens is None:
@@ -85,7 +85,7 @@ class BaseRenderer(ABC):
def __init__(
self,
model_config: ModelConfig,
tokenizer: Optional[AnyTokenizer] = None,
tokenizer: AnyTokenizer | None = None,
):
super().__init__()
self.model_config = model_config
@@ -95,7 +95,7 @@ class BaseRenderer(ABC):
async def render_prompt(
self,
*,
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
prompt_or_prompts: str | list[str] | list[int] | list[list[int]],
config: RenderConfig,
) -> list[EngineTokensPrompt]:
"""
@@ -126,12 +126,10 @@ class BaseRenderer(ABC):
async def render_prompt_and_embeds(
self,
*,
prompt_or_prompts: Optional[
Union[str, list[str], list[int], list[list[int]]]
] = None,
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,
prompt_embeds: bytes | list[bytes] | None = None,
config: RenderConfig,
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
) -> list[EngineTokensPrompt | EngineEmbedsPrompt]:
"""
Convert text/token and/or base64-encoded embeddings inputs into
engine-ready prompt objects using a unified RenderConfig.
@@ -161,9 +159,9 @@ class BaseRenderer(ABC):
@classmethod
def load_prompt_embeds(
cls,
prompt_embeds: Union[bytes, list[bytes]],
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=0)]] = None,
cache_salt: Optional[str] = None,
prompt_embeds: bytes | list[bytes],
truncate_prompt_tokens: Annotated[int, Field(ge=0)] | None = None,
cache_salt: str | None = None,
) -> list[EngineEmbedsPrompt]:
"""Load and validate base64-encoded embeddings into prompt objects."""
@@ -199,19 +197,18 @@ class CompletionRenderer(BaseRenderer):
def __init__(
self,
model_config: ModelConfig,
tokenizer: Optional[AnyTokenizer] = None,
async_tokenizer_pool: Optional[
dict[AnyTokenizer, AsyncMicrobatchTokenizer]
] = None,
tokenizer: AnyTokenizer | None = None,
async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer]
| None = None,
):
super().__init__(model_config, tokenizer)
self.async_tokenizer_pool = async_tokenizer_pool
self.async_tokenizer: Optional[AsyncMicrobatchTokenizer] = None
self.async_tokenizer: AsyncMicrobatchTokenizer | None = None
async def render_prompt(
self,
*,
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
prompt_or_prompts: str | list[str] | list[int] | list[list[int]],
config: RenderConfig,
) -> list[EngineTokensPrompt]:
"""Implementation of prompt rendering for completion-style requests.
@@ -237,12 +234,10 @@ class CompletionRenderer(BaseRenderer):
async def render_prompt_and_embeds(
self,
*,
prompt_or_prompts: Optional[
Union[str, list[str], list[int], list[list[int]]]
] = None,
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,
prompt_embeds: bytes | list[bytes] | None = None,
config: RenderConfig,
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
) -> list[EngineTokensPrompt | EngineEmbedsPrompt]:
"""
Render text/token prompts and/or precomputed embedding prompts. At
least one of `prompt_or_prompts` or `prompt_embeds` must be provided.
@@ -251,7 +246,7 @@ class CompletionRenderer(BaseRenderer):
if truncate_prompt_tokens == 0:
return []
rendered: list[Union[EngineTokensPrompt, EngineEmbedsPrompt]] = []
rendered: list[EngineTokensPrompt | EngineEmbedsPrompt] = []
if prompt_embeds is not None:
rendered.extend(
@@ -271,7 +266,7 @@ class CompletionRenderer(BaseRenderer):
return rendered
def _maybe_apply_truncation(
self, token_ids: list[int], truncate_prompt_tokens: Optional[int]
self, token_ids: list[int], truncate_prompt_tokens: int | None
) -> list[int]:
"""Apply truncation to token sequence."""
if truncate_prompt_tokens is None:
@@ -283,9 +278,9 @@ class CompletionRenderer(BaseRenderer):
async def _create_prompt(
self,
prompt_input: Union[EngineTextPrompt, EngineTokensPrompt],
prompt_input: EngineTextPrompt | EngineTokensPrompt,
config: RenderConfig,
truncate_prompt_tokens: Optional[int],
truncate_prompt_tokens: int | None,
) -> EngineTokensPrompt:
prompt, prompt_token_ids, _ = get_prompt_components(prompt_input)
@@ -315,10 +310,10 @@ class CompletionRenderer(BaseRenderer):
async def _create_prompt_from_text(
self,
text: str,
max_length: Optional[int],
truncate_prompt_tokens: Optional[int],
add_special_tokens: Optional[bool],
cache_salt: Optional[str],
max_length: int | None,
truncate_prompt_tokens: int | None,
add_special_tokens: bool | None,
cache_salt: str | None,
) -> EngineTokensPrompt:
"""Tokenize text input asynchronously."""
async_tokenizer = self._get_async_tokenizer()
@@ -348,10 +343,10 @@ class CompletionRenderer(BaseRenderer):
async def _create_prompt_from_token_ids(
self,
token_ids: list[int],
max_length: Optional[int],
truncate_prompt_tokens: Optional[int],
cache_salt: Optional[str],
needs_detokenization: Optional[bool] = False,
max_length: int | None,
truncate_prompt_tokens: int | None,
cache_salt: str | None,
needs_detokenization: bool | None = False,
) -> EngineTokensPrompt:
"""Optionally detokenize token IDs and build a tokens prompt."""
token_ids = self._maybe_apply_truncation(token_ids, truncate_prompt_tokens)
@@ -391,9 +386,9 @@ class CompletionRenderer(BaseRenderer):
def _create_tokens_prompt(
self,
token_ids: list[int],
max_length: Optional[int] = None,
cache_salt: Optional[str] = None,
prompt: Optional[str] = None,
max_length: int | None = None,
cache_salt: str | None = None,
prompt: str | None = None,
) -> EngineTokensPrompt:
"""Create validated EngineTokensPrompt."""
if max_length is not None and len(token_ids) > max_length:
+17 -17
View File
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Optional, Union, cast
from typing import Any, TypeAlias, cast
from torch.nn import CosineSimilarity
from typing_extensions import Required, TypeAlias, TypedDict
from typing_extensions import Required, TypedDict
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
@@ -25,9 +25,9 @@ from vllm.transformers_utils.tokenizer import (
PreTrainedTokenizerFast,
)
ScoreContentPartParam: TypeAlias = Union[
ChatCompletionContentPartImageParam, ChatCompletionContentPartImageEmbedsParam
]
ScoreContentPartParam: TypeAlias = (
ChatCompletionContentPartImageParam | ChatCompletionContentPartImageEmbedsParam
)
class ScoreMultiModalParam(TypedDict, total=False):
@@ -45,12 +45,12 @@ class ScoreMultiModalParam(TypedDict, total=False):
def _cosine_similarity(
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
embed_1: list[PoolingRequestOutput],
embed_2: list[PoolingRequestOutput],
) -> list[PoolingRequestOutput]:
scorer = CosineSimilarity(0)
scores: Union[list[PoolingRequestOutput]] = []
scores: list[PoolingRequestOutput] = []
for emb_1, emb_2 in zip(embed_1, embed_2):
pair_score = scorer(emb_1.outputs.data, emb_2.outputs.data)
@@ -74,8 +74,8 @@ def _cosine_similarity(
def _validate_score_input_lens(
data_1: Union[list[str], list[ScoreContentPartParam]],
data_2: Union[list[str], list[ScoreContentPartParam]],
data_1: list[str] | list[ScoreContentPartParam],
data_2: list[str] | list[ScoreContentPartParam],
):
len_1 = len(data_1)
len_2 = len(data_2)
@@ -89,18 +89,18 @@ def _validate_score_input_lens(
def parse_score_data(
data_1: Union[str, ScoreContentPartParam],
data_2: Union[str, ScoreContentPartParam],
data_1: str | ScoreContentPartParam,
data_2: str | ScoreContentPartParam,
model_config: ModelConfig,
tokenizer: AnyTokenizer,
) -> tuple[str, str, Optional[MultiModalDataDict]]:
) -> tuple[str, str, MultiModalDataDict | None]:
mm_tracker = MultiModalItemTracker(model_config, tokenizer)
content_1 = _parse_score_content(data_1, mm_tracker)
content_2 = _parse_score_content(data_2, mm_tracker)
def ensure_str(content: Optional[_ContentPart]) -> str:
def ensure_str(content: _ContentPart | None) -> str:
if content is not None and isinstance(content, str):
return cast(str, content)
else:
@@ -113,9 +113,9 @@ def parse_score_data(
def _parse_score_content(
data: Union[str, ScoreContentPartParam],
data: str | ScoreContentPartParam,
mm_tracker: BaseMultiModalItemTracker,
) -> Optional[_ContentPart]:
) -> _ContentPart | None:
if isinstance(data, str):
data = ChatCompletionContentPartTextParam(type="text", text=data)
@@ -182,8 +182,8 @@ def get_score_prompt(
model_config: ModelConfig,
tokenizer: AnyTokenizer,
tokenization_kwargs: dict[str, Any],
data_1: Union[str, ScoreContentPartParam],
data_2: Union[str, ScoreContentPartParam],
data_1: str | ScoreContentPartParam,
data_2: str | ScoreContentPartParam,
) -> tuple[str, TokensPrompt]:
prompt_1, prompt_2, mm_data = parse_score_data(
data_1,
+4 -4
View File
@@ -2,8 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from collections.abc import Callable
from ssl import SSLContext
from typing import Callable, Optional
from watchfiles import Change, awatch
@@ -20,9 +20,9 @@ class SSLCertRefresher:
def __init__(
self,
ssl_context: SSLContext,
key_path: Optional[str] = None,
cert_path: Optional[str] = None,
ca_path: Optional[str] = None,
key_path: str | None = None,
cert_path: str | None = None,
ca_path: str | None = None,
) -> None:
self.ssl = ssl_context
self.key_path = key_path
+6 -6
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
from openai_harmony import ToolDescription, ToolNamespaceConfig
@@ -80,7 +80,7 @@ class ToolServer(ABC):
pass
@abstractmethod
def get_tool_description(self, tool_name: str) -> Optional[ToolNamespaceConfig]:
def get_tool_description(self, tool_name: str) -> ToolNamespaceConfig | None:
"""
Return the tool description for the given tool name.
If the tool is not supported, return None.
@@ -89,7 +89,7 @@ class ToolServer(ABC):
@abstractmethod
def new_session(
self, tool_name: str, session_id: str, headers: Optional[dict[str, str]] = None
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
) -> AbstractAsyncContextManager[Any]:
"""
Create a session for the tool.
@@ -152,7 +152,7 @@ class MCPToolServer(ToolServer):
@asynccontextmanager
async def new_session(
self, tool_name: str, session_id: str, headers: Optional[dict[str, str]] = None
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
):
from mcp import ClientSession
from mcp.client.sse import sse_client
@@ -190,7 +190,7 @@ class DemoToolServer(ToolServer):
def has_tool(self, tool_name: str) -> bool:
return tool_name in self.tools
def get_tool_description(self, tool_name: str) -> Optional[ToolNamespaceConfig]:
def get_tool_description(self, tool_name: str) -> ToolNamespaceConfig | None:
if tool_name not in self.tools:
return None
if tool_name == "browser":
@@ -202,7 +202,7 @@ class DemoToolServer(ToolServer):
@asynccontextmanager
async def new_session(
self, tool_name: str, session_id: str, headers: Optional[dict[str, str]] = None
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
):
if tool_name not in self.tools:
raise KeyError(f"Tool '{tool_name}' is not supported")
+6 -6
View File
@@ -6,7 +6,7 @@ import dataclasses
import functools
import os
from argparse import Namespace
from typing import Any, Optional, Union
from typing import Any
from fastapi import Request
from fastapi.responses import JSONResponse, StreamingResponse
@@ -164,9 +164,9 @@ def cli_env_setup():
def _validate_truncation_size(
max_model_len: int,
truncate_prompt_tokens: Optional[int],
tokenization_kwargs: Optional[dict[str, Any]] = None,
) -> Optional[int]:
truncate_prompt_tokens: int | None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> int | None:
if truncate_prompt_tokens is not None:
if truncate_prompt_tokens <= -1:
truncate_prompt_tokens = max_model_len
@@ -191,7 +191,7 @@ def _validate_truncation_size(
def get_max_tokens(
max_model_len: int,
request: Union[ChatCompletionRequest, CompletionRequest],
request: ChatCompletionRequest | CompletionRequest,
input_length: int,
default_sampling_params: dict,
) -> int:
@@ -211,7 +211,7 @@ def get_max_tokens(
)
def log_non_default_args(args: Union[Namespace, EngineArgs]):
def log_non_default_args(args: Namespace | EngineArgs):
non_default_args = {}
# Handle Namespace