Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -13,7 +13,7 @@ import json
|
||||
import ssl
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
@@ -101,7 +101,7 @@ def build_app(args: Namespace) -> FastAPI:
|
||||
|
||||
async def init_app(
|
||||
args: Namespace,
|
||||
llm_engine: Optional[AsyncLLMEngine] = None,
|
||||
llm_engine: AsyncLLMEngine | None = None,
|
||||
) -> FastAPI:
|
||||
app = build_app(args)
|
||||
|
||||
@@ -120,7 +120,7 @@ async def init_app(
|
||||
|
||||
|
||||
async def run_server(
|
||||
args: Namespace, llm_engine: Optional[AsyncLLMEngine] = None, **uvicorn_kwargs: Any
|
||||
args: Namespace, llm_engine: AsyncLLMEngine | None = None, **uvicorn_kwargs: Any
|
||||
) -> None:
|
||||
logger.info("vLLM API server version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
+104
-107
@@ -5,10 +5,10 @@ import asyncio
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import Counter, defaultdict, deque
|
||||
from collections.abc import Awaitable, Iterable
|
||||
from collections.abc import Awaitable, Callable, Iterable
|
||||
from functools import cached_property, lru_cache, partial
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Generic, Literal, Optional, TypeVar, Union, cast
|
||||
from typing import Any, Generic, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
import jinja2
|
||||
import jinja2.ext
|
||||
@@ -40,7 +40,7 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin
|
||||
|
||||
# pydantic needs the TypedDict from typing_extensions
|
||||
from typing_extensions import Required, TypeAlias, TypedDict
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
@@ -76,7 +76,7 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
|
||||
|
||||
|
||||
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
|
||||
image_embeds: Optional[Union[str, dict[str, str]]]
|
||||
image_embeds: str | dict[str, str] | None
|
||||
"""
|
||||
The image embeddings. It can be either:
|
||||
- A single base64 string.
|
||||
@@ -84,7 +84,7 @@ class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
|
||||
"""
|
||||
type: Required[Literal["image_embeds"]]
|
||||
"""The type of the content part."""
|
||||
uuid: Optional[str]
|
||||
uuid: str | None
|
||||
"""
|
||||
User-provided UUID of a media. User must guarantee that it is properly
|
||||
generated and unique for different medias.
|
||||
@@ -123,8 +123,8 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
|
||||
}
|
||||
"""
|
||||
|
||||
image_pil: Optional[PILImage]
|
||||
uuid: Optional[str]
|
||||
image_pil: PILImage | None
|
||||
uuid: str | None
|
||||
"""
|
||||
User-provided UUID of a media. User must guarantee that it is properly
|
||||
generated and unique for different medias.
|
||||
@@ -141,8 +141,8 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
|
||||
}
|
||||
"""
|
||||
|
||||
image_url: Optional[str]
|
||||
uuid: Optional[str]
|
||||
image_url: str | None
|
||||
uuid: str | None
|
||||
"""
|
||||
User-provided UUID of a media. User must guarantee that it is properly
|
||||
generated and unique for different medias.
|
||||
@@ -158,7 +158,7 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
|
||||
}
|
||||
"""
|
||||
|
||||
audio_url: Optional[str]
|
||||
audio_url: str | None
|
||||
|
||||
|
||||
class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
|
||||
@@ -170,8 +170,8 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
|
||||
}
|
||||
"""
|
||||
|
||||
video_url: Optional[str]
|
||||
uuid: Optional[str]
|
||||
video_url: str | None
|
||||
uuid: str | None
|
||||
"""
|
||||
User-provided UUID of a media. User must guarantee that it is properly
|
||||
generated and unique for different medias.
|
||||
@@ -199,20 +199,20 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
|
||||
"""The thinking type."""
|
||||
|
||||
|
||||
ChatCompletionContentPartParam: TypeAlias = Union[
|
||||
OpenAIChatCompletionContentPartParam,
|
||||
ChatCompletionContentPartAudioParam,
|
||||
ChatCompletionContentPartInputAudioParam,
|
||||
ChatCompletionContentPartVideoParam,
|
||||
ChatCompletionContentPartRefusalParam,
|
||||
CustomChatCompletionContentPILImageParam,
|
||||
CustomChatCompletionContentSimpleImageParam,
|
||||
ChatCompletionContentPartImageEmbedsParam,
|
||||
CustomChatCompletionContentSimpleAudioParam,
|
||||
CustomChatCompletionContentSimpleVideoParam,
|
||||
str,
|
||||
CustomThinkCompletionContentParam,
|
||||
]
|
||||
ChatCompletionContentPartParam: TypeAlias = (
|
||||
OpenAIChatCompletionContentPartParam
|
||||
| ChatCompletionContentPartAudioParam
|
||||
| ChatCompletionContentPartInputAudioParam
|
||||
| ChatCompletionContentPartVideoParam
|
||||
| ChatCompletionContentPartRefusalParam
|
||||
| CustomChatCompletionContentPILImageParam
|
||||
| CustomChatCompletionContentSimpleImageParam
|
||||
| ChatCompletionContentPartImageEmbedsParam
|
||||
| CustomChatCompletionContentSimpleAudioParam
|
||||
| CustomChatCompletionContentSimpleVideoParam
|
||||
| str
|
||||
| CustomThinkCompletionContentParam
|
||||
)
|
||||
|
||||
|
||||
class CustomChatCompletionMessageParam(TypedDict, total=False):
|
||||
@@ -221,7 +221,7 @@ class CustomChatCompletionMessageParam(TypedDict, total=False):
|
||||
role: Required[str]
|
||||
"""The role of the message's author."""
|
||||
|
||||
content: Union[str, list[ChatCompletionContentPartParam]]
|
||||
content: str | list[ChatCompletionContentPartParam]
|
||||
"""The contents of the message."""
|
||||
|
||||
name: str
|
||||
@@ -231,18 +231,18 @@ class CustomChatCompletionMessageParam(TypedDict, total=False):
|
||||
same role.
|
||||
"""
|
||||
|
||||
tool_call_id: Optional[str]
|
||||
tool_call_id: str | None
|
||||
"""Tool call that this message is responding to."""
|
||||
|
||||
tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
|
||||
tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None
|
||||
"""The tool calls generated by the model, such as function calls."""
|
||||
|
||||
|
||||
ChatCompletionMessageParam = Union[
|
||||
OpenAIChatCompletionMessageParam,
|
||||
CustomChatCompletionMessageParam,
|
||||
OpenAIHarmonyMessage,
|
||||
]
|
||||
ChatCompletionMessageParam: TypeAlias = (
|
||||
OpenAIChatCompletionMessageParam
|
||||
| CustomChatCompletionMessageParam
|
||||
| OpenAIHarmonyMessage
|
||||
)
|
||||
|
||||
|
||||
# TODO: Make fields ReadOnly once mypy supports it
|
||||
@@ -250,16 +250,16 @@ class ConversationMessage(TypedDict, total=False):
|
||||
role: Required[str]
|
||||
"""The role of the message's author."""
|
||||
|
||||
content: Union[Optional[str], list[dict[str, str]]]
|
||||
content: str | None | list[dict[str, str]]
|
||||
"""The contents of the message"""
|
||||
|
||||
tool_call_id: Optional[str]
|
||||
tool_call_id: str | None
|
||||
"""Tool call that this message is responding to."""
|
||||
|
||||
name: Optional[str]
|
||||
name: str | None
|
||||
"""The name of the function to call"""
|
||||
|
||||
tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
|
||||
tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None
|
||||
"""The tool calls generated by the model, such as function calls."""
|
||||
|
||||
|
||||
@@ -294,7 +294,7 @@ def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
|
||||
def _is_var_or_elems_access(
|
||||
node: jinja2.nodes.Node,
|
||||
varname: str,
|
||||
key: Optional[str] = None,
|
||||
key: str | None = None,
|
||||
) -> bool:
|
||||
if isinstance(node, jinja2.nodes.Filter):
|
||||
return node.node is not None and _is_var_or_elems_access(
|
||||
@@ -369,7 +369,7 @@ def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
|
||||
break
|
||||
|
||||
|
||||
def _try_extract_ast(chat_template: str) -> Optional[jinja2.nodes.Template]:
|
||||
def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None:
|
||||
try:
|
||||
jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
|
||||
return jinja_compiled.environment.parse(chat_template)
|
||||
@@ -400,9 +400,9 @@ def _detect_content_format(
|
||||
|
||||
|
||||
def resolve_mistral_chat_template(
|
||||
chat_template: Optional[str],
|
||||
chat_template: str | None,
|
||||
**kwargs: Any,
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
if chat_template is not None or kwargs.get("chat_template_kwargs") is not None:
|
||||
raise ValueError(
|
||||
"'chat_template' or 'chat_template_kwargs' cannot be overridden "
|
||||
@@ -412,7 +412,7 @@ def resolve_mistral_chat_template(
|
||||
return None
|
||||
|
||||
|
||||
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], Optional[str]]()
|
||||
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
|
||||
"""
|
||||
Used in `_try_get_processor_chat_template` to avoid calling
|
||||
`cached_get_processor` again if the processor fails to be loaded.
|
||||
@@ -422,9 +422,9 @@ This is needed because `lru_cache` does not cache when an exception happens.
|
||||
|
||||
|
||||
def _try_get_processor_chat_template(
|
||||
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
|
||||
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
|
||||
model_config: ModelConfig,
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)
|
||||
if cache_key in _PROCESSOR_CHAT_TEMPLATES:
|
||||
return _PROCESSOR_CHAT_TEMPLATES[cache_key]
|
||||
@@ -458,12 +458,12 @@ def _try_get_processor_chat_template(
|
||||
|
||||
|
||||
def resolve_hf_chat_template(
|
||||
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
|
||||
chat_template: Optional[str],
|
||||
tools: Optional[list[dict[str, Any]]],
|
||||
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
# 1st priority: The given chat template
|
||||
if chat_template is not None:
|
||||
return chat_template
|
||||
@@ -505,8 +505,8 @@ def resolve_hf_chat_template(
|
||||
|
||||
|
||||
def _resolve_chat_template_content_format(
|
||||
chat_template: Optional[str],
|
||||
tools: Optional[list[dict[str, Any]]],
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
@@ -538,7 +538,7 @@ def _resolve_chat_template_content_format(
|
||||
|
||||
@lru_cache
|
||||
def _log_chat_template_content_format(
|
||||
chat_template: Optional[str],
|
||||
chat_template: str | None,
|
||||
given_format: ChatTemplateContentFormatOption,
|
||||
detected_format: ChatTemplateContentFormatOption,
|
||||
):
|
||||
@@ -561,8 +561,8 @@ def _log_chat_template_content_format(
|
||||
|
||||
|
||||
def resolve_chat_template_content_format(
|
||||
chat_template: Optional[str],
|
||||
tools: Optional[list[dict[str, Any]]],
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
given_format: ChatTemplateContentFormatOption,
|
||||
tokenizer: AnyTokenizer,
|
||||
*,
|
||||
@@ -604,8 +604,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
self._model_config = model_config
|
||||
self._tokenizer = tokenizer
|
||||
|
||||
self._items_by_modality = defaultdict[str, list[Optional[_T]]](list)
|
||||
self._uuids_by_modality = defaultdict[str, list[Optional[str]]](list)
|
||||
self._items_by_modality = defaultdict[str, list[_T | None]](list)
|
||||
self._uuids_by_modality = defaultdict[str, list[str | None]](list)
|
||||
|
||||
@property
|
||||
def model_config(self) -> ModelConfig:
|
||||
@@ -637,9 +637,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
def add(
|
||||
self,
|
||||
modality: ModalityStr,
|
||||
item: Optional[_T],
|
||||
uuid: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
item: _T | None,
|
||||
uuid: str | None = None,
|
||||
) -> str | None:
|
||||
"""
|
||||
Add a multi-modal item to the current prompt and returns the
|
||||
placeholder string to use, if any.
|
||||
@@ -657,7 +657,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
|
||||
return self.model_cls.get_placeholder_str(modality, num_items)
|
||||
|
||||
def all_mm_uuids(self) -> Optional[MultiModalUUIDDict]:
|
||||
def all_mm_uuids(self) -> MultiModalUUIDDict | None:
|
||||
if not self._items_by_modality:
|
||||
return None
|
||||
mm_uuids = {}
|
||||
@@ -684,7 +684,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
|
||||
|
||||
class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
|
||||
def all_mm_data(self) -> Optional[MultiModalDataDict]:
|
||||
def all_mm_data(self) -> MultiModalDataDict | None:
|
||||
if not self._items_by_modality:
|
||||
return None
|
||||
mm_inputs = {}
|
||||
@@ -710,7 +710,7 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
|
||||
|
||||
|
||||
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
|
||||
async def all_mm_data(self) -> Optional[MultiModalDataDict]:
|
||||
async def all_mm_data(self) -> MultiModalDataDict | None:
|
||||
if not self._items_by_modality:
|
||||
return None
|
||||
mm_inputs = {}
|
||||
@@ -756,7 +756,7 @@ class BaseMultiModalContentParser(ABC):
|
||||
# }
|
||||
self._placeholder_storage: dict[str, list] = defaultdict(list)
|
||||
|
||||
def _add_placeholder(self, modality: ModalityStr, placeholder: Optional[str]):
|
||||
def _add_placeholder(self, modality: ModalityStr, placeholder: str | None):
|
||||
mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
|
||||
if placeholder:
|
||||
self._placeholder_storage[mod_placeholder].append(placeholder)
|
||||
@@ -765,35 +765,35 @@ class BaseMultiModalContentParser(ABC):
|
||||
return dict(self._placeholder_storage)
|
||||
|
||||
@abstractmethod
|
||||
def parse_image(self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_image_embeds(
|
||||
self,
|
||||
image_embeds: Union[str, dict[str, str], None],
|
||||
uuid: Optional[str] = None,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_image_pil(
|
||||
self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
|
||||
self, image_pil: Image.Image | None, uuid: str | None = None
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_audio(self, audio_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_input_audio(
|
||||
self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
|
||||
self, input_audio: InputAudio | None, uuid: str | None = None
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -810,7 +810,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
|
||||
def parse_image(self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image = self._connector.fetch_image(image_url) if image_url else None
|
||||
|
||||
placeholder = self._tracker.add("image", image, uuid)
|
||||
@@ -818,8 +818,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_image_embeds(
|
||||
self,
|
||||
image_embeds: Union[str, dict[str, str], None],
|
||||
uuid: Optional[str] = None,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
if isinstance(image_embeds, dict):
|
||||
embeds = {
|
||||
@@ -838,19 +838,19 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
def parse_image_pil(
|
||||
self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
|
||||
self, image_pil: Image.Image | None, uuid: str | None = None
|
||||
) -> None:
|
||||
placeholder = self._tracker.add("image", image_pil, uuid)
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
def parse_audio(self, audio_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
|
||||
audio = self._connector.fetch_audio(audio_url) if audio_url else None
|
||||
|
||||
placeholder = self._tracker.add("audio", audio, uuid)
|
||||
self._add_placeholder("audio", placeholder)
|
||||
|
||||
def parse_input_audio(
|
||||
self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
|
||||
self, input_audio: InputAudio | None, uuid: str | None = None
|
||||
) -> None:
|
||||
if input_audio:
|
||||
audio_data = input_audio.get("data", "")
|
||||
@@ -865,7 +865,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
return self.parse_audio(audio_url, uuid)
|
||||
|
||||
def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
video = self._connector.fetch_video(video_url=video_url) if video_url else None
|
||||
|
||||
placeholder = self._tracker.add("video", video, uuid)
|
||||
@@ -885,7 +885,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
|
||||
def parse_image(self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image_coro = self._connector.fetch_image_async(image_url) if image_url else None
|
||||
|
||||
placeholder = self._tracker.add("image", image_coro, uuid)
|
||||
@@ -893,10 +893,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
def parse_image_embeds(
|
||||
self,
|
||||
image_embeds: Union[str, dict[str, str], None],
|
||||
uuid: Optional[str] = None,
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
future: asyncio.Future[Union[str, dict[str, str], None]] = asyncio.Future()
|
||||
future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()
|
||||
|
||||
if isinstance(image_embeds, dict):
|
||||
embeds = {
|
||||
@@ -916,9 +916,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
def parse_image_pil(
|
||||
self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
|
||||
self, image_pil: Image.Image | None, uuid: str | None = None
|
||||
) -> None:
|
||||
future: asyncio.Future[Optional[Image.Image]] = asyncio.Future()
|
||||
future: asyncio.Future[Image.Image | None] = asyncio.Future()
|
||||
if image_pil:
|
||||
future.set_result(image_pil)
|
||||
else:
|
||||
@@ -927,14 +927,14 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
placeholder = self._tracker.add("image", future, uuid)
|
||||
self._add_placeholder("image", placeholder)
|
||||
|
||||
def parse_audio(self, audio_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
|
||||
audio_coro = self._connector.fetch_audio_async(audio_url) if audio_url else None
|
||||
|
||||
placeholder = self._tracker.add("audio", audio_coro, uuid)
|
||||
self._add_placeholder("audio", placeholder)
|
||||
|
||||
def parse_input_audio(
|
||||
self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
|
||||
self, input_audio: InputAudio | None, uuid: str | None = None
|
||||
) -> None:
|
||||
if input_audio:
|
||||
audio_data = input_audio.get("data", "")
|
||||
@@ -949,7 +949,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
|
||||
return self.parse_audio(audio_url, uuid)
|
||||
|
||||
def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
video = (
|
||||
self._connector.fetch_video_async(video_url=video_url)
|
||||
if video_url
|
||||
@@ -960,7 +960,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
self._add_placeholder("video", placeholder)
|
||||
|
||||
|
||||
def validate_chat_template(chat_template: Optional[Union[Path, str]]):
|
||||
def validate_chat_template(chat_template: Path | str | None):
|
||||
"""Raises if the provided chat template appears invalid."""
|
||||
if chat_template is None:
|
||||
return
|
||||
@@ -984,10 +984,10 @@ def validate_chat_template(chat_template: Optional[Union[Path, str]]):
|
||||
|
||||
|
||||
def _load_chat_template(
|
||||
chat_template: Optional[Union[Path, str]],
|
||||
chat_template: Path | str | None,
|
||||
*,
|
||||
is_literal: bool = False,
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
if chat_template is None:
|
||||
return None
|
||||
|
||||
@@ -1024,10 +1024,10 @@ _cached_load_chat_template = lru_cache(_load_chat_template)
|
||||
|
||||
|
||||
def load_chat_template(
|
||||
chat_template: Optional[Union[Path, str]],
|
||||
chat_template: Path | str | None,
|
||||
*,
|
||||
is_literal: bool = False,
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
return _cached_load_chat_template(chat_template, is_literal=is_literal)
|
||||
|
||||
|
||||
@@ -1107,7 +1107,7 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
|
||||
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
|
||||
|
||||
_ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python
|
||||
_ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio, PILImage]
|
||||
_ContentPart: TypeAlias = str | dict[str, str] | InputAudio | PILImage
|
||||
|
||||
# Define a mapping from part types to their corresponding parsing functions.
|
||||
MM_PARSER_MAP: dict[
|
||||
@@ -1264,7 +1264,7 @@ def _parse_chat_message_content_part(
|
||||
*,
|
||||
wrap_dicts: bool,
|
||||
interleave_strings: bool,
|
||||
) -> Optional[_ContentPart]:
|
||||
) -> _ContentPart | None:
|
||||
"""Parses a single part of a conversation. If wrap_dicts is True,
|
||||
structured dictionary pieces for texts and images will be
|
||||
wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
|
||||
@@ -1310,10 +1310,7 @@ def _parse_chat_message_content_part(
|
||||
mm_parser.parse_image(str_content, uuid)
|
||||
modality = "image"
|
||||
elif part_type == "image_embeds":
|
||||
if content is not None:
|
||||
content = cast(Union[str, dict[str, str]], content)
|
||||
else:
|
||||
content = None
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
mm_parser.parse_image_embeds(content, uuid)
|
||||
modality = "image"
|
||||
elif part_type == "audio_url":
|
||||
@@ -1411,8 +1408,8 @@ def parse_chat_messages(
|
||||
content_format: _ChatTemplateContentFormat,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
Optional[MultiModalDataDict],
|
||||
Optional[MultiModalUUIDDict],
|
||||
MultiModalDataDict | None,
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = MultiModalItemTracker(model_config, tokenizer)
|
||||
@@ -1443,8 +1440,8 @@ def parse_chat_messages_futures(
|
||||
content_format: _ChatTemplateContentFormat,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
Awaitable[Optional[MultiModalDataDict]],
|
||||
Optional[MultiModalUUIDDict],
|
||||
Awaitable[MultiModalDataDict | None],
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
|
||||
@@ -1498,7 +1495,7 @@ _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)
|
||||
|
||||
|
||||
def resolve_chat_template_kwargs(
|
||||
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
|
||||
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
|
||||
chat_template: str,
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
@@ -1518,10 +1515,10 @@ def resolve_chat_template_kwargs(
|
||||
|
||||
|
||||
def apply_hf_chat_template(
|
||||
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
|
||||
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
|
||||
conversation: list[ConversationMessage],
|
||||
chat_template: Optional[str],
|
||||
tools: Optional[list[dict[str, Any]]],
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
tokenize: bool = False, # Different from HF's default
|
||||
@@ -1569,8 +1566,8 @@ def apply_hf_chat_template(
|
||||
def apply_mistral_chat_template(
|
||||
tokenizer: MistralTokenizer,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
chat_template: Optional[str],
|
||||
tools: Optional[list[dict[str, Any]]],
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
**kwargs: Any,
|
||||
) -> list[int]:
|
||||
from mistral_common.exceptions import MistralCommonException
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import typing
|
||||
|
||||
@@ -12,6 +10,8 @@ from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
|
||||
class BenchmarkSubcommand(CLISubcommand):
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import typing
|
||||
|
||||
@@ -11,6 +9,8 @@ from vllm.entrypoints.cli.types import CLISubcommand
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
|
||||
class CollectEnvSubcommand(CLISubcommand):
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
Note that all future modules must be lazily loaded within main
|
||||
to avoid certain eager import breakage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
import sys
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
@@ -16,6 +14,8 @@ from vllm.entrypoints.cli.types import CLISubcommand
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
|
||||
def _register_signal_handlers():
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import importlib.metadata
|
||||
@@ -14,6 +12,8 @@ from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import argparse
|
||||
import signal
|
||||
from typing import Optional
|
||||
|
||||
import uvloop
|
||||
|
||||
@@ -179,7 +178,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
hybrid_dp_lb = parallel_config.data_parallel_hybrid_lb
|
||||
assert external_dp_lb or hybrid_dp_lb or dp_rank == 0
|
||||
|
||||
api_server_manager: Optional[APIServerProcessManager] = None
|
||||
api_server_manager: APIServerProcessManager | None = None
|
||||
|
||||
with launch_core_engines(
|
||||
vllm_config, executor_class, log_stats, num_api_servers
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import typing
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
|
||||
class CLISubcommand:
|
||||
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import AsyncExitStack
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
from openai.types.responses.tool import Mcp
|
||||
from openai_harmony import Author, Message, Role, StreamState, TextContent
|
||||
@@ -82,7 +82,7 @@ class ConversationContext(ABC):
|
||||
@abstractmethod
|
||||
async def init_tool_sessions(
|
||||
self,
|
||||
tool_server: Optional[ToolServer],
|
||||
tool_server: ToolServer | None,
|
||||
exit_stack: AsyncExitStack,
|
||||
request_id: str,
|
||||
mcp_tools: dict[str, Mcp],
|
||||
@@ -122,7 +122,7 @@ class SimpleContext(ConversationContext):
|
||||
|
||||
async def init_tool_sessions(
|
||||
self,
|
||||
tool_server: Optional[ToolServer],
|
||||
tool_server: ToolServer | None,
|
||||
exit_stack: AsyncExitStack,
|
||||
request_id: str,
|
||||
mcp_tools: dict[str, Mcp],
|
||||
@@ -140,9 +140,9 @@ class HarmonyContext(ConversationContext):
|
||||
available_tools: list[str],
|
||||
):
|
||||
self._messages = messages
|
||||
self.finish_reason: Optional[str] = None
|
||||
self.finish_reason: str | None = None
|
||||
self.available_tools = available_tools
|
||||
self._tool_sessions: dict[str, Union[ClientSession, Tool]] = {}
|
||||
self._tool_sessions: dict[str, ClientSession | Tool] = {}
|
||||
self.called_tools: set[str] = set()
|
||||
|
||||
self.parser = get_streamable_parser_for_assistant()
|
||||
@@ -164,7 +164,7 @@ class HarmonyContext(ConversationContext):
|
||||
if self.parser.current_channel in {"analysis", "commentary"}:
|
||||
self.num_reasoning_tokens += 1
|
||||
|
||||
def append_output(self, output: Union[RequestOutput, list[Message]]) -> None:
|
||||
def append_output(self, output: RequestOutput | list[Message]) -> None:
|
||||
if isinstance(output, RequestOutput):
|
||||
output_token_ids = output.outputs[0].token_ids
|
||||
self.parser = get_streamable_parser_for_assistant()
|
||||
@@ -358,7 +358,7 @@ class HarmonyContext(ConversationContext):
|
||||
|
||||
async def init_tool_sessions(
|
||||
self,
|
||||
tool_server: Optional[ToolServer],
|
||||
tool_server: ToolServer | None,
|
||||
exit_stack: AsyncExitStack,
|
||||
request_id: str,
|
||||
mcp_tools: dict[str, Mcp],
|
||||
@@ -446,7 +446,7 @@ class StreamingHarmonyContext(HarmonyContext):
|
||||
def messages(self) -> list:
|
||||
return self._messages
|
||||
|
||||
def append_output(self, output: Union[RequestOutput, list[Message]]) -> None:
|
||||
def append_output(self, output: RequestOutput | list[Message]) -> None:
|
||||
if isinstance(output, RequestOutput):
|
||||
# append_output is called for each output token in streaming case,
|
||||
# so we only want to add the prompt tokens once for each message.
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Literal, Union
|
||||
from typing import Literal
|
||||
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
@@ -122,7 +120,7 @@ def get_system_message(
|
||||
return sys_msg
|
||||
|
||||
|
||||
def create_tool_definition(tool: Union[ChatCompletionToolsParam, Tool]):
|
||||
def create_tool_definition(tool: ChatCompletionToolsParam | Tool):
|
||||
if isinstance(tool, ChatCompletionToolsParam):
|
||||
return ToolDescription.new(
|
||||
name=tool.function.name,
|
||||
@@ -138,13 +136,13 @@ def create_tool_definition(tool: Union[ChatCompletionToolsParam, Tool]):
|
||||
|
||||
def get_developer_message(
|
||||
instructions: str | None = None,
|
||||
tools: list[Union[Tool, ChatCompletionToolsParam]] | None = None,
|
||||
tools: list[Tool | ChatCompletionToolsParam] | None = None,
|
||||
) -> Message:
|
||||
dev_msg_content = DeveloperContent.new()
|
||||
if instructions is not None and not envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
|
||||
dev_msg_content = dev_msg_content.with_instructions(instructions)
|
||||
if tools is not None:
|
||||
function_tools: list[Union[Tool, ChatCompletionToolsParam]] = []
|
||||
function_tools: list[Tool | ChatCompletionToolsParam] = []
|
||||
for tool in tools:
|
||||
if tool.type in (
|
||||
"web_search_preview",
|
||||
@@ -178,7 +176,7 @@ def get_user_message(content: str) -> Message:
|
||||
|
||||
def parse_response_input(
|
||||
response_msg: ResponseInputOutputItem,
|
||||
prev_responses: list[Union[ResponseOutputItem, ResponseReasoningItem]],
|
||||
prev_responses: list[ResponseOutputItem | ResponseReasoningItem],
|
||||
) -> Message:
|
||||
if not isinstance(response_msg, dict):
|
||||
response_msg = response_msg.model_dump()
|
||||
|
||||
@@ -5,7 +5,7 @@ import asyncio
|
||||
import signal
|
||||
import socket
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Request, Response
|
||||
@@ -26,7 +26,7 @@ logger = init_logger(__name__)
|
||||
|
||||
async def serve_http(
|
||||
app: FastAPI,
|
||||
sock: Optional[socket.socket],
|
||||
sock: socket.socket | None,
|
||||
enable_ssl_refresh: bool = False,
|
||||
**uvicorn_kwargs: Any,
|
||||
):
|
||||
|
||||
+113
-121
@@ -2,8 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import cloudpickle
|
||||
import torch.nn as nn
|
||||
@@ -191,36 +191,34 @@ class LLM:
|
||||
*,
|
||||
runner: RunnerOption = "auto",
|
||||
convert: ConvertOption = "auto",
|
||||
tokenizer: Optional[str] = None,
|
||||
tokenizer: str | None = None,
|
||||
tokenizer_mode: TokenizerMode = "auto",
|
||||
skip_tokenizer_init: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
allowed_local_media_path: str = "",
|
||||
allowed_media_domains: Optional[list[str]] = None,
|
||||
allowed_media_domains: list[str] | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
dtype: ModelDType = "auto",
|
||||
quantization: Optional[QuantizationMethods] = None,
|
||||
revision: Optional[str] = None,
|
||||
tokenizer_revision: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
quantization: QuantizationMethods | None = None,
|
||||
revision: str | None = None,
|
||||
tokenizer_revision: str | None = None,
|
||||
seed: int | None = None,
|
||||
gpu_memory_utilization: float = 0.9,
|
||||
swap_space: float = 4,
|
||||
cpu_offload_gb: float = 0,
|
||||
enforce_eager: bool = False,
|
||||
disable_custom_all_reduce: bool = False,
|
||||
hf_token: Optional[Union[bool, str]] = None,
|
||||
hf_overrides: Optional[HfOverrides] = None,
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
||||
pooler_config: Optional[PoolerConfig] = None,
|
||||
override_pooler_config: Optional[PoolerConfig] = None,
|
||||
structured_outputs_config: Optional[
|
||||
Union[dict[str, Any], StructuredOutputsConfig]
|
||||
] = None,
|
||||
kv_cache_memory_bytes: Optional[int] = None,
|
||||
compilation_config: Optional[
|
||||
Union[int, dict[str, Any], CompilationConfig]
|
||||
] = None,
|
||||
logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None,
|
||||
hf_token: bool | str | None = None,
|
||||
hf_overrides: HfOverrides | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
pooler_config: PoolerConfig | None = None,
|
||||
override_pooler_config: PoolerConfig | None = None,
|
||||
structured_outputs_config: dict[str, Any]
|
||||
| StructuredOutputsConfig
|
||||
| None = None,
|
||||
kv_cache_memory_bytes: int | None = None,
|
||||
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
|
||||
logits_processors: list[str | type[LogitsProcessor]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""LLM constructor."""
|
||||
@@ -331,7 +329,7 @@ class LLM:
|
||||
self.engine_class = type(self.llm_engine)
|
||||
|
||||
self.request_counter = Counter()
|
||||
self.default_sampling_params: Union[dict[str, Any], None] = None
|
||||
self.default_sampling_params: dict[str, Any] | None = None
|
||||
|
||||
supported_tasks = self.llm_engine.get_supported_tasks()
|
||||
logger.info("Supported tasks: %s", supported_tasks)
|
||||
@@ -367,14 +365,12 @@ class LLM:
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
sampling_params: Optional[
|
||||
Union[SamplingParams, Sequence[SamplingParams]]
|
||||
] = None,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
sampling_params: SamplingParams | Sequence[SamplingParams] | None = None,
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
priority: Optional[list[int]] = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
priority: list[int] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""Generates the completions for the input prompts.
|
||||
|
||||
@@ -437,8 +433,8 @@ class LLM:
|
||||
|
||||
def _get_modality_specific_lora_reqs(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
lora_request: list[LoRARequest] | LoRARequest | None,
|
||||
):
|
||||
# Grab the lora config off the vllm config on the engine,
|
||||
# since this is the same for both v0 & v1.
|
||||
@@ -474,8 +470,8 @@ class LLM:
|
||||
def _resolve_single_prompt_mm_lora(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
lora_request: Optional[LoRARequest],
|
||||
default_mm_loras: Optional[dict[str, str]],
|
||||
lora_request: LoRARequest | None,
|
||||
default_mm_loras: dict[str, str] | None,
|
||||
):
|
||||
if (
|
||||
not default_mm_loras
|
||||
@@ -526,10 +522,10 @@ class LLM:
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
"""
|
||||
Execute an RPC call on all workers.
|
||||
@@ -571,9 +567,9 @@ class LLM:
|
||||
|
||||
def _get_beam_search_lora_requests(
|
||||
self,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]],
|
||||
prompts: list[Union[TokensPrompt, TextPrompt]],
|
||||
) -> list[Optional[LoRARequest]]:
|
||||
lora_request: list[LoRARequest] | LoRARequest | None,
|
||||
prompts: list[TokensPrompt | TextPrompt],
|
||||
) -> list[LoRARequest | None]:
|
||||
"""Get the optional lora request corresponding to each prompt."""
|
||||
if isinstance(lora_request, Sequence) and len(lora_request) != len(prompts):
|
||||
raise ValueError(
|
||||
@@ -587,11 +583,11 @@ class LLM:
|
||||
|
||||
def beam_search(
|
||||
self,
|
||||
prompts: list[Union[TokensPrompt, TextPrompt]],
|
||||
prompts: list[TokensPrompt | TextPrompt],
|
||||
params: BeamSearchParams,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
use_tqdm: bool = False,
|
||||
concurrency_limit: Optional[int] = None,
|
||||
concurrency_limit: int | None = None,
|
||||
) -> list[BeamSearchOutput]:
|
||||
"""
|
||||
Generate sequences using beam search.
|
||||
@@ -770,16 +766,15 @@ class LLM:
|
||||
|
||||
def preprocess_chat(
|
||||
self,
|
||||
messages: Union[
|
||||
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
|
||||
],
|
||||
chat_template: Optional[str] = None,
|
||||
messages: list[ChatCompletionMessageParam]
|
||||
| list[list[ChatCompletionMessageParam]],
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
continue_final_message: bool = False,
|
||||
tools: Optional[list[dict[str, Any]]] = None,
|
||||
chat_template_kwargs: Optional[dict[str, Any]] = None,
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[TokensPrompt]:
|
||||
"""
|
||||
Generate prompt for a chat conversation. The pre-processed
|
||||
@@ -868,19 +863,18 @@ class LLM:
|
||||
|
||||
def chat(
|
||||
self,
|
||||
messages: Union[
|
||||
list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]
|
||||
],
|
||||
sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
chat_template: Optional[str] = None,
|
||||
messages: list[ChatCompletionMessageParam]
|
||||
| list[list[ChatCompletionMessageParam]],
|
||||
sampling_params: SamplingParams | list[SamplingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: LoRARequest | None = None,
|
||||
chat_template: str | None = None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto",
|
||||
add_generation_prompt: bool = True,
|
||||
continue_final_message: bool = False,
|
||||
tools: Optional[list[dict[str, Any]]] = None,
|
||||
chat_template_kwargs: Optional[dict[str, Any]] = None,
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""
|
||||
Generate responses for a chat conversation.
|
||||
@@ -953,14 +947,14 @@ class LLM:
|
||||
|
||||
def encode(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
pooling_task: PoolingTask = "encode",
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""Apply pooling to the hidden states corresponding to the input
|
||||
prompts.
|
||||
@@ -1086,12 +1080,12 @@ class LLM:
|
||||
|
||||
def embed(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[EmbeddingRequestOutput]:
|
||||
"""
|
||||
Generate an embedding vector for each prompt.
|
||||
@@ -1135,11 +1129,11 @@ class LLM:
|
||||
|
||||
def classify(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ClassificationRequestOutput]:
|
||||
"""
|
||||
Generate class logits for each prompt.
|
||||
@@ -1181,13 +1175,13 @@ class LLM:
|
||||
|
||||
def reward(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType]],
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
/,
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[Union[PoolingParams, Sequence[PoolingParams]]] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""
|
||||
Generate rewards for each prompt.
|
||||
@@ -1220,12 +1214,12 @@ class LLM:
|
||||
def _embedding_score(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
text_1: list[Union[str, TextPrompt, TokensPrompt]],
|
||||
text_2: list[Union[str, TextPrompt, TokensPrompt]],
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
text_1: list[str | TextPrompt | TokensPrompt],
|
||||
text_2: list[str | TextPrompt | TokensPrompt],
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
encoded_output: list[PoolingRequestOutput] = self.encode(
|
||||
text_1 + text_2,
|
||||
@@ -1252,12 +1246,12 @@ class LLM:
|
||||
def _cross_encoding_score(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
data_1: Union[list[str], list[ScoreContentPartParam]],
|
||||
data_2: Union[list[str], list[ScoreContentPartParam]],
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
data_1: list[str] | list[ScoreContentPartParam],
|
||||
data_2: list[str] | list[ScoreContentPartParam],
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
model_config = self.model_config
|
||||
|
||||
@@ -1316,14 +1310,14 @@ class LLM:
|
||||
|
||||
def score(
|
||||
self,
|
||||
data_1: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
|
||||
data_2: Union[SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam],
|
||||
data_1: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
|
||||
data_2: SingletonPrompt | Sequence[SingletonPrompt] | ScoreMultiModalParam,
|
||||
/,
|
||||
*,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
truncate_prompt_tokens: int | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
"""Generate similarity scores for all pairs `<text,text_pair>` or
|
||||
`<multi-modal data, multi-modal data pair>`.
|
||||
@@ -1391,9 +1385,9 @@ class LLM:
|
||||
if not model_config.is_multimodal_model:
|
||||
|
||||
def check_data_type(
|
||||
data: Union[
|
||||
SingletonPrompt, Sequence[SingletonPrompt], ScoreMultiModalParam
|
||||
],
|
||||
data: SingletonPrompt
|
||||
| Sequence[SingletonPrompt]
|
||||
| ScoreMultiModalParam,
|
||||
):
|
||||
if isinstance(data, dict) and "content" in data:
|
||||
raise ValueError(
|
||||
@@ -1470,7 +1464,7 @@ class LLM:
|
||||
def stop_profile(self) -> None:
|
||||
self.llm_engine.stop_profile()
|
||||
|
||||
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
|
||||
def reset_prefix_cache(self, device: Device | None = None) -> bool:
|
||||
return self.llm_engine.reset_prefix_cache(device)
|
||||
|
||||
def sleep(self, level: int = 1):
|
||||
@@ -1495,7 +1489,7 @@ class LLM:
|
||||
self.reset_prefix_cache()
|
||||
self.llm_engine.sleep(level=level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None):
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
"""
|
||||
Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep]
|
||||
method for more details.
|
||||
@@ -1523,17 +1517,15 @@ class LLM:
|
||||
|
||||
def _validate_and_add_requests(
|
||||
self,
|
||||
prompts: Union[PromptType, Sequence[PromptType], DataPrompt],
|
||||
params: Union[
|
||||
SamplingParams,
|
||||
Sequence[SamplingParams],
|
||||
PoolingParams,
|
||||
Sequence[PoolingParams],
|
||||
],
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
params: SamplingParams
|
||||
| Sequence[SamplingParams]
|
||||
| PoolingParams
|
||||
| Sequence[PoolingParams],
|
||||
*,
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
|
||||
priority: Optional[list[int]] = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: Sequence[LoRARequest] | LoRARequest | None,
|
||||
priority: list[int] | None = None,
|
||||
) -> None:
|
||||
if isinstance(prompts, (str, dict)):
|
||||
# Convert a single prompt to a list.
|
||||
@@ -1575,8 +1567,8 @@ class LLM:
|
||||
|
||||
def _validate_mm_data_and_uuids(
|
||||
self,
|
||||
multi_modal_data: Optional[Any], # MultiModalDataDict
|
||||
multi_modal_uuids: Optional[Any], # MultiModalUUIDDict
|
||||
multi_modal_data: Any | None, # MultiModalDataDict
|
||||
multi_modal_uuids: Any | None, # MultiModalUUIDDict
|
||||
):
|
||||
"""
|
||||
Validate that if any multi-modal data is skipped (i.e. None),
|
||||
@@ -1625,9 +1617,9 @@ class LLM:
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: PromptType,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
params: SamplingParams | PoolingParams,
|
||||
*,
|
||||
lora_request: Optional[LoRARequest],
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
) -> tuple[EngineCoreRequest, dict[str, Any]]:
|
||||
"""Use the Processor to process inputs for LLMEngine."""
|
||||
@@ -1651,8 +1643,8 @@ class LLM:
|
||||
def _add_request(
|
||||
self,
|
||||
prompt: PromptType,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
params: SamplingParams | PoolingParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
) -> None:
|
||||
prompt_text, _, _ = get_prompt_components(prompt)
|
||||
@@ -1677,8 +1669,8 @@ class LLM:
|
||||
)
|
||||
|
||||
def _run_engine(
|
||||
self, *, use_tqdm: Union[bool, Callable[..., tqdm]] = True
|
||||
) -> list[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
self, *, use_tqdm: bool | Callable[..., tqdm] = True
|
||||
) -> list[RequestOutput | PoolingRequestOutput]:
|
||||
# Initialize tqdm.
|
||||
if use_tqdm:
|
||||
num_requests = self.llm_engine.get_num_unfinished_requests()
|
||||
@@ -1691,7 +1683,7 @@ class LLM:
|
||||
)
|
||||
|
||||
# Run the engine.
|
||||
outputs: list[Union[RequestOutput, PoolingRequestOutput]] = []
|
||||
outputs: list[RequestOutput | PoolingRequestOutput] = []
|
||||
total_in_toks = 0
|
||||
total_out_toks = 0
|
||||
while self.llm_engine.has_unfinished_requests():
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -15,17 +14,17 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RequestLogger:
|
||||
def __init__(self, *, max_log_len: Optional[int]) -> None:
|
||||
def __init__(self, *, max_log_len: int | None) -> None:
|
||||
self.max_log_len = max_log_len
|
||||
|
||||
def log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: Optional[str],
|
||||
prompt_token_ids: Optional[list[int]],
|
||||
prompt_embeds: Optional[torch.Tensor],
|
||||
params: Optional[Union[SamplingParams, PoolingParams, BeamSearchParams]],
|
||||
lora_request: Optional[LoRARequest],
|
||||
prompt: str | None,
|
||||
prompt_token_ids: list[int] | None,
|
||||
prompt_embeds: torch.Tensor | None,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
max_log_len = self.max_log_len
|
||||
if max_log_len is not None:
|
||||
@@ -52,8 +51,8 @@ class RequestLogger:
|
||||
self,
|
||||
request_id: str,
|
||||
outputs: str,
|
||||
output_token_ids: Optional[Sequence[int]],
|
||||
finish_reason: Optional[str] = None,
|
||||
output_token_ids: Sequence[int] | None,
|
||||
finish_reason: str | None = None,
|
||||
is_streaming: bool = False,
|
||||
delta: bool = False,
|
||||
) -> None:
|
||||
|
||||
@@ -16,10 +16,10 @@ import socket
|
||||
import tempfile
|
||||
import uuid
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated, Any, Callable, Literal, Optional
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import prometheus_client
|
||||
import pydantic
|
||||
@@ -166,8 +166,8 @@ async def build_async_engine_client(
|
||||
args: Namespace,
|
||||
*,
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
disable_frontend_multiprocessing: Optional[bool] = None,
|
||||
client_config: Optional[dict[str, Any]] = None,
|
||||
disable_frontend_multiprocessing: bool | None = None,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
|
||||
# The executor is expected to be mp.
|
||||
@@ -203,7 +203,7 @@ async def build_async_engine_client_from_engine_args(
|
||||
*,
|
||||
usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
|
||||
disable_frontend_multiprocessing: bool = False,
|
||||
client_config: Optional[dict[str, Any]] = None,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
"""
|
||||
Create EngineClient, either:
|
||||
@@ -227,7 +227,7 @@ async def build_async_engine_client_from_engine_args(
|
||||
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
async_llm: Optional[AsyncLLM] = None
|
||||
async_llm: AsyncLLM | None = None
|
||||
|
||||
# Don't mutate the input client_config
|
||||
client_config = dict(client_config) if client_config else {}
|
||||
@@ -308,35 +308,35 @@ def models(request: Request) -> OpenAIServingModels:
|
||||
return request.app.state.openai_serving_models
|
||||
|
||||
|
||||
def responses(request: Request) -> Optional[OpenAIServingResponses]:
|
||||
def responses(request: Request) -> OpenAIServingResponses | None:
|
||||
return request.app.state.openai_serving_responses
|
||||
|
||||
|
||||
def chat(request: Request) -> Optional[OpenAIServingChat]:
|
||||
def chat(request: Request) -> OpenAIServingChat | None:
|
||||
return request.app.state.openai_serving_chat
|
||||
|
||||
|
||||
def completion(request: Request) -> Optional[OpenAIServingCompletion]:
|
||||
def completion(request: Request) -> OpenAIServingCompletion | None:
|
||||
return request.app.state.openai_serving_completion
|
||||
|
||||
|
||||
def pooling(request: Request) -> Optional[OpenAIServingPooling]:
|
||||
def pooling(request: Request) -> OpenAIServingPooling | None:
|
||||
return request.app.state.openai_serving_pooling
|
||||
|
||||
|
||||
def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
|
||||
def embedding(request: Request) -> OpenAIServingEmbedding | None:
|
||||
return request.app.state.openai_serving_embedding
|
||||
|
||||
|
||||
def score(request: Request) -> Optional[ServingScores]:
|
||||
def score(request: Request) -> ServingScores | None:
|
||||
return request.app.state.openai_serving_scores
|
||||
|
||||
|
||||
def classify(request: Request) -> Optional[ServingClassification]:
|
||||
def classify(request: Request) -> ServingClassification | None:
|
||||
return request.app.state.openai_serving_classification
|
||||
|
||||
|
||||
def rerank(request: Request) -> Optional[ServingScores]:
|
||||
def rerank(request: Request) -> ServingScores | None:
|
||||
return request.app.state.openai_serving_scores
|
||||
|
||||
|
||||
@@ -542,8 +542,8 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
|
||||
async def retrieve_responses(
|
||||
response_id: str,
|
||||
raw_request: Request,
|
||||
starting_after: Optional[int] = None,
|
||||
stream: Optional[bool] = False,
|
||||
starting_after: int | None = None,
|
||||
stream: bool | None = False,
|
||||
):
|
||||
handler = responses(raw_request)
|
||||
if handler is None:
|
||||
@@ -1039,7 +1039,7 @@ if envs.VLLM_SERVER_DEV_MODE:
|
||||
# User-defined `method` is responsible for deserialization if needed.
|
||||
args: list[str] = body.get("args", [])
|
||||
kwargs: dict[str, str] = body.get("kwargs", {})
|
||||
timeout: Optional[float] = body.get("timeout")
|
||||
timeout: float | None = body.get("timeout")
|
||||
results = await engine_client(raw_request).collective_rpc(
|
||||
method=method, timeout=timeout, args=tuple(args), kwargs=kwargs
|
||||
)
|
||||
@@ -1120,7 +1120,7 @@ async def is_scaling_elastic_ep(raw_request: Request):
|
||||
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
|
||||
# (requires typing_extensions >= 4.13)
|
||||
RequestType = Any
|
||||
GetHandlerFn = Callable[[Request], Optional[OpenAIServing]]
|
||||
GetHandlerFn = Callable[[Request], OpenAIServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
@@ -1236,7 +1236,7 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
|
||||
return Response(status_code=200, content=response)
|
||||
|
||||
|
||||
def load_log_config(log_config_file: Optional[str]) -> Optional[dict]:
|
||||
def load_log_config(log_config_file: str | None) -> dict | None:
|
||||
if not log_config_file:
|
||||
return None
|
||||
try:
|
||||
@@ -1655,7 +1655,7 @@ async def init_app_state(
|
||||
)
|
||||
|
||||
if args.tool_server == "demo":
|
||||
tool_server: Optional[ToolServer] = DemoToolServer()
|
||||
tool_server: ToolServer | None = DemoToolServer()
|
||||
assert isinstance(tool_server, DemoToolServer)
|
||||
await tool_server.init_and_validate()
|
||||
elif args.tool_server:
|
||||
|
||||
@@ -11,7 +11,7 @@ import json
|
||||
import ssl
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import field
|
||||
from typing import Literal, Optional, Union
|
||||
from typing import Literal
|
||||
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
@@ -39,8 +39,8 @@ class LoRAParserAction(argparse.Action):
|
||||
self,
|
||||
parser: argparse.ArgumentParser,
|
||||
namespace: argparse.Namespace,
|
||||
values: Optional[Union[str, Sequence[str]]],
|
||||
option_string: Optional[str] = None,
|
||||
values: str | Sequence[str] | None,
|
||||
option_string: str | None = None,
|
||||
):
|
||||
if values is None:
|
||||
values = []
|
||||
@@ -73,11 +73,11 @@ class LoRAParserAction(argparse.Action):
|
||||
class FrontendArgs:
|
||||
"""Arguments for the OpenAI-compatible frontend server."""
|
||||
|
||||
host: Optional[str] = None
|
||||
host: str | None = None
|
||||
"""Host name."""
|
||||
port: int = 8000
|
||||
"""Port number."""
|
||||
uds: Optional[str] = None
|
||||
uds: str | None = None
|
||||
"""Unix domain socket path. If set, host and port arguments are ignored."""
|
||||
uvicorn_log_level: Literal[
|
||||
"debug", "info", "warning", "error", "critical", "trace"
|
||||
@@ -93,15 +93,15 @@ class FrontendArgs:
|
||||
"""Allowed methods."""
|
||||
allowed_headers: list[str] = field(default_factory=lambda: ["*"])
|
||||
"""Allowed headers."""
|
||||
api_key: Optional[list[str]] = None
|
||||
api_key: list[str] | None = None
|
||||
"""If provided, the server will require one of these keys to be presented in
|
||||
the header."""
|
||||
lora_modules: Optional[list[LoRAModulePath]] = None
|
||||
lora_modules: list[LoRAModulePath] | None = None
|
||||
"""LoRA modules configurations in either 'name=path' format or JSON format
|
||||
or JSON list format. Example (old format): `'name=path'` Example (new
|
||||
format): `{\"name\": \"name\", \"path\": \"lora_path\",
|
||||
\"base_model_name\": \"id\"}`"""
|
||||
chat_template: Optional[str] = None
|
||||
chat_template: str | None = None
|
||||
"""The file path to the chat template, or the template in single-line form
|
||||
for the specified model."""
|
||||
chat_template_content_format: ChatTemplateContentFormatOption = "auto"
|
||||
@@ -116,17 +116,17 @@ class FrontendArgs:
|
||||
or the ones from tokenizer."""
|
||||
response_role: str = "assistant"
|
||||
"""The role name to return if `request.add_generation_prompt=true`."""
|
||||
ssl_keyfile: Optional[str] = None
|
||||
ssl_keyfile: str | None = None
|
||||
"""The file path to the SSL key file."""
|
||||
ssl_certfile: Optional[str] = None
|
||||
ssl_certfile: str | None = None
|
||||
"""The file path to the SSL cert file."""
|
||||
ssl_ca_certs: Optional[str] = None
|
||||
ssl_ca_certs: str | None = None
|
||||
"""The CA certificates file."""
|
||||
enable_ssl_refresh: bool = False
|
||||
"""Refresh SSL Context when SSL certificate files change"""
|
||||
ssl_cert_reqs: int = int(ssl.CERT_NONE)
|
||||
"""Whether client certificate is required (see stdlib ssl module's)."""
|
||||
root_path: Optional[str] = None
|
||||
root_path: str | None = None
|
||||
"""FastAPI root_path when app is behind a path based routing proxy."""
|
||||
middleware: list[str] = field(default_factory=lambda: [])
|
||||
"""Additional ASGI middleware to apply to the app. We accept multiple
|
||||
@@ -149,7 +149,7 @@ class FrontendArgs:
|
||||
exclude_tools_when_tool_choice_none: bool = False
|
||||
"""If specified, exclude tool definitions in prompts when
|
||||
tool_choice='none'."""
|
||||
tool_call_parser: Optional[str] = None
|
||||
tool_call_parser: str | None = None
|
||||
"""Select the tool call parser depending on the model that you're using.
|
||||
This is used to parse the model-generated tool call into OpenAI API format.
|
||||
Required for `--enable-auto-tool-choice`. You can choose any option from
|
||||
@@ -158,13 +158,13 @@ class FrontendArgs:
|
||||
"""Special the tool parser plugin write to parse the model-generated tool
|
||||
into OpenAI API format, the name register in this plugin can be used in
|
||||
`--tool-call-parser`."""
|
||||
tool_server: Optional[str] = None
|
||||
tool_server: str | None = None
|
||||
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for demo
|
||||
purpose."""
|
||||
log_config_file: Optional[str] = envs.VLLM_LOGGING_CONFIG_PATH
|
||||
log_config_file: str | None = envs.VLLM_LOGGING_CONFIG_PATH
|
||||
"""Path to logging config JSON file for both vllm and uvicorn"""
|
||||
max_log_len: Optional[int] = None
|
||||
max_log_len: int | None = None
|
||||
"""Max number of prompt characters or prompt ID numbers being printed in
|
||||
log. The default of None means unlimited."""
|
||||
disable_fastapi_docs: bool = False
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from collections.abc import Iterable
|
||||
from functools import lru_cache, partial
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -16,8 +15,8 @@ class AllowedTokenIdsLogitsProcessor:
|
||||
specific set of token ids."""
|
||||
|
||||
def __init__(self, allowed_ids: Iterable[int]):
|
||||
self.allowed_ids: Optional[list[int]] = list(allowed_ids)
|
||||
self.mask: Optional[torch.Tensor] = None
|
||||
self.allowed_ids: list[int] | None = list(allowed_ids)
|
||||
self.mask: torch.Tensor | None = None
|
||||
|
||||
def __call__(self, token_ids: list[int], logits: torch.Tensor) -> torch.Tensor:
|
||||
if self.mask is None:
|
||||
@@ -53,8 +52,8 @@ def logit_bias_logits_processor(
|
||||
|
||||
|
||||
def get_logits_processors(
|
||||
logit_bias: Optional[Union[dict[int, float], dict[str, float]]],
|
||||
allowed_token_ids: Optional[list[int]],
|
||||
logit_bias: dict[int, float] | dict[str, float] | None,
|
||||
allowed_token_ids: list[int] | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
) -> list[LogitsProcessor]:
|
||||
logits_processors: list[LogitsProcessor] = []
|
||||
|
||||
+339
-339
File diff suppressed because it is too large
Load Diff
@@ -4,10 +4,9 @@
|
||||
import asyncio
|
||||
import tempfile
|
||||
from argparse import Namespace
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from io import StringIO
|
||||
from typing import Callable, Optional
|
||||
|
||||
import aiohttp
|
||||
import torch
|
||||
@@ -124,7 +123,7 @@ _BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elap
|
||||
class BatchProgressTracker:
|
||||
def __init__(self):
|
||||
self._total = 0
|
||||
self._pbar: Optional[tqdm] = None
|
||||
self._pbar: tqdm | None = None
|
||||
|
||||
def submitted(self):
|
||||
self._total += 1
|
||||
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import Final, Optional, Union
|
||||
from typing import Final
|
||||
|
||||
import jinja2
|
||||
import partial_json_parser
|
||||
@@ -82,15 +82,15 @@ class OpenAIServingChat(OpenAIServing):
|
||||
models: OpenAIServingModels,
|
||||
response_role: str,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
chat_template: Optional[str],
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: Optional[str] = None,
|
||||
tool_parser: str | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
@@ -159,8 +159,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
async def create_chat_completion(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, ErrorResponse]:
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
|
||||
"""
|
||||
Chat Completion API similar to OpenAI's API.
|
||||
|
||||
@@ -281,7 +281,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
)
|
||||
|
||||
sampling_params: Union[SamplingParams, BeamSearchParams]
|
||||
sampling_params: SamplingParams | BeamSearchParams
|
||||
if request.use_beam_search:
|
||||
sampling_params = request.to_beam_search_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
@@ -416,11 +416,11 @@ class OpenAIServingChat(OpenAIServing):
|
||||
def extract_tool_call_required_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: Optional[str],
|
||||
current_text: str | None,
|
||||
delta_text: str,
|
||||
function_name_returned: bool,
|
||||
tool_call_idx: Optional[int] = None,
|
||||
) -> tuple[Optional[DeltaMessage], bool]:
|
||||
tool_call_idx: int | None = None,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
if current_text is None or current_text == "":
|
||||
# if the current text is empty, we cannot parse it
|
||||
return None, function_name_returned
|
||||
@@ -548,7 +548,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
and self._should_stream_with_auto_tool_parsing(request)
|
||||
)
|
||||
|
||||
all_previous_token_ids: Optional[list[list[int]]]
|
||||
all_previous_token_ids: list[list[int]] | None
|
||||
function_name_returned = [False] * num_choices
|
||||
if self.tool_call_id_type == "kimi_k2":
|
||||
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
||||
@@ -583,7 +583,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# Prepare the tool parser if it's needed
|
||||
try:
|
||||
if tool_choice_auto and self.tool_parser:
|
||||
tool_parsers: list[Optional[ToolParser]] = [
|
||||
tool_parsers: list[ToolParser | None] = [
|
||||
self.tool_parser(tokenizer)
|
||||
] * num_choices
|
||||
else:
|
||||
@@ -661,7 +661,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# Send response to echo the input portion of the
|
||||
# last message
|
||||
if request.echo:
|
||||
last_msg_content: Union[str, list[dict[str, str]]] = ""
|
||||
last_msg_content: str | list[dict[str, str]] = ""
|
||||
if (
|
||||
conversation
|
||||
and "content" in conversation[-1]
|
||||
@@ -734,7 +734,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# Chunked prefill case, don't return empty chunks
|
||||
continue
|
||||
|
||||
delta_message: Optional[DeltaMessage]
|
||||
delta_message: DeltaMessage | None
|
||||
|
||||
# just update previous_texts and previous_token_ids
|
||||
if tool_choice_auto or self.reasoning_parser:
|
||||
@@ -1260,9 +1260,9 @@ class OpenAIServingChat(OpenAIServing):
|
||||
conversation: list[ConversationMessage],
|
||||
tokenizer: AnyTokenizer,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> Union[ErrorResponse, ChatCompletionResponse]:
|
||||
) -> ErrorResponse | ChatCompletionResponse:
|
||||
created_time = int(time.time())
|
||||
final_res: Optional[RequestOutput] = None
|
||||
final_res: RequestOutput | None = None
|
||||
|
||||
try:
|
||||
async for res in result_generator:
|
||||
@@ -1512,7 +1512,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
choices.append(choice_data)
|
||||
|
||||
if request.echo:
|
||||
last_msg_content: Union[str, list[dict[str, str]]] = ""
|
||||
last_msg_content: str | list[dict[str, str]] = ""
|
||||
if (
|
||||
conversation
|
||||
and "content" in conversation[-1]
|
||||
@@ -1597,7 +1597,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
def _get_top_logprobs(
|
||||
self,
|
||||
logprobs: dict[int, Logprob],
|
||||
top_logprobs: Optional[int],
|
||||
top_logprobs: int | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
should_return_as_token_id: bool,
|
||||
) -> list[ChatCompletionLogProb]:
|
||||
@@ -1621,10 +1621,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
def _create_chat_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
tokenizer: AnyTokenizer,
|
||||
num_output_top_logprobs: Optional[int] = None,
|
||||
return_as_token_id: Optional[bool] = None,
|
||||
num_output_top_logprobs: int | None = None,
|
||||
return_as_token_id: bool | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs."""
|
||||
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
||||
@@ -1693,7 +1693,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
def _should_check_for_unstreamed_tool_arg_tokens(
|
||||
self,
|
||||
delta_message: Optional[DeltaMessage],
|
||||
delta_message: DeltaMessage | None,
|
||||
output: CompletionOutput,
|
||||
) -> bool:
|
||||
"""
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from http import HTTPStatus
|
||||
from typing import Optional, Union, cast
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
from fastapi import Request
|
||||
@@ -36,7 +36,7 @@ class ClassificationMixin(OpenAIServing):
|
||||
async def _preprocess(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""
|
||||
Process classification inputs: tokenize text, resolve adapters,
|
||||
and prepare model-specific inputs.
|
||||
@@ -70,7 +70,7 @@ class ClassificationMixin(OpenAIServing):
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Union[ClassificationResponse, ErrorResponse]:
|
||||
) -> ClassificationResponse | ErrorResponse:
|
||||
"""
|
||||
Convert model outputs to a formatted classification response
|
||||
with probabilities and labels.
|
||||
@@ -129,7 +129,7 @@ class ServingClassification(ClassificationMixin):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
@@ -143,7 +143,7 @@ class ServingClassification(ClassificationMixin):
|
||||
self,
|
||||
request: ClassificationRequest,
|
||||
raw_request: Request,
|
||||
) -> Union[ClassificationResponse, ErrorResponse]:
|
||||
) -> ClassificationResponse | ErrorResponse:
|
||||
model_name = self.models.model_name()
|
||||
request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
|
||||
|
||||
@@ -160,7 +160,7 @@ class ServingClassification(ClassificationMixin):
|
||||
def _create_pooling_params(
|
||||
self,
|
||||
ctx: ClassificationServeContext,
|
||||
) -> Union[PoolingParams, ErrorResponse]:
|
||||
) -> PoolingParams | ErrorResponse:
|
||||
pooling_params = super()._create_pooling_params(ctx)
|
||||
if isinstance(pooling_params, ErrorResponse):
|
||||
return pooling_params
|
||||
|
||||
@@ -5,7 +5,7 @@ import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import Optional, Union, cast
|
||||
from typing import cast
|
||||
|
||||
import jinja2
|
||||
from fastapi import Request
|
||||
@@ -45,7 +45,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
@@ -73,8 +73,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
async def create_completion(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[AsyncGenerator[str, None], CompletionResponse, ErrorResponse]:
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
|
||||
"""Completion API similar to OpenAI's API.
|
||||
|
||||
See https://platform.openai.com/docs/api-reference/completions/create
|
||||
@@ -166,7 +166,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
)
|
||||
|
||||
sampling_params: Union[SamplingParams, BeamSearchParams]
|
||||
sampling_params: SamplingParams | BeamSearchParams
|
||||
if request.use_beam_search:
|
||||
sampling_params = request.to_beam_search_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
@@ -196,7 +196,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
# Mypy inconsistently requires this second cast in different
|
||||
# environments. It shouldn't be necessary (redundant from above)
|
||||
# but pre-commit in CI fails without it.
|
||||
engine_prompt = cast(Union[EmbedsPrompt, TokensPrompt], engine_prompt)
|
||||
engine_prompt = cast(EmbedsPrompt | TokensPrompt, engine_prompt)
|
||||
if isinstance(sampling_params, BeamSearchParams):
|
||||
generator = self.beam_search(
|
||||
prompt=engine_prompt,
|
||||
@@ -260,7 +260,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
)
|
||||
|
||||
# Non-streaming response
|
||||
final_res_batch: list[Optional[RequestOutput]] = [None] * num_prompts
|
||||
final_res_batch: list[RequestOutput | None] = [None] * num_prompts
|
||||
try:
|
||||
async for i, res in result_generator:
|
||||
final_res_batch[i] = res
|
||||
@@ -312,7 +312,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
async def completion_stream_generator(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
engine_prompts: list[Union[TokensPrompt, EmbedsPrompt]],
|
||||
engine_prompts: list[TokensPrompt | EmbedsPrompt],
|
||||
result_generator: AsyncIterator[tuple[int, RequestOutput]],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
@@ -362,7 +362,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
num_prompt_tokens[prompt_idx] = len(prompt_token_ids)
|
||||
|
||||
delta_token_ids: GenericSequence[int]
|
||||
out_logprobs: Optional[GenericSequence[Optional[dict[int, Logprob]]]]
|
||||
out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
|
||||
|
||||
for output in res.outputs:
|
||||
i = output.index + prompt_idx * num_choices
|
||||
@@ -370,7 +370,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
# Useful when request.return_token_ids is True
|
||||
# Returning prompt token IDs shares the same logic
|
||||
# with the echo implementation.
|
||||
prompt_token_ids_to_return: Optional[list[int]] = None
|
||||
prompt_token_ids_to_return: list[int] | None = None
|
||||
|
||||
assert request.max_tokens is not None
|
||||
if request.echo and not has_echoed[i]:
|
||||
@@ -524,7 +524,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
prompt_text = final_res.prompt
|
||||
|
||||
token_ids: GenericSequence[int]
|
||||
out_logprobs: Optional[GenericSequence[Optional[dict[int, Logprob]]]]
|
||||
out_logprobs: GenericSequence[dict[int, Logprob] | None] | None
|
||||
|
||||
for output in final_res.outputs:
|
||||
assert request.max_tokens is not None
|
||||
@@ -617,17 +617,17 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
def _create_completion_logprobs(
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int,
|
||||
tokenizer: AnyTokenizer,
|
||||
initial_text_offset: int = 0,
|
||||
return_as_token_id: Optional[bool] = None,
|
||||
return_as_token_id: bool | None = None,
|
||||
) -> CompletionLogProbs:
|
||||
"""Create logprobs for OpenAI Completion API."""
|
||||
out_text_offset: list[int] = []
|
||||
out_token_logprobs: list[Optional[float]] = []
|
||||
out_token_logprobs: list[float | None] = []
|
||||
out_tokens: list[str] = []
|
||||
out_top_logprobs: list[Optional[dict[str, float]]] = []
|
||||
out_top_logprobs: list[dict[str, float] | None] = []
|
||||
|
||||
last_token_len = 0
|
||||
|
||||
@@ -695,7 +695,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
def _build_render_config(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
max_input_length: Optional[int] = None,
|
||||
max_input_length: int | None = None,
|
||||
) -> RenderConfig:
|
||||
max_input_tokens_len = self.max_model_len - (request.max_tokens or 0)
|
||||
return RenderConfig(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import base64
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from typing import Any, Final, Literal, Optional, Union, cast
|
||||
from typing import Any, Final, Literal, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -48,7 +48,7 @@ logger = init_logger(__name__)
|
||||
def _get_embedding(
|
||||
output: EmbeddingOutput,
|
||||
encoding_format: Literal["float", "base64"],
|
||||
) -> Union[list[float], str]:
|
||||
) -> list[float] | str:
|
||||
if encoding_format == "float":
|
||||
return output.embedding
|
||||
elif encoding_format == "base64":
|
||||
@@ -80,7 +80,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
async def _preprocess(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
ctx = cast(EmbeddingServeContext, ctx)
|
||||
try:
|
||||
ctx.lora_request = self._maybe_get_adapters(ctx.request)
|
||||
@@ -130,7 +130,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Union[EmbeddingResponse, ErrorResponse]:
|
||||
) -> EmbeddingResponse | ErrorResponse:
|
||||
items: list[EmbeddingResponseData] = []
|
||||
num_prompt_tokens = 0
|
||||
|
||||
@@ -314,9 +314,9 @@ class EmbeddingMixin(OpenAIServing):
|
||||
ctx: EmbeddingServeContext,
|
||||
engine_prompt: EngineTokensPrompt,
|
||||
pooling_params: PoolingParams,
|
||||
trace_headers: Optional[Mapping[str, str]],
|
||||
trace_headers: Mapping[str, str] | None,
|
||||
prompt_index: int,
|
||||
) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
|
||||
) -> AsyncGenerator[RequestOutput | PoolingRequestOutput, None]:
|
||||
"""Create a generator for a single prompt using standard processing."""
|
||||
request_id_item = f"{ctx.request_id}-{prompt_index}"
|
||||
|
||||
@@ -341,7 +341,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
async def _prepare_generators(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""Override to support chunked processing."""
|
||||
ctx = cast(EmbeddingServeContext, ctx)
|
||||
|
||||
@@ -354,7 +354,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
|
||||
# Custom logic for chunked processing
|
||||
generators: list[
|
||||
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
|
||||
AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
|
||||
] = []
|
||||
|
||||
try:
|
||||
@@ -413,7 +413,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
async def _collect_batch(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""Collect and aggregate batch results
|
||||
with support for chunked processing.
|
||||
|
||||
@@ -522,9 +522,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
)
|
||||
|
||||
# Finalize aggregated results
|
||||
final_res_batch: list[
|
||||
Union[PoolingRequestOutput, EmbeddingRequestOutput]
|
||||
] = []
|
||||
final_res_batch: list[PoolingRequestOutput | EmbeddingRequestOutput] = []
|
||||
num_prompts = len(ctx.engine_prompts)
|
||||
|
||||
for prompt_idx in range(num_prompts):
|
||||
@@ -581,7 +579,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
)
|
||||
|
||||
ctx.final_res_batch = cast(
|
||||
list[Union[RequestOutput, PoolingRequestOutput]], final_res_batch
|
||||
list[RequestOutput | PoolingRequestOutput], final_res_batch
|
||||
)
|
||||
|
||||
return None
|
||||
@@ -598,8 +596,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
chat_template: Optional[str],
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
@@ -618,8 +616,8 @@ class OpenAIServingEmbedding(EmbeddingMixin):
|
||||
async def create_embedding(
|
||||
self,
|
||||
request: EmbeddingRequest,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[EmbeddingResponse, ErrorResponse]:
|
||||
raw_request: Request | None = None,
|
||||
) -> EmbeddingResponse | ErrorResponse:
|
||||
"""
|
||||
Embedding API similar to OpenAI's API.
|
||||
|
||||
@@ -647,7 +645,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
|
||||
def _create_pooling_params(
|
||||
self,
|
||||
ctx: ServeContext[EmbeddingRequest],
|
||||
) -> Union[PoolingParams, ErrorResponse]:
|
||||
) -> PoolingParams | ErrorResponse:
|
||||
pooling_params = super()._create_pooling_params(ctx)
|
||||
if isinstance(pooling_params, ErrorResponse):
|
||||
return pooling_params
|
||||
@@ -662,7 +660,7 @@ class OpenAIServingEmbedding(EmbeddingMixin):
|
||||
async def _preprocess(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
if isinstance(ctx.request, EmbeddingChatRequest):
|
||||
error_check_ret = self._validate_chat_template(
|
||||
request_chat_template=ctx.request.chat_template,
|
||||
|
||||
@@ -5,10 +5,10 @@ import json
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
|
||||
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
from fastapi import Request
|
||||
@@ -102,38 +102,38 @@ from vllm.v1.engine import EngineCoreRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
CompletionLikeRequest = Union[
|
||||
CompletionRequest,
|
||||
DetokenizeRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
RerankRequest,
|
||||
ClassificationRequest,
|
||||
ScoreRequest,
|
||||
TokenizeCompletionRequest,
|
||||
]
|
||||
CompletionLikeRequest: TypeAlias = (
|
||||
CompletionRequest
|
||||
| DetokenizeRequest
|
||||
| EmbeddingCompletionRequest
|
||||
| RerankRequest
|
||||
| ClassificationRequest
|
||||
| ScoreRequest
|
||||
| TokenizeCompletionRequest
|
||||
)
|
||||
|
||||
ChatLikeRequest = Union[
|
||||
ChatCompletionRequest, EmbeddingChatRequest, TokenizeChatRequest
|
||||
]
|
||||
SpeechToTextRequest = Union[TranscriptionRequest, TranslationRequest]
|
||||
AnyRequest = Union[
|
||||
CompletionLikeRequest,
|
||||
ChatLikeRequest,
|
||||
SpeechToTextRequest,
|
||||
ResponsesRequest,
|
||||
IOProcessorRequest,
|
||||
]
|
||||
ChatLikeRequest: TypeAlias = (
|
||||
ChatCompletionRequest | EmbeddingChatRequest | TokenizeChatRequest
|
||||
)
|
||||
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
|
||||
AnyRequest: TypeAlias = (
|
||||
CompletionLikeRequest
|
||||
| ChatLikeRequest
|
||||
| SpeechToTextRequest
|
||||
| ResponsesRequest
|
||||
| IOProcessorRequest
|
||||
)
|
||||
|
||||
AnyResponse = Union[
|
||||
CompletionResponse,
|
||||
ChatCompletionResponse,
|
||||
EmbeddingResponse,
|
||||
TranscriptionResponse,
|
||||
TokenizeResponse,
|
||||
PoolingResponse,
|
||||
ClassificationResponse,
|
||||
ScoreResponse,
|
||||
]
|
||||
AnyResponse: TypeAlias = (
|
||||
CompletionResponse
|
||||
| ChatCompletionResponse
|
||||
| EmbeddingResponse
|
||||
| TranscriptionResponse
|
||||
| TokenizeResponse
|
||||
| PoolingResponse
|
||||
| ClassificationResponse
|
||||
| ScoreResponse
|
||||
)
|
||||
|
||||
|
||||
class TextTokensPrompt(TypedDict):
|
||||
@@ -145,7 +145,7 @@ class EmbedsPrompt(TypedDict):
|
||||
prompt_embeds: torch.Tensor
|
||||
|
||||
|
||||
RequestPrompt = Union[list[int], str, TextTokensPrompt, EmbedsPrompt]
|
||||
RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt
|
||||
|
||||
|
||||
def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]:
|
||||
@@ -173,8 +173,8 @@ class RequestProcessingMixin(BaseModel):
|
||||
handling prompt preparation and engine input.
|
||||
"""
|
||||
|
||||
request_prompts: Optional[Sequence[RequestPrompt]] = []
|
||||
engine_prompts: Optional[list[EngineTokensPrompt]] = []
|
||||
request_prompts: Sequence[RequestPrompt] | None = []
|
||||
engine_prompts: list[EngineTokensPrompt] | None = []
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
@@ -185,10 +185,10 @@ class ResponseGenerationMixin(BaseModel):
|
||||
managing result generators and final batch results.
|
||||
"""
|
||||
|
||||
result_generator: Optional[
|
||||
AsyncGenerator[tuple[int, Union[RequestOutput, PoolingRequestOutput]], None]
|
||||
] = None
|
||||
final_res_batch: list[Union[RequestOutput, PoolingRequestOutput]] = Field(
|
||||
result_generator: (
|
||||
AsyncGenerator[tuple[int, RequestOutput | PoolingRequestOutput], None] | None
|
||||
) = None
|
||||
final_res_batch: list[RequestOutput | PoolingRequestOutput] = Field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
@@ -203,14 +203,14 @@ class ServeContext(
|
||||
):
|
||||
# Shared across all requests
|
||||
request: RequestT
|
||||
raw_request: Optional[Request] = None
|
||||
raw_request: Request | None = None
|
||||
model_name: str
|
||||
request_id: str
|
||||
created_time: int = Field(default_factory=lambda: int(time.time()))
|
||||
lora_request: Optional[LoRARequest] = None
|
||||
lora_request: LoRARequest | None = None
|
||||
|
||||
# Shared across most requests
|
||||
tokenizer: Optional[AnyTokenizer] = None
|
||||
tokenizer: AnyTokenizer | None = None
|
||||
|
||||
# `protected_namespaces` resolves Pydantic v2's warning
|
||||
# on conflict with protected namespace "model_"
|
||||
@@ -224,7 +224,7 @@ ClassificationServeContext = ServeContext[ClassificationRequest]
|
||||
|
||||
|
||||
class EmbeddingServeContext(ServeContext[EmbeddingRequest]):
|
||||
chat_template: Optional[str] = None
|
||||
chat_template: str | None = None
|
||||
chat_template_content_format: ChatTemplateContentFormatOption
|
||||
|
||||
|
||||
@@ -247,7 +247,7 @@ class OpenAIServing:
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
@@ -276,8 +276,8 @@ class OpenAIServing:
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
|
||||
def _get_tool_parser(
|
||||
self, tool_parser_name: Optional[str] = None, enable_auto_tools: bool = False
|
||||
) -> Optional[Callable[[AnyTokenizer], ToolParser]]:
|
||||
self, tool_parser_name: str | None = None, enable_auto_tools: bool = False
|
||||
) -> Callable[[AnyTokenizer], ToolParser] | None:
|
||||
"""Get the tool parser based on the name."""
|
||||
parser = None
|
||||
if not enable_auto_tools or tool_parser_name is None:
|
||||
@@ -307,7 +307,7 @@ class OpenAIServing:
|
||||
def _get_reasoning_parser(
|
||||
self,
|
||||
reasoning_parser_name: str,
|
||||
) -> Optional[Callable[[AnyTokenizer], ReasoningParser]]:
|
||||
) -> Callable[[AnyTokenizer], ReasoningParser] | None:
|
||||
"""Get the reasoning parser based on the name."""
|
||||
parser = None
|
||||
if not reasoning_parser_name:
|
||||
@@ -328,7 +328,7 @@ class OpenAIServing:
|
||||
prompt: PromptType,
|
||||
request_id: str,
|
||||
params: BeamSearchParams,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
beam_width = params.beam_width
|
||||
max_tokens = params.max_tokens
|
||||
@@ -364,9 +364,9 @@ class OpenAIServing:
|
||||
# this happens again in generation, so the double expansion causes
|
||||
# a mismatch.
|
||||
# TODO - would be ideal to handle this more gracefully.
|
||||
prompt_text: Optional[str]
|
||||
prompt_text: str | None
|
||||
prompt_token_ids: list[int]
|
||||
multi_modal_data: Optional[MultiModalDataDict]
|
||||
multi_modal_data: MultiModalDataDict | None
|
||||
if isinstance(prompt, str):
|
||||
prompt_text = prompt
|
||||
prompt_token_ids = []
|
||||
@@ -376,7 +376,7 @@ class OpenAIServing:
|
||||
prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore
|
||||
multi_modal_data = prompt.get("multi_modal_data") # type: ignore
|
||||
|
||||
mm_processor_kwargs: Optional[dict[str, Any]] = processed_inputs.get(
|
||||
mm_processor_kwargs: dict[str, Any] | None = processed_inputs.get(
|
||||
"mm_processor_kwargs"
|
||||
) # type: ignore
|
||||
|
||||
@@ -507,7 +507,7 @@ class OpenAIServing:
|
||||
prompt_logprobs=None,
|
||||
)
|
||||
|
||||
def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:
|
||||
def _get_renderer(self, tokenizer: AnyTokenizer | None) -> BaseRenderer:
|
||||
"""
|
||||
Get a Renderer instance with the provided tokenizer.
|
||||
Uses shared async tokenizer pool for efficiency.
|
||||
@@ -545,7 +545,7 @@ class OpenAIServing:
|
||||
async def _preprocess(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""
|
||||
Default preprocessing hook. Subclasses may override
|
||||
to prepare `ctx` (classification, embedding, etc.).
|
||||
@@ -555,7 +555,7 @@ class OpenAIServing:
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Union[AnyResponse, ErrorResponse]:
|
||||
) -> AnyResponse | ErrorResponse:
|
||||
"""
|
||||
Default response builder. Subclass may override this method
|
||||
to return the appropriate response object.
|
||||
@@ -565,8 +565,8 @@ class OpenAIServing:
|
||||
async def handle(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Union[AnyResponse, ErrorResponse]:
|
||||
generation: AsyncGenerator[Union[AnyResponse, ErrorResponse], None]
|
||||
) -> AnyResponse | ErrorResponse:
|
||||
generation: AsyncGenerator[AnyResponse | ErrorResponse, None]
|
||||
generation = self._pipeline(ctx)
|
||||
|
||||
async for response in generation:
|
||||
@@ -577,7 +577,7 @@ class OpenAIServing:
|
||||
async def _pipeline(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> AsyncGenerator[Union[AnyResponse, ErrorResponse], None]:
|
||||
) -> AsyncGenerator[AnyResponse | ErrorResponse, None]:
|
||||
"""Execute the request processing pipeline yielding responses."""
|
||||
if error := await self._check_model(ctx.request):
|
||||
yield error
|
||||
@@ -598,7 +598,7 @@ class OpenAIServing:
|
||||
|
||||
yield self._build_response(ctx)
|
||||
|
||||
def _validate_request(self, ctx: ServeContext) -> Optional[ErrorResponse]:
|
||||
def _validate_request(self, ctx: ServeContext) -> ErrorResponse | None:
|
||||
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
|
||||
|
||||
if (
|
||||
@@ -615,7 +615,7 @@ class OpenAIServing:
|
||||
def _create_pooling_params(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Union[PoolingParams, ErrorResponse]:
|
||||
) -> PoolingParams | ErrorResponse:
|
||||
if not hasattr(ctx.request, "to_pooling_params"):
|
||||
return self.create_error_response(
|
||||
"Request type does not support pooling parameters"
|
||||
@@ -626,10 +626,10 @@ class OpenAIServing:
|
||||
async def _prepare_generators(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""Schedule the request and get the result generator."""
|
||||
generators: list[
|
||||
AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]
|
||||
AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
|
||||
] = []
|
||||
|
||||
try:
|
||||
@@ -678,14 +678,14 @@ class OpenAIServing:
|
||||
async def _collect_batch(
|
||||
self,
|
||||
ctx: ServeContext,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""Collect batch results from the result generator."""
|
||||
try:
|
||||
if ctx.engine_prompts is None:
|
||||
return self.create_error_response("Engine prompts not available")
|
||||
|
||||
num_prompts = len(ctx.engine_prompts)
|
||||
final_res_batch: list[Optional[Union[RequestOutput, PoolingRequestOutput]]]
|
||||
final_res_batch: list[RequestOutput | PoolingRequestOutput | None]
|
||||
final_res_batch = [None] * num_prompts
|
||||
|
||||
if ctx.result_generator is None:
|
||||
@@ -738,7 +738,7 @@ class OpenAIServing:
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyRequest,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
error_response = None
|
||||
|
||||
if self._is_model_supported(request.model):
|
||||
@@ -764,9 +764,7 @@ class OpenAIServing:
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
)
|
||||
|
||||
def _get_active_default_mm_loras(
|
||||
self, request: AnyRequest
|
||||
) -> Optional[LoRARequest]:
|
||||
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
@@ -793,7 +791,7 @@ class OpenAIServing:
|
||||
self,
|
||||
request: AnyRequest,
|
||||
supports_default_mm_loras: bool = False,
|
||||
) -> Optional[LoRARequest]:
|
||||
) -> LoRARequest | None:
|
||||
if request.model in self.models.lora_requests:
|
||||
return self.models.lora_requests[request.model]
|
||||
|
||||
@@ -877,7 +875,7 @@ class OpenAIServing:
|
||||
self,
|
||||
request: AnyRequest,
|
||||
prompt_ids: list[int],
|
||||
tokenizer: Optional[AnyTokenizer],
|
||||
tokenizer: AnyTokenizer | None,
|
||||
) -> TextTokensPrompt:
|
||||
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
|
||||
|
||||
@@ -972,7 +970,7 @@ class OpenAIServing:
|
||||
self,
|
||||
request: AnyRequest,
|
||||
tokenizer: AnyTokenizer,
|
||||
prompt_input: Union[str, list[int]],
|
||||
prompt_input: str | list[int],
|
||||
add_special_tokens: bool = True,
|
||||
) -> TextTokensPrompt:
|
||||
"""
|
||||
@@ -991,7 +989,7 @@ class OpenAIServing:
|
||||
self,
|
||||
request: AnyRequest,
|
||||
tokenizer: AnyTokenizer,
|
||||
prompt_inputs: Iterable[Union[str, list[int]]],
|
||||
prompt_inputs: Iterable[str | list[int]],
|
||||
add_special_tokens: bool = True,
|
||||
) -> AsyncGenerator[TextTokensPrompt, None]:
|
||||
"""
|
||||
@@ -1014,10 +1012,10 @@ class OpenAIServing:
|
||||
|
||||
def _validate_chat_template(
|
||||
self,
|
||||
request_chat_template: Optional[str],
|
||||
chat_template_kwargs: Optional[dict[str, Any]],
|
||||
request_chat_template: str | None,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
trust_request_chat_template: bool,
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
if not trust_request_chat_template and (
|
||||
request_chat_template is not None
|
||||
or (
|
||||
@@ -1034,17 +1032,17 @@ class OpenAIServing:
|
||||
|
||||
async def _preprocess_chat(
|
||||
self,
|
||||
request: Union[ChatLikeRequest, ResponsesRequest],
|
||||
request: ChatLikeRequest | ResponsesRequest,
|
||||
tokenizer: AnyTokenizer,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
chat_template: Optional[str],
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
add_generation_prompt: bool = True,
|
||||
continue_final_message: bool = False,
|
||||
tool_dicts: Optional[list[dict[str, Any]]] = None,
|
||||
documents: Optional[list[dict[str, str]]] = None,
|
||||
chat_template_kwargs: Optional[dict[str, Any]] = None,
|
||||
tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
documents: list[dict[str, str]] | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
tool_parser: Callable[[AnyTokenizer], ToolParser] | None = None,
|
||||
add_special_tokens: bool = False,
|
||||
) -> tuple[
|
||||
list[ConversationMessage],
|
||||
@@ -1076,7 +1074,7 @@ class OpenAIServing:
|
||||
)
|
||||
_chat_template_kwargs.update(chat_template_kwargs or {})
|
||||
|
||||
request_prompt: Union[str, list[int]]
|
||||
request_prompt: str | list[int]
|
||||
|
||||
if tokenizer is None:
|
||||
request_prompt = "placeholder"
|
||||
@@ -1158,10 +1156,10 @@ class OpenAIServing:
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: PromptType,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
params: SamplingParams | PoolingParams,
|
||||
*,
|
||||
lora_request: Optional[LoRARequest],
|
||||
trace_headers: Optional[Mapping[str, str]],
|
||||
lora_request: LoRARequest | None,
|
||||
trace_headers: Mapping[str, str] | None,
|
||||
priority: int,
|
||||
) -> tuple[EngineCoreRequest, dict[str, Any]]:
|
||||
"""Use the Processor to process inputs for AsyncLLM."""
|
||||
@@ -1188,7 +1186,7 @@ class OpenAIServing:
|
||||
engine_prompt: EngineTokensPrompt,
|
||||
sampling_params: SamplingParams,
|
||||
context: ConversationContext,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -1250,7 +1248,7 @@ class OpenAIServing:
|
||||
|
||||
def _get_prompt_components(
|
||||
self,
|
||||
prompt: Union[RequestPrompt, PromptType],
|
||||
prompt: RequestPrompt | PromptType,
|
||||
) -> PromptComponents:
|
||||
if isinstance(prompt, list):
|
||||
return PromptComponents(token_ids=prompt)
|
||||
@@ -1260,9 +1258,9 @@ class OpenAIServing:
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
inputs: Union[RequestPrompt, PromptType],
|
||||
params: Optional[Union[SamplingParams, PoolingParams, BeamSearchParams]],
|
||||
lora_request: Optional[LoRARequest],
|
||||
inputs: RequestPrompt | PromptType,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
if self.request_logger is None:
|
||||
return
|
||||
@@ -1281,7 +1279,7 @@ class OpenAIServing:
|
||||
async def _get_trace_headers(
|
||||
self,
|
||||
headers: Headers,
|
||||
) -> Optional[Mapping[str, str]]:
|
||||
) -> Mapping[str, str] | None:
|
||||
is_tracing_enabled = await self.engine_client.is_tracing_enabled()
|
||||
|
||||
if is_tracing_enabled:
|
||||
@@ -1294,8 +1292,8 @@ class OpenAIServing:
|
||||
|
||||
@staticmethod
|
||||
def _base_request_id(
|
||||
raw_request: Optional[Request], default: Optional[str] = None
|
||||
) -> Optional[str]:
|
||||
raw_request: Request | None, default: str | None = None
|
||||
) -> str | None:
|
||||
"""Pulls the request id to use from a header, if provided"""
|
||||
default = default or random_uuid()
|
||||
if raw_request is None:
|
||||
@@ -1317,15 +1315,15 @@ class OpenAIServing:
|
||||
return logprob.decoded_token
|
||||
return tokenizer.decode(token_id)
|
||||
|
||||
def _is_model_supported(self, model_name: Optional[str]) -> bool:
|
||||
def _is_model_supported(self, model_name: str | None) -> bool:
|
||||
if not model_name:
|
||||
return True
|
||||
return self.models.is_base_model(model_name)
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
prompt_logprobs: Union[PromptLogprobs, None],
|
||||
) -> Union[PromptLogprobs, None]:
|
||||
prompt_logprobs: PromptLogprobs | None,
|
||||
) -> PromptLogprobs | None:
|
||||
if prompt_logprobs is None:
|
||||
return prompt_logprobs
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from asyncio import Lock
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import Optional, Union
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
@@ -35,7 +34,7 @@ class BaseModelPath:
|
||||
class LoRAModulePath:
|
||||
name: str
|
||||
path: str
|
||||
base_model_name: Optional[str] = None
|
||||
base_model_name: str | None = None
|
||||
|
||||
|
||||
class OpenAIServingModels:
|
||||
@@ -52,7 +51,7 @@ class OpenAIServingModels:
|
||||
engine_client: EngineClient,
|
||||
base_model_paths: list[BaseModelPath],
|
||||
*,
|
||||
lora_modules: Optional[list[LoRAModulePath]] = None,
|
||||
lora_modules: list[LoRAModulePath] | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -93,7 +92,7 @@ class OpenAIServingModels:
|
||||
def is_base_model(self, model_name) -> bool:
|
||||
return any(model.name == model_name for model in self.base_model_paths)
|
||||
|
||||
def model_name(self, lora_request: Optional[LoRARequest] = None) -> str:
|
||||
def model_name(self, lora_request: LoRARequest | None = None) -> str:
|
||||
"""Returns the appropriate model name depending on the availability
|
||||
and support of the LoRA or base model.
|
||||
Parameters:
|
||||
@@ -132,8 +131,8 @@ class OpenAIServingModels:
|
||||
return ModelList(data=model_cards)
|
||||
|
||||
async def load_lora_adapter(
|
||||
self, request: LoadLoRAAdapterRequest, base_model_name: Optional[str] = None
|
||||
) -> Union[ErrorResponse, str]:
|
||||
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
|
||||
) -> ErrorResponse | str:
|
||||
lora_name = request.lora_name
|
||||
|
||||
# Ensure atomicity based on the lora name
|
||||
@@ -173,7 +172,7 @@ class OpenAIServingModels:
|
||||
|
||||
async def unload_lora_adapter(
|
||||
self, request: UnloadLoRAAdapterRequest
|
||||
) -> Union[ErrorResponse, str]:
|
||||
) -> ErrorResponse | str:
|
||||
lora_name = request.lora_name
|
||||
|
||||
# Ensure atomicity based on the lora name
|
||||
@@ -189,7 +188,7 @@ class OpenAIServingModels:
|
||||
|
||||
async def _check_load_lora_adapter_request(
|
||||
self, request: LoadLoRAAdapterRequest
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
# Check if both 'lora_name' and 'lora_path' are provided
|
||||
if not request.lora_name or not request.lora_path:
|
||||
return create_error_response(
|
||||
@@ -211,7 +210,7 @@ class OpenAIServingModels:
|
||||
|
||||
async def _check_unload_lora_adapter_request(
|
||||
self, request: UnloadLoRAAdapterRequest
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
# Check if 'lora_name' is not provided return an error
|
||||
if not request.lora_name:
|
||||
return create_error_response(
|
||||
@@ -230,7 +229,7 @@ class OpenAIServingModels:
|
||||
|
||||
return None
|
||||
|
||||
async def resolve_lora(self, lora_name: str) -> Union[LoRARequest, ErrorResponse]:
|
||||
async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse:
|
||||
"""Attempt to resolve a LoRA adapter using available resolvers.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -5,7 +5,7 @@ import asyncio
|
||||
import base64
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Final, Literal, Optional, Union, cast
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
import jinja2
|
||||
import numpy as np
|
||||
@@ -41,7 +41,7 @@ logger = init_logger(__name__)
|
||||
def _get_data(
|
||||
output: PoolingOutput,
|
||||
encoding_format: Literal["float", "base64"],
|
||||
) -> Union[list[float], str]:
|
||||
) -> list[float] | str:
|
||||
if encoding_format == "float":
|
||||
return output.data.tolist()
|
||||
elif encoding_format == "base64":
|
||||
@@ -60,8 +60,8 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
chat_template: Optional[str],
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
@@ -80,8 +80,8 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
async def create_pooling(
|
||||
self,
|
||||
request: PoolingRequest,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[PoolingResponse, IOProcessorResponse, ErrorResponse]:
|
||||
raw_request: Request | None = None,
|
||||
) -> PoolingResponse | IOProcessorResponse | ErrorResponse:
|
||||
"""
|
||||
See https://platform.openai.com/docs/api-reference/embeddings/create
|
||||
for the API specification. This API mimics the OpenAI Embedding API.
|
||||
@@ -219,7 +219,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
num_prompts = len(engine_prompts)
|
||||
|
||||
# Non-streaming response
|
||||
final_res_batch: list[Optional[PoolingRequestOutput]]
|
||||
final_res_batch: list[PoolingRequestOutput | None]
|
||||
final_res_batch = [None] * num_prompts
|
||||
try:
|
||||
async for i, res in result_generator:
|
||||
|
||||
@@ -6,11 +6,11 @@ import json
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Sequence
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
|
||||
from contextlib import AsyncExitStack
|
||||
from copy import copy
|
||||
from http import HTTPStatus
|
||||
from typing import Callable, Final, Optional, Union
|
||||
from typing import Final
|
||||
|
||||
import jinja2
|
||||
from fastapi import Request
|
||||
@@ -109,14 +109,14 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
chat_template: Optional[str],
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
reasoning_parser: str = "",
|
||||
enable_auto_tools: bool = False,
|
||||
tool_parser: Optional[str] = None,
|
||||
tool_server: Optional[ToolServer] = None,
|
||||
tool_parser: str | None = None,
|
||||
tool_server: ToolServer | None = None,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
enable_log_outputs: bool = False,
|
||||
@@ -210,7 +210,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
|
||||
def _validate_generator_input(
|
||||
self, engine_prompt: EngineTokensPrompt
|
||||
) -> Optional[ErrorResponse]:
|
||||
) -> ErrorResponse | None:
|
||||
"""Add validations to the input to the generator here."""
|
||||
if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
|
||||
error_message = (
|
||||
@@ -229,12 +229,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
async def create_responses(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[
|
||||
AsyncGenerator[StreamingResponsesResponse, None],
|
||||
ResponsesResponse,
|
||||
ErrorResponse,
|
||||
]:
|
||||
raw_request: Request | None = None,
|
||||
) -> (
|
||||
AsyncGenerator[StreamingResponsesResponse, None]
|
||||
| ResponsesResponse
|
||||
| ErrorResponse
|
||||
):
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
@@ -460,7 +460,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
async def _make_request(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: Optional[ResponsesResponse],
|
||||
prev_response: ResponsesResponse | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
):
|
||||
if len(request.tools) > 0:
|
||||
@@ -481,7 +481,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: Optional[ResponsesResponse],
|
||||
prev_response: ResponsesResponse | None,
|
||||
):
|
||||
if request.tool_choice != "auto":
|
||||
raise NotImplementedError(
|
||||
@@ -522,8 +522,8 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
model_name: str,
|
||||
tokenizer: AnyTokenizer,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
created_time: Optional[int] = None,
|
||||
) -> Union[ErrorResponse, ResponsesResponse]:
|
||||
created_time: int | None = None,
|
||||
) -> ErrorResponse | ResponsesResponse:
|
||||
if created_time is None:
|
||||
created_time = int(time.time())
|
||||
|
||||
@@ -642,9 +642,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
def _create_response_logprobs(
|
||||
self,
|
||||
token_ids: Sequence[int],
|
||||
logprobs: Optional[SampleLogprobs],
|
||||
logprobs: SampleLogprobs | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
top_logprobs: Optional[int] = None,
|
||||
top_logprobs: int | None = None,
|
||||
) -> list[Logprob]:
|
||||
assert logprobs is not None, "logprobs must be provided"
|
||||
assert len(token_ids) == len(logprobs), (
|
||||
@@ -676,9 +676,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
def _create_stream_response_logprobs(
|
||||
self,
|
||||
token_ids: Sequence[int],
|
||||
logprobs: Optional[SampleLogprobs],
|
||||
logprobs: SampleLogprobs | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
top_logprobs: Optional[int] = None,
|
||||
top_logprobs: int | None = None,
|
||||
) -> list[response_text_delta_event.Logprob]:
|
||||
lgs = self._create_response_logprobs(
|
||||
token_ids=token_ids,
|
||||
@@ -793,7 +793,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
def _construct_input_messages(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: Optional[ResponsesResponse] = None,
|
||||
prev_response: ResponsesResponse | None = None,
|
||||
) -> list[ChatCompletionMessageParam]:
|
||||
messages: list[ChatCompletionMessageParam] = []
|
||||
if request.instructions:
|
||||
@@ -833,7 +833,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
def _construct_input_messages_with_harmony(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: Optional[ResponsesResponse],
|
||||
prev_response: ResponsesResponse | None,
|
||||
) -> list[OpenAIHarmonyMessage]:
|
||||
messages: list[OpenAIHarmonyMessage] = []
|
||||
if prev_response is None:
|
||||
@@ -986,7 +986,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
async def responses_background_stream_generator(
|
||||
self,
|
||||
response_id: str,
|
||||
starting_after: Optional[int] = None,
|
||||
starting_after: int | None = None,
|
||||
) -> AsyncGenerator[StreamingResponsesResponse, None]:
|
||||
if response_id not in self.event_store:
|
||||
raise ValueError(f"Unknown response_id: {response_id}")
|
||||
@@ -1011,13 +1011,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
async def retrieve_responses(
|
||||
self,
|
||||
response_id: str,
|
||||
starting_after: Optional[int],
|
||||
stream: Optional[bool],
|
||||
) -> Union[
|
||||
ErrorResponse,
|
||||
ResponsesResponse,
|
||||
AsyncGenerator[StreamingResponsesResponse, None],
|
||||
]:
|
||||
starting_after: int | None,
|
||||
stream: bool | None,
|
||||
) -> (
|
||||
ErrorResponse
|
||||
| ResponsesResponse
|
||||
| AsyncGenerator[StreamingResponsesResponse, None]
|
||||
):
|
||||
async with self.response_store_lock:
|
||||
response = self.response_store.get(response_id)
|
||||
|
||||
@@ -1034,7 +1034,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
async def cancel_responses(
|
||||
self,
|
||||
response_id: str,
|
||||
) -> Union[ErrorResponse, ResponsesResponse]:
|
||||
) -> ErrorResponse | ResponsesResponse:
|
||||
async with self.response_store_lock:
|
||||
response = self.response_store.get(response_id)
|
||||
if response is None:
|
||||
@@ -1082,7 +1082,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
sampling_params: SamplingParams,
|
||||
result_generator: AsyncIterator[Optional[ConversationContext]],
|
||||
result_generator: AsyncIterator[ConversationContext | None],
|
||||
context: ConversationContext,
|
||||
model_name: str,
|
||||
tokenizer: AnyTokenizer,
|
||||
@@ -1385,7 +1385,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
sampling_params: SamplingParams,
|
||||
result_generator: AsyncIterator[Optional[ConversationContext]],
|
||||
result_generator: AsyncIterator[ConversationContext | None],
|
||||
context: ConversationContext,
|
||||
model_name: str,
|
||||
tokenizer: AnyTokenizer,
|
||||
@@ -1772,12 +1772,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
sampling_params: SamplingParams,
|
||||
result_generator: AsyncIterator[Optional[ConversationContext]],
|
||||
result_generator: AsyncIterator[ConversationContext | None],
|
||||
context: ConversationContext,
|
||||
model_name: str,
|
||||
tokenizer: AnyTokenizer,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
created_time: Optional[int] = None,
|
||||
created_time: int | None = None,
|
||||
) -> AsyncGenerator[StreamingResponsesResponse, None]:
|
||||
# TODO:
|
||||
# 1. Handle disconnect
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
@@ -48,7 +48,7 @@ class ServingScores(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
@@ -63,12 +63,12 @@ class ServingScores(OpenAIServing):
|
||||
tokenizer: AnyTokenizer,
|
||||
texts_1: list[str],
|
||||
texts_2: list[str],
|
||||
request: Union[RerankRequest, ScoreRequest],
|
||||
request: RerankRequest | ScoreRequest,
|
||||
request_id: str,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
lora_request: Optional[Union[LoRARequest, None]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
lora_request: LoRARequest | None | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
) -> list[PoolingRequestOutput] | ErrorResponse:
|
||||
input_texts = texts_1 + texts_2
|
||||
|
||||
engine_prompts: list[TokensPrompt] = []
|
||||
@@ -125,7 +125,7 @@ class ServingScores(OpenAIServing):
|
||||
# Non-streaming response
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
embeddings: list[Optional[PoolingRequestOutput]] = [None] * len(engine_prompts)
|
||||
embeddings: list[PoolingRequestOutput | None] = [None] * len(engine_prompts)
|
||||
|
||||
async for i, res in result_generator:
|
||||
embeddings[i] = res
|
||||
@@ -152,11 +152,11 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
def _preprocess_score(
|
||||
self,
|
||||
request: Union[RerankRequest, ScoreRequest],
|
||||
request: RerankRequest | ScoreRequest,
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenization_kwargs: dict[str, Any],
|
||||
data_1: Union[str, ScoreContentPartParam],
|
||||
data_2: Union[str, ScoreContentPartParam],
|
||||
data_1: str | ScoreContentPartParam,
|
||||
data_2: str | ScoreContentPartParam,
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
model_config = self.model_config
|
||||
|
||||
@@ -176,14 +176,14 @@ class ServingScores(OpenAIServing):
|
||||
async def _cross_encoding_score(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
data_1: Union[list[str], list[ScoreContentPartParam]],
|
||||
data_2: Union[list[str], list[ScoreContentPartParam]],
|
||||
request: Union[RerankRequest, ScoreRequest],
|
||||
data_1: list[str] | list[ScoreContentPartParam],
|
||||
data_2: list[str] | list[ScoreContentPartParam],
|
||||
request: RerankRequest | ScoreRequest,
|
||||
request_id: str,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
lora_request: Optional[Union[LoRARequest, None]] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
lora_request: LoRARequest | None | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
) -> list[PoolingRequestOutput] | ErrorResponse:
|
||||
request_prompts: list[str] = []
|
||||
engine_prompts: list[TokensPrompt] = []
|
||||
|
||||
@@ -259,7 +259,7 @@ class ServingScores(OpenAIServing):
|
||||
result_generator = merge_async_iterators(*generators)
|
||||
|
||||
# Non-streaming response
|
||||
final_res_batch: list[Optional[PoolingRequestOutput]] = [None] * len(
|
||||
final_res_batch: list[PoolingRequestOutput | None] = [None] * len(
|
||||
engine_prompts
|
||||
)
|
||||
|
||||
@@ -270,12 +270,12 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
async def _run_scoring(
|
||||
self,
|
||||
data_1: Union[list[str], str, ScoreMultiModalParam],
|
||||
data_2: Union[list[str], str, ScoreMultiModalParam],
|
||||
request: Union[ScoreRequest, RerankRequest],
|
||||
data_1: list[str] | str | ScoreMultiModalParam,
|
||||
data_2: list[str] | str | ScoreMultiModalParam,
|
||||
request: ScoreRequest | RerankRequest,
|
||||
request_id: str,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[list[PoolingRequestOutput], ErrorResponse]:
|
||||
raw_request: Request | None = None,
|
||||
) -> list[PoolingRequestOutput] | ErrorResponse:
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
@@ -339,8 +339,8 @@ class ServingScores(OpenAIServing):
|
||||
async def create_score(
|
||||
self,
|
||||
request: ScoreRequest,
|
||||
raw_request: Optional[Request] = None,
|
||||
) -> Union[ScoreResponse, ErrorResponse]:
|
||||
raw_request: Request | None = None,
|
||||
) -> ScoreResponse | ErrorResponse:
|
||||
"""
|
||||
Score API similar to Sentence Transformers cross encoder
|
||||
|
||||
@@ -377,8 +377,8 @@ class ServingScores(OpenAIServing):
|
||||
return self.create_error_response(str(e))
|
||||
|
||||
async def do_rerank(
|
||||
self, request: RerankRequest, raw_request: Optional[Request] = None
|
||||
) -> Union[RerankResponse, ErrorResponse]:
|
||||
self, request: RerankRequest, raw_request: Request | None = None
|
||||
) -> RerankResponse | ErrorResponse:
|
||||
"""
|
||||
Rerank API based on JinaAI's rerank API; implements the same
|
||||
API interface. Designed for compatibility with off-the-shelf
|
||||
@@ -468,7 +468,7 @@ class ServingScores(OpenAIServing):
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
documents: Union[list[str], ScoreMultiModalParam],
|
||||
documents: list[str] | ScoreMultiModalParam,
|
||||
top_n: int,
|
||||
) -> RerankResponse:
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Final, Optional, Union
|
||||
from typing import Any, Final
|
||||
|
||||
import jinja2
|
||||
from fastapi import Request
|
||||
@@ -33,8 +33,8 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
chat_template: Optional[str],
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
@@ -54,7 +54,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
self,
|
||||
request: TokenizeRequest,
|
||||
raw_request: Request,
|
||||
) -> Union[TokenizeResponse, ErrorResponse]:
|
||||
) -> TokenizeResponse | ErrorResponse:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
@@ -129,7 +129,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
self,
|
||||
request: DetokenizeRequest,
|
||||
raw_request: Request,
|
||||
) -> Union[DetokenizeResponse, ErrorResponse]:
|
||||
) -> DetokenizeResponse | ErrorResponse:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
@@ -155,7 +155,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
|
||||
async def get_tokenizer_info(
|
||||
self,
|
||||
) -> Union[TokenizerInfoResponse, ErrorResponse]:
|
||||
) -> TokenizerInfoResponse | ErrorResponse:
|
||||
"""Get comprehensive tokenizer information."""
|
||||
try:
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
@@ -171,7 +171,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
@dataclass
|
||||
class TokenizerInfo:
|
||||
tokenizer: AnyTokenizer
|
||||
chat_template: Optional[str]
|
||||
chat_template: str | None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Return the tokenizer configuration."""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Optional, Union
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
@@ -35,7 +34,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
):
|
||||
@@ -50,7 +49,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
|
||||
|
||||
async def create_transcription(
|
||||
self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request
|
||||
) -> Union[TranscriptionResponse, AsyncGenerator[str, None], ErrorResponse]:
|
||||
) -> TranscriptionResponse | AsyncGenerator[str, None] | ErrorResponse:
|
||||
"""Transcription API similar to OpenAI's API.
|
||||
|
||||
See https://platform.openai.com/docs/api-reference/audio/createTranscription
|
||||
@@ -94,7 +93,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
):
|
||||
@@ -109,7 +108,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
|
||||
|
||||
async def create_translation(
|
||||
self, audio_data: bytes, request: TranslationRequest, raw_request: Request
|
||||
) -> Union[TranslationResponse, AsyncGenerator[str, None], ErrorResponse]:
|
||||
) -> TranslationResponse | AsyncGenerator[str, None] | ErrorResponse:
|
||||
"""Translation API similar to OpenAI's API.
|
||||
|
||||
See https://platform.openai.com/docs/api-reference/audio/createTranslation
|
||||
|
||||
@@ -4,9 +4,9 @@ import asyncio
|
||||
import io
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from functools import cached_property
|
||||
from typing import Callable, Literal, Optional, TypeVar, Union, cast
|
||||
from typing import Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
from fastapi import Request
|
||||
@@ -39,7 +39,7 @@ try:
|
||||
except ImportError:
|
||||
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
|
||||
|
||||
SpeechToTextResponse = Union[TranscriptionResponse, TranslationResponse]
|
||||
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
|
||||
T = TypeVar("T", bound=SpeechToTextResponse)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -54,7 +54,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: Optional[RequestLogger],
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
task_type: Literal["transcribe", "translate"] = "transcribe",
|
||||
log_error_stack: bool = False,
|
||||
@@ -140,7 +140,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
raw_request: Request,
|
||||
response_class: type[T],
|
||||
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
|
||||
) -> Union[T, AsyncGenerator[str, None], ErrorResponse]:
|
||||
) -> T | AsyncGenerator[str, None] | ErrorResponse:
|
||||
"""Base method for speech-to-text operations like transcription and
|
||||
translation."""
|
||||
error_check_ret = await self._check_model(request)
|
||||
@@ -181,9 +181,7 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
logger.exception("Error in preprocessing prompt inputs")
|
||||
return self.create_error_response(str(e))
|
||||
|
||||
list_result_generator: Optional[list[AsyncGenerator[RequestOutput, None]]] = (
|
||||
None
|
||||
)
|
||||
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
|
||||
try:
|
||||
# Unlike most decoder-only models, whisper generation length is not
|
||||
# constrained by the size of the input audio, which is mapped to a
|
||||
@@ -252,13 +250,10 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
request_metadata: RequestResponseMetadata,
|
||||
audio_duration_s: float,
|
||||
chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
|
||||
response_stream_choice_class: Union[
|
||||
type[TranscriptionResponseStreamChoice],
|
||||
type[TranslationResponseStreamChoice],
|
||||
],
|
||||
stream_response_class: Union[
|
||||
type[TranscriptionStreamResponse], type[TranslationStreamResponse]
|
||||
],
|
||||
response_stream_choice_class: type[TranscriptionResponseStreamChoice]
|
||||
| type[TranslationResponseStreamChoice],
|
||||
stream_response_class: type[TranscriptionStreamResponse]
|
||||
| type[TranslationStreamResponse],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
created_time = int(time.time())
|
||||
model_name = request.model
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from functools import cached_property
|
||||
from typing import Callable, Optional, Union
|
||||
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
@@ -69,7 +68,7 @@ class ToolParser:
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Instance method that should be implemented for extracting tool calls
|
||||
from an incomplete response; for use when handling tool calls and
|
||||
@@ -101,7 +100,7 @@ class ToolParserManager:
|
||||
def _register_module(
|
||||
cls,
|
||||
module: type,
|
||||
module_name: Optional[Union[str, list[str]]] = None,
|
||||
module_name: str | list[str] | None = None,
|
||||
force: bool = True,
|
||||
) -> None:
|
||||
if not issubclass(module, ToolParser):
|
||||
@@ -123,10 +122,10 @@ class ToolParserManager:
|
||||
@classmethod
|
||||
def register_module(
|
||||
cls,
|
||||
name: Optional[Union[str, list[str]]] = None,
|
||||
name: str | list[str] | None = None,
|
||||
force: bool = True,
|
||||
module: Union[type, None] = None,
|
||||
) -> Union[type, Callable]:
|
||||
module: type | None = None,
|
||||
) -> type | Callable:
|
||||
"""
|
||||
Register module with the given name or name list. it can be used as a
|
||||
decoder(with module as None) or normal function(with module as not
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -129,7 +128,7 @@ class DeepSeekV31ToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
logger.debug("delta_text: %s", delta_text)
|
||||
logger.debug("delta_token_ids: %s", delta_token_ids)
|
||||
# check to see if we should be streaming a tool call - is there a
|
||||
@@ -272,7 +271,7 @@ class DeepSeekV31ToolParser(ToolParser):
|
||||
if not self.current_tool_name_sent:
|
||||
if current_tool_call is None:
|
||||
return None
|
||||
function_name: Union[str, None] = current_tool_call.get("name")
|
||||
function_name: str | None = current_tool_call.get("name")
|
||||
if function_name:
|
||||
self.current_tool_name_sent = True
|
||||
return DeltaMessage(
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -129,7 +128,7 @@ class DeepSeekV3ToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
logger.debug("delta_text: %s", delta_text)
|
||||
logger.debug("delta_token_ids: %s", delta_token_ids)
|
||||
# check to see if we should be streaming a tool call - is there a
|
||||
@@ -272,7 +271,7 @@ class DeepSeekV3ToolParser(ToolParser):
|
||||
if not self.current_tool_name_sent:
|
||||
if current_tool_call is None:
|
||||
return None
|
||||
function_name: Union[str, None] = current_tool_call.get("name")
|
||||
function_name: str | None = current_tool_call.get("name")
|
||||
if function_name:
|
||||
self.current_tool_name_sent = True
|
||||
return DeltaMessage(
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import ast
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -66,7 +66,7 @@ class Glm4MoeModelToolParser(ToolParser):
|
||||
def _is_string_type(
|
||||
tool_name: str,
|
||||
arg_name: str,
|
||||
tools: Optional[list[ChatCompletionToolsParam]],
|
||||
tools: list[ChatCompletionToolsParam] | None,
|
||||
) -> bool:
|
||||
if tools is None:
|
||||
return False
|
||||
@@ -144,7 +144,7 @@ class Glm4MoeModelToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
self._buffer += delta_text
|
||||
cur_text = self._buffer
|
||||
start_idx = cur_text.find(self.tool_call_start_token)
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from json import JSONDecoder
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
@@ -121,7 +120,7 @@ class Granite20bFCToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
if len(current_text) < len(self.bot_token) and self.bot_token.startswith(
|
||||
current_text
|
||||
):
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
from partial_json_parser.core.options import Allow
|
||||
@@ -108,7 +107,7 @@ class GraniteToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
start_idx = consume_space(0, current_text)
|
||||
if current_text[start_idx:].startswith(self.bot_token):
|
||||
start_idx = consume_space(start_idx + len(self.bot_token), current_text)
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
@@ -181,7 +180,7 @@ class Hermes2ProToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
# 1. All tokens are parsed based on _text, not token_ids.
|
||||
# 2. All incoming text data is processed by the tool_call_delta_buffer
|
||||
# function for buffering before being used for parsing.
|
||||
@@ -333,7 +332,7 @@ class Hermes2ProToolParser(ToolParser):
|
||||
if not self.current_tool_name_sent:
|
||||
if current_tool_call is None:
|
||||
return None
|
||||
function_name: Union[str, None] = current_tool_call.get("name")
|
||||
function_name: str | None = current_tool_call.get("name")
|
||||
if function_name:
|
||||
self.current_tool_name_sent = True
|
||||
return DeltaMessage(
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -73,7 +73,7 @@ class HunyuanA13BToolParser(ToolParser):
|
||||
|
||||
def preprocess_model_output(
|
||||
self, model_output: str
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
) -> tuple[str | None, str | None]:
|
||||
# find the location tool call
|
||||
for match in self.answer_tool_calls_pattern.finditer(model_output):
|
||||
start, end = match.span()
|
||||
@@ -176,7 +176,7 @@ class HunyuanA13BToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Extract tool calls for streaming mode.
|
||||
"""
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
from partial_json_parser.core.options import Allow
|
||||
@@ -59,7 +58,7 @@ class Internlm2ToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
if "<|action_start|>" not in current_text:
|
||||
self.position = len(current_text)
|
||||
return DeltaMessage(content=delta_text)
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
@@ -129,7 +128,7 @@ class JambaToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
# if the tool call token is not in the tokens generated so far, append
|
||||
# output to contents since it's not a tool
|
||||
if self.tool_calls_start_token not in current_text:
|
||||
@@ -190,7 +189,7 @@ class JambaToolParser(ToolParser):
|
||||
# auto-generated due to JSON completions, but wasn't
|
||||
# streamed to the client yet.
|
||||
if self.current_tool_id >= 0:
|
||||
diff: Union[str, None] = current_tool_call.get("arguments")
|
||||
diff: str | None = current_tool_call.get("arguments")
|
||||
|
||||
if diff:
|
||||
diff = json.dumps(diff, ensure_ascii=False).replace(
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
# code modified from deepseekv3_tool_parser.py
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -131,7 +130,7 @@ class KimiK2ToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
logger.debug("delta_text: %s", delta_text)
|
||||
logger.debug("delta_token_ids: %s", delta_token_ids)
|
||||
# check to see if we should be streaming a tool call - is there a
|
||||
@@ -278,7 +277,7 @@ class KimiK2ToolParser(ToolParser):
|
||||
if not self.current_tool_name_sent:
|
||||
if current_tool_call is None:
|
||||
return None
|
||||
function_name: Union[str, None] = current_tool_call.get("name")
|
||||
function_name: str | None = current_tool_call.get("name")
|
||||
tool_id = current_tool_call.get("id")
|
||||
if function_name:
|
||||
self.current_tool_name_sent = True
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import ast
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
@@ -128,7 +128,7 @@ class Llama4PythonicToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
if not current_text.startswith("[") and not current_text.startswith(
|
||||
"<|python_start|>"
|
||||
):
|
||||
@@ -245,7 +245,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
|
||||
)
|
||||
|
||||
|
||||
def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
|
||||
def _make_valid_python(text: str) -> tuple[str, str] | None:
|
||||
bracket_stack = []
|
||||
for index, char in enumerate(text):
|
||||
if char in {"[", "(", "{"}:
|
||||
@@ -317,7 +317,7 @@ def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
|
||||
|
||||
def _compute_tool_delta(
|
||||
previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
|
||||
) -> Union[DeltaToolCall, None]:
|
||||
) -> DeltaToolCall | None:
|
||||
new_call_args = new_call.function.arguments
|
||||
if withheld_suffix:
|
||||
assert new_call_args.endswith(withheld_suffix)
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
@@ -134,7 +133,7 @@ class Llama3JsonToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
if not (
|
||||
current_text.startswith(self.bot_token) or current_text.startswith("{")
|
||||
):
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -509,7 +509,7 @@ class MinimaxToolParser(ToolParser):
|
||||
|
||||
def _get_current_tool_content(
|
||||
self, text: str, tool_index: int
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Get the content of a specific tool by index.
|
||||
|
||||
@@ -545,7 +545,7 @@ class MinimaxToolParser(ToolParser):
|
||||
|
||||
def _handle_tool_name_streaming(
|
||||
self, tool_content: str, tool_count: int
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Handle streaming of tool names.
|
||||
|
||||
@@ -595,7 +595,7 @@ class MinimaxToolParser(ToolParser):
|
||||
|
||||
def _handle_tool_args_streaming(
|
||||
self, tool_content: str, tool_count: int
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
"""
|
||||
Handle streaming of tool arguments.
|
||||
|
||||
@@ -702,7 +702,7 @@ class MinimaxToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
self._update_thinking_state(current_text)
|
||||
|
||||
if self.in_thinking_tag:
|
||||
@@ -776,7 +776,7 @@ class MinimaxToolParser(ToolParser):
|
||||
)
|
||||
return None
|
||||
|
||||
def _find_tool_start_outside_thinking(self, current_text: str) -> Optional[int]:
|
||||
def _find_tool_start_outside_thinking(self, current_text: str) -> int | None:
|
||||
"""
|
||||
Find the start position of tool calls outside of thinking tags.
|
||||
|
||||
@@ -809,7 +809,7 @@ class MinimaxToolParser(ToolParser):
|
||||
|
||||
def _extract_content_before_tools(
|
||||
self, current_text: str, delta_text: str, tool_start: int
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
"""
|
||||
Extract content that appears before tool calls.
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import json
|
||||
from collections.abc import Sequence
|
||||
from random import choices
|
||||
from string import ascii_letters, digits
|
||||
from typing import Union
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
@@ -194,7 +193,7 @@ class MistralToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
# if the tool call token is not in the tokens generated so far, append
|
||||
# output to contents since it's not a tool
|
||||
if self.bot_token not in current_text:
|
||||
@@ -252,7 +251,7 @@ class MistralToolParser(ToolParser):
|
||||
# auto-generated due to JSON completions, but wasn't
|
||||
# streamed to the client yet.
|
||||
if self.current_tool_id >= 0:
|
||||
diff: Union[str, None] = current_tool_call.get("arguments")
|
||||
diff: str | None = current_tool_call.get("arguments")
|
||||
|
||||
if diff:
|
||||
diff = json.dumps(diff, ensure_ascii=False).replace(
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -22,13 +20,15 @@ from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
else:
|
||||
AnyTokenizer = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@ToolParserManager.register_module("openai")
|
||||
class OpenAIToolParser(ToolParser):
|
||||
def __init__(self, tokenizer: AnyTokenizer):
|
||||
def __init__(self, tokenizer: "AnyTokenizer"):
|
||||
super().__init__(tokenizer)
|
||||
|
||||
def extract_tool_calls(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
@@ -118,5 +118,5 @@ class Phi4MiniJsonToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Optional[DeltaMessage]:
|
||||
) -> DeltaMessage | None:
|
||||
return None
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import ast
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
@@ -124,7 +124,7 @@ class PythonicToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
if not current_text.startswith("["):
|
||||
return DeltaMessage(content=delta_text)
|
||||
|
||||
@@ -236,7 +236,7 @@ def _handle_single_tool(call: ast.Call) -> ToolCall:
|
||||
)
|
||||
|
||||
|
||||
def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
|
||||
def _make_valid_python(text: str) -> tuple[str, str] | None:
|
||||
bracket_stack = []
|
||||
for index, char in enumerate(text):
|
||||
if char in {"[", "(", "{"}:
|
||||
@@ -308,7 +308,7 @@ def _make_valid_python(text: str) -> Union[tuple[str, str], None]:
|
||||
|
||||
def _compute_tool_delta(
|
||||
previously_sent_args: str, new_call: ToolCall, index: int, withheld_suffix: str
|
||||
) -> Union[DeltaToolCall, None]:
|
||||
) -> DeltaToolCall | None:
|
||||
new_call_args = new_call.function.arguments
|
||||
if withheld_suffix:
|
||||
assert new_call_args.endswith(withheld_suffix)
|
||||
|
||||
@@ -4,7 +4,7 @@ import ast
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -36,7 +36,7 @@ class Qwen3CoderToolParser(ToolParser):
|
||||
self.current_tool_name_sent: bool = False
|
||||
self.prev_tool_call_arr: list[dict] = []
|
||||
# Override base class type - we use string IDs for tool calls
|
||||
self.current_tool_id: Optional[str] = None # type: ignore
|
||||
self.current_tool_id: str | None = None # type: ignore
|
||||
self.streamed_args_for_tool: list[str] = []
|
||||
|
||||
# Sentinel tokens for streaming mode
|
||||
@@ -110,7 +110,7 @@ class Qwen3CoderToolParser(ToolParser):
|
||||
self.streaming_request = None
|
||||
|
||||
def _get_arguments_config(
|
||||
self, func_name: str, tools: Optional[list[ChatCompletionToolsParam]]
|
||||
self, func_name: str, tools: list[ChatCompletionToolsParam] | None
|
||||
) -> dict:
|
||||
"""Extract argument configuration for a function."""
|
||||
if tools is None:
|
||||
@@ -240,8 +240,8 @@ class Qwen3CoderToolParser(ToolParser):
|
||||
return param_value
|
||||
|
||||
def _parse_xml_function_call(
|
||||
self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
|
||||
) -> Optional[ToolCall]:
|
||||
self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None
|
||||
) -> ToolCall | None:
|
||||
# Extract function name
|
||||
end_index = function_call_str.index(">")
|
||||
function_name = function_call_str[:end_index]
|
||||
@@ -349,7 +349,7 @@ class Qwen3CoderToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
# Store request for type conversion
|
||||
if not previous_text:
|
||||
self._reset_streaming_state()
|
||||
|
||||
@@ -4,7 +4,7 @@ import ast
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
from xml.parsers.expat import ParserCreate
|
||||
|
||||
import regex as re
|
||||
@@ -39,7 +39,7 @@ class StreamingXMLToolCallParser:
|
||||
self.reset_streaming_state()
|
||||
|
||||
# Tool configuration information
|
||||
self.tools: Union[list[ChatCompletionToolsParam], None] = None
|
||||
self.tools: list[ChatCompletionToolsParam] | None = None
|
||||
self.tool_call_start_token: str = "<tool_call>"
|
||||
self.tool_call_end_token: str = "</tool_call>"
|
||||
self.function_start_token: str = "<function="
|
||||
@@ -341,7 +341,7 @@ class StreamingXMLToolCallParser:
|
||||
# Skip blank content
|
||||
return not element
|
||||
|
||||
def _find_next_complete_element(self, start_pos: int) -> tuple[Optional[str], int]:
|
||||
def _find_next_complete_element(self, start_pos: int) -> tuple[str | None, int]:
|
||||
"""
|
||||
Find next complete XML element from specified position
|
||||
|
||||
@@ -584,7 +584,7 @@ class StreamingXMLToolCallParser:
|
||||
"""Emit Delta response (streaming output)"""
|
||||
self.deltas.append(delta)
|
||||
|
||||
def _auto_close_open_parameter_if_needed(self, incoming_tag: Optional[str] = None):
|
||||
def _auto_close_open_parameter_if_needed(self, incoming_tag: str | None = None):
|
||||
"""Before starting to process new elements,
|
||||
if there are unclosed tags from before,
|
||||
automatically complete their endings to the parser.
|
||||
@@ -953,7 +953,7 @@ class StreamingXMLToolCallParser:
|
||||
self.parser.EndElementHandler = self._end_element
|
||||
self.parser.CharacterDataHandler = self._char_data
|
||||
|
||||
def set_tools(self, tools: Union[list[ChatCompletionToolsParam], None]):
|
||||
def set_tools(self, tools: list[ChatCompletionToolsParam] | None):
|
||||
"""Set tool configuration information"""
|
||||
self.tools = tools
|
||||
|
||||
@@ -961,7 +961,7 @@ class StreamingXMLToolCallParser:
|
||||
"""Generate unique call ID"""
|
||||
return f"call_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
def _extract_function_name(self, name: str, attrs: dict[str, str]) -> Optional[str]:
|
||||
def _extract_function_name(self, name: str, attrs: dict[str, str]) -> str | None:
|
||||
"""Extract function name from various formats"""
|
||||
if attrs and "name" in attrs:
|
||||
return attrs["name"]
|
||||
@@ -973,9 +973,7 @@ class StreamingXMLToolCallParser:
|
||||
|
||||
return None
|
||||
|
||||
def _extract_parameter_name(
|
||||
self, name: str, attrs: dict[str, str]
|
||||
) -> Optional[str]:
|
||||
def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> str | None:
|
||||
"""Extract parameter name from various formats"""
|
||||
if attrs and "name" in attrs:
|
||||
return attrs["name"]
|
||||
@@ -1218,7 +1216,7 @@ class Qwen3XMLToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
if not previous_text:
|
||||
self.parser.reset_streaming_state()
|
||||
if request:
|
||||
|
||||
@@ -7,7 +7,7 @@ import ast
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -109,8 +109,8 @@ class SeedOssToolParser(ToolParser):
|
||||
self.json_closed = False
|
||||
|
||||
def _parse_xml_function_call(
|
||||
self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
|
||||
) -> Optional[ToolCall]:
|
||||
self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None
|
||||
) -> ToolCall | None:
|
||||
def get_arguments_config(func_name: str) -> dict:
|
||||
if tools is None:
|
||||
return {}
|
||||
@@ -357,7 +357,7 @@ class SeedOssToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
# If no delta text, return None unless
|
||||
# it's an EOS token after tool calls
|
||||
if not delta_text:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -58,7 +58,7 @@ class Step3ToolParser(ToolParser):
|
||||
@staticmethod
|
||||
def _parse_steptml_invoke(
|
||||
action_text: str,
|
||||
) -> tuple[Optional[str], Optional[dict[str, str]]]:
|
||||
) -> tuple[str | None, dict[str, str] | None]:
|
||||
func_name_match = re.search(r'<steptml:invoke name="([^"]+)">', action_text)
|
||||
if not func_name_match:
|
||||
return None, None
|
||||
@@ -117,7 +117,7 @@ class Step3ToolParser(ToolParser):
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> Union[DeltaMessage, None]:
|
||||
) -> DeltaMessage | None:
|
||||
# The main loop processes the stream from the last known position.
|
||||
while True:
|
||||
if self.position >= len(current_text):
|
||||
|
||||
@@ -5,7 +5,7 @@ import asyncio
|
||||
import io
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Optional, Union
|
||||
from typing import Annotated
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
@@ -24,25 +24,25 @@ from vllm.utils import AsyncMicrobatchTokenizer
|
||||
class RenderConfig:
|
||||
"""Configuration to control how prompts are prepared."""
|
||||
|
||||
max_length: Optional[int] = None
|
||||
max_length: int | None = None
|
||||
"""Maximum allowable total input token length. If provided,
|
||||
token inputs longer than this raise ``ValueError``."""
|
||||
|
||||
truncate_prompt_tokens: Optional[int] = None
|
||||
truncate_prompt_tokens: int | None = None
|
||||
"""Number of tokens to keep. ``None`` means no truncation.
|
||||
``0`` yields an empty list (and skips embeds).
|
||||
``-1`` maps to ``model_config.max_model_len``."""
|
||||
|
||||
add_special_tokens: Optional[bool] = True
|
||||
add_special_tokens: bool | None = True
|
||||
"""Whether to add model-specific special tokens during tokenization."""
|
||||
|
||||
cache_salt: Optional[str] = None
|
||||
cache_salt: str | None = None
|
||||
"""String to disambiguate prefix cache entries."""
|
||||
|
||||
needs_detokenization: Optional[bool] = False
|
||||
needs_detokenization: bool | None = False
|
||||
"""If True, detokenize IDs back to text for inclusion in outputs."""
|
||||
|
||||
def verify_truncate_prompt_tokens(self, model_config: ModelConfig) -> Optional[int]:
|
||||
def verify_truncate_prompt_tokens(self, model_config: ModelConfig) -> int | None:
|
||||
"""Validate and normalize `truncate_prompt_tokens` parameter."""
|
||||
truncate_prompt_tokens = self.truncate_prompt_tokens
|
||||
if truncate_prompt_tokens is None:
|
||||
@@ -85,7 +85,7 @@ class BaseRenderer(ABC):
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
tokenizer: Optional[AnyTokenizer] = None,
|
||||
tokenizer: AnyTokenizer | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.model_config = model_config
|
||||
@@ -95,7 +95,7 @@ class BaseRenderer(ABC):
|
||||
async def render_prompt(
|
||||
self,
|
||||
*,
|
||||
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
|
||||
prompt_or_prompts: str | list[str] | list[int] | list[list[int]],
|
||||
config: RenderConfig,
|
||||
) -> list[EngineTokensPrompt]:
|
||||
"""
|
||||
@@ -126,12 +126,10 @@ class BaseRenderer(ABC):
|
||||
async def render_prompt_and_embeds(
|
||||
self,
|
||||
*,
|
||||
prompt_or_prompts: Optional[
|
||||
Union[str, list[str], list[int], list[list[int]]]
|
||||
] = None,
|
||||
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
|
||||
prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,
|
||||
prompt_embeds: bytes | list[bytes] | None = None,
|
||||
config: RenderConfig,
|
||||
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
|
||||
) -> list[EngineTokensPrompt | EngineEmbedsPrompt]:
|
||||
"""
|
||||
Convert text/token and/or base64-encoded embeddings inputs into
|
||||
engine-ready prompt objects using a unified RenderConfig.
|
||||
@@ -161,9 +159,9 @@ class BaseRenderer(ABC):
|
||||
@classmethod
|
||||
def load_prompt_embeds(
|
||||
cls,
|
||||
prompt_embeds: Union[bytes, list[bytes]],
|
||||
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=0)]] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
prompt_embeds: bytes | list[bytes],
|
||||
truncate_prompt_tokens: Annotated[int, Field(ge=0)] | None = None,
|
||||
cache_salt: str | None = None,
|
||||
) -> list[EngineEmbedsPrompt]:
|
||||
"""Load and validate base64-encoded embeddings into prompt objects."""
|
||||
|
||||
@@ -199,19 +197,18 @@ class CompletionRenderer(BaseRenderer):
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
tokenizer: Optional[AnyTokenizer] = None,
|
||||
async_tokenizer_pool: Optional[
|
||||
dict[AnyTokenizer, AsyncMicrobatchTokenizer]
|
||||
] = None,
|
||||
tokenizer: AnyTokenizer | None = None,
|
||||
async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer]
|
||||
| None = None,
|
||||
):
|
||||
super().__init__(model_config, tokenizer)
|
||||
self.async_tokenizer_pool = async_tokenizer_pool
|
||||
self.async_tokenizer: Optional[AsyncMicrobatchTokenizer] = None
|
||||
self.async_tokenizer: AsyncMicrobatchTokenizer | None = None
|
||||
|
||||
async def render_prompt(
|
||||
self,
|
||||
*,
|
||||
prompt_or_prompts: Union[str, list[str], list[int], list[list[int]]],
|
||||
prompt_or_prompts: str | list[str] | list[int] | list[list[int]],
|
||||
config: RenderConfig,
|
||||
) -> list[EngineTokensPrompt]:
|
||||
"""Implementation of prompt rendering for completion-style requests.
|
||||
@@ -237,12 +234,10 @@ class CompletionRenderer(BaseRenderer):
|
||||
async def render_prompt_and_embeds(
|
||||
self,
|
||||
*,
|
||||
prompt_or_prompts: Optional[
|
||||
Union[str, list[str], list[int], list[list[int]]]
|
||||
] = None,
|
||||
prompt_embeds: Optional[Union[bytes, list[bytes]]] = None,
|
||||
prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None,
|
||||
prompt_embeds: bytes | list[bytes] | None = None,
|
||||
config: RenderConfig,
|
||||
) -> list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]:
|
||||
) -> list[EngineTokensPrompt | EngineEmbedsPrompt]:
|
||||
"""
|
||||
Render text/token prompts and/or precomputed embedding prompts. At
|
||||
least one of `prompt_or_prompts` or `prompt_embeds` must be provided.
|
||||
@@ -251,7 +246,7 @@ class CompletionRenderer(BaseRenderer):
|
||||
if truncate_prompt_tokens == 0:
|
||||
return []
|
||||
|
||||
rendered: list[Union[EngineTokensPrompt, EngineEmbedsPrompt]] = []
|
||||
rendered: list[EngineTokensPrompt | EngineEmbedsPrompt] = []
|
||||
|
||||
if prompt_embeds is not None:
|
||||
rendered.extend(
|
||||
@@ -271,7 +266,7 @@ class CompletionRenderer(BaseRenderer):
|
||||
return rendered
|
||||
|
||||
def _maybe_apply_truncation(
|
||||
self, token_ids: list[int], truncate_prompt_tokens: Optional[int]
|
||||
self, token_ids: list[int], truncate_prompt_tokens: int | None
|
||||
) -> list[int]:
|
||||
"""Apply truncation to token sequence."""
|
||||
if truncate_prompt_tokens is None:
|
||||
@@ -283,9 +278,9 @@ class CompletionRenderer(BaseRenderer):
|
||||
|
||||
async def _create_prompt(
|
||||
self,
|
||||
prompt_input: Union[EngineTextPrompt, EngineTokensPrompt],
|
||||
prompt_input: EngineTextPrompt | EngineTokensPrompt,
|
||||
config: RenderConfig,
|
||||
truncate_prompt_tokens: Optional[int],
|
||||
truncate_prompt_tokens: int | None,
|
||||
) -> EngineTokensPrompt:
|
||||
prompt, prompt_token_ids, _ = get_prompt_components(prompt_input)
|
||||
|
||||
@@ -315,10 +310,10 @@ class CompletionRenderer(BaseRenderer):
|
||||
async def _create_prompt_from_text(
|
||||
self,
|
||||
text: str,
|
||||
max_length: Optional[int],
|
||||
truncate_prompt_tokens: Optional[int],
|
||||
add_special_tokens: Optional[bool],
|
||||
cache_salt: Optional[str],
|
||||
max_length: int | None,
|
||||
truncate_prompt_tokens: int | None,
|
||||
add_special_tokens: bool | None,
|
||||
cache_salt: str | None,
|
||||
) -> EngineTokensPrompt:
|
||||
"""Tokenize text input asynchronously."""
|
||||
async_tokenizer = self._get_async_tokenizer()
|
||||
@@ -348,10 +343,10 @@ class CompletionRenderer(BaseRenderer):
|
||||
async def _create_prompt_from_token_ids(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
max_length: Optional[int],
|
||||
truncate_prompt_tokens: Optional[int],
|
||||
cache_salt: Optional[str],
|
||||
needs_detokenization: Optional[bool] = False,
|
||||
max_length: int | None,
|
||||
truncate_prompt_tokens: int | None,
|
||||
cache_salt: str | None,
|
||||
needs_detokenization: bool | None = False,
|
||||
) -> EngineTokensPrompt:
|
||||
"""Optionally detokenize token IDs and build a tokens prompt."""
|
||||
token_ids = self._maybe_apply_truncation(token_ids, truncate_prompt_tokens)
|
||||
@@ -391,9 +386,9 @@ class CompletionRenderer(BaseRenderer):
|
||||
def _create_tokens_prompt(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
max_length: Optional[int] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
prompt: Optional[str] = None,
|
||||
max_length: int | None = None,
|
||||
cache_salt: str | None = None,
|
||||
prompt: str | None = None,
|
||||
) -> EngineTokensPrompt:
|
||||
"""Create validated EngineTokensPrompt."""
|
||||
if max_length is not None and len(token_ids) > max_length:
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any, Optional, Union, cast
|
||||
from typing import Any, TypeAlias, cast
|
||||
|
||||
from torch.nn import CosineSimilarity
|
||||
from typing_extensions import Required, TypeAlias, TypedDict
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -25,9 +25,9 @@ from vllm.transformers_utils.tokenizer import (
|
||||
PreTrainedTokenizerFast,
|
||||
)
|
||||
|
||||
ScoreContentPartParam: TypeAlias = Union[
|
||||
ChatCompletionContentPartImageParam, ChatCompletionContentPartImageEmbedsParam
|
||||
]
|
||||
ScoreContentPartParam: TypeAlias = (
|
||||
ChatCompletionContentPartImageParam | ChatCompletionContentPartImageEmbedsParam
|
||||
)
|
||||
|
||||
|
||||
class ScoreMultiModalParam(TypedDict, total=False):
|
||||
@@ -45,12 +45,12 @@ class ScoreMultiModalParam(TypedDict, total=False):
|
||||
|
||||
|
||||
def _cosine_similarity(
|
||||
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
|
||||
tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
|
||||
embed_1: list[PoolingRequestOutput],
|
||||
embed_2: list[PoolingRequestOutput],
|
||||
) -> list[PoolingRequestOutput]:
|
||||
scorer = CosineSimilarity(0)
|
||||
scores: Union[list[PoolingRequestOutput]] = []
|
||||
scores: list[PoolingRequestOutput] = []
|
||||
|
||||
for emb_1, emb_2 in zip(embed_1, embed_2):
|
||||
pair_score = scorer(emb_1.outputs.data, emb_2.outputs.data)
|
||||
@@ -74,8 +74,8 @@ def _cosine_similarity(
|
||||
|
||||
|
||||
def _validate_score_input_lens(
|
||||
data_1: Union[list[str], list[ScoreContentPartParam]],
|
||||
data_2: Union[list[str], list[ScoreContentPartParam]],
|
||||
data_1: list[str] | list[ScoreContentPartParam],
|
||||
data_2: list[str] | list[ScoreContentPartParam],
|
||||
):
|
||||
len_1 = len(data_1)
|
||||
len_2 = len(data_2)
|
||||
@@ -89,18 +89,18 @@ def _validate_score_input_lens(
|
||||
|
||||
|
||||
def parse_score_data(
|
||||
data_1: Union[str, ScoreContentPartParam],
|
||||
data_2: Union[str, ScoreContentPartParam],
|
||||
data_1: str | ScoreContentPartParam,
|
||||
data_2: str | ScoreContentPartParam,
|
||||
model_config: ModelConfig,
|
||||
tokenizer: AnyTokenizer,
|
||||
) -> tuple[str, str, Optional[MultiModalDataDict]]:
|
||||
) -> tuple[str, str, MultiModalDataDict | None]:
|
||||
mm_tracker = MultiModalItemTracker(model_config, tokenizer)
|
||||
|
||||
content_1 = _parse_score_content(data_1, mm_tracker)
|
||||
|
||||
content_2 = _parse_score_content(data_2, mm_tracker)
|
||||
|
||||
def ensure_str(content: Optional[_ContentPart]) -> str:
|
||||
def ensure_str(content: _ContentPart | None) -> str:
|
||||
if content is not None and isinstance(content, str):
|
||||
return cast(str, content)
|
||||
else:
|
||||
@@ -113,9 +113,9 @@ def parse_score_data(
|
||||
|
||||
|
||||
def _parse_score_content(
|
||||
data: Union[str, ScoreContentPartParam],
|
||||
data: str | ScoreContentPartParam,
|
||||
mm_tracker: BaseMultiModalItemTracker,
|
||||
) -> Optional[_ContentPart]:
|
||||
) -> _ContentPart | None:
|
||||
if isinstance(data, str):
|
||||
data = ChatCompletionContentPartTextParam(type="text", text=data)
|
||||
|
||||
@@ -182,8 +182,8 @@ def get_score_prompt(
|
||||
model_config: ModelConfig,
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenization_kwargs: dict[str, Any],
|
||||
data_1: Union[str, ScoreContentPartParam],
|
||||
data_2: Union[str, ScoreContentPartParam],
|
||||
data_1: str | ScoreContentPartParam,
|
||||
data_2: str | ScoreContentPartParam,
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
prompt_1, prompt_2, mm_data = parse_score_data(
|
||||
data_1,
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from ssl import SSLContext
|
||||
from typing import Callable, Optional
|
||||
|
||||
from watchfiles import Change, awatch
|
||||
|
||||
@@ -20,9 +20,9 @@ class SSLCertRefresher:
|
||||
def __init__(
|
||||
self,
|
||||
ssl_context: SSLContext,
|
||||
key_path: Optional[str] = None,
|
||||
cert_path: Optional[str] = None,
|
||||
ca_path: Optional[str] = None,
|
||||
key_path: str | None = None,
|
||||
cert_path: str | None = None,
|
||||
ca_path: str | None = None,
|
||||
) -> None:
|
||||
self.ssl = ssl_context
|
||||
self.key_path = key_path
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from openai_harmony import ToolDescription, ToolNamespaceConfig
|
||||
|
||||
@@ -80,7 +80,7 @@ class ToolServer(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_tool_description(self, tool_name: str) -> Optional[ToolNamespaceConfig]:
|
||||
def get_tool_description(self, tool_name: str) -> ToolNamespaceConfig | None:
|
||||
"""
|
||||
Return the tool description for the given tool name.
|
||||
If the tool is not supported, return None.
|
||||
@@ -89,7 +89,7 @@ class ToolServer(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def new_session(
|
||||
self, tool_name: str, session_id: str, headers: Optional[dict[str, str]] = None
|
||||
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
|
||||
) -> AbstractAsyncContextManager[Any]:
|
||||
"""
|
||||
Create a session for the tool.
|
||||
@@ -152,7 +152,7 @@ class MCPToolServer(ToolServer):
|
||||
|
||||
@asynccontextmanager
|
||||
async def new_session(
|
||||
self, tool_name: str, session_id: str, headers: Optional[dict[str, str]] = None
|
||||
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
|
||||
):
|
||||
from mcp import ClientSession
|
||||
from mcp.client.sse import sse_client
|
||||
@@ -190,7 +190,7 @@ class DemoToolServer(ToolServer):
|
||||
def has_tool(self, tool_name: str) -> bool:
|
||||
return tool_name in self.tools
|
||||
|
||||
def get_tool_description(self, tool_name: str) -> Optional[ToolNamespaceConfig]:
|
||||
def get_tool_description(self, tool_name: str) -> ToolNamespaceConfig | None:
|
||||
if tool_name not in self.tools:
|
||||
return None
|
||||
if tool_name == "browser":
|
||||
@@ -202,7 +202,7 @@ class DemoToolServer(ToolServer):
|
||||
|
||||
@asynccontextmanager
|
||||
async def new_session(
|
||||
self, tool_name: str, session_id: str, headers: Optional[dict[str, str]] = None
|
||||
self, tool_name: str, session_id: str, headers: dict[str, str] | None = None
|
||||
):
|
||||
if tool_name not in self.tools:
|
||||
raise KeyError(f"Tool '{tool_name}' is not supported")
|
||||
|
||||
@@ -6,7 +6,7 @@ import dataclasses
|
||||
import functools
|
||||
import os
|
||||
from argparse import Namespace
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
@@ -164,9 +164,9 @@ def cli_env_setup():
|
||||
|
||||
def _validate_truncation_size(
|
||||
max_model_len: int,
|
||||
truncate_prompt_tokens: Optional[int],
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[int]:
|
||||
truncate_prompt_tokens: int | None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> int | None:
|
||||
if truncate_prompt_tokens is not None:
|
||||
if truncate_prompt_tokens <= -1:
|
||||
truncate_prompt_tokens = max_model_len
|
||||
@@ -191,7 +191,7 @@ def _validate_truncation_size(
|
||||
|
||||
def get_max_tokens(
|
||||
max_model_len: int,
|
||||
request: Union[ChatCompletionRequest, CompletionRequest],
|
||||
request: ChatCompletionRequest | CompletionRequest,
|
||||
input_length: int,
|
||||
default_sampling_params: dict,
|
||||
) -> int:
|
||||
@@ -211,7 +211,7 @@ def get_max_tokens(
|
||||
)
|
||||
|
||||
|
||||
def log_non_default_args(args: Union[Namespace, EngineArgs]):
|
||||
def log_non_default_args(args: Namespace | EngineArgs):
|
||||
non_default_args = {}
|
||||
|
||||
# Handle Namespace
|
||||
|
||||
Reference in New Issue
Block a user