Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+33 -33
View File
@@ -21,13 +21,13 @@ import logging
import math
import random
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping
from collections.abc import Callable, Iterator, Mapping
from contextlib import suppress
from copy import deepcopy
from dataclasses import dataclass
from functools import cache
from io import BytesIO
from typing import Any, Callable, Optional, Union, cast
from typing import Any, cast
import numpy as np
from PIL import Image
@@ -75,12 +75,12 @@ class SampleRequest:
Represents a single inference request for benchmarking.
"""
prompt: Union[str, list[str]]
prompt: str | list[str]
prompt_len: int
expected_output_len: int
multi_modal_data: Optional[Union[MultiModalDataDict, dict, list[dict]]] = None
lora_request: Optional[LoRARequest] = None
request_id: Optional[str] = None
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
lora_request: LoRARequest | None = None
request_id: str | None = None
# -----------------------------------------------------------------------------
@@ -94,7 +94,7 @@ class BenchmarkDataset(ABC):
def __init__(
self,
dataset_path: Optional[str] = None,
dataset_path: str | None = None,
random_seed: int = DEFAULT_SEED,
disable_shuffle: bool = False,
**kwargs,
@@ -119,7 +119,7 @@ class BenchmarkDataset(ABC):
def apply_multimodal_chat_transformation(
self,
prompt: str,
mm_content: Optional[Union[MultiModalDataDict, dict, list[dict]]] = None,
mm_content: MultiModalDataDict | dict | list[dict] | None = None,
) -> list[dict]:
"""
Transform a prompt and optional multimodal content into a chat format.
@@ -154,9 +154,9 @@ class BenchmarkDataset(ABC):
def get_random_lora_request(
self,
max_loras: Optional[int] = None,
lora_path: Optional[str] = None,
) -> Optional[LoRARequest]:
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a random LoRA request.
@@ -384,7 +384,7 @@ def gen_prompt_decode_to_target_len(
target_token_len: int,
max_retry: int = 10,
add_special_tokens: bool = False,
rng: Optional[np.random.Generator] = None,
rng: np.random.Generator | None = None,
) -> tuple[str, list[int]]:
"""
Ensure decoded-then-encoded prompt length matches the target token length.
@@ -1054,9 +1054,9 @@ class ShareGPTDataset(BenchmarkDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
lora_path: Optional[str] = None,
max_loras: Optional[int] = None,
output_len: Optional[int] = None,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -1766,9 +1766,9 @@ class CustomDataset(BenchmarkDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
lora_path: Optional[str] = None,
max_loras: Optional[int] = None,
output_len: Optional[int] = None,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
request_id_prefix: str = "",
@@ -1997,8 +1997,8 @@ class BurstGPTDataset(BenchmarkDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
max_loras: Optional[int] = None,
lora_path: Optional[str] = None,
max_loras: int | None = None,
lora_path: str | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2034,15 +2034,15 @@ class BurstGPTDataset(BenchmarkDataset):
class HuggingFaceDataset(BenchmarkDataset):
"""Base class for datasets hosted on HuggingFace."""
SUPPORTED_DATASET_PATHS: Union[set[str], dict[str, Callable]] = set()
SUPPORTED_DATASET_PATHS: set[str] | dict[str, Callable] = set()
def __init__(
self,
dataset_path: str,
dataset_split: str,
no_stream: bool = False,
dataset_subset: Optional[str] = None,
hf_name: Optional[str] = None,
dataset_subset: str | None = None,
hf_name: str | None = None,
**kwargs,
) -> None:
super().__init__(dataset_path=dataset_path, **kwargs)
@@ -2083,7 +2083,7 @@ class ConversationDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2152,7 +2152,7 @@ class VisionArenaDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2206,7 +2206,7 @@ class MMVUDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2267,7 +2267,7 @@ class InstructCoderDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
request_id_prefix: str = "",
@@ -2331,7 +2331,7 @@ class MTBenchDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
request_id_prefix: str = "",
@@ -2397,7 +2397,7 @@ class BlazeditDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
skip_chat_template: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2478,7 +2478,7 @@ class AIMODataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2660,7 +2660,7 @@ class ASRDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2738,7 +2738,7 @@ class MLPerfDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2902,7 +2902,7 @@ class MMStarDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
+2 -2
View File
@@ -7,7 +7,7 @@ import dataclasses
import json
import os
import time
from typing import Any, Optional
from typing import Any
import numpy as np
from tqdm import tqdm
@@ -127,7 +127,7 @@ def main(args: argparse.Namespace):
),
)
def run_to_completion(profile_dir: Optional[str] = None):
def run_to_completion(profile_dir: str | None = None):
if profile_dir:
llm.start_profile()
llm_generate()
+20 -20
View File
@@ -10,7 +10,7 @@ import time
import traceback
from collections.abc import Awaitable
from dataclasses import dataclass, field
from typing import Any, Literal, Optional, Protocol, Union
from typing import Any, Literal, Protocol
import aiohttp
import regex as re
@@ -69,14 +69,14 @@ class RequestFuncInput:
prompt_len: int
output_len: int
model: str
model_name: Optional[str] = None
logprobs: Optional[int] = None
extra_headers: Optional[dict] = None
extra_body: Optional[dict] = None
multi_modal_content: Optional[Union[dict, list[dict]]] = None
model_name: str | None = None
logprobs: int | None = None
extra_headers: dict | None = None
extra_body: dict | None = None
multi_modal_content: dict | list[dict] | None = None
ignore_eos: bool = False
language: Optional[str] = None
request_id: Optional[str] = None
language: str | None = None
request_id: str | None = None
@dataclass
@@ -100,14 +100,14 @@ class RequestFunc(Protocol):
self,
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> Awaitable[RequestFuncOutput]: ...
def _validate_api_url(
api_url: str,
api_name: str,
expected_suffixes: Union[str, set[str]],
expected_suffixes: str | set[str],
) -> None:
if isinstance(expected_suffixes, str):
expected_suffixes = {expected_suffixes}
@@ -141,7 +141,7 @@ def _update_headers_common(
async def async_request_openai_completions(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
"""The async request function for the OpenAI Completions API.
@@ -279,7 +279,7 @@ def _get_chat_content(
async def async_request_openai_chat_completions(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
mm_position: Literal["first", "last"] = "last",
) -> RequestFuncOutput:
api_url = request_func_input.api_url
@@ -376,7 +376,7 @@ async def async_request_openai_chat_completions(
async def async_request_openai_audio(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
# Lazy import without PlaceholderModule to avoid vllm dep.
import soundfile
@@ -489,7 +489,7 @@ async def _run_openai_embeddings(
api_url: str,
payload: dict[str, Any],
headers: dict[str, Any],
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
output = RequestFuncOutput()
st = time.perf_counter()
@@ -517,7 +517,7 @@ async def _run_openai_embeddings(
async def async_request_openai_embeddings(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Embeddings API", "embeddings")
@@ -548,7 +548,7 @@ async def async_request_openai_embeddings(
async def async_request_openai_embeddings_chat(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
mm_position: Literal["first", "last"] = "last",
) -> RequestFuncOutput:
api_url = request_func_input.api_url
@@ -627,7 +627,7 @@ def _preprocess_vlm2vec(request_func_input: RequestFuncInput):
async def async_request_openai_embeddings_clip(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
_preprocess_clip(request_func_input)
@@ -641,7 +641,7 @@ async def async_request_openai_embeddings_clip(
async def async_request_openai_embeddings_vlm2vec(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
_preprocess_vlm2vec(request_func_input)
@@ -656,7 +656,7 @@ async def async_request_openai_embeddings_vlm2vec(
async def async_request_infinity_embeddings(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
_validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
@@ -697,7 +697,7 @@ async def async_request_infinity_embeddings(
async def async_request_infinity_embeddings_clip(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
_preprocess_clip(request_func_input)
+15 -15
View File
@@ -31,7 +31,7 @@ from collections.abc import AsyncGenerator, Iterable
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any, Literal, Optional
from typing import Any, Literal
import aiohttp
import numpy as np
@@ -107,9 +107,9 @@ class EmbedBenchmarkMetrics:
def _get_current_request_rate(
ramp_up_strategy: Optional[Literal["linear", "exponential"]],
ramp_up_start_rps: Optional[int],
ramp_up_end_rps: Optional[int],
ramp_up_strategy: Literal["linear", "exponential"] | None,
ramp_up_start_rps: int | None,
ramp_up_end_rps: int | None,
request_index: int,
total_requests: int,
request_rate: float,
@@ -135,9 +135,9 @@ async def get_request(
input_requests: list[SampleRequest],
request_rate: float,
burstiness: float = 1.0,
ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
ramp_up_start_rps: Optional[int] = None,
ramp_up_end_rps: Optional[int] = None,
ramp_up_strategy: Literal["linear", "exponential"] | None = None,
ramp_up_start_rps: int | None = None,
ramp_up_end_rps: int | None = None,
) -> AsyncGenerator[tuple[SampleRequest, float], None]:
"""
Asynchronously generates requests at a specified rate
@@ -474,7 +474,7 @@ async def benchmark(
model_name: str,
tokenizer: PreTrainedTokenizerBase,
input_requests: list[SampleRequest],
logprobs: Optional[int],
logprobs: int | None,
request_rate: float,
burstiness: float,
disable_tqdm: bool,
@@ -483,13 +483,13 @@ async def benchmark(
selected_percentiles: list[float],
ignore_eos: bool,
goodput_config_dict: dict[str, float],
max_concurrency: Optional[int],
lora_modules: Optional[Iterable[str]],
extra_headers: Optional[dict],
extra_body: Optional[dict],
ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
ramp_up_start_rps: Optional[int] = None,
ramp_up_end_rps: Optional[int] = None,
max_concurrency: int | None,
lora_modules: Iterable[str] | None,
extra_headers: dict | None,
extra_body: dict | None,
ramp_up_strategy: Literal["linear", "exponential"] | None = None,
ramp_up_start_rps: int | None = None,
ramp_up_end_rps: int | None = None,
ready_check_timeout_sec: int = 600,
):
try:
+7 -7
View File
@@ -9,7 +9,7 @@ import os
import random
import time
import warnings
from typing import Any, Optional, Union
from typing import Any
import torch
import uvloop
@@ -43,7 +43,7 @@ def run_vllm(
engine_args: EngineArgs,
do_profile: bool,
disable_detokenize: bool = False,
) -> tuple[float, Optional[list[RequestOutput]]]:
) -> tuple[float, list[RequestOutput] | None]:
from vllm import LLM, SamplingParams
llm = LLM(**dataclasses.asdict(engine_args))
@@ -56,7 +56,7 @@ def run_vllm(
" prompt_len and expected_output_len for all requests."
)
# Add the requests to the engine.
prompts: list[Union[TextPrompt, TokensPrompt]] = []
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
for request in requests:
prompt = (
@@ -79,7 +79,7 @@ def run_vllm(
detokenize=not disable_detokenize,
)
)
lora_requests: Optional[list[LoRARequest]] = None
lora_requests: list[LoRARequest] | None = None
if engine_args.enable_lora:
lora_requests = [request.lora_request for request in requests]
@@ -197,9 +197,9 @@ async def run_vllm_async(
)
# Add the requests to the engine.
prompts: list[Union[TextPrompt, TokensPrompt]] = []
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[Optional[LoRARequest]] = []
lora_requests: list[LoRARequest | None] = []
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
@@ -696,7 +696,7 @@ def main(args: argparse.Namespace):
)
requests = get_requests(args, tokenizer)
is_multi_modal = any(request.multi_modal_data is not None for request in requests)
request_outputs: Optional[list[RequestOutput]] = None
request_outputs: list[RequestOutput] | None = None
if args.backend == "vllm":
if args.async_engine:
elapsed_time = uvloop.run(