Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
"""Tests for v1 attention backends without GPUModelRunner dependency."""
|
||||
|
||||
from functools import partial
|
||||
from typing import Optional, Union
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -202,7 +201,7 @@ def run_attention_backend(
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
sliding_window: Optional[int] = None,
|
||||
sliding_window: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run attention computation using the specified backend's AttentionImpl."""
|
||||
|
||||
@@ -289,7 +288,7 @@ def run_attention_backend(
|
||||
def _test_backend_correctness(
|
||||
batch_spec: BatchSpec,
|
||||
model: str,
|
||||
backend_to_test: list[Union[_Backend, str]],
|
||||
backend_to_test: list[_Backend | str],
|
||||
mask_mod,
|
||||
*,
|
||||
block_size: int = 16,
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for v1 MLA backends without GPUModelRunner dependency."""
|
||||
|
||||
from typing import Optional, Union
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -81,8 +79,8 @@ def create_and_prepopulate_kv_cache(
|
||||
num_blocks: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
randomize_blocks: bool = True,
|
||||
kv_cache_dtype: Optional[str] = None,
|
||||
scale: Union[float, torch.Tensor] = 1.0,
|
||||
kv_cache_dtype: str | None = None,
|
||||
scale: float | torch.Tensor = 1.0,
|
||||
) -> torch.Tensor:
|
||||
"""Create and prepopulate an MLA KV cache with context data.
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
"""Utility functions for attention-related v1 tests."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Union
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -150,14 +149,14 @@ def create_vllm_config(
|
||||
model_name: str = "meta-llama/Meta-Llama-3-8B",
|
||||
tensor_parallel_size: int = 1,
|
||||
max_model_len: int = 1024,
|
||||
dtype: Union[ModelDType, torch.dtype] = "auto",
|
||||
dtype: ModelDType | torch.dtype = "auto",
|
||||
num_gpu_blocks: int = 1000,
|
||||
block_size: int = 16,
|
||||
max_num_seqs: int = 256,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
enable_chunked_prefill: bool = True,
|
||||
add_mock_model_methods: bool = True,
|
||||
hf_config_override: Optional[dict] = None,
|
||||
hf_config_override: dict | None = None,
|
||||
) -> VllmConfig:
|
||||
"""Create a VllmConfig for testing with reasonable defaults."""
|
||||
|
||||
@@ -252,7 +251,7 @@ class BackendConfig:
|
||||
name: str
|
||||
env_vars: dict
|
||||
comp_config: dict # compilation config
|
||||
specific_gpu_arch: Optional[tuple] = None
|
||||
specific_gpu_arch: tuple | None = None
|
||||
|
||||
|
||||
# Define all backend configurations of full cudagraph to be tested
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib
|
||||
from typing import Callable, Optional
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -62,9 +62,9 @@ def make_request(
|
||||
prompt_token_ids: list[int],
|
||||
block_size: int = 3,
|
||||
hash_fn: Callable = hash,
|
||||
mm_positions: Optional[list[PlaceholderRange]] = None,
|
||||
mm_hashes: Optional[list[str]] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
mm_positions: list[PlaceholderRange] | None = None,
|
||||
mm_hashes: list[str] | None = None,
|
||||
cache_salt: str | None = None,
|
||||
):
|
||||
mm_features = []
|
||||
if mm_positions is not None:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Compare the with and without prefix caching."""
|
||||
|
||||
import copy
|
||||
from typing import Callable, Optional
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -55,10 +55,10 @@ def make_request(
|
||||
prompt_token_ids: list[int],
|
||||
block_size: int,
|
||||
hash_fn: Callable,
|
||||
mm_positions: Optional[list[PlaceholderRange]] = None,
|
||||
mm_hashes: Optional[list[str]] = None,
|
||||
prompt_logprobs: Optional[int] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
mm_positions: list[PlaceholderRange] | None = None,
|
||||
mm_hashes: list[str] | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
cache_salt: str | None = None,
|
||||
):
|
||||
mm_features = []
|
||||
if mm_positions is not None:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from typing import Optional
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
@@ -78,9 +77,7 @@ def test_get_num_unfinished_requests():
|
||||
(True, 5),
|
||||
],
|
||||
)
|
||||
def test_schedule(
|
||||
enable_prefix_caching: Optional[bool], prompt_logprobs: Optional[int]
|
||||
):
|
||||
def test_schedule(enable_prefix_caching: bool | None, prompt_logprobs: int | None):
|
||||
"""Test scheduling.
|
||||
Two cases: default APC/no prompt logprobs; APC=True + prompt logprobs
|
||||
"""
|
||||
@@ -595,7 +592,7 @@ def test_check_stop_min_tokens():
|
||||
],
|
||||
)
|
||||
def test_schedule_concurrent_batches(
|
||||
enable_prefix_caching: Optional[bool], prompt_logprobs: Optional[int]
|
||||
enable_prefix_caching: bool | None, prompt_logprobs: int | None
|
||||
):
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=1024,
|
||||
@@ -1323,14 +1320,14 @@ def create_scheduler_with_priority(
|
||||
model: str = "facebook/opt-125m",
|
||||
max_num_seqs: int = 16,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
enable_prefix_caching: Optional[bool] = None,
|
||||
enable_prefix_caching: bool | None = None,
|
||||
long_prefill_token_threshold: int = 0,
|
||||
disable_chunked_mm_input: bool = False,
|
||||
use_kv_connector: bool = False,
|
||||
num_blocks: int = 10000,
|
||||
block_size: int = 16,
|
||||
max_model_len: Optional[int] = None,
|
||||
num_speculative_tokens: Optional[int] = None,
|
||||
max_model_len: int | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
) -> Scheduler:
|
||||
"""Create scheduler with priority policy enabled.
|
||||
|
||||
@@ -1385,7 +1382,7 @@ def create_scheduler_with_priority(
|
||||
else None
|
||||
)
|
||||
|
||||
speculative_config: Optional[SpeculativeConfig] = None
|
||||
speculative_config: SpeculativeConfig | None = None
|
||||
if num_speculative_tokens is not None:
|
||||
speculative_config = SpeculativeConfig(
|
||||
model="ngram", num_speculative_tokens=num_speculative_tokens
|
||||
@@ -1420,12 +1417,12 @@ def create_scheduler_with_priority(
|
||||
def create_requests_with_priority(
|
||||
num_requests: int,
|
||||
priorities: list[int],
|
||||
arrival_times: Optional[list[float]] = None,
|
||||
arrival_times: list[float] | None = None,
|
||||
num_tokens: int = 10,
|
||||
mm_positions: Optional[list[list[PlaceholderRange]]] = None,
|
||||
mm_positions: list[list[PlaceholderRange]] | None = None,
|
||||
max_tokens: int = 16,
|
||||
stop_token_ids: Optional[list[int]] = None,
|
||||
prompt_logprobs: Optional[int] = None,
|
||||
stop_token_ids: list[int] | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
starting_idx: int = 0,
|
||||
):
|
||||
"""Create requests with specified priorities and arrival times."""
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -37,17 +36,17 @@ def create_scheduler(
|
||||
model: str = "facebook/opt-125m",
|
||||
max_num_seqs: int = 16,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
enable_prefix_caching: Optional[bool] = None,
|
||||
enable_prefix_caching: bool | None = None,
|
||||
long_prefill_token_threshold: int = 0,
|
||||
disable_chunked_mm_input: bool = False,
|
||||
use_kv_connector: bool = False,
|
||||
num_blocks: int = 10000,
|
||||
block_size: int = 16,
|
||||
max_model_len: Optional[int] = None,
|
||||
num_speculative_tokens: Optional[int] = None,
|
||||
max_model_len: int | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
skip_tokenizer_init: bool = False,
|
||||
async_scheduling: bool = False,
|
||||
) -> Union[Scheduler, AsyncScheduler]:
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
Args:
|
||||
@@ -102,7 +101,7 @@ def create_scheduler(
|
||||
else None
|
||||
)
|
||||
|
||||
speculative_config: Optional[SpeculativeConfig] = None
|
||||
speculative_config: SpeculativeConfig | None = None
|
||||
if num_speculative_tokens is not None:
|
||||
speculative_config = SpeculativeConfig(
|
||||
model="ngram", num_speculative_tokens=num_speculative_tokens
|
||||
@@ -141,10 +140,10 @@ _none_hash_initialized = False
|
||||
def create_requests(
|
||||
num_requests: int,
|
||||
num_tokens: int = 10,
|
||||
mm_positions: Optional[list[list[PlaceholderRange]]] = None,
|
||||
mm_positions: list[list[PlaceholderRange]] | None = None,
|
||||
max_tokens: int = 16,
|
||||
stop_token_ids: Optional[list[int]] = None,
|
||||
prompt_logprobs: Optional[int] = None,
|
||||
stop_token_ids: list[int] | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
same_prompt: bool = False,
|
||||
block_size: int = 16,
|
||||
) -> list[Request]:
|
||||
|
||||
@@ -5,7 +5,6 @@ import asyncio
|
||||
import os
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -35,8 +34,8 @@ async def generate(
|
||||
prompt: PromptType,
|
||||
output_kind: RequestOutputKind,
|
||||
max_tokens: int,
|
||||
prompt_logprobs: Optional[int] = None,
|
||||
data_parallel_rank: Optional[int] = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
data_parallel_rank: int | None = None,
|
||||
) -> tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
@@ -91,9 +90,9 @@ async def test_load(
|
||||
|
||||
def record(
|
||||
self,
|
||||
scheduler_stats: Optional[SchedulerStats],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
mm_cache_stats: Optional[MultiModalCacheStats] = None,
|
||||
scheduler_stats: SchedulerStats | None,
|
||||
iteration_stats: IterationStats | None,
|
||||
mm_cache_stats: MultiModalCacheStats | None = None,
|
||||
engine_idx: int = 0,
|
||||
):
|
||||
if iteration_stats:
|
||||
|
||||
@@ -5,7 +5,7 @@ import os
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from typing import Optional, cast
|
||||
from typing import cast
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
@@ -46,7 +46,7 @@ class MultinodeInternalLBServerManager:
|
||||
self.tp_size = tp_size
|
||||
self.api_server_count = api_server_count
|
||||
self.base_server_args = base_server_args
|
||||
self.servers: list[Optional[tuple[RemoteOpenAIServer, list[str]]]] = [None] * (
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]] | None] = [None] * (
|
||||
dp_size // dp_per_node
|
||||
)
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
@@ -175,7 +175,7 @@ class APIOnlyServerManager:
|
||||
self.tp_size = tp_size
|
||||
self.api_server_count = api_server_count
|
||||
self.base_server_args = base_server_args
|
||||
self.servers: list[Optional[tuple[RemoteOpenAIServer, list[str]]]] = [None] * 2
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]] | None] = [None] * 2
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
|
||||
@@ -13,8 +13,6 @@ Covers:
|
||||
5) Multiple stop conditions
|
||||
"""
|
||||
|
||||
from typing import Optional, Union
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
@@ -33,9 +31,9 @@ class MinTokensTestCase:
|
||||
name: str,
|
||||
min_tokens: int,
|
||||
max_tokens: int,
|
||||
stop: Optional[Union[str, list[str]]] = None,
|
||||
expected_min_len: Optional[int] = None,
|
||||
expected_exact_len: Optional[int] = None,
|
||||
stop: str | list[str] | None = None,
|
||||
expected_min_len: int | None = None,
|
||||
expected_exact_len: int | None = None,
|
||||
):
|
||||
self.name = name
|
||||
self.min_tokens = min_tokens
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Any, Union
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -34,7 +32,7 @@ def get_test_prompts(mm_enabled: bool):
|
||||
for kind in random_prompt_type_choices:
|
||||
word_choices = ["test", "temp", "hello", "where"]
|
||||
word = random.choice(word_choices)
|
||||
prompt: Union[str, list[dict[str, Any]]] = ""
|
||||
prompt: str | list[dict[str, Any]] = ""
|
||||
if kind == "repeat":
|
||||
prompt = f"""
|
||||
please repeat the word '{word}' 10 times.
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import asyncio
|
||||
from contextlib import ExitStack
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -53,8 +52,8 @@ async def generate(
|
||||
output_kind: RequestOutputKind,
|
||||
max_tokens: int,
|
||||
n: int = 1,
|
||||
prompt_logprobs: Optional[int] = None,
|
||||
cancel_after: Optional[int] = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
cancel_after: int | None = None,
|
||||
) -> tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
@@ -545,9 +544,9 @@ async def collect_outputs(
|
||||
prompt: PromptType,
|
||||
sampling_params: SamplingParams,
|
||||
outputs_list: list[RequestOutput],
|
||||
) -> Optional[RequestOutput]:
|
||||
) -> RequestOutput | None:
|
||||
"""Helper to collect outputs and return the final one."""
|
||||
final_output: Optional[RequestOutput] = None
|
||||
final_output: RequestOutput | None = None
|
||||
async for output in engine.generate(
|
||||
request_id=request_id, prompt=prompt, sampling_params=sampling_params
|
||||
):
|
||||
|
||||
@@ -8,7 +8,7 @@ import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from threading import Thread
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -41,7 +41,7 @@ PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids
|
||||
|
||||
|
||||
def make_request(
|
||||
params: SamplingParams, prompt_tokens_ids: Optional[list[int]] = None
|
||||
params: SamplingParams, prompt_tokens_ids: list[int] | None = None
|
||||
) -> EngineCoreRequest:
|
||||
if not prompt_tokens_ids:
|
||||
prompt_tokens_ids = PROMPT_TOKENS
|
||||
@@ -113,9 +113,7 @@ async def loop_until_fully_done_async(client: EngineCoreClient, outputs: dict):
|
||||
|
||||
|
||||
# Dummy utility function to monkey-patch into engine core.
|
||||
def echo(
|
||||
self, msg: str, err_msg: Optional[str] = None, sleep: Optional[float] = None
|
||||
) -> str:
|
||||
def echo(self, msg: str, err_msg: str | None = None, sleep: float | None = None) -> str:
|
||||
print(f"echo util function called: {msg}, {err_msg}")
|
||||
if sleep is not None:
|
||||
time.sleep(sleep)
|
||||
@@ -317,7 +315,7 @@ def echo_dc(
|
||||
self,
|
||||
msg: str,
|
||||
return_list: bool = False,
|
||||
) -> Union[MyDataclass, list[MyDataclass]]:
|
||||
) -> MyDataclass | list[MyDataclass]:
|
||||
print(f"echo dc util function called: {msg}")
|
||||
val = None if msg is None else MyDataclass(msg)
|
||||
# Return dataclass to verify support for returning custom types
|
||||
@@ -330,7 +328,7 @@ def echo_dc_dict(
|
||||
self,
|
||||
msg: str,
|
||||
return_dict: bool = False,
|
||||
) -> Union[MyDataclass, dict[str, MyDataclass]]:
|
||||
) -> MyDataclass | dict[str, MyDataclass]:
|
||||
print(f"echo dc dict util function called: {msg}")
|
||||
val = None if msg is None else MyDataclass(msg)
|
||||
# Return dict of dataclasses to verify support for returning dicts
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -13,6 +11,8 @@ from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tests.conftest import VllmRunner
|
||||
else:
|
||||
VllmRunner = object
|
||||
|
||||
MODEL = "facebook/opt-125m"
|
||||
DTYPE = "half"
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import math
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -118,13 +117,13 @@ def test_incremental_detokenization(
|
||||
|
||||
def _validate_logprobs(
|
||||
gen_tokens: dict[str, list[int]],
|
||||
gen_logprobs: dict[str, Optional[SampleLogprobs]],
|
||||
gen_prompt_logprobs: dict[str, Optional[PromptLogprobs]],
|
||||
gen_logprobs: dict[str, SampleLogprobs | None],
|
||||
gen_prompt_logprobs: dict[str, PromptLogprobs | None],
|
||||
gen_cumulative_logprob: dict[str, float],
|
||||
dtv: DummyOutputProcessorTestVectors,
|
||||
request_id_list: list[str],
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_prompt_logprobs: Optional[int],
|
||||
num_sample_logprobs: int | None,
|
||||
num_prompt_logprobs: int | None,
|
||||
) -> None:
|
||||
for req_idx, req_id in enumerate(request_id_list):
|
||||
new_tokens = gen_tokens[req_id]
|
||||
@@ -413,8 +412,8 @@ def _validate_logprobs(
|
||||
@pytest.mark.parametrize("num_prompt_logprobs", [None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
|
||||
def test_logprobs_processor(
|
||||
request_output_kind: RequestOutputKind,
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_prompt_logprobs: Optional[int],
|
||||
num_sample_logprobs: int | None,
|
||||
num_prompt_logprobs: int | None,
|
||||
dummy_test_vectors,
|
||||
):
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=False)
|
||||
@@ -530,7 +529,7 @@ def test_logprobs_processor(
|
||||
)
|
||||
def test_stop_token(
|
||||
include_stop_str_in_output: bool,
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_sample_logprobs: int | None,
|
||||
stop_token_type: str,
|
||||
ignore_eos: bool,
|
||||
dummy_test_vectors,
|
||||
@@ -696,7 +695,7 @@ def test_stop_token(
|
||||
@pytest.mark.parametrize("num_sample_logprobs", [None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
|
||||
def test_stop_string(
|
||||
include_stop_str_in_output: bool,
|
||||
num_sample_logprobs: Optional[int],
|
||||
num_sample_logprobs: int | None,
|
||||
dummy_test_vectors,
|
||||
):
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=False)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Union
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
@@ -12,7 +12,7 @@ from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.engine import EngineCoreOutput, FinishReason
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
|
||||
|
||||
GeneralTokenizerType = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
|
||||
GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
|
||||
|
||||
# Number of sample logprobs to request when testing sample logprobs
|
||||
NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5
|
||||
@@ -332,16 +332,15 @@ class MockEngineCore:
|
||||
# For each request, for each sampled token offset,
|
||||
# a tuple of
|
||||
# (list of topk token ids, list of sample logprob vals, rank)
|
||||
generated_logprobs_raw: Optional[
|
||||
list[list[tuple[list[int], list[float], int]]]
|
||||
] = None,
|
||||
generated_logprobs_raw: list[list[tuple[list[int], list[float], int]]]
|
||||
| None = None,
|
||||
# For each request, a tuple of
|
||||
# (prompt logprob val matrix, prompt logprob tok id matrix);
|
||||
# each matrix has dimensions
|
||||
# (num prompt toks) x (num prompt logprobs+1)
|
||||
prompt_logprobs_raw: Optional[list[LogprobsTensors]] = None,
|
||||
eos_token_id: Optional[int] = None,
|
||||
stop_token_ids: Optional[list[int]] = None,
|
||||
prompt_logprobs_raw: list[LogprobsTensors] | None = None,
|
||||
eos_token_id: int | None = None,
|
||||
stop_token_ids: list[int] | None = None,
|
||||
ignore_eos: bool = False,
|
||||
) -> None:
|
||||
self.num_requests = len(tokens_list)
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import fields
|
||||
from enum import Enum
|
||||
@@ -30,6 +28,8 @@ from vllm.sampling_params import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import TokenizerMode
|
||||
else:
|
||||
TokenizerMode = str
|
||||
|
||||
NGRAM_SPEC_CONFIG = {
|
||||
"model": "[ngram]",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
@@ -195,7 +194,7 @@ async def test_too_many_completion_logprobs(
|
||||
[(MODEL_NAME, -1), (MODEL_NAME, 0), (MODEL_NAME, 1), (MODEL_NAME, None)],
|
||||
)
|
||||
async def test_prompt_logprobs_completion(
|
||||
client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: Optional[int]
|
||||
client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: int | None
|
||||
):
|
||||
params: dict = {
|
||||
"prompt": ["A robot may not injure another robot", "My name is"],
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from typing import Any, Callable, Optional, Union
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -20,12 +21,12 @@ class Mock: ...
|
||||
class CustomMultiprocExecutor(MultiprocExecutor):
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable,
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None,
|
||||
kwargs: dict | None = None,
|
||||
non_block: bool = False,
|
||||
unique_reply_rank: Optional[int] = None,
|
||||
unique_reply_rank: int | None = None,
|
||||
) -> list[Any]:
|
||||
# Drop marker to show that this was run
|
||||
with open(".marker", "w"):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Callable
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -9,7 +9,6 @@ import textwrap
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -154,7 +153,7 @@ class FakeNixlWrapper:
|
||||
local_block_descs_ids: list[int],
|
||||
remote_xfer_side_handle: int,
|
||||
remote_block_descs_ids: list[int],
|
||||
notif_msg: Optional[bytes] = None,
|
||||
notif_msg: bytes | None = None,
|
||||
) -> int:
|
||||
return uuid.uuid4().int
|
||||
|
||||
@@ -982,7 +981,7 @@ class FakePlatform(Platform):
|
||||
return {"oot": ("oot",)}
|
||||
|
||||
@classmethod
|
||||
def get_nixl_memory_type(cls) -> Optional[str]:
|
||||
def get_nixl_memory_type(cls) -> str | None:
|
||||
"""
|
||||
Returns the nixl memory type for the current platform.
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from concurrent.futures import Future
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -14,9 +13,9 @@ pytestmark = pytest.mark.cpu_test
|
||||
class DummyModelRunnerOutput(ModelRunnerOutput):
|
||||
def __init__(
|
||||
self,
|
||||
finished_sending: Optional[set[str]] = None,
|
||||
finished_recving: Optional[set[str]] = None,
|
||||
invalid_block_ids: Optional[set[int]] = None,
|
||||
finished_sending: set[str] | None = None,
|
||||
finished_recving: set[str] | None = None,
|
||||
invalid_block_ids: set[int] | None = None,
|
||||
):
|
||||
self.kv_connector_output = KVConnectorOutput(
|
||||
finished_sending=finished_sending,
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from itertools import count
|
||||
from typing import Any, Callable, Optional
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -147,7 +148,7 @@ _none_hash_initialized = False
|
||||
|
||||
|
||||
def create_request(
|
||||
request_id: Optional[int] = None,
|
||||
request_id: int | None = None,
|
||||
num_tokens: int = 10,
|
||||
common_prefix_len=0,
|
||||
max_tokens: int = 16,
|
||||
@@ -168,7 +169,7 @@ def create_request(
|
||||
init_none_hash(hash_fn)
|
||||
_none_hash_initialized = True
|
||||
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
if do_remote_decode:
|
||||
assert not do_remote_prefill
|
||||
@@ -205,9 +206,9 @@ def create_request(
|
||||
|
||||
def create_model_runner_output(
|
||||
reqs: list[Request],
|
||||
finished_sending: Optional[set[str]] = None,
|
||||
finished_recving: Optional[set[str]] = None,
|
||||
invalid_block_ids: Optional[set[int]] = None,
|
||||
finished_sending: set[str] | None = None,
|
||||
finished_recving: set[str] | None = None,
|
||||
invalid_block_ids: set[int] | None = None,
|
||||
use_eos: bool = False,
|
||||
token_id: int = 0,
|
||||
) -> ModelRunnerOutput:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -29,7 +28,7 @@ def to_hashes(int_hashes: list[int]) -> list[BlockHash]:
|
||||
|
||||
|
||||
def verify_store_output(
|
||||
prepare_store_output: Optional[PrepareStoreOutput],
|
||||
prepare_store_output: PrepareStoreOutput | None,
|
||||
expected_prepare_store_output: ExpectedPrepareStoreOutput,
|
||||
):
|
||||
assert prepare_store_output is not None
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import random
|
||||
from collections.abc import Callable
|
||||
from typing import NamedTuple, Optional, Union
|
||||
from typing import NamedTuple, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -48,7 +48,7 @@ REQS_PER_LOGITPROC = 50
|
||||
STR_NO_LOGITPROC = "none"
|
||||
|
||||
# LogitsProcessor subclass or "none"
|
||||
LogitprocType = Union[type[LogitsProcessor], str]
|
||||
LogitprocType: TypeAlias = type[LogitsProcessor] | str
|
||||
|
||||
|
||||
class LogitsProcsRequestParams:
|
||||
@@ -435,7 +435,7 @@ class LogitsprocTestHelpers(NamedTuple):
|
||||
"""Supports setting up and validating logitsprocs unit tests."""
|
||||
|
||||
eval_fxn: Callable
|
||||
gen_request_fxn: Optional[Callable] = None
|
||||
gen_request_fxn: Callable | None = None
|
||||
|
||||
|
||||
logitsprocs_test_mapping = {
|
||||
@@ -471,7 +471,7 @@ def _generate_fake_step_update(
|
||||
workload_params: list[LogitsProcsRequestParams],
|
||||
wdx: int,
|
||||
batch_update_builder: BatchUpdateBuilder,
|
||||
) -> tuple[Optional[BatchUpdate], int, int]:
|
||||
) -> tuple[BatchUpdate | None, int, int]:
|
||||
batch_size = len(persistent_batch)
|
||||
workload_size = len(workload_params)
|
||||
workload_reqs_remaining = workload_size - wdx
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
import sys
|
||||
from typing import Any, Union
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -159,7 +159,7 @@ def test_custom_logitsprocs(monkeypatch, logitproc_source: CustomLogitprocSource
|
||||
_run_test({}, logitproc_loaded=True)
|
||||
return
|
||||
|
||||
kwargs: dict[str, list[Union[str, type[LogitsProcessor]]]] = {}
|
||||
kwargs: dict[str, list[str | type[LogitsProcessor]]] = {}
|
||||
if logitproc_source == CustomLogitprocSource.LOGITPROC_SOURCE_FQCN:
|
||||
# Scenario: load logitproc based on fully-qualified class name (FQCN)
|
||||
# Inject dummy module which defines logitproc
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
@@ -25,7 +25,7 @@ from tests.v1.logits_processors.utils import entry_points as fake_entry_points
|
||||
|
||||
|
||||
def _server_with_logitproc_entrypoint(
|
||||
env_dict: Optional[dict[str, str]],
|
||||
env_dict: dict[str, str] | None,
|
||||
model: str,
|
||||
vllm_serve_args: list[str],
|
||||
) -> None:
|
||||
@@ -48,7 +48,7 @@ def _server_with_logitproc_entrypoint(
|
||||
|
||||
|
||||
def _server_with_logitproc_module(
|
||||
env_dict: Optional[dict[str, str]],
|
||||
env_dict: dict[str, str] | None,
|
||||
model: str,
|
||||
vllm_serve_args: list[str],
|
||||
) -> None:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import types
|
||||
from enum import Enum, auto
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -61,7 +61,7 @@ class DummyLogitsProcessor(LogitsProcessor):
|
||||
"""Never impacts greedy sampling"""
|
||||
return False
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
process_dict_updates(
|
||||
self.req_info,
|
||||
batch_update,
|
||||
@@ -145,7 +145,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
def new_req_logits_processor(
|
||||
self,
|
||||
params: SamplingParams,
|
||||
) -> Optional[RequestLogitsProcessor]:
|
||||
) -> RequestLogitsProcessor | None:
|
||||
"""This method returns a new request-level logits processor, customized
|
||||
to the `target_token` value associated with a particular request.
|
||||
|
||||
@@ -159,7 +159,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
Returns:
|
||||
`Callable` request logits processor, or None
|
||||
"""
|
||||
target_token: Optional[Any] = params.extra_args and params.extra_args.get(
|
||||
target_token: Any | None = params.extra_args and params.extra_args.get(
|
||||
"target_token"
|
||||
)
|
||||
if target_token is None:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -24,7 +24,7 @@ def rejection_sampler():
|
||||
def create_logits_tensor(
|
||||
output_token_ids: list[list[int]],
|
||||
vocab_size: int = 100,
|
||||
token_idx_to_override: Optional[int] = None,
|
||||
token_idx_to_override: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Helper function to create logits tensor that
|
||||
will produce desired token ids on argmax"""
|
||||
@@ -43,18 +43,18 @@ def create_logits_tensor(
|
||||
|
||||
def create_sampling_metadata(
|
||||
all_greedy: bool,
|
||||
output_token_ids: Optional[list[list[int]]] = None,
|
||||
prompt_token_ids: Optional[torch.Tensor] = None,
|
||||
spec_token_ids: Optional[torch.Tensor] = None,
|
||||
temperature: Optional[torch.Tensor] = None,
|
||||
top_k: Optional[torch.Tensor] = None,
|
||||
top_p: Optional[torch.Tensor] = None,
|
||||
generators: Optional[dict[int, Any]] = None,
|
||||
frequency_penalties: Optional[list[float]] = None,
|
||||
presence_penalties: Optional[list[float]] = None,
|
||||
repetition_penalties: Optional[list[float]] = None,
|
||||
bad_words_token_ids: Optional[dict[int, list[list[int]]]] = None,
|
||||
allowed_token_ids_mask: Optional[torch.Tensor] = None,
|
||||
output_token_ids: list[list[int]] | None = None,
|
||||
prompt_token_ids: torch.Tensor | None = None,
|
||||
spec_token_ids: torch.Tensor | None = None,
|
||||
temperature: torch.Tensor | None = None,
|
||||
top_k: torch.Tensor | None = None,
|
||||
top_p: torch.Tensor | None = None,
|
||||
generators: dict[int, Any] | None = None,
|
||||
frequency_penalties: list[float] | None = None,
|
||||
presence_penalties: list[float] | None = None,
|
||||
repetition_penalties: list[float] | None = None,
|
||||
bad_words_token_ids: dict[int, list[list[int]]] | None = None,
|
||||
allowed_token_ids_mask: torch.Tensor | None = None,
|
||||
) -> SamplingMetadata:
|
||||
"""Create a v1 sampling metadata object with all_greedy set
|
||||
to the given value. Either all greedy or all random sampling
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
from collections.abc import Iterator
|
||||
from enum import Enum
|
||||
from typing import NamedTuple, Optional
|
||||
from typing import NamedTuple
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
@@ -23,7 +23,7 @@ class BatchLogprobsComposition(Enum):
|
||||
SAMPLE_PROMPT = 3
|
||||
|
||||
|
||||
BatchLogprobsSpecType = list[tuple[Optional[int], Optional[int]]]
|
||||
BatchLogprobsSpecType = list[tuple[int | None, int | None]]
|
||||
|
||||
|
||||
def get_test_batch(
|
||||
@@ -222,8 +222,8 @@ def create_allowed_token_ids(
|
||||
vocab_size: int,
|
||||
num_allowed_token_ids: int,
|
||||
device: torch.device,
|
||||
) -> Optional[torch.Tensor]:
|
||||
mask: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor | None:
|
||||
mask: torch.Tensor | None = None
|
||||
for i in range(batch_size):
|
||||
if i % 2 == 1:
|
||||
continue
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Optional
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
@@ -39,7 +38,7 @@ eagle3_dir = "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B"
|
||||
def _create_proposer(
|
||||
method: str,
|
||||
num_speculative_tokens: int,
|
||||
speculative_token_tree: Optional[list[tuple[int, ...]]] = None,
|
||||
speculative_token_tree: list[tuple[int, ...]] | None = None,
|
||||
) -> EagleProposer:
|
||||
model_config = ModelConfig(model=model_dir, runner="generate", max_model_len=100)
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -37,7 +36,7 @@ def forward_attention(
|
||||
slot_mapping: torch.Tensor,
|
||||
seqlen_k: int,
|
||||
backend: _Backend,
|
||||
spec_token_tree: Optional[str] = None,
|
||||
spec_token_tree: str | None = None,
|
||||
num_spec_tokens: int = 0,
|
||||
) -> torch.Tensor:
|
||||
batch_size, q_len, num_heads, dim_per_head = q.shape
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import UserDict
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -100,7 +99,7 @@ def test_encode_decode(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
class MyRequest(msgspec.Struct):
|
||||
mm: Optional[list[MultiModalKwargsItems]]
|
||||
mm: list[MultiModalKwargsItems] | None
|
||||
|
||||
|
||||
def test_multimodal_kwargs():
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
Run `pytest tests/v1/tpu/test_basic.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
@@ -16,6 +14,8 @@ from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tests.conftest import VllmRunner
|
||||
else:
|
||||
VllmRunner = object
|
||||
|
||||
MODELS = [
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
Run `pytest tests/v1/tpu/test_perf.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -20,6 +18,8 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tests.conftest import VllmRunner
|
||||
else:
|
||||
VllmRunner = object
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa
|
||||
# type: ignore
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections.abc import Iterable
|
||||
from concurrent import futures
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import inspect
|
||||
from collections.abc import Sequence
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -271,7 +270,7 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
|
||||
reqs, req_ids_retained, input_batch.req_id_to_index, device=torch.device(device)
|
||||
)
|
||||
|
||||
def same(t1: Optional[torch.Tensor], t2: Optional[torch.Tensor]) -> bool:
|
||||
def same(t1: torch.Tensor | None, t2: torch.Tensor | None) -> bool:
|
||||
return (t1 is None and t2 is None) or (
|
||||
t1 is not None and t2 is not None and torch.allclose(t1, t2)
|
||||
)
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import tempfile
|
||||
from multiprocessing import Queue
|
||||
from typing import Optional
|
||||
from multiprocessing.queues import Queue
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -16,7 +15,7 @@ from vllm.utils import MemorySnapshot
|
||||
from vllm.v1.worker.gpu_worker import Worker, init_worker_distributed_environment
|
||||
|
||||
# Global queue to track operation order across processes
|
||||
_QUEUE: Optional[Queue] = None
|
||||
_QUEUE: Queue | None = None
|
||||
|
||||
|
||||
def track_operation(operation: str, rank: int):
|
||||
|
||||
Reference in New Issue
Block a user