Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+2 -3
View File
@@ -1,9 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# vllm/_bc_linter.py
from __future__ import annotations
from typing import Any, Callable, TypeVar, overload
from collections.abc import Callable
from typing import Any, TypeVar, overload
T = TypeVar("T")
+103 -105
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Literal, Optional, Union
from typing import TYPE_CHECKING, Literal
import torch
@@ -37,7 +37,7 @@ def paged_attention_v1(
seq_lens: torch.Tensor,
block_size: int,
max_seq_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
@@ -84,7 +84,7 @@ def paged_attention_v2(
seq_lens: torch.Tensor,
block_size: int,
max_seq_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
@@ -132,14 +132,14 @@ def paged_attention_rocm(
scale: float,
block_tables: torch.Tensor,
seq_lens: torch.Tensor,
query_start_loc: Optional[torch.Tensor],
query_start_loc: torch.Tensor | None,
block_size: int,
max_seq_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
fp8_out_scale: Optional[torch.Tensor] = None,
fp8_out_scale: torch.Tensor | None = None,
mfma_type: str = "fp8" if envs.VLLM_ROCM_FP8_MFMA_PAGE_ATTN else "f16",
) -> None:
torch.ops._rocm_C.paged_attention(
@@ -186,7 +186,7 @@ def merge_attn_states(
prefix_lse: torch.Tensor,
suffix_output: torch.Tensor,
suffix_lse: torch.Tensor,
output_lse: Optional[torch.Tensor] = None,
output_lse: torch.Tensor | None = None,
) -> None:
torch.ops._C.merge_attn_states(
output, output_lse, prefix_output, prefix_lse, suffix_output, suffix_lse
@@ -314,7 +314,7 @@ def convert_vertical_slash_indexes_mergehead(
def rotary_embedding(
positions: torch.Tensor,
query: torch.Tensor,
key: Optional[torch.Tensor],
key: torch.Tensor | None,
head_size: int,
cos_sin_cache: torch.Tensor,
is_neox: bool,
@@ -408,8 +408,8 @@ def rms_norm_dynamic_per_token_quant(
weight: torch.Tensor,
epsilon: float,
quant_dtype: torch.dtype,
scale_ub: Optional[torch.Tensor] = None,
residual: Optional[torch.Tensor] = None,
scale_ub: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
output = torch.empty_like(input, dtype=quant_dtype)
scales = torch.empty(
@@ -527,14 +527,14 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
@register_fake("_C::gptq_marlin_gemm")
def _gptq_marlin_gemm_fake(
a: torch.Tensor,
c: Optional[torch.Tensor],
c: torch.Tensor | None,
b_q_weight: torch.Tensor,
b_bias: Optional[torch.Tensor],
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
global_scale: Optional[torch.Tensor],
b_zeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
global_scale: torch.Tensor | None,
b_zeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
b_q_type_id: int,
size_m: torch.SymInt,
@@ -582,13 +582,13 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
# b_q Should be the tensor returned by machete_prepack_B
b_q: torch.Tensor,
b_type: ScalarType,
out_type: Optional[torch.dtype] = None,
b_group_scales: Optional[torch.Tensor] = None,
b_group_zeros: Optional[torch.Tensor] = None,
b_group_size: Optional[int] = None,
b_channel_scales: Optional[torch.Tensor] = None,
a_token_scales: Optional[torch.Tensor] = None,
schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
b_group_scales: torch.Tensor | None = None,
b_group_zeros: torch.Tensor | None = None,
b_group_size: int | None = None,
b_channel_scales: torch.Tensor | None = None,
a_token_scales: torch.Tensor | None = None,
schedule: str | None = None,
) -> torch.Tensor:
m = a.size(0)
n = b_q.size(1)
@@ -599,7 +599,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
b_q_weight: torch.Tensor,
a_type: torch.dtype,
b_type: ScalarType,
group_scales_type: Optional[torch.dtype],
group_scales_type: torch.dtype | None,
) -> torch.Tensor:
return torch.empty_like(b_q_weight, memory_format=torch.contiguous_format)
@@ -612,8 +612,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
b_group_size: int,
b_channel_scales: torch.Tensor,
a_token_scales: torch.Tensor,
out_type: Optional[torch.dtype] = None,
maybe_schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
maybe_schedule: str | None = None,
) -> torch.Tensor:
m = a.size(0)
n = b_q.size(1)
@@ -636,7 +636,7 @@ if hasattr(torch.ops._C, "allspark_w8a16_gemm"):
a: torch.Tensor,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
n: torch.SymInt,
group_size: torch.SymInt,
sm_count: torch.SymInt,
@@ -657,7 +657,7 @@ if hasattr(torch.ops._C, "ggml_dequantize"):
quant_type: int,
m: torch.SymInt,
n: torch.SymInt,
dtype: Optional[torch.dtype] = None,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
return torch.empty((m, n), dtype=torch.float16, device=W.device)
@@ -760,7 +760,7 @@ def cutlass_scaled_mm(
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out_dtype: torch.dtype,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
`cutlass_scaled_mm` implements a fused version of
@@ -812,8 +812,8 @@ def cutlass_scaled_mm_azp(
scale_b: torch.Tensor,
out_dtype: torch.dtype,
azp_adj: torch.Tensor,
azp: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
azp: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
:param azp_adj: In the per-tensor case, this should include the azp.
@@ -890,7 +890,7 @@ def cutlass_scaled_sparse_mm(
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out_dtype: torch.dtype,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Performs a scaled sparse matrix multiplication using Cutlass.
@@ -940,7 +940,7 @@ def get_cutlass_moe_mm_data(
num_experts: int,
n: int,
k: int,
blockscale_offsets: Optional[torch.Tensor] = None,
blockscale_offsets: torch.Tensor | None = None,
):
"""
Prepare data necessary to perform CUTLASS grouped matrix multiplications
@@ -986,7 +986,7 @@ def get_cutlass_moe_mm_problem_sizes(
num_experts: int,
n: int,
k: int,
blockscale_offsets: Optional[torch.Tensor] = None,
blockscale_offsets: torch.Tensor | None = None,
):
"""
Compute only the per-expert problem sizes needed by the two grouped matrix
@@ -1195,14 +1195,14 @@ def awq_marlin_moe_repack(
def gptq_marlin_gemm(
a: torch.Tensor,
c: Optional[torch.Tensor],
c: torch.Tensor | None,
b_q_weight: torch.Tensor,
b_bias: Optional[torch.Tensor],
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
global_scale: Optional[torch.Tensor],
b_zeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
global_scale: torch.Tensor | None,
b_zeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
b_q_type: ScalarType,
size_m: int,
@@ -1239,11 +1239,11 @@ def gptq_marlin_gemm(
def machete_supported_schedules(
a_type: torch.dtype,
b_type: ScalarType,
group_scales_type: Optional[torch.dtype],
group_zeros_type: Optional[torch.dtype] = None,
channel_scales_type: Optional[torch.dtype] = None,
token_scales_type: Optional[torch.dtype] = None,
out_type: Optional[torch.dtype] = None,
group_scales_type: torch.dtype | None,
group_zeros_type: torch.dtype | None = None,
channel_scales_type: torch.dtype | None = None,
token_scales_type: torch.dtype | None = None,
out_type: torch.dtype | None = None,
) -> list[str]:
return torch.ops._C.machete_supported_schedules(
a_type,
@@ -1261,13 +1261,13 @@ def machete_mm(
# b_q Should be the tensor returned by machete_prepack_B
b_q: torch.Tensor,
b_type: ScalarType,
out_type: Optional[torch.dtype] = None,
b_group_scales: Optional[torch.Tensor] = None,
b_group_zeros: Optional[torch.Tensor] = None,
b_group_size: Optional[int] = None,
b_channel_scales: Optional[torch.Tensor] = None,
a_token_scales: Optional[torch.Tensor] = None,
schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
b_group_scales: torch.Tensor | None = None,
b_group_zeros: torch.Tensor | None = None,
b_group_size: int | None = None,
b_channel_scales: torch.Tensor | None = None,
a_token_scales: torch.Tensor | None = None,
schedule: str | None = None,
) -> torch.Tensor:
return torch.ops._C.machete_mm(
a,
@@ -1287,7 +1287,7 @@ def machete_prepack_B(
b_q_weight: torch.Tensor,
a_type: torch.dtype,
b_type: ScalarType,
group_scales_type: Optional[torch.dtype],
group_scales_type: torch.dtype | None,
) -> torch.Tensor:
return torch.ops._C.machete_prepack_B(
b_q_weight, a_type, b_type.id, group_scales_type
@@ -1303,8 +1303,8 @@ def cutlass_w4a8_mm(
b_group_size: int,
b_channel_scales: torch.Tensor,
a_token_scales: torch.Tensor,
out_type: Optional[torch.dtype] = None,
maybe_schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
maybe_schedule: str | None = None,
) -> torch.Tensor:
return torch.ops._C.cutlass_w4a8_mm(
a,
@@ -1458,11 +1458,11 @@ def scaled_fp4_experts_quant(
# fp8
def scaled_fp8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
num_token_padding: Optional[int] = None,
scale_ub: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
num_token_padding: int | None = None,
scale_ub: torch.Tensor | None = None,
use_per_token_if_dynamic: bool = False,
output: Optional[torch.Tensor] = None,
output: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to FP8 and return quantized tensor and scale.
@@ -1489,7 +1489,7 @@ def scaled_fp8_quant(
"""
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
shape: Union[tuple[int, int], torch.Size] = input.shape
shape: tuple[int, int] | torch.Size = input.shape
# For ROCm on MI300, the output fp8 dtype is torch.float_e3m3fnuz
out_dtype: torch.dtype = current_platform.fp8_dtype()
if num_token_padding:
@@ -1520,7 +1520,7 @@ def scaled_fp8_quant(
def allspark_repack_weight(
qweight: torch.Tensor,
scale: torch.Tensor,
zero_point: Optional[torch.Tensor] = None,
zero_point: torch.Tensor | None = None,
has_zp: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
@@ -1576,7 +1576,7 @@ def allspark_w8a16_gemm(
a: torch.Tensor,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
n: int,
group_size: int,
sm_count: int,
@@ -1603,10 +1603,10 @@ def allspark_w8a16_gemm(
# int8
def scaled_int8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
azp: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
azp: torch.Tensor | None = None,
symmetric: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
@@ -1643,7 +1643,7 @@ def scaled_int8_quant(
# gguf
def ggml_dequantize(
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: Optional[torch.dtype]
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None
) -> torch.Tensor:
return torch.ops._C.ggml_dequantize(W, quant_type, m, n, dtype)
@@ -1713,13 +1713,13 @@ def selective_scan_fwd(
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D_: Optional[torch.Tensor],
z_: Optional[torch.Tensor],
delta_bias_: Optional[torch.Tensor],
D_: torch.Tensor | None,
z_: torch.Tensor | None,
delta_bias_: torch.Tensor | None,
delta_softplus: bool,
query_start_loc: Optional[torch.Tensor],
cache_indices: Optional[torch.Tensor],
has_initial_state: Optional[torch.Tensor],
query_start_loc: torch.Tensor | None,
cache_indices: torch.Tensor | None,
has_initial_state: torch.Tensor | None,
ssm_states: torch.Tensor,
pad_slot_id: int,
):
@@ -1794,8 +1794,8 @@ def moe_wna16_gemm(
output: torch.Tensor,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
topk_weights: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
topk_weights: torch.Tensor | None,
sorted_token_ids: torch.Tensor,
experts_ids: torch.Tensor,
num_tokens_post_pad: torch.Tensor,
@@ -1864,14 +1864,14 @@ def grouped_topk(
def moe_wna16_marlin_gemm(
input: torch.Tensor,
output: Optional[torch.Tensor],
output: torch.Tensor | None,
b_qweight: torch.Tensor,
b_bias: Optional[torch.Tensor],
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
global_scale: Optional[torch.Tensor],
b_qzeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
global_scale: torch.Tensor | None,
b_qzeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
@@ -1950,12 +1950,12 @@ if hasattr(torch.ops, "_moe_C") and hasattr(torch.ops._moe_C, "marlin_gemm_moe")
@register_fake("_moe_C::moe_wna16_marlin_gemm")
def moe_wna16_marlin_gemm_fake(
input: torch.Tensor,
output: Optional[torch.Tensor],
output: torch.Tensor | None,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
@@ -2068,7 +2068,7 @@ def gather_and_maybe_dequant_cache(
batch_size: int,
kv_cache_dtype: str,
scale: torch.Tensor,
seq_starts: Optional[torch.Tensor] = None,
seq_starts: torch.Tensor | None = None,
) -> None:
torch.ops._C_cache_ops.gather_and_maybe_dequant_cache(
src_cache,
@@ -2088,7 +2088,7 @@ def cp_gather_cache(
block_table: torch.Tensor,
cu_seq_lens: torch.Tensor,
batch_size: int,
seq_starts: Optional[torch.Tensor] = None,
seq_starts: torch.Tensor | None = None,
) -> None:
torch.ops._C_cache_ops.cp_gather_cache(
src_cache, dst, block_table, cu_seq_lens, batch_size, seq_starts
@@ -2187,9 +2187,7 @@ def free_shared_buffer(ptr: int) -> None:
# quick all reduce
def init_custom_qr(
rank: int, world_size: int, qr_max_size: Optional[int] = None
) -> int:
def init_custom_qr(rank: int, world_size: int, qr_max_size: int | None = None) -> int:
return torch.ops._C_custom_ar.init_custom_qr(rank, world_size, qr_max_size)
@@ -2247,7 +2245,7 @@ def flash_mla_with_kvcache(
head_dim_v: int,
tile_scheduler_metadata: torch.Tensor,
num_splits: torch.Tensor,
softmax_scale: Optional[float] = None,
softmax_scale: float | None = None,
causal: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
@@ -2324,7 +2322,7 @@ if hasattr(torch.ops._C, "weight_packed_linear"):
def weight_packed_linear_fake(
mat1: torch.Tensor,
mat2: torch.Tensor,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
is_vnni: bool,
) -> torch.Tensor:
return torch.empty(
@@ -2344,11 +2342,11 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
inplace: bool,
use_int8_w8a8: bool,
use_fp8_w8a16: bool,
w1_scale: Optional[torch.Tensor],
w2_scale: Optional[torch.Tensor],
block_size: Optional[list[int]],
a1_scale: Optional[torch.Tensor],
a2_scale: Optional[torch.Tensor],
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
block_size: list[int] | None,
a1_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
is_vnni: bool,
) -> torch.Tensor:
return torch.empty_like(hidden_states)
@@ -2361,7 +2359,7 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
mat1: torch.Tensor,
mat2: torch.Tensor,
scales2: torch.Tensor,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
out_dtype: torch.dtype,
is_vnni: bool,
) -> torch.Tensor:
@@ -2372,7 +2370,7 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
class CPUDNNLGEMMHandler:
def __init__(self) -> None:
self.handler: Optional[int] = None
self.handler: int | None = None
self.n = -1
self.k = -1
@@ -2403,7 +2401,7 @@ def create_onednn_mm(
def onednn_mm(
dnnl_handler: CPUDNNLGEMMHandler,
x: torch.Tensor,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
) -> torch.Tensor:
output = torch.empty((*x.shape[0:-1], dnnl_handler.n), dtype=x.dtype)
torch.ops._C.onednn_mm(
@@ -2431,8 +2429,8 @@ def create_onednn_scaled_mm(
def onednn_scaled_int8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
azp: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
azp: torch.Tensor | None = None,
symmetric: bool = True,
):
"""
@@ -2471,10 +2469,10 @@ def onednn_scaled_mm(
dnnl_handler: CPUDNNLGEMMHandler,
x: torch.Tensor,
output: torch.Tensor,
input_scale: Optional[torch.Tensor],
input_zp: Optional[torch.Tensor],
input_zp_adj: Optional[torch.Tensor],
bias: Optional[torch.Tensor],
input_scale: torch.Tensor | None,
input_zp: torch.Tensor | None,
input_zp_adj: torch.Tensor | None,
bias: torch.Tensor | None,
) -> torch.Tensor:
torch.ops._C.onednn_scaled_mm(
output, x, input_scale, input_zp, input_zp_adj, bias, dnnl_handler.handler
+19 -20
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
import torch
@@ -65,7 +64,7 @@ class ipex_ops:
context_lens: torch.Tensor,
block_size: int,
max_context_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: float,
v_scale: float,
@@ -107,7 +106,7 @@ class ipex_ops:
context_lens: torch.Tensor,
block_size: int,
max_context_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: float,
v_scale: float,
@@ -174,7 +173,7 @@ class ipex_ops:
out: torch.Tensor,
seqlen_q: torch.Tensor,
seqlen_k: torch.Tensor,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
max_seqlen_q: int,
max_seqlen_k: int,
pdropout: float,
@@ -254,8 +253,8 @@ class ipex_ops:
value_cache: torch.Tensor,
slot_mapping: torch.Tensor,
kv_cache_dtype: str,
k_scale: Optional[torch.Tensor] = None,
v_scale: Optional[torch.Tensor] = None,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
k_scale_float: float = 1.0,
v_scale_float: float = 1.0,
) -> None:
@@ -283,10 +282,10 @@ class ipex_ops:
softmax_scale: float,
causal: bool,
block_table: torch.Tensor,
alibi_slopes: Optional[torch.Tensor],
window_size: Optional[list[int]] = None,
softcap: Optional[float] = 0.0,
cu_seqlens_k: Optional[torch.Tensor] = None,
alibi_slopes: torch.Tensor | None,
window_size: list[int] | None = None,
softcap: float | None = 0.0,
cu_seqlens_k: torch.Tensor | None = None,
# The following parameters are not used in ipex kernel currently,
# we keep API compatible to CUDA's.
scheduler_metadata=None,
@@ -295,7 +294,7 @@ class ipex_ops:
k_descale=None,
v_descale=None,
num_splits=0,
s_aux: Optional[torch.Tensor] = None,
s_aux: torch.Tensor | None = None,
):
if cu_seqlens_k is None:
# cu_seqlens_k is not used in ipex kernel.
@@ -344,10 +343,10 @@ class ipex_ops:
cache_seqlens: torch.Tensor,
qkv_dtype=torch.bfloat16,
headdim_v=None,
cu_seqlens_q: Optional[torch.Tensor] = None,
cu_seqlens_k_new: Optional[torch.Tensor] = None,
cache_leftpad: Optional[torch.Tensor] = None,
page_size: Optional[int] = None,
cu_seqlens_q: torch.Tensor | None = None,
cu_seqlens_k_new: torch.Tensor | None = None,
cache_leftpad: torch.Tensor | None = None,
page_size: int | None = None,
max_seqlen_k_new=0,
causal=False,
window_size=(-1, -1), # -1 means infinite context window
@@ -382,11 +381,11 @@ class ipex_ops:
@staticmethod
def scaled_fp8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
num_token_padding: Optional[int] = None,
scale_ub: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
num_token_padding: int | None = None,
scale_ub: torch.Tensor | None = None,
use_per_token_if_dynamic: bool = False,
output: Optional[torch.Tensor] = None,
output: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to FP8 and return quantized tensor and scale.
@@ -414,7 +413,7 @@ class ipex_ops:
"""
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
shape: Union[tuple[int, int], torch.Size] = input.shape
shape: tuple[int, int] | torch.Size = input.shape
out_dtype: torch.dtype = current_platform.fp8_dtype()
if num_token_padding:
shape = (max(num_token_padding, input.shape[0]), shape[1])
+1 -2
View File
@@ -3,7 +3,6 @@
from functools import lru_cache
from pathlib import Path
from typing import Optional
import vllm.envs as envs
from vllm.connections import global_http_connection
@@ -20,7 +19,7 @@ def get_cache_dir() -> Path:
@lru_cache
def get_vllm_public_assets(filename: str, s3_prefix: Optional[str] = None) -> Path:
def get_vllm_public_assets(filename: str, s3_prefix: str | None = None) -> Path:
"""
Download an asset file from ``s3://vllm-public-assets``
and return the path to the downloaded file.
+2 -2
View File
@@ -3,7 +3,7 @@
from dataclasses import dataclass
from functools import lru_cache
from typing import Any, ClassVar, Literal, Optional
from typing import Any, ClassVar, Literal
import cv2
import numpy as np
@@ -137,7 +137,7 @@ class VideoAsset:
ret = video_get_metadata(self.video_path, self.num_frames)
return ret
def get_audio(self, sampling_rate: Optional[float] = None) -> npt.NDArray:
def get_audio(self, sampling_rate: float | None = None) -> npt.NDArray:
"""
Read audio data from the video asset, used in Qwen2.5-Omni examples.
+20 -20
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Generic, Optional, Protocol, TypeVar, Union
from typing import Generic, Protocol, TypeVar
import torch
@@ -65,7 +65,7 @@ class AttentionBackend(ABC):
raise NotImplementedError
@classmethod
def get_supported_kernel_block_size(cls) -> list[Union[int, MultipleOf]]:
def get_supported_kernel_block_size(cls) -> list[int | MultipleOf]:
return cls.get_impl_cls().get_supported_kernel_block_size()
@classmethod
@@ -158,18 +158,18 @@ class AttentionImpl(ABC, Generic[T]):
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: Optional[int] = None,
alibi_slopes: Optional[list[float]] = None,
sliding_window: Optional[int] = None,
num_kv_heads: int | None = None,
alibi_slopes: list[float] | None = None,
sliding_window: int | None = None,
kv_cache_dtype: str = "auto",
logits_soft_cap: Optional[float] = None,
logits_soft_cap: float | None = None,
attn_type: str = AttentionType.DECODER,
kv_sharing_target_layer_name: Optional[str] = None,
kv_sharing_target_layer_name: str | None = None,
) -> None:
raise NotImplementedError
@staticmethod
def get_supported_kernel_block_size() -> list[Union[int, MultipleOf]]:
def get_supported_kernel_block_size() -> list[int | MultipleOf]:
# TODO: implement this function for all backends.
return [MultipleOf(1)]
@@ -182,9 +182,9 @@ class AttentionImpl(ABC, Generic[T]):
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: T,
output: Optional[torch.Tensor] = None,
output_scale: Optional[torch.Tensor] = None,
output_block_scale: Optional[torch.Tensor] = None,
output: torch.Tensor | None = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError
@@ -208,21 +208,21 @@ class MLAAttentionImpl(AttentionImpl[T], Generic[T]):
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: Optional[list[float]],
sliding_window: Optional[int],
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: Optional[float],
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: Optional[str],
kv_sharing_target_layer_name: str | None,
# MLA Specific Arguments
q_lora_rank: Optional[int],
q_lora_rank: int | None,
kv_lora_rank: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
qk_head_dim: int,
v_head_dim: int,
kv_b_proj: ColumnParallelLinear,
indexer: Optional[object] = None,
indexer: object | None = None,
) -> None:
raise NotImplementedError
@@ -235,9 +235,9 @@ class MLAAttentionImpl(AttentionImpl[T], Generic[T]):
k_pe: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: T,
output: Optional[torch.Tensor] = None,
output_scale: Optional[torch.Tensor] = None,
output_block_scale: Optional[torch.Tensor] = None,
output: torch.Tensor | None = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError
+2 -3
View File
@@ -3,7 +3,6 @@
"""Attention backend registry"""
import enum
from typing import Optional
from vllm.utils import resolve_obj_by_qualname
@@ -53,7 +52,7 @@ BACKEND_MAP = {
}
def register_attn_backend(backend: _Backend, class_path: Optional[str] = None):
def register_attn_backend(backend: _Backend, class_path: str | None = None):
"""
Decorator: register a custom attention backend into BACKEND_MAPPING.
- If class_path is provided, use it.
@@ -98,7 +97,7 @@ def backend_to_class(backend: _Backend) -> type:
return resolve_obj_by_qualname(backend_class_name)
def backend_name_to_enum(backend_name: str) -> Optional[_Backend]:
def backend_name_to_enum(backend_name: str) -> _Backend | None:
"""
Convert a string backend name to a _Backend enum value.
+1 -2
View File
@@ -3,7 +3,6 @@
"""Attention backend utils"""
from dataclasses import dataclass
from typing import Optional
from vllm.config import ModelConfig
from vllm.logger import init_logger
@@ -15,7 +14,7 @@ PAD_SLOT_ID = -1
@dataclass
class MLADims:
q_lora_rank: Optional[int]
q_lora_rank: int | None
kv_lora_rank: int
qk_nope_head_dim: int
qk_rope_head_dim: int
+27 -26
View File
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention layer."""
from typing import Callable, Optional, cast
from collections.abc import Callable
from typing import cast
import torch
import torch.nn as nn
@@ -128,16 +129,16 @@ class Attention(nn.Module, AttentionLayerBase):
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: Optional[int] = None,
alibi_slopes: Optional[list[float]] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
logits_soft_cap: Optional[float] = None,
per_layer_sliding_window: Optional[int] = None,
num_kv_heads: int | None = None,
alibi_slopes: list[float] | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
logits_soft_cap: float | None = None,
per_layer_sliding_window: int | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
kv_sharing_target_layer_name: Optional[str] = None,
attn_backend: Optional[type[AttentionBackend]] = None,
kv_sharing_target_layer_name: str | None = None,
attn_backend: type[AttentionBackend] | None = None,
**extra_impl_args,
) -> None:
"""
@@ -191,7 +192,7 @@ class Attention(nn.Module, AttentionLayerBase):
# The output scale on host memory. This should be the input scale of
# the quant op after this attention layer.
self._o_scale_float: Optional[float] = None
self._o_scale_float: float | None = None
self.num_heads = num_heads
self.head_size = head_size
@@ -319,7 +320,7 @@ class Attention(nn.Module, AttentionLayerBase):
# For some alternate attention backends like MLA the attention output
# shape does not match the query shape, so we optionally let the model
# definition specify the output tensor shape.
output_shape: Optional[torch.Size] = None,
output_shape: torch.Size | None = None,
) -> torch.Tensor:
"""
The KV cache is stored inside this class and is accessed via
@@ -427,7 +428,7 @@ class MultiHeadAttention(nn.Module):
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: Optional[int] = None,
num_kv_heads: int | None = None,
# This has no effect, it is only here to make it easier to swap
# between Attention and MultiHeadAttention
prefix: str = "",
@@ -582,14 +583,14 @@ class MLAAttention(nn.Module, AttentionLayerBase):
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: Optional[int],
q_lora_rank: int | None,
kv_lora_rank: int,
kv_b_proj: ColumnParallelLinear,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_sparse: bool = False,
indexer: Optional[object] = None,
indexer: object | None = None,
):
super().__init__()
self.num_heads = num_heads
@@ -670,7 +671,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self._q_scale_float = 1.0
self._k_scale_float = 1.0
self._v_scale_float = 1.0
self._o_scale_float: Optional[float] = None
self._o_scale_float: float | None = None
self.use_sparse = use_sparse
@@ -688,7 +689,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
output_shape: Optional[torch.Size] = None,
output_shape: torch.Size | None = None,
) -> torch.Tensor:
if self.use_direct_call:
forward_context: ForwardContext = get_forward_context()
@@ -888,8 +889,8 @@ def unified_attention_with_output(
value: torch.Tensor,
output: torch.Tensor,
layer_name: str,
output_scale: Optional[torch.Tensor] = None,
output_block_scale: Optional[torch.Tensor] = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> None:
wait_for_kv_layer_from_connector(layer_name)
forward_context: ForwardContext = get_forward_context()
@@ -919,8 +920,8 @@ def unified_attention_with_output_fake(
value: torch.Tensor,
output: torch.Tensor,
layer_name: str,
output_scale: Optional[torch.Tensor] = None,
output_block_scale: Optional[torch.Tensor] = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> None:
return
@@ -978,8 +979,8 @@ def unified_mla_attention_with_output(
k_pe: torch.Tensor,
output: torch.Tensor,
layer_name: str,
output_scale: Optional[torch.Tensor] = None,
output_block_scale: Optional[torch.Tensor] = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> None:
wait_for_kv_layer_from_connector(layer_name)
forward_context: ForwardContext = get_forward_context()
@@ -1009,8 +1010,8 @@ def unified_mla_attention_with_output_fake(
k_pe: torch.Tensor,
output: torch.Tensor,
layer_name: str,
output_scale: Optional[torch.Tensor] = None,
output_block_scale: Optional[torch.Tensor] = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> None:
return
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from typing import ClassVar, Optional
from typing import ClassVar
import torch
@@ -60,11 +60,11 @@ class ChunkedLocalAttention(Attention):
head_size: int,
scale: float,
attention_chunk_size: int,
num_kv_heads: Optional[int] = None,
alibi_slopes: Optional[list[float]] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
kv_sharing_target_layer_name: Optional[str] = None,
num_kv_heads: int | None = None,
alibi_slopes: list[float] | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
kv_sharing_target_layer_name: str | None = None,
prefix: str = "",
):
dtype = torch.get_default_dtype()
+2 -3
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from copy import copy
from typing import Optional
import numpy as np
import torch
@@ -138,8 +137,8 @@ class CrossAttention(Attention):
num_heads: int,
head_size: int,
scale: float,
cache_config: Optional[CacheConfig] = None,
attn_type: Optional[str] = None,
cache_config: CacheConfig | None = None,
attn_type: str | None = None,
**kwargs,
):
dtype = torch.get_default_dtype()
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from copy import copy
from typing import Optional
import torch
@@ -60,8 +59,8 @@ class EncoderOnlyAttention(Attention):
num_heads: int,
head_size: int,
scale: float,
cache_config: Optional[CacheConfig] = None,
attn_type: Optional[str] = None,
cache_config: CacheConfig | None = None,
attn_type: str | None = None,
**kwargs,
):
dtype = torch.get_default_dtype()
+9 -10
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# adapted from: https://github.com/deepseek-ai/FlashMLA/blob/main/flash_mla/flash_mla_interface.py
from typing import Optional
import torch
@@ -31,7 +30,7 @@ else:
_flashmla_extension_C_AVAILABLE = False
def _is_flashmla_available() -> tuple[bool, Optional[str]]:
def _is_flashmla_available() -> tuple[bool, str | None]:
if not _flashmla_C_AVAILABLE:
return (
False,
@@ -49,7 +48,7 @@ def _is_flashmla_available() -> tuple[bool, Optional[str]]:
return True, None
def is_flashmla_dense_supported() -> tuple[bool, Optional[str]]:
def is_flashmla_dense_supported() -> tuple[bool, str | None]:
"""
Return: is_supported_flag, unsupported_reason (optional).
"""
@@ -61,7 +60,7 @@ def is_flashmla_dense_supported() -> tuple[bool, Optional[str]]:
return True, None
def is_flashmla_sparse_supported() -> tuple[bool, Optional[str]]:
def is_flashmla_sparse_supported() -> tuple[bool, str | None]:
"""
Return: is_supported_flag, unsupported_reason (optional).
"""
@@ -80,9 +79,9 @@ def get_mla_metadata(
cache_seqlens: torch.Tensor,
num_q_tokens_per_head_k: int,
num_heads_k: int,
num_heads_q: Optional[int] = None,
num_heads_q: int | None = None,
is_fp8_kvcache: bool = False,
topk: Optional[int] = None,
topk: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Arguments:
@@ -121,12 +120,12 @@ def flash_mla_with_kvcache(
head_dim_v: int,
tile_scheduler_metadata: torch.Tensor,
num_splits: torch.Tensor,
softmax_scale: Optional[float] = None,
softmax_scale: float | None = None,
causal: bool = False,
descale_q: Optional[torch.Tensor] = None,
descale_k: Optional[torch.Tensor] = None,
descale_q: torch.Tensor | None = None,
descale_k: torch.Tensor | None = None,
is_fp8_kvcache: bool = False,
indices: Optional[torch.Tensor] = None,
indices: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Arguments:
+1 -2
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
@@ -13,7 +12,7 @@ def merge_attn_states(
prefix_lse: torch.Tensor,
suffix_output: torch.Tensor,
suffix_lse: torch.Tensor,
output_lse: Optional[torch.Tensor] = None,
output_lse: torch.Tensor | None = None,
) -> None:
# NOTE(DefTruth): Currently, custom merge_attn_states CUDA kernel
# is not support for FP8 dtype, fallback to use Triton kernel.
+5 -6
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Optional
import torch
@@ -27,7 +26,7 @@ class PagedAttentionMetadata:
# (batch_size,). The length of sequences (entire tokens seen so far) per
# sequence.
seq_lens_tensor: Optional[torch.Tensor]
seq_lens_tensor: torch.Tensor | None
# Maximum sequence length in the batch. 0 if it is prefill-only batch.
max_decode_seq_len: int
# (batch_size, max_blocks_per_seq).
@@ -36,7 +35,7 @@ class PagedAttentionMetadata:
# in the kv cache. Each block can contain up to block_size tokens.
# 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
# captured.
block_tables: Optional[torch.Tensor]
block_tables: torch.Tensor | None
class PagedAttention:
@@ -102,7 +101,7 @@ class PagedAttention:
kv_cache_dtype: str,
num_kv_heads: int,
scale: float,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
tp_rank: int = 0,
@@ -211,8 +210,8 @@ class PagedAttention:
query_start_loc: torch.Tensor,
seq_lens_tensor: torch.Tensor,
max_query_len: int,
alibi_slopes: Optional[torch.Tensor],
sliding_window: Optional[int],
alibi_slopes: torch.Tensor | None,
sliding_window: int | None,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
) -> torch.Tensor:
+9 -10
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
@@ -30,9 +29,9 @@ def aiter_mla_decode_fwd(
sm_scale: float,
qo_indptr: torch.Tensor,
max_seqlen_qo: int,
kv_indptr: Optional[torch.Tensor] = None,
kv_indices: Optional[torch.Tensor] = None,
kv_last_page_lens: Optional[torch.Tensor] = None,
kv_indptr: torch.Tensor | None = None,
kv_indices: torch.Tensor | None = None,
kv_last_page_lens: torch.Tensor | None = None,
logit_cap: float = 0.0,
):
torch.ops.vllm.rocm_aiter_mla_decode_fwd(
@@ -55,9 +54,9 @@ def mla_decode_fwd_impl(
o: torch.Tensor,
qo_indptr: torch.Tensor,
max_seqlen_qo: int,
kv_indptr: Optional[torch.Tensor] = None,
kv_indices: Optional[torch.Tensor] = None,
kv_last_page_lens: Optional[torch.Tensor] = None,
kv_indptr: torch.Tensor | None = None,
kv_indices: torch.Tensor | None = None,
kv_last_page_lens: torch.Tensor | None = None,
sm_scale: float = 1.0,
logit_cap: float = 0.0,
) -> None:
@@ -83,9 +82,9 @@ def mla_decode_fwd_fake(
o: torch.Tensor,
qo_indptr: torch.Tensor,
max_seqlen_qo: int,
kv_indptr: Optional[torch.Tensor] = None,
kv_indices: Optional[torch.Tensor] = None,
kv_last_page_lens: Optional[torch.Tensor] = None,
kv_indptr: torch.Tensor | None = None,
kv_indices: torch.Tensor | None = None,
kv_last_page_lens: torch.Tensor | None = None,
sm_scale: float = 1.0,
logit_cap: float = 0.0,
) -> None:
+1 -2
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import aiter as rocm_aiter
import torch
@@ -62,7 +61,7 @@ class AITERPagedAttention(PagedAttention):
kv_cache_dtype: str,
num_kv_heads: int,
scale: float,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
tp_rank: int = 0,
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
@@ -15,7 +14,7 @@ def merge_attn_states(
prefix_lse: torch.Tensor,
suffix_output: torch.Tensor,
suffix_lse: torch.Tensor,
output_lse: Optional[torch.Tensor] = None,
output_lse: torch.Tensor | None = None,
) -> None:
num_tokens = output.shape[0]
num_query_heads = output.shape[1]
+9 -10
View File
@@ -6,7 +6,6 @@ from collections.abc import Generator
from contextlib import contextmanager
from dataclasses import dataclass
from functools import cache
from typing import Optional, Union
import torch
@@ -19,7 +18,7 @@ from vllm.utils import STR_BACKEND_ENV_VAR, resolve_obj_by_qualname
logger = init_logger(__name__)
def get_env_variable_attn_backend() -> Optional[_Backend]:
def get_env_variable_attn_backend() -> _Backend | None:
"""
Get the backend override specified by the vLLM attention
backend environment variable, if one is specified.
@@ -40,10 +39,10 @@ def get_env_variable_attn_backend() -> Optional[_Backend]:
#
# THIS SELECTION TAKES PRECEDENCE OVER THE
# VLLM_ATTENTION_BACKEND ENVIRONMENT VARIABLE
forced_attn_backend: Optional[_Backend] = None
forced_attn_backend: _Backend | None = None
def global_force_attn_backend(attn_backend: Optional[_Backend]) -> None:
def global_force_attn_backend(attn_backend: _Backend | None) -> None:
"""
Force all attention operations to use a specified backend.
@@ -58,7 +57,7 @@ def global_force_attn_backend(attn_backend: Optional[_Backend]) -> None:
forced_attn_backend = attn_backend
def get_global_forced_attn_backend() -> Optional[_Backend]:
def get_global_forced_attn_backend() -> _Backend | None:
"""
Get the currently-forced choice of attention backend,
or None if auto-selection is currently enabled.
@@ -77,7 +76,7 @@ class _IsSupported:
def is_attn_backend_supported(
attn_backend: Union[str, type[AttentionBackend]],
attn_backend: str | type[AttentionBackend],
head_size: int,
dtype: torch.dtype,
*,
@@ -127,7 +126,7 @@ def is_attn_backend_supported(
def get_attn_backend(
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: Optional[str],
kv_cache_dtype: str | None,
block_size: int,
use_mla: bool = False,
has_sink: bool = False,
@@ -154,7 +153,7 @@ def get_attn_backend(
def _cached_get_attn_backend(
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: Optional[str],
kv_cache_dtype: str | None,
block_size: int,
use_v1: bool = False,
use_mla: bool = False,
@@ -167,12 +166,12 @@ def _cached_get_attn_backend(
# THIS SELECTION OVERRIDES THE VLLM_ATTENTION_BACKEND
# ENVIRONMENT VARIABLE.
selected_backend = None
backend_by_global_setting: Optional[_Backend] = get_global_forced_attn_backend()
backend_by_global_setting: _Backend | None = get_global_forced_attn_backend()
if backend_by_global_setting is not None:
selected_backend = backend_by_global_setting
else:
# Check the environment variable and override if specified
backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
backend_by_env_var: str | None = envs.VLLM_ATTENTION_BACKEND
if backend_by_env_var is not None:
if backend_by_env_var.endswith("_VLLM_V1"):
logger.warning(
+1 -2
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
from vllm import envs
from vllm.logger import init_logger
@@ -21,7 +20,7 @@ elif current_platform.is_xpu():
get_scheduler_metadata = ops.get_scheduler_metadata
def get_flash_attn_version(requires_alibi: bool = False) -> Optional[int]:
def get_flash_attn_version(requires_alibi: bool = False) -> int | None:
# import here to avoid circular dependencies
from vllm.platforms import current_platform
+8 -8
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any, Optional
from vllm.logprobs import Logprob
from vllm.lora.request import LoRARequest
@@ -22,13 +22,13 @@ class BeamSearchSequence:
# The tokens include the prompt.
tokens: list[int]
logprobs: list[dict[int, Logprob]]
lora_request: Optional[LoRARequest] = None
lora_request: LoRARequest | None = None
cum_logprob: float = 0.0
text: Optional[str] = None
finish_reason: Optional[str] = None
stop_reason: Union[int, str, None] = None
text: str | None = None
finish_reason: str | None = None
stop_reason: int | str | None = None
multi_modal_data: Optional["MultiModalDataDict"] = None
mm_processor_kwargs: Optional[dict[str, Any]] = None
mm_processor_kwargs: dict[str, Any] | None = None
@dataclass
@@ -45,8 +45,8 @@ class BeamSearchInstance:
def __init__(
self,
prompt_tokens: list[int],
lora_request: Optional[LoRARequest] = None,
logprobs: Optional[list[dict[int, Logprob]]] = None,
lora_request: LoRARequest | None = None,
logprobs: list[dict[int, Logprob]] | None = None,
**kwargs,
):
self.beams: list[BeamSearchSequence] = [
+33 -33
View File
@@ -21,13 +21,13 @@ import logging
import math
import random
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping
from collections.abc import Callable, Iterator, Mapping
from contextlib import suppress
from copy import deepcopy
from dataclasses import dataclass
from functools import cache
from io import BytesIO
from typing import Any, Callable, Optional, Union, cast
from typing import Any, cast
import numpy as np
from PIL import Image
@@ -75,12 +75,12 @@ class SampleRequest:
Represents a single inference request for benchmarking.
"""
prompt: Union[str, list[str]]
prompt: str | list[str]
prompt_len: int
expected_output_len: int
multi_modal_data: Optional[Union[MultiModalDataDict, dict, list[dict]]] = None
lora_request: Optional[LoRARequest] = None
request_id: Optional[str] = None
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
lora_request: LoRARequest | None = None
request_id: str | None = None
# -----------------------------------------------------------------------------
@@ -94,7 +94,7 @@ class BenchmarkDataset(ABC):
def __init__(
self,
dataset_path: Optional[str] = None,
dataset_path: str | None = None,
random_seed: int = DEFAULT_SEED,
disable_shuffle: bool = False,
**kwargs,
@@ -119,7 +119,7 @@ class BenchmarkDataset(ABC):
def apply_multimodal_chat_transformation(
self,
prompt: str,
mm_content: Optional[Union[MultiModalDataDict, dict, list[dict]]] = None,
mm_content: MultiModalDataDict | dict | list[dict] | None = None,
) -> list[dict]:
"""
Transform a prompt and optional multimodal content into a chat format.
@@ -154,9 +154,9 @@ class BenchmarkDataset(ABC):
def get_random_lora_request(
self,
max_loras: Optional[int] = None,
lora_path: Optional[str] = None,
) -> Optional[LoRARequest]:
max_loras: int | None = None,
lora_path: str | None = None,
) -> LoRARequest | None:
"""
Optionally select a random LoRA request.
@@ -384,7 +384,7 @@ def gen_prompt_decode_to_target_len(
target_token_len: int,
max_retry: int = 10,
add_special_tokens: bool = False,
rng: Optional[np.random.Generator] = None,
rng: np.random.Generator | None = None,
) -> tuple[str, list[int]]:
"""
Ensure decoded-then-encoded prompt length matches the target token length.
@@ -1054,9 +1054,9 @@ class ShareGPTDataset(BenchmarkDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
lora_path: Optional[str] = None,
max_loras: Optional[int] = None,
output_len: Optional[int] = None,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -1766,9 +1766,9 @@ class CustomDataset(BenchmarkDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
lora_path: Optional[str] = None,
max_loras: Optional[int] = None,
output_len: Optional[int] = None,
lora_path: str | None = None,
max_loras: int | None = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
request_id_prefix: str = "",
@@ -1997,8 +1997,8 @@ class BurstGPTDataset(BenchmarkDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
max_loras: Optional[int] = None,
lora_path: Optional[str] = None,
max_loras: int | None = None,
lora_path: str | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2034,15 +2034,15 @@ class BurstGPTDataset(BenchmarkDataset):
class HuggingFaceDataset(BenchmarkDataset):
"""Base class for datasets hosted on HuggingFace."""
SUPPORTED_DATASET_PATHS: Union[set[str], dict[str, Callable]] = set()
SUPPORTED_DATASET_PATHS: set[str] | dict[str, Callable] = set()
def __init__(
self,
dataset_path: str,
dataset_split: str,
no_stream: bool = False,
dataset_subset: Optional[str] = None,
hf_name: Optional[str] = None,
dataset_subset: str | None = None,
hf_name: str | None = None,
**kwargs,
) -> None:
super().__init__(dataset_path=dataset_path, **kwargs)
@@ -2083,7 +2083,7 @@ class ConversationDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2152,7 +2152,7 @@ class VisionArenaDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2206,7 +2206,7 @@ class MMVUDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2267,7 +2267,7 @@ class InstructCoderDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
request_id_prefix: str = "",
@@ -2331,7 +2331,7 @@ class MTBenchDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
skip_chat_template: bool = False,
request_id_prefix: str = "",
@@ -2397,7 +2397,7 @@ class BlazeditDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
skip_chat_template: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
@@ -2478,7 +2478,7 @@ class AIMODataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2660,7 +2660,7 @@ class ASRDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2738,7 +2738,7 @@ class MLPerfDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
@@ -2902,7 +2902,7 @@ class MMStarDataset(HuggingFaceDataset):
self,
tokenizer: PreTrainedTokenizerBase,
num_requests: int,
output_len: Optional[int] = None,
output_len: int | None = None,
enable_multimodal_chat: bool = False,
request_id_prefix: str = "",
no_oversample: bool = False,
+2 -2
View File
@@ -7,7 +7,7 @@ import dataclasses
import json
import os
import time
from typing import Any, Optional
from typing import Any
import numpy as np
from tqdm import tqdm
@@ -127,7 +127,7 @@ def main(args: argparse.Namespace):
),
)
def run_to_completion(profile_dir: Optional[str] = None):
def run_to_completion(profile_dir: str | None = None):
if profile_dir:
llm.start_profile()
llm_generate()
+20 -20
View File
@@ -10,7 +10,7 @@ import time
import traceback
from collections.abc import Awaitable
from dataclasses import dataclass, field
from typing import Any, Literal, Optional, Protocol, Union
from typing import Any, Literal, Protocol
import aiohttp
import regex as re
@@ -69,14 +69,14 @@ class RequestFuncInput:
prompt_len: int
output_len: int
model: str
model_name: Optional[str] = None
logprobs: Optional[int] = None
extra_headers: Optional[dict] = None
extra_body: Optional[dict] = None
multi_modal_content: Optional[Union[dict, list[dict]]] = None
model_name: str | None = None
logprobs: int | None = None
extra_headers: dict | None = None
extra_body: dict | None = None
multi_modal_content: dict | list[dict] | None = None
ignore_eos: bool = False
language: Optional[str] = None
request_id: Optional[str] = None
language: str | None = None
request_id: str | None = None
@dataclass
@@ -100,14 +100,14 @@ class RequestFunc(Protocol):
self,
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> Awaitable[RequestFuncOutput]: ...
def _validate_api_url(
api_url: str,
api_name: str,
expected_suffixes: Union[str, set[str]],
expected_suffixes: str | set[str],
) -> None:
if isinstance(expected_suffixes, str):
expected_suffixes = {expected_suffixes}
@@ -141,7 +141,7 @@ def _update_headers_common(
async def async_request_openai_completions(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
"""The async request function for the OpenAI Completions API.
@@ -279,7 +279,7 @@ def _get_chat_content(
async def async_request_openai_chat_completions(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
mm_position: Literal["first", "last"] = "last",
) -> RequestFuncOutput:
api_url = request_func_input.api_url
@@ -376,7 +376,7 @@ async def async_request_openai_chat_completions(
async def async_request_openai_audio(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
# Lazy import without PlaceholderModule to avoid vllm dep.
import soundfile
@@ -489,7 +489,7 @@ async def _run_openai_embeddings(
api_url: str,
payload: dict[str, Any],
headers: dict[str, Any],
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
output = RequestFuncOutput()
st = time.perf_counter()
@@ -517,7 +517,7 @@ async def _run_openai_embeddings(
async def async_request_openai_embeddings(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Embeddings API", "embeddings")
@@ -548,7 +548,7 @@ async def async_request_openai_embeddings(
async def async_request_openai_embeddings_chat(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
mm_position: Literal["first", "last"] = "last",
) -> RequestFuncOutput:
api_url = request_func_input.api_url
@@ -627,7 +627,7 @@ def _preprocess_vlm2vec(request_func_input: RequestFuncInput):
async def async_request_openai_embeddings_clip(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
_preprocess_clip(request_func_input)
@@ -641,7 +641,7 @@ async def async_request_openai_embeddings_clip(
async def async_request_openai_embeddings_vlm2vec(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
_preprocess_vlm2vec(request_func_input)
@@ -656,7 +656,7 @@ async def async_request_openai_embeddings_vlm2vec(
async def async_request_infinity_embeddings(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
_validate_api_url(api_url, "Infinity Embeddings API", "embeddings")
@@ -697,7 +697,7 @@ async def async_request_infinity_embeddings(
async def async_request_infinity_embeddings_clip(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: Optional[tqdm] = None,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
_preprocess_clip(request_func_input)
+15 -15
View File
@@ -31,7 +31,7 @@ from collections.abc import AsyncGenerator, Iterable
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any, Literal, Optional
from typing import Any, Literal
import aiohttp
import numpy as np
@@ -107,9 +107,9 @@ class EmbedBenchmarkMetrics:
def _get_current_request_rate(
ramp_up_strategy: Optional[Literal["linear", "exponential"]],
ramp_up_start_rps: Optional[int],
ramp_up_end_rps: Optional[int],
ramp_up_strategy: Literal["linear", "exponential"] | None,
ramp_up_start_rps: int | None,
ramp_up_end_rps: int | None,
request_index: int,
total_requests: int,
request_rate: float,
@@ -135,9 +135,9 @@ async def get_request(
input_requests: list[SampleRequest],
request_rate: float,
burstiness: float = 1.0,
ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
ramp_up_start_rps: Optional[int] = None,
ramp_up_end_rps: Optional[int] = None,
ramp_up_strategy: Literal["linear", "exponential"] | None = None,
ramp_up_start_rps: int | None = None,
ramp_up_end_rps: int | None = None,
) -> AsyncGenerator[tuple[SampleRequest, float], None]:
"""
Asynchronously generates requests at a specified rate
@@ -474,7 +474,7 @@ async def benchmark(
model_name: str,
tokenizer: PreTrainedTokenizerBase,
input_requests: list[SampleRequest],
logprobs: Optional[int],
logprobs: int | None,
request_rate: float,
burstiness: float,
disable_tqdm: bool,
@@ -483,13 +483,13 @@ async def benchmark(
selected_percentiles: list[float],
ignore_eos: bool,
goodput_config_dict: dict[str, float],
max_concurrency: Optional[int],
lora_modules: Optional[Iterable[str]],
extra_headers: Optional[dict],
extra_body: Optional[dict],
ramp_up_strategy: Optional[Literal["linear", "exponential"]] = None,
ramp_up_start_rps: Optional[int] = None,
ramp_up_end_rps: Optional[int] = None,
max_concurrency: int | None,
lora_modules: Iterable[str] | None,
extra_headers: dict | None,
extra_body: dict | None,
ramp_up_strategy: Literal["linear", "exponential"] | None = None,
ramp_up_start_rps: int | None = None,
ramp_up_end_rps: int | None = None,
ready_check_timeout_sec: int = 600,
):
try:
+7 -7
View File
@@ -9,7 +9,7 @@ import os
import random
import time
import warnings
from typing import Any, Optional, Union
from typing import Any
import torch
import uvloop
@@ -43,7 +43,7 @@ def run_vllm(
engine_args: EngineArgs,
do_profile: bool,
disable_detokenize: bool = False,
) -> tuple[float, Optional[list[RequestOutput]]]:
) -> tuple[float, list[RequestOutput] | None]:
from vllm import LLM, SamplingParams
llm = LLM(**dataclasses.asdict(engine_args))
@@ -56,7 +56,7 @@ def run_vllm(
" prompt_len and expected_output_len for all requests."
)
# Add the requests to the engine.
prompts: list[Union[TextPrompt, TokensPrompt]] = []
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
for request in requests:
prompt = (
@@ -79,7 +79,7 @@ def run_vllm(
detokenize=not disable_detokenize,
)
)
lora_requests: Optional[list[LoRARequest]] = None
lora_requests: list[LoRARequest] | None = None
if engine_args.enable_lora:
lora_requests = [request.lora_request for request in requests]
@@ -197,9 +197,9 @@ async def run_vllm_async(
)
# Add the requests to the engine.
prompts: list[Union[TextPrompt, TokensPrompt]] = []
prompts: list[TextPrompt | TokensPrompt] = []
sampling_params: list[SamplingParams] = []
lora_requests: list[Optional[LoRARequest]] = []
lora_requests: list[LoRARequest | None] = []
for request in requests:
prompt = (
TokensPrompt(prompt_token_ids=request.prompt["prompt_token_ids"])
@@ -696,7 +696,7 @@ def main(args: argparse.Namespace):
)
requests = get_requests(args, tokenizer)
is_multi_modal = any(request.multi_modal_data is not None for request in requests)
request_outputs: Optional[list[RequestOutput]] = None
request_outputs: list[RequestOutput] | None = None
if args.backend == "vllm":
if args.async_engine:
elapsed_time = uvloop.run(
+9 -8
View File
@@ -7,9 +7,9 @@ import hashlib
import os
import pprint
import time
from collections.abc import Sequence
from collections.abc import Callable, Sequence
from contextlib import contextmanager
from typing import Any, Callable, Optional
from typing import Any
import torch
import torch.fx as fx
@@ -75,7 +75,7 @@ class CompilerManager:
"""
def __init__(self, compilation_config: CompilationConfig):
self.cache: dict[tuple[Optional[int], int, str], Any] = dict()
self.cache: dict[tuple[int | None, int, str], Any] = dict()
self.is_cache_updated = False
self.compilation_config = compilation_config
self.compiler = make_compiler(compilation_config)
@@ -84,7 +84,7 @@ class CompilerManager:
return self.compiler.compute_hash(vllm_config)
@contextmanager
def compile_context(self, runtime_shape: Optional[int] = None):
def compile_context(self, runtime_shape: int | None = None):
"""Provide compilation context for the duration of compilation to set
any torch global properties we want to scope to a single Inductor
compilation (e.g. partition rules, pass context)."""
@@ -145,8 +145,8 @@ class CompilerManager:
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
) -> Optional[Callable]:
runtime_shape: int | None = None,
) -> Callable | None:
if (runtime_shape, graph_index, self.compiler.name) not in self.cache:
return None
handle = self.cache[(runtime_shape, graph_index, self.compiler.name)]
@@ -178,7 +178,7 @@ class CompilerManager:
compilation_config: CompilationConfig,
graph_index: int = 0,
num_graphs: int = 1,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Any:
if graph_index == 0:
# before compiling the first graph, record the start time
@@ -656,7 +656,8 @@ class VllmBackend:
graph_path = os.path.join(local_cache_dir, "computation_graph.py")
if not os.path.exists(graph_path):
# code adapted from https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30 # noqa
# code adapted from
# https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30
# use `print_readable` because it can include submodules
src = (
"from __future__ import annotations\nimport torch\n"
+2 -1
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Callable, Protocol
from collections.abc import Callable
from typing import Any, Protocol
from vllm.config import CUDAGraphMode, VllmConfig
+9 -10
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from importlib.util import find_spec
from typing import Optional
import torch
import torch._inductor.pattern_matcher as pm
@@ -432,7 +431,7 @@ class AsyncTPPass(VllmPatternMatcherPass):
self.dump_patterns(config, self.patterns)
def is_applicable_for_shape(self, shape: Optional[int]) -> bool:
def is_applicable_for_shape(self, shape: int | None) -> bool:
# only do replace for specific shapes
tp_size = get_tensor_model_parallel_world_size()
return shape is not None and shape % tp_size == 0
@@ -485,10 +484,10 @@ if flashinfer_comm is not None:
max_token_num: int,
pattern_code: int,
fuse_rms_quant: bool,
norm_out: Optional[torch.Tensor] = None,
quant_out: Optional[torch.Tensor] = None,
scale_out: Optional[torch.Tensor] = None,
scale_factor: Optional[torch.Tensor] = None,
norm_out: torch.Tensor | None = None,
quant_out: torch.Tensor | None = None,
scale_out: torch.Tensor | None = None,
scale_factor: torch.Tensor | None = None,
) -> None:
num_tokens, hidden_size = allreduce_in.shape
element_size = allreduce_in.element_size()
@@ -589,10 +588,10 @@ if flashinfer_comm is not None:
max_token_num: int,
pattern_code: int,
fuse_rms_quant: bool,
norm_out: Optional[torch.Tensor] = None,
quant_out: Optional[torch.Tensor] = None,
scale_out: Optional[torch.Tensor] = None,
scale_factor: Optional[torch.Tensor] = None,
norm_out: torch.Tensor | None = None,
quant_out: torch.Tensor | None = None,
scale_out: torch.Tensor | None = None,
scale_factor: torch.Tensor | None = None,
) -> None:
pass
+17 -16
View File
@@ -4,8 +4,9 @@ import contextlib
import copy
import hashlib
import os
from collections.abc import Callable
from contextlib import ExitStack
from typing import Any, Callable, Optional
from typing import Any
from unittest.mock import patch
import torch
@@ -62,9 +63,9 @@ class CompilerInterface:
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
"""
Compile the graph with the given example inputs and compiler config,
with a runtime shape. If the `runtime_shape` is None, it means
@@ -97,7 +98,7 @@ class CompilerInterface:
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Callable:
"""
Load the compiled function from the handle.
@@ -191,9 +192,9 @@ class InductorStandaloneAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
compilation_counter.num_inductor_compiles += 1
current_config = {}
if compiler_config is not None:
@@ -229,7 +230,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Callable:
assert isinstance(handle, tuple)
assert isinstance(handle[0], str)
@@ -293,9 +294,9 @@ class InductorAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
compilation_counter.num_inductor_compiles += 1
from torch._inductor.compile_fx import compile_fx
@@ -492,7 +493,7 @@ class InductorAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Callable:
assert isinstance(handle, tuple)
assert isinstance(handle[0], str)
@@ -610,9 +611,9 @@ class EagerAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
compilation_counter.num_eager_compiles += 1
# we don't need to compile the graph, just return the graph itself.
# It does not support caching, return None for the handle.
+6 -5
View File
@@ -2,8 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from collections.abc import Callable
from contextlib import ExitStack
from typing import Any, Callable, Optional
from typing import Any
from unittest.mock import patch
import torch
@@ -24,12 +25,12 @@ logger = init_logger(__name__)
@dataclasses.dataclass
class CUDAGraphEntry:
batch_descriptor: BatchDescriptor
cudagraph: Optional[torch.cuda.CUDAGraph] = None
output: Optional[Any] = None
cudagraph: torch.cuda.CUDAGraph | None = None
output: Any | None = None
# for cudagraph debugging, track the input addresses
# during capture, and check if they are the same during replay
input_addresses: Optional[list[int]] = None
input_addresses: list[int] | None = None
@dataclasses.dataclass
@@ -69,7 +70,7 @@ class CUDAGraphWrapper:
runnable: Callable,
vllm_config: VllmConfig,
runtime_mode: CUDAGraphMode,
cudagraph_options: Optional[CUDAGraphOptions] = None,
cudagraph_options: CUDAGraphOptions | None = None,
):
self.runnable = runnable
self.vllm_config = vllm_config
+12 -11
View File
@@ -6,7 +6,8 @@ import hashlib
import inspect
import os
import sys
from typing import Callable, Optional, TypeVar, Union, overload
from collections.abc import Callable
from typing import TypeVar, overload
from unittest.mock import patch
import torch
@@ -61,14 +62,14 @@ def _should_ignore_torch_compile(cls) -> bool:
@overload
def support_torch_compile(
*,
enable_if: Optional[Callable[[VllmConfig], bool]] = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
) -> Callable[[_T], _T]: ...
@overload
def support_torch_compile(
*,
dynamic_arg_dims: Optional[dict[str, Union[int, list[int]]]],
dynamic_arg_dims: dict[str, int | list[int]] | None,
) -> Callable[[_T], _T]: ...
@@ -77,11 +78,11 @@ def support_torch_compile(cls: _T) -> _T: ...
def support_torch_compile(
cls: Optional[_T] = None,
cls: _T | None = None,
*,
dynamic_arg_dims: Optional[dict[str, Union[int, list[int]]]] = None,
enable_if: Optional[Callable[[VllmConfig], bool]] = None,
) -> Union[Callable[[_T], _T], _T]:
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
) -> Callable[[_T], _T] | _T:
"""
A decorator to add support for compiling the forward method of a class.
@@ -147,9 +148,9 @@ def support_torch_compile(
for k, v in sig.parameters.items():
if v.annotation in [
torch.Tensor,
Optional[torch.Tensor],
torch.Tensor | None,
IntermediateTensors,
Optional[IntermediateTensors],
IntermediateTensors | None,
]:
inferred_dynamic_arg_dims[k] = 0
@@ -209,8 +210,8 @@ def _verify_source_unchanged(source_info, vllm_config) -> None:
def _support_torch_compile(
cls: _T,
dynamic_arg_dims: dict[str, Union[int, list[int]]],
enable_if: Optional[Callable[[VllmConfig], bool]] = None,
dynamic_arg_dims: dict[str, int | list[int]],
enable_if: Callable[[VllmConfig], bool] | None = None,
) -> _T:
"""
A decorator to add support for compiling the forward method of a class.
+5 -6
View File
@@ -3,7 +3,6 @@
import operator
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
@@ -150,7 +149,7 @@ class FixFunctionalizationPass(VllmInductorPass):
)
self.nodes_to_remove.clear()
def _remove(self, node_or_nodes: Union[torch.fx.Node, Iterable[torch.fx.Node]]):
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]):
"""
Stage a node (or nodes) for removal at the end of the pass.
"""
@@ -163,8 +162,8 @@ class FixFunctionalizationPass(VllmInductorPass):
self,
graph: torch.fx.Graph,
node: torch.fx.Node,
mutated_args: dict[int, Union[torch.fx.Node, str]],
args: Optional[tuple[Union[torch.fx.Node, str], ...]] = None,
mutated_args: dict[int, torch.fx.Node | str],
args: tuple[torch.fx.Node | str, ...] | None = None,
):
"""
De-functionalize a node by replacing it with a call to the original.
@@ -176,7 +175,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self._remove(node)
def replace_users_with_mutated_args(
self, node: torch.fx.Node, mutated_args: dict[int, Union[torch.fx.Node, str]]
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
):
"""
Replace all getitem users of the auto-functionalized node with the
@@ -207,7 +206,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self,
graph: torch.fx.Graph,
node: torch.fx.Node,
args: Optional[tuple[Union[torch.fx.Node, str], ...]] = None,
args: tuple[torch.fx.Node | str, ...] | None = None,
):
"""
Insert a new defunctionalized node into the graph before node.
+3 -6
View File
@@ -3,7 +3,6 @@
import operator
from collections.abc import Iterable, Iterator
from typing import Optional
from torch import fx
from torch._higher_order_ops.auto_functionalize import auto_functionalized
@@ -19,9 +18,7 @@ def is_auto_func(node: fx.Node, op: OpOverload) -> bool:
# Returns the first specified node with the given op (if it exists)
def find_specified_fn_maybe(
nodes: Iterable[fx.Node], op: OpOverload
) -> Optional[fx.Node]:
def find_specified_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node | None:
for node in nodes:
if node.target == op:
return node
@@ -36,7 +33,7 @@ def find_specified_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node:
# Returns the first auto_functionalized node with the given op (if it exists)
def find_auto_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> Optional[fx.Node]:
def find_auto_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node | None:
for node in nodes:
if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
return node
@@ -52,7 +49,7 @@ def find_auto_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node:
# Returns the getitem node that extracts the idx-th element from node
# (if it exists)
def find_getitem_maybe(node: fx.Node, idx: int) -> Optional[fx.Node]:
def find_getitem_maybe(node: fx.Node, idx: int) -> fx.Node | None:
for user in node.users:
if is_func(user, operator.getitem) and user.args[1] == idx:
return user
+7 -8
View File
@@ -6,8 +6,9 @@ import hashlib
import inspect
import json
import types
from collections.abc import Callable
from contextlib import contextmanager
from typing import Any, Callable, Optional, Union
from typing import Any
import torch
from torch import fx
@@ -27,7 +28,7 @@ _pass_context = None
class PassContext:
def __init__(self, runtime_shape: Optional[int]):
def __init__(self, runtime_shape: int | None):
self.runtime_shape = runtime_shape
@@ -38,7 +39,7 @@ def get_pass_context() -> PassContext:
@contextmanager
def pass_context(runtime_shape: Optional[int]):
def pass_context(runtime_shape: int | None):
"""A context manager that stores the current pass context,
usually it is a list of sizes to specialize.
"""
@@ -67,7 +68,7 @@ class InductorPass(CustomGraphPass):
return InductorPass.hash_source(self)
@staticmethod
def hash_source(*srcs: Union[str, Any]):
def hash_source(*srcs: str | Any):
"""
Utility method to hash the sources of functions or objects.
:param srcs: strings or objects to add to the hash.
@@ -95,7 +96,7 @@ class InductorPass(CustomGraphPass):
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def is_applicable_for_shape(self, shape: Optional[int]):
def is_applicable_for_shape(self, shape: int | None):
return True
@@ -105,9 +106,7 @@ class CallableInductorPass(InductorPass):
implementation of the UUID.
"""
def __init__(
self, callable: Callable[[fx.Graph], None], uuid: Optional[Any] = None
):
def __init__(self, callable: Callable[[fx.Graph], None], uuid: Any | None = None):
self.callable = callable
self._uuid = self.hash_source(callable) if uuid is None else uuid
+2 -5
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Union
import torch.fx
from torch import SymInt
@@ -105,9 +104,7 @@ class NoOpEliminationPass(VllmInductorPass):
logger.debug("Removed %s no-op reshapes and slices", count)
# ---------------------- Shape comparison helpers ----------------------
def dims_equivalent(
self, dim: Union[int, SymInt], i_dim: Union[int, SymInt]
) -> bool:
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
"""
This function checks if two dimensions are equivalent.
:param dim: The dimension arg to reshape/slice
@@ -127,7 +124,7 @@ class NoOpEliminationPass(VllmInductorPass):
return False
def all_dims_equivalent(
self, dims: Iterable[Union[int, SymInt]], i_dims: Iterable[Union[int, SymInt]]
self, dims: Iterable[int | SymInt], i_dims: Iterable[int | SymInt]
) -> bool:
dims_ = list(dims)
i_dims_ = list(i_dims)
+2 -4
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import contextlib
from typing import TYPE_CHECKING
@@ -16,7 +14,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def resolve_defined_ops(op_names: list[str]) -> list[torch._ops.OpOverload]:
def resolve_defined_ops(op_names: list[str]) -> list["torch._ops.OpOverload"]:
"""Resolve operator names to OpOverload objects.
Skips operators that fail to resolve (e.g., operators not registered or
@@ -49,7 +47,7 @@ def resolve_defined_ops(op_names: list[str]) -> list[torch._ops.OpOverload]:
@contextlib.contextmanager
def inductor_partition_rule_context(overloads: list[torch._ops.OpOverload]):
def inductor_partition_rule_context(overloads: list["torch._ops.OpOverload"]):
"""Context manager to temporarily register Inductor partition rules.
Registers custom partition rules for specified operators, forcing the
+2 -1
View File
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from typing import Any, Callable
from collections.abc import Callable
from typing import Any
import torch.fx as fx
+3 -4
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
import torch._inductor.pattern_matcher as pm
@@ -27,7 +26,7 @@ class _RMSNormAndQuantOpHelper:
epsilon: float,
dtype: torch.dtype,
device: str,
quant_op: Optional[torch._ops.OpOverload] = None,
quant_op: torch._ops.OpOverload | None = None,
**kwargs,
):
self.epsilon = epsilon
@@ -110,7 +109,7 @@ class _SequenceParallelPatternHelper(_RMSNormAndQuantOpHelper):
epsilon: float,
dtype: torch.dtype,
device: str,
quant_op: Optional[torch._ops.OpOverload] = None,
quant_op: torch._ops.OpOverload | None = None,
**kwargs,
):
super().__init__(epsilon, dtype, device, quant_op=quant_op, **kwargs)
@@ -483,7 +482,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
).register(self.patterns)
self.dump_patterns(config, self.patterns)
def is_applicable_for_shape(self, shape: Optional[int]) -> bool:
def is_applicable_for_shape(self, shape: int | None) -> bool:
tp_size = get_tensor_model_parallel_world_size()
return shape is not None and shape % tp_size == 0
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Any, Optional
from typing import Any
import torch
@@ -23,7 +23,7 @@ class Torch25CustomGraphPass(ABC): # noqa (redefinition)
"""
@abstractmethod
def uuid(self) -> Optional[Any]:
def uuid(self) -> Any | None:
"""
Return an ID to uniquely identify your custom pass implementation.
Return None to skip inductor code caching entirely.
+2 -2
View File
@@ -3,7 +3,7 @@
import functools
import operator
import time
from typing import ClassVar, Optional
from typing import ClassVar
import regex as re
import torch
@@ -24,7 +24,7 @@ class VllmInductorPass(InductorPass):
It provides timing, logging, and dumping utilities.
"""
dump_prefix: ClassVar[Optional[int]] = None
dump_prefix: ClassVar[int | None] = None
"""Keep track of pass index for debug dump ordering."""
def __init__(self, config: VllmConfig):
+2 -2
View File
@@ -4,9 +4,9 @@
import os
import sys
from abc import abstractmethod
from collections.abc import Callable
from contextlib import contextmanager
from types import CodeType
from typing import Callable, Optional
import torch
@@ -31,7 +31,7 @@ class TorchCompileWrapperWithCustomDispatcher:
"""
def __init__(
self, compiled_callable: Optional[Callable] = None, compilation_level: int = 0
self, compiled_callable: Callable | None = None, compilation_level: int = 0
):
vllm_config = get_current_vllm_config()
self.vllm_config = vllm_config
+10 -10
View File
@@ -3,7 +3,7 @@
import hashlib
from dataclasses import field
from typing import TYPE_CHECKING, Any, Literal, Optional
from typing import TYPE_CHECKING, Any, Literal
from pydantic import Field, SkipValidation, field_validator
from pydantic.dataclasses import dataclass
@@ -58,13 +58,13 @@ class CacheConfig:
is_attention_free: bool = False
"""Whether the model is attention-free. This is primarily set in
`ModelConfig` and that value should be manually duplicated here."""
num_gpu_blocks_override: Optional[int] = None
num_gpu_blocks_override: int | None = None
"""Number of GPU blocks to use. This overrides the profiled `num_gpu_blocks`
if specified. Does nothing if `None`. Used for testing preemption."""
sliding_window: Optional[int] = None
sliding_window: int | None = None
"""Sliding window size for the KV cache. This is primarily set in
`ModelConfig` and that value should be manually duplicated here."""
enable_prefix_caching: Optional[bool] = None
enable_prefix_caching: bool | None = None
"""Whether to enable prefix caching. Enabled by default for V1."""
prefix_caching_hash_algo: PrefixCachingHashAlgo = "sha256"
"""Set the hash algorithm for prefix caching:\n
@@ -84,12 +84,12 @@ class CacheConfig:
"""This enables dynamic calculation of `k_scale` and `v_scale` when
kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model
checkpoint if available. Otherwise, the scales will default to 1.0."""
cpu_kvcache_space_bytes: Optional[int] = None
cpu_kvcache_space_bytes: int | None = None
"""(CPU backend only) CPU key-value cache space."""
mamba_page_size_padded: Optional[int] = None
mamba_page_size_padded: int | None = None
""" Optional override for mamba page size; used by hybrid mamba/attention
models to ensure exact alignment with attention page size."""
mamba_block_size: Optional[int] = None
mamba_block_size: int | None = None
"""Size of a contiguous cache block in number of tokens for mamba cache."""
mamba_cache_dtype: MambaDType = "auto"
"""The data type to use for the Mamba cache (both the conv as well as the
@@ -101,9 +101,9 @@ class CacheConfig:
for the ssm state will be determined by mamba_cache_dtype."""
# Will be set after profiling.
num_gpu_blocks: Optional[int] = field(default=None, init=False)
num_gpu_blocks: int | None = field(default=None, init=False)
"""The number of blocks to allocate for GPU memory."""
num_cpu_blocks: Optional[int] = field(default=None, init=False)
num_cpu_blocks: int | None = field(default=None, init=False)
"""The number of blocks to allocate for CPU memory."""
kv_sharing_fast_prefill: bool = False
@@ -116,7 +116,7 @@ class CacheConfig:
necessary for implementing this optimization in some models (e.g. Gemma3n)
"""
kv_cache_memory_bytes: Optional[int] = None
kv_cache_memory_bytes: int | None = None
"""Size of KV Cache per GPU in bytes. By default, this is set to None
and vllm can automatically infer the kv cache size based on
gpu_memory_utilization. However, users may want to manually specify
+10 -9
View File
@@ -4,9 +4,10 @@
import enum
import hashlib
from collections import Counter
from collections.abc import Callable
from dataclasses import asdict, field
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union
from typing import TYPE_CHECKING, Any, ClassVar
from pydantic import TypeAdapter, field_validator
from pydantic.dataclasses import dataclass
@@ -168,7 +169,7 @@ class CompilationConfig:
"""
# Top-level Compilation control
level: Optional[int] = None
level: int | None = None
"""The level of compilation:
- None: If None, we will select the default compilation level.
@@ -177,7 +178,7 @@ class CompilationConfig:
- 1: dynamo as is.
- 2: dynamo once.
- 3: piecewise compilation."""
debug_dump_path: Optional[Path] = None
debug_dump_path: Path | None = None
"""The path to dump the debug information."""
cache_dir: str = ""
"""The directory to store the compiled graph, to accelerate Inductor
@@ -208,7 +209,7 @@ class CompilationConfig:
By default, all custom ops are enabled when running without Inductor and
disabled when running with Inductor: level>=PIECEWISE and use_inductor=True.
Inductor generates (fused) Triton kernels for disabled custom ops."""
splitting_ops: Optional[list[str]] = None
splitting_ops: list[str] | None = None
"""A list of ops to exclude from cudagraphs, used in piecewise compilation.
The behavior depends on use_inductor_graph_partition:
@@ -238,7 +239,7 @@ class CompilationConfig:
are compiled using configurations in inductor_compile_config.
This setting is ignored if level<PIECEWISE."""
compile_sizes: Optional[list[Union[int, str]]] = None
compile_sizes: list[int | str] | None = None
"""Sizes to compile for inductor. In addition
to integers, it also supports "cudagraph_capture_sizes" to
specify the sizes for cudagraph capture."""
@@ -253,7 +254,7 @@ class CompilationConfig:
constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""
# CudaGraph compilation
cudagraph_mode: Optional[CUDAGraphMode] = None
cudagraph_mode: CUDAGraphMode | None = None
"""
The mode of the cudagraph:
@@ -308,7 +309,7 @@ class CompilationConfig:
It means the first several runs will be treated as warmup runs.
Only after that, the execution will be recorded, and the recorded
cudagraph will be used for subsequent runs."""
cudagraph_capture_sizes: Optional[list[int]] = None
cudagraph_capture_sizes: list[int] | None = None
"""Sizes to capture cudagraph.
- None (default): capture sizes are inferred from vllm config.
- list[int]: capture sizes are specified as given."""
@@ -320,7 +321,7 @@ class CompilationConfig:
internally managed buffer. Default is False.
Note that this flag is only effective when cudagraph_mode is PIECEWISE.
"""
full_cuda_graph: Optional[bool] = False
full_cuda_graph: bool | None = False
"""whether to use a full cuda graph for the entire forward pass rather than
splitting certain operations such as attention into subgraphs. Thus this
flag cannot be used together with splitting_ops. This may provide
@@ -544,7 +545,7 @@ class CompilationConfig:
"(where 'op' is the registered op name)"
)
def init_backend(self, vllm_config: "VllmConfig") -> Union[str, Callable]:
def init_backend(self, vllm_config: "VllmConfig") -> str | Callable:
if self.level == CompilationLevel.NO_COMPILATION:
raise ValueError("No compilation level is set.")
+2 -2
View File
@@ -3,7 +3,7 @@
import hashlib
from dataclasses import field
from typing import Any, Literal, Optional, Union
from typing import Any, Literal
import torch
from pydantic import ConfigDict, SkipValidation
@@ -19,7 +19,7 @@ Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]
class DeviceConfig:
"""Configuration for the device to use for vLLM execution."""
device: SkipValidation[Optional[Union[Device, torch.device]]] = "auto"
device: SkipValidation[Device | torch.device | None] = "auto"
"""Device type for vLLM execution.
This parameter is deprecated and will be
removed in a future release.
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
from pydantic.dataclasses import dataclass
@@ -26,7 +25,7 @@ class KVEventsConfig:
"""The zmq endpoint to use for publishing kv events.
"""
replay_endpoint: Optional[str] = None
replay_endpoint: str | None = None
"""The zmq endpoint to use for replaying kv events.
"""
+7 -7
View File
@@ -4,7 +4,7 @@
import hashlib
import uuid
from dataclasses import field
from typing import Any, Literal, Optional, get_args
from typing import Any, Literal, get_args
from pydantic.dataclasses import dataclass
@@ -20,14 +20,14 @@ KVRole = Literal[KVProducer, KVConsumer]
class KVTransferConfig:
"""Configuration for distributed KV cache transfer."""
kv_connector: Optional[str] = None
kv_connector: str | None = None
"""The KV connector for vLLM to transmit KV caches between vLLM instances.
"""
engine_id: Optional[str] = None
engine_id: str | None = None
"""The engine id for KV transfers."""
kv_buffer_device: Optional[str] = "cuda"
kv_buffer_device: str | None = "cuda"
"""The device used by kv connector to buffer the KV cache. Choices are
'cuda' and 'cpu'."""
@@ -35,11 +35,11 @@ class KVTransferConfig:
"""The buffer size for TorchDistributedConnector. Measured in number of
bytes. Recommended value: 1e9 (about 1GB)."""
kv_role: Optional[KVRole] = None
kv_role: KVRole | None = None
"""Whether this vLLM instance produces, consumes KV cache, or both. Choices
are 'kv_producer', 'kv_consumer', and 'kv_both'."""
kv_rank: Optional[int] = None
kv_rank: int | None = None
"""The rank of this vLLM instance in the KV cache transfer. Typical value:
0 for prefill instance, 1 for decode instance.
Currently only 1P1D is supported."""
@@ -57,7 +57,7 @@ class KVTransferConfig:
kv_connector_extra_config: dict[str, Any] = field(default_factory=dict)
"""any extra config that the connector may need."""
kv_connector_module_path: Optional[str] = None
kv_connector_module_path: str | None = None
"""The Python module path to dynamically load the KV connector from.
Only supported in V1."""
+9 -13
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any
from pydantic import Field, field_validator
from pydantic.dataclasses import dataclass
@@ -25,7 +25,7 @@ logger = init_logger(__name__)
class LoadConfig:
"""Configuration for loading the model weights."""
load_format: Union[str, LoadFormats] = "auto"
load_format: str | LoadFormats = "auto"
"""The format of the model weights to load:\n
- "auto" will try to load the weights in the safetensors format and fall
back to the pytorch bin format if safetensors format is not available.\n
@@ -48,7 +48,7 @@ class LoadConfig:
- "mistral" will load weights from consolidated safetensors files used by
Mistral models.
- Other custom values can be supported via plugins."""
download_dir: Optional[str] = None
download_dir: str | None = None
"""Directory to download and load the weights, default to the default
cache directory of Hugging Face."""
safetensors_load_strategy: str = "lazy"
@@ -64,23 +64,19 @@ class LoadConfig:
was quantized using torchao and saved using safetensors.
Needs torchao >= 0.14.0
"""
model_loader_extra_config: Union[dict, TensorizerConfig] = Field(
default_factory=dict
)
model_loader_extra_config: dict | TensorizerConfig = Field(default_factory=dict)
"""Extra config for model loader. This will be passed to the model loader
corresponding to the chosen load_format."""
device: Optional[str] = None
device: str | None = None
"""Device to which model weights will be loaded, default to
device_config.device"""
ignore_patterns: Union[list[str], str] = Field(
default_factory=lambda: ["original/**/*"]
)
ignore_patterns: list[str] | str = Field(default_factory=lambda: ["original/**/*"])
"""The list of patterns to ignore when loading the model. Default to
"original/**/*" to avoid repeated loading of llama's checkpoints."""
use_tqdm_on_load: bool = True
"""Whether to enable tqdm for showing progress bar when loading model
weights."""
pt_load_map_location: Union[str, dict[str, str]] = "cpu"
pt_load_map_location: str | dict[str, str] = "cpu"
"""
pt_load_map_location: the map location for loading pytorch checkpoint, to
support loading checkpoints can only be loaded on certain devices like
@@ -115,8 +111,8 @@ class LoadConfig:
@field_validator("ignore_patterns", mode="after")
def _validate_ignore_patterns(
cls, ignore_patterns: Union[list[str], str]
) -> Union[list[str], str]:
cls, ignore_patterns: list[str] | str
) -> list[str] | str:
if ignore_patterns != ["original/**/*"] and len(ignore_patterns) > 0:
logger.info(
"Ignoring the following patterns when downloading weights: %s",
+4 -4
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union
from typing import TYPE_CHECKING, Any, ClassVar, Literal
import torch
from pydantic import ConfigDict, Field, model_validator
@@ -42,10 +42,10 @@ class LoRAConfig:
parallelism. Enabling this will use the fully sharded layers. At high
sequence length, max rank or tensor parallel size, this is likely faster.
"""
max_cpu_loras: Optional[int] = None
max_cpu_loras: int | None = None
"""Maximum number of LoRAs to store in CPU memory. Must be >= than
`max_loras`."""
lora_dtype: Union[torch.dtype, LoRADType] = "auto"
lora_dtype: torch.dtype | LoRADType = "auto"
"""Data type for LoRA. If auto, will default to base model dtype."""
lora_extra_vocab_size: LoRAExtraVocabSize = Field(
default=256,
@@ -60,7 +60,7 @@ class LoRAConfig:
lora_vocab_padding_size: ClassVar[int] = (
current_platform.get_lora_vocab_padding_size()
)
default_mm_loras: Optional[dict[str, str]] = None
default_mm_loras: dict[str, str] | None = None
"""Dictionary mapping specific modalities to LoRA model paths; this field
is only applicable to multimodal models and should be leveraged when a
model always expects a LoRA to be active when a given modality is present.
+62 -72
View File
@@ -4,18 +4,10 @@
import hashlib
import json
import warnings
from collections.abc import Callable
from dataclasses import InitVar, field
from importlib.util import find_spec
from typing import (
TYPE_CHECKING,
Any,
Callable,
Literal,
Optional,
Union,
cast,
get_args,
)
from typing import TYPE_CHECKING, Any, Literal, cast, get_args
import torch
from pydantic import ConfigDict, SkipValidation, field_validator, model_validator
@@ -89,7 +81,7 @@ ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
LogprobsMode = Literal[
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
]
HfOverrides = Union[dict[str, Any], Callable[[PretrainedConfig], PretrainedConfig]]
HfOverrides = dict[str, Any] | Callable[[PretrainedConfig], PretrainedConfig]
ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"]
_RUNNER_TASKS: dict[RunnerType, list[TaskOption]] = {
@@ -121,7 +113,7 @@ class ModelConfig:
"""Convert the model using adapters defined in
[vllm.model_executor.models.adapters][]. The most common use case is to
adapt a text generation model to be used for pooling tasks."""
task: Optional[TaskOption] = None
task: TaskOption | None = None
"""[DEPRECATED] The task to use the model for. If the model supports more
than one model runner, this is used to select which model runner to run.
@@ -139,7 +131,7 @@ class ModelConfig:
trust_remote_code: bool = False
"""Trust remote code (e.g., from HuggingFace) when downloading the model
and tokenizer."""
dtype: Union[ModelDType, torch.dtype] = "auto"
dtype: ModelDType | torch.dtype = "auto"
"""Data type for model weights and activations:\n
- "auto" will use FP16 precision for FP32 and FP16 models, and BF16
precision for BF16 models.\n
@@ -148,33 +140,33 @@ class ModelConfig:
- "bfloat16" for a balance between precision and range.\n
- "float" is shorthand for FP32 precision.\n
- "float32" for FP32 precision."""
seed: Optional[int] = None
seed: int | None = None
"""Random seed for reproducibility. Initialized to None in V0, but
initialized to 0 in V1."""
hf_config_path: Optional[str] = None
hf_config_path: str | None = None
"""Name or path of the Hugging Face config to use. If unspecified, model
name or path will be used."""
allowed_local_media_path: str = ""
"""Allowing API requests to read local images or videos from directories
specified by the server file system. This is a security risk. Should only
be enabled in trusted environments."""
allowed_media_domains: Optional[list[str]] = None
allowed_media_domains: list[str] | None = None
"""If set, only media URLs that belong to this domain can be used for
multi-modal inputs. """
revision: Optional[str] = None
revision: str | None = None
"""The specific model version to use. It can be a branch name, a tag name,
or a commit id. If unspecified, will use the default version."""
code_revision: Optional[str] = None
code_revision: str | None = None
"""The specific revision to use for the model code on the Hugging Face Hub.
It can be a branch name, a tag name, or a commit id. If unspecified, will
use the default version."""
rope_scaling: dict[str, Any] = field(default_factory=dict)
"""RoPE scaling configuration. For example,
`{"rope_type":"dynamic","factor":2.0}`."""
rope_theta: Optional[float] = None
rope_theta: float | None = None
"""RoPE theta. Use with `rope_scaling`. In some cases, changing the RoPE
theta improves the performance of the scaled model."""
tokenizer_revision: Optional[str] = None
tokenizer_revision: str | None = None
"""The specific revision to use for the tokenizer on the Hugging Face Hub.
It can be a branch name, a tag name, or a commit id. If unspecified, will
use the default version."""
@@ -187,9 +179,9 @@ class ModelConfig:
- 1k -> 1000\n
- 1K -> 1024\n
- 25.6k -> 25,600"""
spec_target_max_model_len: Optional[int] = None
spec_target_max_model_len: int | None = None
"""Specify the maximum length for spec decoding draft models."""
quantization: SkipValidation[Optional[QuantizationMethods]] = None
quantization: SkipValidation[QuantizationMethods | None] = None
"""Method used to quantize the weights. If `None`, we first check the
`quantization_config` attribute in the model config file. If that is
`None`, we assume the model weights are not quantized and use `dtype` to
@@ -230,7 +222,7 @@ class ModelConfig:
"""If `True`, enables passing text embeddings as inputs via the
`prompt_embeds` key. Note that enabling this will double the time required
for graph compilation."""
served_model_name: Optional[Union[str, list[str]]] = None
served_model_name: str | list[str] | None = None
"""The model name(s) used in the API. If multiple names are provided, the
server will respond to any of the provided names. The model name in the
model field of a response will be the first name in this list. If not
@@ -238,20 +230,20 @@ class ModelConfig:
that this name(s) will also be used in `model_name` tag content of
prometheus metrics, if multiple names provided, metrics tag will take the
first one."""
config_format: Union[str, ConfigFormat] = "auto"
config_format: str | ConfigFormat = "auto"
"""The format of the model config to load:\n
- "auto" will try to load the config in hf format if available else it
will try to load in mistral format.\n
- "hf" will load the config in hf format.\n
- "mistral" will load the config in mistral format."""
hf_token: Optional[Union[bool, str]] = None
hf_token: bool | str | None = None
"""The token to use as HTTP bearer authorization for remote files . If
`True`, will use the token generated when running `huggingface-cli login`
(stored in `~/.huggingface`)."""
hf_overrides: HfOverrides = field(default_factory=dict)
"""If a dictionary, contains arguments to be forwarded to the Hugging Face
config. If a callable, it is called to update the HuggingFace config."""
logits_processor_pattern: Optional[str] = None
logits_processor_pattern: str | None = None
"""Optional regex pattern specifying valid logits processor qualified names
that can be passed with the `logits_processors` extra completion argument.
Defaults to `None`, which allows no processors."""
@@ -269,7 +261,7 @@ class ModelConfig:
`--generation-config vllm`, only the override parameters are used."""
enable_sleep_mode: bool = False
"""Enable sleep mode for the engine (only cuda platform is supported)."""
model_impl: Union[str, ModelImpl] = "auto"
model_impl: str | ModelImpl = "auto"
"""Which implementation of the model to use:\n
- "auto" will try to use the vLLM implementation, if it exists, and fall
back to the Transformers implementation if no vLLM implementation is
@@ -278,36 +270,36 @@ class ModelConfig:
- "transformers" will use the Transformers model implementation.\n
- "terratorch" will use the TerraTorch model implementation.
"""
override_attention_dtype: Optional[str] = None
override_attention_dtype: str | None = None
"""Override dtype for attention"""
logits_processors: Optional[list[Union[str, type[LogitsProcessor]]]] = None
logits_processors: list[str | type[LogitsProcessor]] | None = None
"""One or more logits processors' fully-qualified class names or class
definitions"""
io_processor_plugin: Optional[str] = None
io_processor_plugin: str | None = None
"""IOProcessor plugin name to load at model startup"""
# Pooler config
pooler_config: Optional[PoolerConfig] = None
pooler_config: PoolerConfig | None = None
"""Pooler config which controls the behaviour of output pooling in pooling
models."""
override_pooler_config: Optional[Union[dict, PoolerConfig]] = None
override_pooler_config: dict | PoolerConfig | None = None
"""[DEPRECATED] Use `pooler_config` instead. This field will be removed in
v0.12.0 or v1.0.0, whichever is sooner."""
# Multimodal config and init vars
multimodal_config: Optional[MultiModalConfig] = None
multimodal_config: MultiModalConfig | None = None
"""Configuration for multimodal model. If `None`, this will be inferred
from the architecture of `self.model`."""
limit_mm_per_prompt: InitVar[Optional[dict[str, Union[int, dict[str, int]]]]] = None
media_io_kwargs: InitVar[Optional[dict[str, dict[str, Any]]]] = None
mm_processor_kwargs: InitVar[Optional[dict[str, Any]]] = None
mm_processor_cache_gb: InitVar[Optional[float]] = None
mm_processor_cache_type: InitVar[Optional[MMCacheType]] = None
mm_shm_cache_max_object_size_mb: InitVar[Optional[int]] = None
mm_encoder_tp_mode: InitVar[Optional[MMEncoderTPMode]] = None
interleave_mm_strings: InitVar[Optional[bool]] = None
skip_mm_profiling: InitVar[Optional[bool]] = None
video_pruning_rate: InitVar[Optional[float]] = None
limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None
media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None
mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
mm_processor_cache_gb: InitVar[float | None] = None
mm_processor_cache_type: InitVar[MMCacheType | None] = None
mm_shm_cache_max_object_size_mb: InitVar[int | None] = None
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
interleave_mm_strings: InitVar[bool | None] = None
skip_mm_profiling: InitVar[bool | None] = None
video_pruning_rate: InitVar[float | None] = None
def compute_hash(self) -> str:
"""
@@ -369,7 +361,7 @@ class ModelConfig:
def _update_nested(
self,
target: Union["PretrainedConfig", dict[str, Any]],
target: PretrainedConfig | dict[str, Any],
updates: dict[str, Any],
) -> None:
"""Recursively updates a config or dict with nested updates."""
@@ -397,7 +389,7 @@ class ModelConfig:
def _apply_dict_overrides(
self,
config: "PretrainedConfig",
config: PretrainedConfig,
overrides: dict[str, Any],
) -> None:
"""Apply dict overrides, handling both nested configs and dict values."""
@@ -415,16 +407,16 @@ class ModelConfig:
def __post_init__(
self,
# Multimodal config init vars
limit_mm_per_prompt: Optional[dict[str, int]],
media_io_kwargs: Optional[dict[str, dict[str, Any]]],
mm_processor_kwargs: Optional[dict[str, Any]],
mm_processor_cache_gb: Optional[float],
mm_processor_cache_type: Optional[MMCacheType],
mm_shm_cache_max_object_size_mb: Optional[int],
mm_encoder_tp_mode: Optional[MMEncoderTPMode],
interleave_mm_strings: Optional[bool],
skip_mm_profiling: Optional[bool],
video_pruning_rate: Optional[float],
limit_mm_per_prompt: dict[str, int] | None,
media_io_kwargs: dict[str, dict[str, Any]] | None,
mm_processor_kwargs: dict[str, Any] | None,
mm_processor_cache_gb: float | None,
mm_processor_cache_type: MMCacheType | None,
mm_shm_cache_max_object_size_mb: int | None,
mm_encoder_tp_mode: MMEncoderTPMode | None,
interleave_mm_strings: bool | None,
skip_mm_profiling: bool | None,
video_pruning_rate: float | None,
) -> None:
# Set the default seed to 0 in V1.
# NOTE(woosuk): In V0, we set the default seed to None because the
@@ -1209,7 +1201,7 @@ class ModelConfig:
"Supported models implement the `SupportsPP` interface."
)
def get_sliding_window(self) -> Optional[int]:
def get_sliding_window(self) -> int | None:
"""Get the sliding window size from the HF text config if present."""
return getattr(self.hf_text_config, "sliding_window", None)
@@ -1479,7 +1471,7 @@ class ModelConfig:
f"{block_type.value} layers"
)
def get_mamba_chunk_size(self) -> Optional[int]:
def get_mamba_chunk_size(self) -> int | None:
"""
Returns the mamba chunk size if it exists
"""
@@ -1715,9 +1707,7 @@ class ModelConfig:
return max_model_len
def get_served_model_name(
model: str, served_model_name: Optional[Union[str, list[str]]]
):
def get_served_model_name(model: str, served_model_name: str | list[str] | None):
"""
If the input is a non-empty list, the first model_name in
`served_model_name` is taken.
@@ -1761,9 +1751,9 @@ def iter_architecture_defaults():
def try_match_architecture_defaults(
architecture: str,
*,
runner_type: Optional[RunnerType] = None,
convert_type: Optional[ConvertType] = None,
) -> Optional[tuple[str, tuple[RunnerType, ConvertType]]]:
runner_type: RunnerType | None = None,
convert_type: ConvertType | None = None,
) -> tuple[str, tuple[RunnerType, ConvertType]] | None:
for suffix, (
default_runner_type,
default_convert_type,
@@ -1817,7 +1807,7 @@ def _find_dtype(
model_id: str,
config: PretrainedConfig,
*,
revision: Optional[str],
revision: str | None,
):
# NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
# because config.torch_dtype can be None.
@@ -1902,10 +1892,10 @@ def _resolve_auto_dtype(
def _get_and_verify_dtype(
model_id: str,
config: PretrainedConfig,
dtype: Union[str, torch.dtype],
dtype: str | torch.dtype,
*,
is_pooling_model: bool,
revision: Optional[str] = None,
revision: str | None = None,
) -> torch.dtype:
config_dtype = _find_dtype(model_id, config, revision=revision)
model_type = config.model_type
@@ -1947,7 +1937,7 @@ def _get_and_verify_dtype(
def _get_head_dtype(
config: PretrainedConfig, dtype: torch.dtype, runner_type: str
) -> torch.dtype:
head_dtype: Optional[Union[str, torch.dtype]] = getattr(config, "head_dtype", None)
head_dtype: str | torch.dtype | None = getattr(config, "head_dtype", None)
if head_dtype == "model":
return dtype
@@ -1970,12 +1960,12 @@ def _get_head_dtype(
def _get_and_verify_max_len(
hf_config: PretrainedConfig,
tokenizer_config: Optional[dict],
max_model_len: Optional[int],
tokenizer_config: dict | None,
max_model_len: int | None,
disable_sliding_window: bool,
sliding_window: Optional[int],
spec_target_max_model_len: Optional[int] = None,
encoder_config: Optional[Any] = None,
sliding_window: int | None,
spec_target_max_model_len: int | None = None,
encoder_config: Any | None = None,
) -> int:
"""Get and verify the model's maximum length."""
derived_max_model_len = float("inf")
+14 -14
View File
@@ -4,7 +4,7 @@
import hashlib
from collections.abc import Mapping
from dataclasses import field
from typing import Any, Literal, Optional, Union
from typing import Any, Literal, TypeAlias
from pydantic import ConfigDict, Field, field_validator
from pydantic.dataclasses import dataclass
@@ -23,31 +23,31 @@ class BaseDummyOptions:
class VideoDummyOptions(BaseDummyOptions):
"""Options for generating dummy video data during profiling."""
num_frames: Optional[int] = Field(None, gt=0)
width: Optional[int] = Field(None, gt=0)
height: Optional[int] = Field(None, gt=0)
num_frames: int | None = Field(None, gt=0)
width: int | None = Field(None, gt=0)
height: int | None = Field(None, gt=0)
@dataclass(config=ConfigDict(extra="forbid"))
class ImageDummyOptions(BaseDummyOptions):
"""Options for generating dummy image data during profiling."""
width: Optional[int] = Field(None, gt=0)
height: Optional[int] = Field(None, gt=0)
width: int | None = Field(None, gt=0)
height: int | None = Field(None, gt=0)
@dataclass(config=ConfigDict(extra="forbid"))
class AudioDummyOptions(BaseDummyOptions):
"""Options for generating dummy audio data during profiling."""
length: Optional[int] = Field(None, gt=0)
length: int | None = Field(None, gt=0)
MMEncoderTPMode = Literal["weights", "data"]
MMCacheType = Literal["shm", "lru"]
DummyOptions = Union[
BaseDummyOptions, VideoDummyOptions, ImageDummyOptions, AudioDummyOptions
]
DummyOptions: TypeAlias = (
BaseDummyOptions | VideoDummyOptions | ImageDummyOptions | AudioDummyOptions
)
@config
@@ -75,7 +75,7 @@ class MultiModalConfig:
"""Additional args passed to process media inputs, keyed by modalities.
For example, to set num_frames for video, set
`--media-io-kwargs '{"video": {"num_frames": 40} }'`"""
mm_processor_kwargs: Optional[dict[str, object]] = None
mm_processor_kwargs: dict[str, object] | None = None
"""Arguments to be forwarded to the model's processor for multi-modal data,
e.g., image processor. Overrides for the multi-modal processor obtained
from `transformers.AutoProcessor.from_pretrained`.
@@ -123,7 +123,7 @@ class MultiModalConfig:
This reduces engine startup time but shifts the responsibility to users for
estimating the peak memory usage of the activation of multimodal encoder and
embedding cache."""
video_pruning_rate: Optional[float] = None
video_pruning_rate: float | None = None
"""Sets pruning rate for video pruning via Efficient Video Sampling.
Value sits in range [0;1) and determines fraction of media tokens
from each video to be pruned.
@@ -132,7 +132,7 @@ class MultiModalConfig:
@field_validator("limit_per_prompt", mode="before")
@classmethod
def _validate_limit_per_prompt(
cls, value: dict[str, Union[int, dict[str, int]]]
cls, value: dict[str, int | dict[str, int]]
) -> dict[str, DummyOptions]:
for k, v in value.items():
# Handle legacy format where only count is specified
@@ -179,7 +179,7 @@ class MultiModalConfig:
return 999
return limit_data.count
def get_dummy_options(self, modality: str) -> Optional[BaseDummyOptions]:
def get_dummy_options(self, modality: str) -> BaseDummyOptions | None:
"""
Get the configurable dummy data options for a modality.
Returns None if no options are configured for this modality.
+4 -4
View File
@@ -3,7 +3,7 @@
import hashlib
from functools import cached_property
from typing import Any, Literal, Optional, cast
from typing import Any, Literal, cast
from pydantic.dataclasses import dataclass
@@ -18,7 +18,7 @@ DetailedTraceModules = Literal["model", "worker", "all"]
class ObservabilityConfig:
"""Configuration for observability - metrics and tracing."""
show_hidden_metrics_for_version: Optional[str] = None
show_hidden_metrics_for_version: str | None = None
"""Enable deprecated Prometheus metrics that have been hidden since the
specified version. For example, if a previously deprecated metric has been
hidden since the v0.7.0 release, you use
@@ -33,10 +33,10 @@ class ObservabilityConfig:
return False
return version._prev_minor_version_was(self.show_hidden_metrics_for_version)
otlp_traces_endpoint: Optional[str] = None
otlp_traces_endpoint: str | None = None
"""Target URL to which OpenTelemetry traces will be sent."""
collect_detailed_traces: Optional[list[DetailedTraceModules]] = None
collect_detailed_traces: list[DetailedTraceModules] | None = None
"""It makes sense to set this only if `--otlp-traces-endpoint` is set. If
set, it will collect detailed traces for the specified modules. This
involves use of possibly costly and or blocking operations and hence might
+13 -13
View File
@@ -3,7 +3,7 @@
import hashlib
import os
from typing import TYPE_CHECKING, Any, Literal, Optional, Union
from typing import TYPE_CHECKING, Any, Literal
import torch
from pydantic import Field, model_validator
@@ -75,7 +75,7 @@ class ParallelConfig:
"""Number of local data parallel groups."""
data_parallel_rank: int = 0
"""Rank of the data parallel group."""
data_parallel_rank_local: Optional[int] = None
data_parallel_rank_local: int | None = None
"""Local rank of the data parallel group,
set only in SPMD mode."""
data_parallel_master_ip: str = "127.0.0.1"
@@ -113,24 +113,24 @@ class ParallelConfig:
with 4 experts and 2 ranks, rank 0 will have experts [0, 2] and rank 1
will have experts [1, 3]. This strategy can help improve load balancing
for grouped expert models with no redundant experts."""
num_redundant_experts: Optional[int] = None
num_redundant_experts: int | None = None
"""`num_redundant_experts` is deprecated and has been replaced with
`eplb_config.num_redundant_experts`. This will be removed in v0.12.0.
Please use `eplb_config.num_redundant_experts` instead."""
eplb_window_size: Optional[int] = None
eplb_window_size: int | None = None
"""`eplb_window_size` is deprecated and has been replaced with
`eplb_config.window_size`. This will be removed in v0.12.0.
Please use `eplb_config.window_size` instead."""
eplb_step_interval: Optional[int] = None
eplb_step_interval: int | None = None
"""`eplb_step_interval` is deprecated and has been replaced with
`eplb_config.step_interval`. This will be removed in v0.12.0.
Please use `eplb_config.step_interval` instead."""
eplb_log_balancedness: Optional[bool] = None
eplb_log_balancedness: bool | None = None
"""`eplb_log_balancedness` is deprecated and has been replaced with
`eplb_config.log_balancedness`. This will be removed in v0.12.0.
Please use `eplb_config.log_balancedness` instead."""
max_parallel_loading_workers: Optional[int] = None
max_parallel_loading_workers: int | None = None
"""Maximum number of parallel loading workers when loading model
sequentially in multiple batches. To avoid RAM OOM when using tensor
parallel and large models."""
@@ -159,15 +159,15 @@ class ParallelConfig:
ray_workers_use_nsight: bool = False
"""Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler."""
ray_runtime_env: Optional[RuntimeEnv] = None
ray_runtime_env: RuntimeEnv | None = None
"""Ray runtime environment to pass to distributed workers."""
placement_group: Optional[PlacementGroup] = None
placement_group: PlacementGroup | None = None
"""ray distributed model workers placement group."""
distributed_executor_backend: Optional[
Union[str, DistributedExecutorBackend, type[ExecutorBase]]
] = None
distributed_executor_backend: (
str | DistributedExecutorBackend | type[ExecutorBase] | None
) = None
"""Backend to use for distributed model
workers, either "ray" or "mp" (multiprocessing). If the product
of pipeline_parallel_size and tensor_parallel_size is less than
@@ -306,7 +306,7 @@ class ParallelConfig:
)
max_retries = 5
last_exc: Optional[Exception] = None
last_exc: Exception | None = None
for _ in range(max_retries):
try:
# use gloo since the engine process might not have cuda device
+11 -11
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
from typing import Any, Optional
from typing import Any
from pydantic.dataclasses import dataclass
@@ -14,23 +14,23 @@ from vllm.config.utils import config
class PoolerConfig:
"""Controls the behavior of output pooling in pooling models."""
pooling_type: Optional[str] = None
pooling_type: str | None = None
"""
The pooling method of the pooling model. This should be a key in
[`vllm.model_executor.layers.pooler.PoolingType`][].
"""
## for embeddings models
normalize: Optional[bool] = None
normalize: bool | None = None
"""
Whether to normalize the embeddings outputs. Defaults to True.
"""
dimensions: Optional[int] = None
dimensions: int | None = None
"""
Reduce the dimensions of embeddings if model
support matryoshka representation. Defaults to None.
"""
enable_chunked_processing: Optional[bool] = None
enable_chunked_processing: bool | None = None
"""
Whether to enable chunked processing for long inputs that exceed the model's
maximum position embeddings. When enabled, long inputs will be split into
@@ -38,7 +38,7 @@ class PoolerConfig:
This allows embedding models to handle arbitrarily long text without CUDA
errors. Defaults to False.
"""
max_embed_len: Optional[int] = None
max_embed_len: int | None = None
"""
Maximum input length allowed for embedding generation. When set, allows
inputs longer than max_embed_len to be accepted for embedding models.
@@ -48,29 +48,29 @@ class PoolerConfig:
"""
## for classification models
activation: Optional[bool] = None
activation: bool | None = None
"""
Whether to apply activation function to the classification outputs.
Defaults to True.
"""
logit_bias: Optional[float] = None
logit_bias: float | None = None
"""
If provided, apply classification logit biases. Defaults to None.
"""
## for reward models
softmax: Optional[bool] = None
softmax: bool | None = None
"""
Whether to apply softmax to the reward outputs.
Defaults to True.
"""
step_tag_id: Optional[int] = None
step_tag_id: int | None = None
"""
If set, only the score corresponding to the ``step_tag_id`` in the
generated sentence should be returned. Otherwise, the scores for all tokens
are returned.
"""
returned_token_ids: Optional[list[int]] = None
returned_token_ids: list[int] | None = None
"""
A list of indices for the vocabulary dimensions to be extracted,
such as the token IDs of ``good_token`` and ``bad_token`` in the
+2 -2
View File
@@ -3,7 +3,7 @@
import hashlib
from dataclasses import InitVar, field
from typing import Any, Literal, Union
from typing import Any, Literal
from pydantic import SkipValidation, model_validator
from pydantic.dataclasses import dataclass
@@ -133,7 +133,7 @@ class SchedulerConfig:
# scheduler class or path. "vllm.core.scheduler.Scheduler" (default)
# or "mod.custom_class".
scheduler_cls: Union[str, type[object]] = "vllm.core.scheduler.Scheduler"
scheduler_cls: str | type[object] = "vllm.core.scheduler.Scheduler"
"""The scheduler class to use. "vllm.core.scheduler.Scheduler" is the
default scheduler. Can be a class directly or the path to a class of form
"mod.custom_class"."""
+15 -15
View File
@@ -3,7 +3,7 @@
import ast
import hashlib
from typing import TYPE_CHECKING, Any, Literal, Optional
from typing import TYPE_CHECKING, Any, Literal
from pydantic import SkipValidation, model_validator
from pydantic.dataclasses import dataclass
@@ -59,16 +59,16 @@ MTP_MODEL_TYPES = (
class SpeculativeConfig:
"""Configuration for speculative decoding."""
enforce_eager: Optional[bool] = None
enforce_eager: bool | None = None
"""Override the default enforce_eager from model_config"""
# General speculative decoding control
num_speculative_tokens: SkipValidation[int] = None # type: ignore
"""The number of speculative tokens, if provided. It will default to the
number in the draft model config if present, otherwise, it is required."""
model: Optional[str] = None
model: str | None = None
"""The name of the draft model, eagle head, or additional weights, if
provided."""
method: Optional[SpeculativeMethod] = None
method: SpeculativeMethod | None = None
"""The name of the speculative method to use. If users provide and set the
`model` param, the speculative method type will be detected automatically
if possible, if `model` param is not provided, the method name must be
@@ -76,7 +76,7 @@ class SpeculativeConfig:
If using `ngram` method, the related configuration `prompt_lookup_max` and
`prompt_lookup_min` should be considered."""
draft_tensor_parallel_size: Optional[int] = None
draft_tensor_parallel_size: int | None = None
"""The degree of the tensor parallelism for the draft model. Can only be 1
or the same as the target model's tensor parallel size."""
disable_logprobs: bool = True
@@ -85,24 +85,24 @@ class SpeculativeConfig:
according to the log probability settings in SamplingParams."""
# Draft model configuration
quantization: Optional[me_quant.QuantizationMethods] = None
quantization: me_quant.QuantizationMethods | None = None
"""Quantization method that was used to quantize the draft model weights.
If `None`, we assume the model weights are not quantized. Note that it only
takes effect when using the draft model-based speculative method."""
max_model_len: Optional[int] = None
max_model_len: int | None = None
"""The maximum model length of the draft model. Used when testing the
ability to skip speculation for some sequences."""
revision: Optional[str] = None
revision: str | None = None
"""The specific model version to use for the draft model. It can be a
branch name, a tag name, or a commit id. If unspecified, will use the
default version."""
code_revision: Optional[str] = None
code_revision: str | None = None
"""The specific revision to use for the draft model code on Hugging Face
Hub. It can be a branch name, a tag name, or a commit id. If unspecified,
will use the default version."""
# Advanced control
disable_by_batch_size: Optional[int] = None
disable_by_batch_size: int | None = None
"""Disable speculative decoding for new incoming requests when the number
of enqueued requests is larger than this value, if provided."""
disable_padded_drafter_batch: bool = False
@@ -112,14 +112,14 @@ class SpeculativeConfig:
only affects the EAGLE method of speculation."""
# Ngram proposer configuration
prompt_lookup_max: Optional[int] = None
prompt_lookup_max: int | None = None
"""Maximum size of ngram token window when using Ngram proposer, required
when method is set to ngram."""
prompt_lookup_min: Optional[int] = None
prompt_lookup_min: int | None = None
"""Minimum size of ngram token window when using Ngram proposer, if
provided. Defaults to 1."""
speculative_token_tree: Optional[str] = None
speculative_token_tree: str | None = None
"""Specifies the tree structure for speculative token generation.
"""
# required configuration params passed from engine
@@ -449,7 +449,7 @@ class SpeculativeConfig:
@staticmethod
def _maybe_override_draft_max_model_len(
speculative_max_model_len: Optional[int],
speculative_max_model_len: int | None,
draft_max_model_len: int,
target_max_model_len: int,
) -> int:
@@ -488,7 +488,7 @@ class SpeculativeConfig:
@staticmethod
def _verify_and_get_draft_tp(
target_parallel_config: ParallelConfig,
speculative_draft_tensor_parallel_size: Optional[int],
speculative_draft_tensor_parallel_size: int | None,
draft_hf_config: PretrainedConfig,
) -> int:
"""
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
from pydantic.dataclasses import dataclass
@@ -28,7 +27,7 @@ class SpeechToTextConfig:
splitting long audio. This helps maintain context across chunk boundaries
and improves transcription quality at split points."""
min_energy_split_window_size: Optional[int] = 1600
min_energy_split_window_size: int | None = 1600
"""Window size in samples for finding low-energy (quiet) regions to split
audio chunks. The algorithm looks for the quietest moment within this
window to minimize cutting through speech. Default 1600 samples ≈ 100ms
+16 -16
View File
@@ -10,7 +10,7 @@ from contextlib import contextmanager
from dataclasses import field, replace
from functools import lru_cache
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
from typing import TYPE_CHECKING, Any, TypeVar
import torch
from pydantic import ConfigDict
@@ -69,17 +69,17 @@ class VllmConfig:
"""Device configuration."""
load_config: LoadConfig = field(default_factory=LoadConfig)
"""Load configuration."""
lora_config: Optional[LoRAConfig] = None
lora_config: LoRAConfig | None = None
"""LoRA configuration."""
speculative_config: Optional[SpeculativeConfig] = None
speculative_config: SpeculativeConfig | None = None
"""Speculative decoding configuration."""
structured_outputs_config: StructuredOutputsConfig = field(
default_factory=StructuredOutputsConfig
)
"""Structured outputs configuration."""
observability_config: Optional[ObservabilityConfig] = None
observability_config: ObservabilityConfig | None = None
"""Observability configuration."""
quant_config: Optional[QuantizationConfig] = None
quant_config: QuantizationConfig | None = None
"""Quantization configuration."""
compilation_config: CompilationConfig = field(default_factory=CompilationConfig)
"""`torch.compile` and cudagraph capture configuration for the model.
@@ -96,14 +96,14 @@ class VllmConfig:
You can specify the full compilation config like so:
`{"level": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`
"""
kv_transfer_config: Optional[KVTransferConfig] = None
kv_transfer_config: KVTransferConfig | None = None
"""The configurations for distributed KV cache transfer."""
kv_events_config: Optional[KVEventsConfig] = None
kv_events_config: KVEventsConfig | None = None
"""The configurations for event publishing."""
# some opaque config, only used to provide additional information
# for the hash computation, mainly used for testing, debugging or out of
# tree config registration.
additional_config: Union[dict, SupportsHash] = field(default_factory=dict)
additional_config: dict | SupportsHash = field(default_factory=dict)
"""Additional config for specified platform. Different platforms may
support different configs. Make sure the configs are valid for the platform
you are using. Contents must be hashable."""
@@ -212,7 +212,7 @@ class VllmConfig:
@staticmethod
def _get_quantization_config(
model_config: ModelConfig, load_config: LoadConfig
) -> Optional[QuantizationConfig]:
) -> QuantizationConfig | None:
"""Get the quantization config."""
from vllm.platforms import current_platform
@@ -245,7 +245,7 @@ class VllmConfig:
@staticmethod
def get_quantization_config(
model_config: ModelConfig, load_config: LoadConfig
) -> Optional[QuantizationConfig]:
) -> QuantizationConfig | None:
import copy
# For some reason, the _ version of this modifies the model_config
@@ -257,7 +257,7 @@ class VllmConfig:
def with_hf_config(
self,
hf_config: PretrainedConfig,
architectures: Optional[list[str]] = None,
architectures: list[str] | None = None,
) -> "VllmConfig":
if architectures is not None:
hf_config = copy.deepcopy(hf_config)
@@ -740,7 +740,7 @@ class VllmConfig:
f"Model: {self.model_config.model}"
)
def compile_debug_dump_path(self) -> Optional[Path]:
def compile_debug_dump_path(self) -> Path | None:
"""Returns a rank-aware path for dumping
torch.compile debug information.
"""
@@ -790,13 +790,13 @@ class VllmConfig:
)
_current_vllm_config: Optional[VllmConfig] = None
_current_prefix: Optional[str] = None
_current_vllm_config: VllmConfig | None = None
_current_prefix: str | None = None
@contextmanager
def set_current_vllm_config(
vllm_config: VllmConfig, check_compile=False, prefix: Optional[str] = None
vllm_config: VllmConfig, check_compile=False, prefix: str | None = None
):
"""
Temporarily set the current vLLM config.
@@ -866,7 +866,7 @@ T = TypeVar("T")
def get_layers_from_vllm_config(
vllm_config: VllmConfig,
layer_type: type[T],
layer_names: Optional[list[str]] = None,
layer_names: list[str] | None = None,
) -> dict[str, T]:
"""
Get layers from the vLLM config.
+14 -15
View File
@@ -3,7 +3,6 @@
from collections.abc import Mapping, MutableMapping
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse
import aiohttp
@@ -20,8 +19,8 @@ class HTTPConnection:
self.reuse_client = reuse_client
self._sync_client: Optional[requests.Session] = None
self._async_client: Optional[aiohttp.ClientSession] = None
self._sync_client: requests.Session | None = None
self._async_client: aiohttp.ClientSession | None = None
def get_sync_client(self) -> requests.Session:
if self._sync_client is None or not self.reuse_client:
@@ -53,8 +52,8 @@ class HTTPConnection:
url: str,
*,
stream: bool = False,
timeout: Optional[float] = None,
extra_headers: Optional[Mapping[str, str]] = None,
timeout: float | None = None,
extra_headers: Mapping[str, str] | None = None,
allow_redirects: bool = True,
):
self._validate_http_url(url)
@@ -74,8 +73,8 @@ class HTTPConnection:
self,
url: str,
*,
timeout: Optional[float] = None,
extra_headers: Optional[Mapping[str, str]] = None,
timeout: float | None = None,
extra_headers: Mapping[str, str] | None = None,
allow_redirects: bool = True,
):
self._validate_http_url(url)
@@ -91,7 +90,7 @@ class HTTPConnection:
)
def get_bytes(
self, url: str, *, timeout: Optional[float] = None, allow_redirects: bool = True
self, url: str, *, timeout: float | None = None, allow_redirects: bool = True
) -> bytes:
with self.get_response(
url, timeout=timeout, allow_redirects=allow_redirects
@@ -104,7 +103,7 @@ class HTTPConnection:
self,
url: str,
*,
timeout: Optional[float] = None,
timeout: float | None = None,
allow_redirects: bool = True,
) -> bytes:
async with await self.get_async_response(
@@ -114,7 +113,7 @@ class HTTPConnection:
return await r.read()
def get_text(self, url: str, *, timeout: Optional[float] = None) -> str:
def get_text(self, url: str, *, timeout: float | None = None) -> str:
with self.get_response(url, timeout=timeout) as r:
r.raise_for_status()
@@ -124,14 +123,14 @@ class HTTPConnection:
self,
url: str,
*,
timeout: Optional[float] = None,
timeout: float | None = None,
) -> str:
async with await self.get_async_response(url, timeout=timeout) as r:
r.raise_for_status()
return await r.text()
def get_json(self, url: str, *, timeout: Optional[float] = None) -> str:
def get_json(self, url: str, *, timeout: float | None = None) -> str:
with self.get_response(url, timeout=timeout) as r:
r.raise_for_status()
@@ -141,7 +140,7 @@ class HTTPConnection:
self,
url: str,
*,
timeout: Optional[float] = None,
timeout: float | None = None,
) -> str:
async with await self.get_async_response(url, timeout=timeout) as r:
r.raise_for_status()
@@ -153,7 +152,7 @@ class HTTPConnection:
url: str,
save_path: Path,
*,
timeout: Optional[float] = None,
timeout: float | None = None,
chunk_size: int = 128,
) -> Path:
with self.get_response(url, timeout=timeout) as r:
@@ -170,7 +169,7 @@ class HTTPConnection:
url: str,
save_path: Path,
*,
timeout: Optional[float] = None,
timeout: float | None = None,
chunk_size: int = 128,
) -> Path:
async with await self.get_async_response(url, timeout=timeout) as r:
+7 -6
View File
@@ -11,8 +11,9 @@
import dataclasses
import gc
import os
from collections.abc import Callable
from contextlib import contextmanager
from typing import Any, Callable, Optional, Union
from typing import Any
import torch
@@ -22,7 +23,7 @@ from vllm.utils import is_pin_memory_available
logger = init_logger(__name__)
def find_loaded_library(lib_name) -> Optional[str]:
def find_loaded_library(lib_name) -> str | None:
"""
According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
the file `/proc/self/maps` contains the memory maps of the process, which includes the
@@ -78,7 +79,7 @@ HandleType = tuple[int, int, int, int]
class AllocationData:
handle: HandleType
tag: str
cpu_backup_tensor: Optional[torch.Tensor] = None
cpu_backup_tensor: torch.Tensor | None = None
def create_and_map(allocation_handle: HandleType) -> None:
@@ -197,7 +198,7 @@ class CuMemAllocator:
)
return data.handle
def sleep(self, offload_tags: Optional[Union[tuple[str, ...], str]] = None) -> None:
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None:
"""
Put the allocator in sleep mode.
All data in the memory allocation with the specified tag will be
@@ -247,7 +248,7 @@ class CuMemAllocator:
gc.collect()
torch.cuda.empty_cache()
def wake_up(self, tags: Optional[list[str]] = None) -> None:
def wake_up(self, tags: list[str] | None = None) -> None:
"""
Wake up the allocator from sleep mode.
All data that is previously offloaded will be loaded back to GPU
@@ -272,7 +273,7 @@ class CuMemAllocator:
data.cpu_backup_tensor = None
@contextmanager
def use_memory_pool(self, tag: Optional[str] = None):
def use_memory_pool(self, tag: str | None = None):
"""
A context manager to use the memory pool.
All memory allocation created inside the context will be allocated
+3 -3
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Optional, Union
from typing import Any
import torch
import torch.distributed
@@ -30,13 +30,13 @@ def tensor_model_parallel_reduce_scatter(
def tensor_model_parallel_gather(
input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Gather the input tensor across model parallel group."""
return get_tp_group().gather(input_, dst, dim)
def broadcast_tensor_dict(
tensor_dict: Optional[dict[Any, Union[torch.Tensor, Any]]] = None, src: int = 0
tensor_dict: dict[Any, torch.Tensor | Any] | None = None, src: int = 0
):
if not torch.distributed.is_initialized():
return tensor_dict
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Optional
from typing import Any
import torch
import torch.distributed as dist
@@ -368,7 +368,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
return handle
# DeepEP LL uses RDMA so no SMs are used for communication
def max_sms_used(self) -> Optional[int]:
def max_sms_used(self) -> int | None:
return 0
@@ -10,7 +10,7 @@ import sys
import tempfile
from collections.abc import Sequence
from itertools import product
from typing import Any, Optional
from typing import Any
import torch
import torch.distributed as dist
@@ -86,7 +86,7 @@ def producer(
producer_queue,
consumer_queue,
result_queue,
cuda_visible_devices: Optional[str] = None,
cuda_visible_devices: str | None = None,
):
if cuda_visible_devices is not None:
update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
@@ -120,7 +120,7 @@ def consumer(
producer_queue,
consumer_queue,
result_queue,
cuda_visible_devices: Optional[str] = None,
cuda_visible_devices: str | None = None,
):
if cuda_visible_devices is not None:
update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
@@ -253,7 +253,7 @@ def can_actually_p2p(
# e.g. used by different vllm engines. The device id in the cache file is a
# **local** device id, i.e. from 0 to num_dev-1, where num_dev is the number
# of visible devices in the vllm engine.
_gpu_p2p_access_cache: Optional[dict[str, bool]] = None
_gpu_p2p_access_cache: dict[str, bool] | None = None
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from typing import Optional, Union
from weakref import WeakValueDictionary
import torch
@@ -75,7 +74,7 @@ class All2AllManagerBase:
def set_num_sms(self, num_sms: int):
pass
def max_sms_used(self) -> Optional[int]:
def max_sms_used(self) -> int | None:
return None # None means it could use the whole GPU
def combine(self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False):
@@ -96,8 +95,8 @@ class DeviceCommunicatorBase:
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
self.device = device or torch.device("cpu")
@@ -123,7 +122,7 @@ class DeviceCommunicatorBase:
self.is_ep_communicator = "ep" in unique_name
self.use_all2all = self.is_ep_communicator and use_ep
self.all2all_manager: Optional[All2AllManagerBase] = None
self.all2all_manager: All2AllManagerBase | None = None
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
dist.all_reduce(input_, group=self.device_group)
@@ -156,10 +155,10 @@ class DeviceCommunicatorBase:
def all_gatherv(
self,
input_: Union[torch.Tensor, list[torch.Tensor]],
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: Optional[list[int]] = None,
) -> Union[torch.Tensor, list[torch.Tensor]]:
sizes: list[int] | None = None,
) -> torch.Tensor | list[torch.Tensor]:
raise NotImplementedError
def reduce_scatter(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
@@ -196,13 +195,13 @@ class DeviceCommunicatorBase:
return output_tensor.movedim(0, dim).contiguous()
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: Optional[list[int]] = None
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
) -> torch.Tensor:
raise NotImplementedError
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
all the ranks.
@@ -231,7 +230,7 @@ class DeviceCommunicatorBase:
output_tensor = None
return output_tensor
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
@@ -239,7 +238,7 @@ class DeviceCommunicatorBase:
torch.distributed.send(tensor, self.ranks[dst], self.device_group)
def recv(
self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None
self, size: torch.Size, dtype: torch.dtype, src: int | None = None
) -> torch.Tensor:
"""Receives a tensor from the source rank."""
"""NOTE: `src` is the local rank of the source rank."""
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import Any, Optional, Union
from typing import Any
import torch
from torch.distributed import ProcessGroup
@@ -18,8 +18,8 @@ class CpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -38,7 +38,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
all the ranks.
@@ -99,7 +99,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def send_tensor_dict(
self,
tensor_dict: dict[str, Union[torch.Tensor, Any]],
tensor_dict: dict[str, torch.Tensor | Any],
dst: int,
) -> None:
return self.dist_module.send_tensor_dict(tensor_dict, dst)
@@ -107,7 +107,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def recv_tensor_dict(
self,
src: int,
) -> dict[str, Union[torch.Tensor, Any]]:
) -> dict[str, torch.Tensor | Any]:
return self.dist_module.recv_tensor_dict(src)
@@ -140,16 +140,16 @@ class _CPUSHMDistributed:
return handle
def all_reduce(
self, input: torch.Tensor, group: Optional[ProcessGroup] = None
self, input: torch.Tensor, group: ProcessGroup | None = None
) -> None:
torch.ops._C.shm_allreduce(self.handle, input)
def gather(
self,
input: torch.Tensor,
gather_list: Optional[list[torch.Tensor]],
gather_list: list[torch.Tensor] | None,
dst: int = -1,
group: Optional[ProcessGroup] = None,
group: ProcessGroup | None = None,
) -> None:
# Note: different from the torch gather, here we use local dst rank.
torch.ops._C.shm_gather(
@@ -163,13 +163,13 @@ class _CPUSHMDistributed:
self,
output: torch.Tensor,
input: torch.Tensor,
group: Optional[ProcessGroup] = None,
group: ProcessGroup | None = None,
) -> None:
torch.ops._C.shm_all_gather(self.handle, input, output)
def send_tensor_dict(
self,
tensor_dict: dict[str, Union[torch.Tensor, Any]],
tensor_dict: dict[str, torch.Tensor | Any],
dst: int,
) -> None:
key_list = list(tensor_dict.keys())
@@ -191,7 +191,7 @@ class _CPUSHMDistributed:
def recv_tensor_dict(
self,
src: int,
) -> dict[str, Union[torch.Tensor, Any]]:
) -> dict[str, torch.Tensor | Any]:
tensor_list = torch.ops._C.shm_recv_tensor_list(self.handle, src)
value_list: list[torch.Tensor] = tensor_list[:-1]
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
import torch
from torch.distributed import ProcessGroup
@@ -26,8 +25,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -54,7 +53,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
)
from vllm.distributed.device_communicators.symm_mem import SymmMemCommunicator
self.pynccl_comm: Optional[PyNcclCommunicator] = None
self.pynccl_comm: PyNcclCommunicator | None = None
if self.world_size > 1:
self.pynccl_comm = PyNcclCommunicator(
group=self.cpu_group,
@@ -63,9 +62,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
if is_symmetric_memory_enabled():
register_nccl_symmetric_ops(self.pynccl_comm)
self.ca_comm: Optional[CustomAllreduce] = None
self.qr_comm: Optional[QuickAllReduce] = None
self.symm_mem_comm: Optional[SymmMemCommunicator] = None
self.ca_comm: CustomAllreduce | None = None
self.qr_comm: QuickAllReduce | None = None
self.symm_mem_comm: SymmMemCommunicator | None = None
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
group=self.cpu_group,
@@ -201,7 +200,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
return output.movedim(0, dim).contiguous()
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: Optional[list[int]] = None
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
):
world_size = self.world_size
pynccl_comm = self.pynccl_comm
@@ -235,7 +234,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
# Reshape before returning
return output.movedim(0, dim).contiguous()
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
@@ -248,7 +247,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
torch.distributed.send(tensor, self.ranks[dst], self.device_group)
def recv(
self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None
self, size: torch.Size, dtype: torch.dtype, src: int | None = None
) -> torch.Tensor:
"""Receives a tensor from the source rank."""
"""NOTE: `src` is the local rank of the source rank."""
@@ -274,9 +273,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
def all_gatherv(
self,
input_: Union[torch.Tensor, list[torch.Tensor]],
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: Optional[list[int]] = None,
sizes: list[int] | None = None,
):
if dim != 0:
raise NotImplementedError("only dim 0 all-gatherv is supported")
@@ -289,7 +288,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
def _all_gather_single(input_: torch.Tensor, sizes: Optional[list[int]] = None):
def _all_gather_single(input_: torch.Tensor, sizes: list[int] | None = None):
input_size = input_.size()
if sizes is not None:
assert len(sizes) == world_size
@@ -7,7 +7,7 @@ convenient for use when we just need to call a few functions.
import ctypes
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
# this line makes it possible to directly load `libcudart.so` using `ctypes`
import torch # noqa
@@ -36,7 +36,7 @@ class Function:
argtypes: list[Any]
def find_loaded_library(lib_name) -> Optional[str]:
def find_loaded_library(lib_name) -> str | None:
"""
According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
the file `/proc/self/maps` contains the memory maps of the process, which includes the
@@ -113,7 +113,7 @@ class CudaRTLibrary:
# to the corresponding dictionary
path_to_dict_mapping: dict[str, dict[str, Any]] = {}
def __init__(self, so_file: Optional[str] = None):
def __init__(self, so_file: str | None = None):
if so_file is None:
so_file = find_loaded_library("libcudart")
if so_file is None:
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Optional, Union
import torch
import torch.distributed as dist
@@ -55,7 +54,7 @@ class CustomAllreduce:
def __init__(
self,
group: ProcessGroup,
device: Union[int, str, torch.device],
device: int | str | torch.device,
max_size=8192 * 1024,
symm_mem_enabled=False,
) -> None:
@@ -260,7 +259,7 @@ class CustomAllreduce:
)
return out
def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None:
"""The main allreduce API that provides support for cuda graph."""
# When custom allreduce is disabled, this will be None.
if self.disabled or not self.should_custom_ar(input):
@@ -292,8 +291,8 @@ class CustomAllreduce:
@staticmethod
def create_shared_buffer(
size_in_bytes: int,
group: Optional[ProcessGroup] = None,
uncached: Optional[bool] = False,
group: ProcessGroup | None = None,
uncached: bool | None = False,
) -> list[int]:
pointer, handle = ops.allocate_shared_buffer_and_handle(size_in_bytes)
@@ -313,8 +312,8 @@ class CustomAllreduce:
@staticmethod
def free_shared_buffer(
pointers: list[int],
group: Optional[ProcessGroup] = None,
rank: Optional[int] = None,
group: ProcessGroup | None = None,
rank: int | None = None,
) -> None:
if rank is None:
rank = dist.get_rank(group=group)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
# ===================== import region =====================
import torch
@@ -59,9 +58,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
class PyNcclCommunicator:
def __init__(
self,
group: Union[ProcessGroup, StatelessProcessGroup],
device: Union[int, str, torch.device],
library_path: Optional[str] = None,
group: ProcessGroup | StatelessProcessGroup,
device: int | str | torch.device,
library_path: str | None = None,
):
"""
Args:
@@ -3,7 +3,7 @@
import atexit
import contextlib
import tempfile
from typing import Any, Optional
from typing import Any
import torch
from packaging import version
@@ -141,7 +141,7 @@ class nccl_symm_mem_context:
or version.parse(torch.__version__) < version.parse("2.8.0.a0")
)
if self.disabled:
self.pynccl_comm: Optional[PyNcclCommunicator] = None
self.pynccl_comm: PyNcclCommunicator | None = None
self._mem_pool_ctx: contextlib.AbstractContextManager[Any] = (
contextlib.nullcontext()
)
@@ -25,7 +25,7 @@
import ctypes
import platform
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
import torch
from torch.distributed import ReduceOp
@@ -305,7 +305,7 @@ class NCCLLibrary:
# to the corresponding dictionary
path_to_dict_mapping: dict[str, dict[str, Any]] = {}
def __init__(self, so_file: Optional[str] = None):
def __init__(self, so_file: str | None = None):
so_file = so_file or find_nccl_library()
try:
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Union
import torch
import torch.distributed as dist
@@ -58,9 +57,7 @@ class QuickAllReduce:
(torch.bfloat16, 8): [16 * MB, 2048 * MB, 2048 * MB, 2048 * MB],
}
def __init__(
self, group: ProcessGroup, device: Union[int, str, torch.device]
) -> None:
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
"""
Custom allreduce provides non-destructive acceleration and is
available for CUDA and ROCm MI300 series.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
from typing import Any, Optional
from typing import Any
import ray
import torch
@@ -27,15 +27,15 @@ class RayPPCommunicator(Communicator):
This class is not thread-safe.
"""
_comm: Optional[DeviceCommunicatorBase]
_comm: DeviceCommunicatorBase | None
def __init__(
self,
world_size: int,
comm_id: Any,
rank: Optional[int],
rank: int | None,
actor_handles: list["ray.actor.ActorHandle"],
cuda_stream: Optional[torch.cuda.Stream],
cuda_stream: torch.cuda.Stream | None,
use_communication_streams: bool = False,
):
"""
@@ -56,7 +56,7 @@ class RayPPCommunicator(Communicator):
This is not supported.
"""
self._world_size = world_size
self._rank: Optional[int] = None
self._rank: int | None = None
self._actor_handles = actor_handles
if use_communication_streams:
raise NotImplementedError("use_communication_streams is not supported")
@@ -143,7 +143,7 @@ class RayPPCommunicator(Communicator):
else:
raise ValueError(f"Actor {actor} not found in communicator group")
def get_self_rank(self) -> Optional[int]:
def get_self_rank(self) -> int | None:
"""
Return this actor's rank.
"""
@@ -7,7 +7,7 @@ from contextlib import contextmanager
from dataclasses import dataclass, field
from multiprocessing import shared_memory
from threading import Event
from typing import Any, Optional, Union
from typing import Any
from unittest.mock import patch
import torch
@@ -80,7 +80,7 @@ class ShmRingBuffer:
n_reader: int,
max_chunk_bytes: int,
max_chunks: int,
name: Optional[str] = None,
name: str | None = None,
):
"""
A shared memory ring buffer implementation for broadcast communication.
@@ -213,9 +213,9 @@ class ShmRingBuffer:
class Handle:
local_reader_ranks: list[int] = field(default_factory=list)
buffer_handle: Optional[tuple[int, int, int, str]] = None
local_subscribe_addr: Optional[str] = None
remote_subscribe_addr: Optional[str] = None
buffer_handle: tuple[int, int, int, str] | None = None
local_subscribe_addr: str | None = None
remote_subscribe_addr: str | None = None
remote_addr_ipv6: bool = False
@@ -224,10 +224,10 @@ class MessageQueue:
self,
n_reader, # number of all readers
n_local_reader, # number of local readers through shared memory
local_reader_ranks: Optional[list[int]] = None,
local_reader_ranks: list[int] | None = None,
max_chunk_bytes: int = 1024 * 1024 * 10,
max_chunks: int = 10,
connect_ip: Optional[str] = None,
connect_ip: str | None = None,
):
if local_reader_ranks is None:
local_reader_ranks = list(range(n_local_reader))
@@ -384,7 +384,7 @@ class MessageQueue:
assert recv == b"READY"
@contextmanager
def acquire_write(self, timeout: Optional[float] = None):
def acquire_write(self, timeout: float | None = None):
assert self._is_writer, "Only writers can acquire write"
start_time = time.monotonic()
n_warning = 1
@@ -444,8 +444,8 @@ class MessageQueue:
@contextmanager
def acquire_read(
self,
timeout: Optional[float] = None,
cancel: Optional[Event] = None,
timeout: float | None = None,
cancel: Event | None = None,
indefinite: bool = False,
):
assert self._is_local_reader, "Only readers can acquire read"
@@ -502,7 +502,7 @@ class MessageQueue:
self._read_spin_timer.record_activity()
break
def enqueue(self, obj, timeout: Optional[float] = None):
def enqueue(self, obj, timeout: float | None = None):
"""Write to message queue with optional timeout (in seconds)"""
assert self._is_writer, "Only writers can enqueue"
serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
@@ -520,8 +520,8 @@ class MessageQueue:
def dequeue(
self,
timeout: Optional[float] = None,
cancel: Optional[Event] = None,
timeout: float | None = None,
cancel: Event | None = None,
indefinite: bool = False,
):
"""Read from message queue with optional timeout (in seconds)"""
@@ -542,7 +542,7 @@ class MessageQueue:
return obj
@staticmethod
def recv(socket: zmq.Socket, timeout: Optional[float]) -> Any:
def recv(socket: zmq.Socket, timeout: float | None) -> Any:
timeout_ms = None if timeout is None else int(timeout * 1000)
if not socket.poll(timeout=timeout_ms):
raise TimeoutError
@@ -558,7 +558,7 @@ class MessageQueue:
@staticmethod
def create_from_process_group(
pg: Union[ProcessGroup, StatelessProcessGroup],
pg: ProcessGroup | StatelessProcessGroup,
max_chunk_bytes,
max_chunks,
writer_rank=0,
@@ -3,13 +3,13 @@
import pickle
from abc import ABC, abstractmethod
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from contextlib import contextmanager
from dataclasses import dataclass
from itertools import chain
from multiprocessing import shared_memory
from multiprocessing.synchronize import Lock as LockType
from typing import Any, Callable, Optional, Union
from typing import Any
from unittest.mock import patch
import torch
@@ -109,7 +109,7 @@ class SingleWriterShmRingBuffer:
def __init__(
self,
data_buffer_size: int,
name: Optional[str] = None,
name: str | None = None,
create: bool = False,
):
self.data_buffer_size = data_buffer_size
@@ -252,7 +252,7 @@ class SingleWriterShmRingBuffer:
def free_buf(
self,
is_free_fn: Callable[[int, memoryview], bool],
nbytes: Optional[int] = None,
nbytes: int | None = None,
) -> Iterable[int]:
"""
Free a buffer of the given size. This is a no-op in shared memory,
@@ -340,9 +340,7 @@ class MsgpackSerde(ObjectSerde):
self.mm_decoder = MsgpackDecoder(MultiModalKwargsItem)
self._mm_kwargs_item_cls = MultiModalKwargsItem
def serialize(
self, value: Any
) -> tuple[Union[bytes, list[bytes]], int, bytes, int]:
def serialize(self, value: Any) -> tuple[bytes | list[bytes], int, bytes, int]:
len_arr = None
if isinstance(value, (torch.Tensor, self._mm_kwargs_item_cls)):
type_name = type(value).__name__
@@ -396,7 +394,7 @@ class ShmObjectStorageHandle:
n_readers: int
ring_buffer_handle: tuple[int, str]
serde_class: type[ObjectSerde]
reader_lock: Optional[LockType]
reader_lock: LockType | None
class SingleWriterShmObjectStorage:
@@ -444,7 +442,7 @@ class SingleWriterShmObjectStorage:
n_readers: int,
ring_buffer: SingleWriterShmRingBuffer,
serde_class: type[ObjectSerde] = MsgpackSerde,
reader_lock: Optional[LockType] = None,
reader_lock: LockType | None = None,
):
"""
Initialize the object storage.
@@ -492,7 +490,7 @@ class SingleWriterShmObjectStorage:
def copy_to_buffer(
self,
data: Union[bytes, list[bytes]],
data: bytes | list[bytes],
data_bytes: int,
metadata: bytes,
md_bytes: int,
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
import torch
import torch.distributed as dist
@@ -31,10 +30,10 @@ class SymmMemCommunicator:
def __init__(
self,
group: ProcessGroup,
device: Union[int, str, torch.device],
device: int | str | torch.device,
# add options for testing
force_multimem: Optional[bool] = None,
max_size_override: Optional[int] = None,
force_multimem: bool | None = None,
max_size_override: int | None = None,
):
self.disabled = True
@@ -108,8 +107,8 @@ class SymmMemCommunicator:
return inp_size < self.max_size
def all_reduce(
self, inp: torch.Tensor, *, out: Optional[torch.Tensor] = None
) -> Optional[torch.Tensor]:
self, inp: torch.Tensor, *, out: torch.Tensor | None = None
) -> torch.Tensor | None:
if not self.should_use_symm_mem(inp):
return None
if out is None:
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import Optional
import torch
from torch.distributed import ProcessGroup
@@ -39,8 +38,8 @@ class TpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
import torch.distributed as dist
@@ -19,8 +18,8 @@ class XpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -45,7 +44,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
assert -input_.dim() <= dim < input_.dim(), (
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
)
+7 -8
View File
@@ -29,7 +29,6 @@ physical experts.
import time
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Optional, Union
import torch
from torch.distributed import ProcessGroup, all_reduce
@@ -186,9 +185,9 @@ class EplbState:
model: MixtureOfExperts,
device: torch.device,
parallel_config: ParallelConfig,
global_expert_load: Optional[torch.Tensor] = None,
old_global_expert_indices: Optional[torch.Tensor] = None,
rank_mapping: Optional[dict[int, int]] = None,
global_expert_load: torch.Tensor | None = None,
old_global_expert_indices: torch.Tensor | None = None,
rank_mapping: dict[int, int] | None = None,
) -> "EplbState":
"""
Build the initial EPLB state.
@@ -439,9 +438,9 @@ class EplbState:
model: MixtureOfExperts,
is_profile: bool = False,
execute_shuffle: bool = True,
global_expert_load: Optional[torch.Tensor] = None,
rank_mapping: Optional[dict[int, int]] = None,
) -> Optional[torch.Tensor]:
global_expert_load: torch.Tensor | None = None,
rank_mapping: dict[int, int] | None = None,
) -> torch.Tensor | None:
"""
Rearrange the experts according to the current load.
"""
@@ -611,7 +610,7 @@ class EplbState:
def _node_count_with_rank_mapping(
pg: Union[ProcessGroup, StatelessProcessGroup],
pg: ProcessGroup | StatelessProcessGroup,
rank_mapping: dict[int, int],
) -> int:
if isinstance(pg, ProcessGroup):
+1 -2
View File
@@ -8,7 +8,6 @@ This involves the exchange of expert weights between GPUs.
from collections.abc import Iterable, MutableSequence, Sequence
from functools import partial
from typing import Optional
import torch
from torch.distributed import (
@@ -253,7 +252,7 @@ def rearrange_expert_weights_inplace(
expert_weights: Sequence[Iterable[torch.Tensor]],
ep_group: ProcessGroup,
is_profile: bool = False,
rank_mapping: Optional[dict[int, int]] = None,
rank_mapping: dict[int, int] | None = None,
) -> None:
"""
Rearranges the expert weights in place according to the new expert indices.
+15 -14
View File
@@ -6,10 +6,11 @@ import threading
import time
from abc import ABC, abstractmethod
from collections import deque
from collections.abc import Callable
from dataclasses import asdict
from itertools import count
from queue import Queue
from typing import Any, Callable, Optional, Union
from typing import Any
import msgspec
import zmq
@@ -29,7 +30,7 @@ class EventBatch(
):
ts: float
events: list[Any]
data_parallel_rank: Optional[int] = None
data_parallel_rank: int | None = None
class KVCacheEvent(
@@ -47,16 +48,16 @@ MEDIUM_GPU = "GPU"
class BlockStored(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
parent_block_hash: Optional[ExternalBlockHash]
parent_block_hash: ExternalBlockHash | None
token_ids: list[int]
block_size: int
lora_id: Optional[int]
medium: Optional[str]
lora_id: int | None
medium: str | None
class BlockRemoved(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
medium: Optional[str]
medium: str | None
class AllBlocksCleared(KVCacheEvent):
@@ -64,7 +65,7 @@ class AllBlocksCleared(KVCacheEvent):
class KVEventBatch(EventBatch):
events: list[Union[BlockStored, BlockRemoved, AllBlocksCleared]]
events: list[BlockStored | BlockRemoved | AllBlocksCleared]
class EventPublisher(ABC):
@@ -139,7 +140,7 @@ class ZmqEventPublisher(EventPublisher):
self,
data_parallel_rank: int,
endpoint: str = "tcp://*:5557",
replay_endpoint: Optional[str] = None,
replay_endpoint: str | None = None,
buffer_steps: int = 10_000,
hwm: int = 100_000,
max_queue_size: int = 100_000,
@@ -147,13 +148,13 @@ class ZmqEventPublisher(EventPublisher):
) -> None:
# Storage
super().__init__(data_parallel_rank)
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
self._event_queue = Queue[EventBatch | None](maxsize=max_queue_size)
self._buffer = deque[tuple[int, bytes]](maxlen=buffer_steps)
# ZMQ sockets
self._ctx = zmq.Context.instance()
self._pub: Optional[zmq.Socket] = None
self._replay: Optional[zmq.Socket] = None
self._pub: zmq.Socket | None = None
self._replay: zmq.Socket | None = None
self._dp_rank = data_parallel_rank
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
@@ -303,8 +304,8 @@ class ZmqEventPublisher(EventPublisher):
@staticmethod
def offset_endpoint_port(
endpoint: Optional[str], data_parallel_rank: int
) -> Optional[str]:
endpoint: str | None, data_parallel_rank: int
) -> str | None:
"""Helper function to offset the port in an endpoint by
the data parallel rank.
@@ -349,7 +350,7 @@ class EventPublisherFactory:
@classmethod
def create(
cls, config: Optional[KVEventsConfig], data_parallel_rank: int = 0
cls, config: KVEventsConfig | None, data_parallel_rank: int = 0
) -> EventPublisher:
"""Create publisher from a config mapping."""
if not config:
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from typing import TYPE_CHECKING, Callable
from collections.abc import Callable
from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.distributed.kv_transfer.kv_connector.base import (
@@ -7,7 +7,7 @@ KV cache helper for store.
from collections import defaultdict
from collections.abc import Sequence
from concurrent.futures import CancelledError, Future
from typing import Literal, Optional, Union, cast
from typing import Literal, cast
import torch
@@ -136,7 +136,7 @@ class KVOutputAggregator:
# Aggregate kv_connector_output from all workers
def update_finished_set(
req_ids: Optional[set[str]],
req_ids: set[str] | None,
remaining_count_dict: dict[str, int],
finished_set: set[str],
) -> None:
@@ -197,7 +197,7 @@ class KVOutputAggregator:
to the respective list of outputs."""
result_future: Future[ModelRunnerOutput] = Future()
outputs: list[Optional[ModelRunnerOutput]] = [None] * len(output_futures)
outputs: list[ModelRunnerOutput | None] = [None] * len(output_futures)
def make_callback(idx):
def callback(fut):
@@ -230,8 +230,8 @@ class KVOutputAggregator:
def _make_src_and_dst_indices(
src_block_ids: list[int],
dst_block_ids: list[int],
src_device: Union[torch.device, str],
dst_device: Union[torch.device, str],
src_device: torch.device | str,
dst_device: torch.device | str,
) -> tuple[torch.Tensor, torch.Tensor]:
src_indices = torch.tensor(src_block_ids, device=src_device, dtype=torch.int64)
dst_indices = torch.tensor(dst_block_ids, device=dst_device, dtype=torch.int64)
@@ -37,8 +37,8 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional
from collections.abc import Callable, Iterable
from typing import TYPE_CHECKING, Any, Literal, Optional
import torch
@@ -93,7 +93,7 @@ class KVConnectorBase_V1(ABC):
"Initializing KVConnectorBase_V1. This API is experimental and "
"subject to change in the future as we iterate the design."
)
self._connector_metadata: Optional[KVConnectorMetadata] = None
self._connector_metadata: KVConnectorMetadata | None = None
self._vllm_config = vllm_config
self._role = role
@@ -222,7 +222,7 @@ class KVConnectorBase_V1(ABC):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
@@ -281,7 +281,7 @@ class KVConnectorBase_V1(ABC):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -361,7 +361,7 @@ class KVConnectorBase_V1(ABC):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
freed.
@@ -388,7 +388,7 @@ class KVConnectorBase_V1(ABC):
return ()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> Optional[str]:
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
Args:
@@ -406,7 +406,7 @@ class KVConnectorBase_V1(ABC):
)
return None
def get_finished_count(self) -> Optional[int]:
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
via this connector.
@@ -419,7 +419,7 @@ class KVConnectorBase_V1(ABC):
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
cls, data: dict[str, Any] | None = None
) -> Optional["KVConnectorStats"]:
"""
KVConnectorStats resolution method. This method allows dynamically
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import torch
from lmcache.integration.vllm.vllm_v1_adapter import LMCacheConnectorV1Impl
@@ -96,7 +96,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -117,7 +117,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -161,7 +161,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any, Optional, Union
from typing import Any
from vllm.config.kv_transfer import KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
@@ -32,7 +32,7 @@ class KVConnectorStats:
"""
raise NotImplementedError
def reduce(self) -> dict[str, Union[int, float]]:
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
@@ -58,7 +58,7 @@ class KVConnectorLogging:
self.reset()
def reset(self):
self.transfer_stats_accumulator: Optional[KVConnectorStats] = None
self.transfer_stats_accumulator: KVConnectorStats | None = None
def observe(self, transfer_stats_data: dict[str, Any]):
# Should not be called when a KVConnector is not configured.
@@ -3,7 +3,7 @@
import copy
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import torch
@@ -33,7 +33,7 @@ logger = init_logger(__name__)
@dataclass
class MultiKVConnectorMetadata(KVConnectorMetadata):
metadata: tuple[KVConnectorMetadata, ...]
extra_async_saves: Optional[dict[str, int]] = None
extra_async_saves: dict[str, int] | None = None
@dataclass
@@ -130,7 +130,7 @@ class MultiConnector(KVConnectorBase_V1):
c.clear_connector_metadata()
def shutdown(self):
exception: Optional[Exception] = None
exception: Exception | None = None
for c in self._connectors:
try:
c.shutdown()
@@ -169,7 +169,7 @@ class MultiConnector(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
finished_sending: set[str] = set()
finished_recving: set[str] = set()
for c in self._connectors:
@@ -207,7 +207,7 @@ class MultiConnector(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
to_return = (0, False)
for i, c in enumerate(self._connectors):
toks, load_async = c.get_num_new_matched_tokens(
@@ -258,7 +258,7 @@ class MultiConnector(KVConnectorBase_V1):
self,
request: "Request",
blocks: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
async_saves = 0
kv_txfer_params = None
for c in self._connectors:
@@ -286,7 +286,7 @@ class MultiConnector(KVConnectorBase_V1):
yield from c.take_events()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> Optional[str]:
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
Args:
@@ -323,17 +323,17 @@ class MultiConnector(KVConnectorBase_V1):
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
) -> Optional[KVConnectorStats]:
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MultiKVConnectorStats(data=data)
if data is not None
else MultiKVConnectorStats()
)
def get_kv_connector_stats(self) -> Optional[MultiKVConnectorStats]:
def get_kv_connector_stats(self) -> MultiKVConnectorStats | None:
# Group connector stats by connector type.
stats_by_connector: Optional[MultiKVConnectorStats] = None
stats_by_connector: MultiKVConnectorStats | None = None
for c in self._connectors:
stats = c.get_kv_connector_stats()
if stats is None:
@@ -13,7 +13,7 @@ from collections import defaultdict
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any
import msgspec
import numpy as np
@@ -153,10 +153,10 @@ class NixlConnector(KVConnectorBase_V1):
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler: Optional[NixlConnectorScheduler] = (
self.connector_scheduler: NixlConnectorScheduler | None = (
NixlConnectorScheduler(vllm_config, self.engine_id)
)
self.connector_worker: Optional[NixlConnectorWorker] = None
self.connector_worker: NixlConnectorWorker | None = None
elif role == KVConnectorRole.WORKER:
self.connector_scheduler = None
self.connector_worker = NixlConnectorWorker(vllm_config, self.engine_id)
@@ -189,7 +189,7 @@ class NixlConnector(KVConnectorBase_V1):
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -214,7 +214,7 @@ class NixlConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -234,14 +234,14 @@ class NixlConnector(KVConnectorBase_V1):
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_kv_connector_stats(self) -> Optional[KVConnectorStats]:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
assert self.connector_worker is not None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
) -> Optional[KVConnectorStats]:
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
NixlKVConnectorStats(data=data)
if data is not None
@@ -445,7 +445,7 @@ class NixlConnectorScheduler:
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
@@ -584,7 +584,7 @@ class NixlConnectorWorker:
)
# Note: host xfer buffer ops when use_host_buffer is True
self.copy_blocks: Optional[CopyBlocksOp] = None
self.copy_blocks: CopyBlocksOp | None = None
# Map of engine_id -> kv_caches_base_addr. For TP case, each local
# rank will still only pull from a single remote TP worker.
@@ -615,7 +615,7 @@ class NixlConnectorWorker:
self._reqs_to_process: set[ReqId] = set()
# Background thread for handling new handshake requests.
self._nixl_handshake_listener_t: Optional[threading.Thread] = None
self._nixl_handshake_listener_t: threading.Thread | None = None
# Background thread for initializing new NIXL handshakes.
self._handshake_initiation_executor = ThreadPoolExecutor(
# NIXL is not guaranteed to be thread-safe, limit 1 worker.
@@ -635,7 +635,7 @@ class NixlConnectorWorker:
# TODO(mgoin): remove this once we have hybrid memory allocator
# Optimization for models with local attention (Llama 4)
# List of block window sizes for each layer for local attention
self.block_window_per_layer: list[Optional[int]] = []
self.block_window_per_layer: list[int | None] = []
self.use_mla = self.model_config.use_mla
backend = get_attn_backend(
@@ -1472,7 +1472,7 @@ class NixlConnectorWorker:
self._recving_transfers[request_id].append((handle, time.perf_counter()))
def _get_block_descs_ids(
self, engine_id: str, block_ids: list[int], layer_idx: Optional[int] = None
self, engine_id: str, block_ids: list[int], layer_idx: int | None = None
) -> np.ndarray:
"""
Get the descs ids for a set of block ids.
@@ -1518,7 +1518,7 @@ class NixlConnectorWorker:
block_len = self.block_len_per_layer[layer_idx]
return block_len
def get_kv_connector_stats(self) -> Optional[KVConnectorStats]:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
@@ -1559,7 +1559,7 @@ def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: Optional[zmq.Context] = None
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
@@ -1611,7 +1611,7 @@ class NixlKVConnectorStats(KVConnectorStats):
accumulator.extend(v)
return self
def reduce(self) -> dict[str, Union[int, float]]:
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging
if self.is_empty():
return {
@@ -4,7 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import islice
from typing import Any, Optional
from typing import Any
import torch
@@ -46,8 +46,8 @@ class OffloadingConnector(KVConnectorBase_V1):
spec = OffloadingSpecFactory.create_spec(vllm_config)
self.connector_scheduler: Optional[OffloadingConnectorScheduler] = None
self.connector_worker: Optional[OffloadingConnectorWorker] = None
self.connector_scheduler: OffloadingConnectorScheduler | None = None
self.connector_worker: OffloadingConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = OffloadingConnectorScheduler(spec)
elif role == KVConnectorRole.WORKER:
@@ -113,7 +113,7 @@ class OffloadingConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -148,7 +148,7 @@ class OffloadingConnectorScheduler:
self,
req: Request,
start_idx: int = 0,
end_idx: Optional[int] = None,
end_idx: int | None = None,
) -> Iterable[BlockHash]:
return islice(
req.block_hashes,
@@ -354,7 +354,7 @@ class OffloadingConnectorScheduler:
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import regex as re
import torch
@@ -304,7 +304,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str], **kwargs: Any
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -466,7 +466,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -5,11 +5,10 @@ import logging
import os
import threading
import time
import typing
from collections import deque
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
import msgpack
import torch
@@ -77,7 +76,7 @@ class P2pNcclEngine:
config: KVTransferConfig,
hostname: str = "",
port_offset: int = 0,
library_path: Optional[str] = None,
library_path: str | None = None,
) -> None:
self.config = config
self.rank = port_offset
@@ -187,7 +186,7 @@ class P2pNcclEngine:
self.nccl_num_channels,
)
def create_connect(self, remote_address: typing.Optional[str] = None):
def create_connect(self, remote_address: str | None = None):
assert remote_address is not None
if remote_address not in self.socks:
sock = self.context.socket(zmq.DEALER)
@@ -224,7 +223,7 @@ class P2pNcclEngine:
self,
tensor_id: str,
tensor: torch.Tensor,
remote_address: typing.Optional[str] = None,
remote_address: str | None = None,
) -> bool:
if remote_address is None:
with self.recv_store_cv:
@@ -296,7 +295,7 @@ class P2pNcclEngine:
def recv_tensor(
self,
tensor_id: str,
remote_address: typing.Optional[str] = None,
remote_address: str | None = None,
) -> torch.Tensor:
if self.send_type == "PUT" or self.send_type == "PUT_ASYNC":
start_time = time.time()
@@ -527,7 +526,7 @@ class P2pNcclEngine:
def get_finished(
self, finished_req_ids: set[str], no_compile_layers
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -3,7 +3,7 @@
import hashlib
import os
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import safetensors
import torch
@@ -249,7 +249,7 @@ class SharedStorageConnector(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -14,7 +14,6 @@ These classes above are abstracted behind class `KVCacheBufferBase`.
"""
from abc import ABC, abstractmethod
from typing import Optional
import torch
@@ -98,8 +97,8 @@ class KVLookupBufferBase(KVCacheBufferBase):
@abstractmethod
def drop_select(
self, input_tokens: Optional[torch.Tensor], roi: Optional[torch.Tensor]
) -> list[Optional[torch.Tensor]]:
self, input_tokens: torch.Tensor | None, roi: torch.Tensor | None
) -> list[torch.Tensor | None]:
"""Select and *drop* KV cache entries from the lookup buffer.
The functionality is similar to the following python statements
@@ -143,7 +142,7 @@ class KVStoreBufferBase(KVCacheBufferBase):
def put(
self,
key: str,
value: Optional[torch.Tensor],
value: torch.Tensor | None,
) -> None:
"""Store a key-value pair in the buffer.
@@ -163,7 +162,7 @@ class KVStoreBufferBase(KVCacheBufferBase):
def get(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Retrieve a value from the buffer by key.
Args:
@@ -10,7 +10,6 @@ from this remote lookup buffer.
import json
import os
from dataclasses import dataclass
from typing import Optional
import torch
from safetensors.torch import load as safetensors_load
@@ -110,7 +109,7 @@ class MooncakeStore(KVStoreBufferBase):
def put(
self,
key: str,
value: Optional[torch.Tensor],
value: torch.Tensor | None,
) -> None:
# A message queue needs to be introduced before making it asynchronous.
if value is not None:
@@ -119,7 +118,7 @@ class MooncakeStore(KVStoreBufferBase):
def get(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
# A message queue needs to be introduced before making it asynchronous.
value = self._get_impl(key)
return value
@@ -142,7 +141,7 @@ class MooncakeStore(KVStoreBufferBase):
def _get_impl(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Get KVCache from Mooncake Store"""
try:
data = self.store.get(key)
@@ -13,7 +13,6 @@ Key Features:
import threading
from collections import deque
from typing import Optional, Union
import torch
@@ -46,7 +45,7 @@ class SimpleBuffer(KVLookupBufferBase):
self.buffer_cv = threading.Condition()
self.signal_pipe = signal_pipe
self.data_pipe = data_pipe
self.request_handling_thread: Optional[threading.Thread] = None
self.request_handling_thread: threading.Thread | None = None
self.normal_signal = torch.tensor([0], device="cpu")
self.end_signal = None
@@ -81,14 +80,14 @@ class SimpleBuffer(KVLookupBufferBase):
return 0
def _send_tensor_and_dec_size(self, tensor: Optional[torch.Tensor]) -> None:
def _send_tensor_and_dec_size(self, tensor: torch.Tensor | None) -> None:
assert tensor is not None, "Use self.data_pipe.send(None) instead"
self.buffer_size -= tensor.element_size() * tensor.numel()
if tensor.dtype == torch.bool:
tensor = tensor.float()
self.data_pipe.send_tensor(tensor)
def _get_element_size(self, data: Optional[Union[list, torch.Tensor]]):
def _get_element_size(self, data: list | torch.Tensor | None):
if isinstance(data, torch.Tensor):
return data.element_size() * data.numel()
if not data:
@@ -184,8 +183,8 @@ class SimpleBuffer(KVLookupBufferBase):
logger.debug("Closing drop_select_handler")
def drop_select(
self, input_tokens: Optional[torch.Tensor], roi: Optional[torch.Tensor]
) -> list[Optional[torch.Tensor]]:
self, input_tokens: torch.Tensor | None, roi: torch.Tensor | None
) -> list[torch.Tensor | None]:
assert self.request_handling_thread is None, (
"drop_select should be called by the KV cache consumer "
"(e.g. the decode vLLM instance)"
+2 -3
View File
@@ -12,7 +12,6 @@ you can bypass this interface and directly start from `kv_lookup_buffer`.
"""
from abc import ABC, abstractmethod
from typing import Optional
import torch
@@ -24,7 +23,7 @@ class KVPipeBase(ABC):
"""
@abstractmethod
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""Send a tensor, or None, via the pipe.
Need to support sending None -- important for error handling.
@@ -42,7 +41,7 @@ class KVPipeBase(ABC):
raise NotImplementedError
@abstractmethod
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""Receive a tensor (can be None) from the pipeline.
Returns:
@@ -6,7 +6,6 @@ import os
import struct
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Optional, Union
import torch
import zmq
@@ -26,7 +25,7 @@ NONE_INT = -150886311
class MooncakeTransferEngineConfig:
prefill_url: str
decode_url: str
metadata_backend: Union[str, None]
metadata_backend: str | None
metadata_server: str
protocol: str
device_name: str
@@ -143,7 +142,7 @@ class MooncakeTransferEngine:
metadata_server: str,
protocol: str,
device_name: str,
metadata_backend: Union[str, None],
metadata_backend: str | None,
) -> None:
"""Initialize the mooncake instance."""
if metadata_backend is None:
@@ -231,7 +230,7 @@ class MooncakePipe(KVPipeBase):
"""MooncakeTransferEngine based Pipe implementation."""
def __init__(
self, local_rank: int, config: KVTransferConfig, device: Optional[str] = None
self, local_rank: int, config: KVTransferConfig, device: str | None = None
):
"""Initialize the mooncake pipe and set related parameters."""
self.config = config
@@ -243,7 +242,7 @@ class MooncakePipe(KVPipeBase):
self.device = self._select_device(device)
self.transfer_engine = MooncakeTransferEngine(self.kv_rank, self.local_rank)
self.transport_thread: Optional[ThreadPoolExecutor] = None
self.transport_thread: ThreadPoolExecutor | None = None
self.none_tensor = torch.tensor([NONE_INT], device=self.device)
def _select_device(self, device: str) -> torch.device:
@@ -267,7 +266,7 @@ class MooncakePipe(KVPipeBase):
data = self.transfer_engine.recv_bytes()
return safetensors_load(data)["tensor"].to(self.device)
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""Send tensor to the target process."""
if self.transport_thread is None:
self.transport_thread = ThreadPoolExecutor(max_workers=1)
@@ -275,7 +274,7 @@ class MooncakePipe(KVPipeBase):
assert len(tensor.shape) > 0
self.transport_thread.submit(self._send_impl, tensor)
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""Receive tensor from other processes."""
if self.transport_thread is None:
self.transport_thread = ThreadPoolExecutor(max_workers=1)
@@ -15,8 +15,8 @@ Key Features:
import threading
import time
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Optional
import torch
@@ -35,7 +35,7 @@ class BrokenPipeException(Exception):
super().__init__(self.message)
Metadata = dict[str, Optional[torch.Tensor]]
Metadata = dict[str, torch.Tensor | None]
class PyNcclPipe(KVPipeBase):
@@ -47,7 +47,7 @@ class PyNcclPipe(KVPipeBase):
self,
local_rank: int,
config: KVTransferConfig,
device: Optional[str] = None,
device: str | None = None,
port_offset: int = 0,
):
self.config = config
@@ -77,7 +77,7 @@ class PyNcclPipe(KVPipeBase):
self.target_rank_for_recv = (self.kv_rank - 1) % self.kv_parallel_size
# transportation-related variables
self.transport_thread: Optional[ThreadPoolExecutor] = None
self.transport_thread: ThreadPoolExecutor | None = None
self.buffer_size = 0
self.buffer_size_lock = threading.Lock()
self.buffer_size_thresh = self.config.kv_buffer_size
@@ -115,7 +115,7 @@ class PyNcclPipe(KVPipeBase):
else:
return torch.device("cpu")
def _make_metadata(self, tensor: Optional[torch.Tensor]) -> Metadata:
def _make_metadata(self, tensor: torch.Tensor | None) -> Metadata:
"""
Create the metadata as a dictionary based on the input tensor.
@@ -167,7 +167,7 @@ class PyNcclPipe(KVPipeBase):
"""
return self.group.recv_obj(self.target_rank_for_recv)
def _send_impl(self, tensor: Optional[torch.Tensor]) -> None:
def _send_impl(self, tensor: torch.Tensor | None) -> None:
"""
The actual implementation of sending the tensor and its metadata to the
target rank.
@@ -181,7 +181,7 @@ class PyNcclPipe(KVPipeBase):
if tensor is not None:
self.device_send_func(tensor.to(self.device), self.target_rank_for_send)
def _recv_impl(self) -> Optional[torch.Tensor]:
def _recv_impl(self) -> torch.Tensor | None:
"""
The actual implementation of receiving a tensor and its metadata from
the target rank.
@@ -198,7 +198,7 @@ class PyNcclPipe(KVPipeBase):
return buffer
def send_tensor_wrapper(
self, tensor: Optional[torch.Tensor], tensor_size: int
self, tensor: torch.Tensor | None, tensor_size: int
) -> None:
"""
Wrapper for _send_impl to handle exceptions and update buffer size.
@@ -228,7 +228,7 @@ class PyNcclPipe(KVPipeBase):
logger.debug("KV cache transfer pipe is full. Waiting...")
time.sleep(0.05)
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""
Sends a tensor and its metadata to the destination rank in a
non-blocking way.
@@ -251,7 +251,7 @@ class PyNcclPipe(KVPipeBase):
self.transport_thread.submit(self.send_tensor_wrapper, tensor, tensor_size)
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""
Receives a tensor and its metadata from the source rank. Blocking call.
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING
from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType
@@ -13,7 +13,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
if TYPE_CHECKING:
from vllm.config import VllmConfig
_KV_CONNECTOR_AGENT: Optional[KVConnectorBaseType] = None
_KV_CONNECTOR_AGENT: KVConnectorBaseType | None = None
def get_kv_transfer_group() -> KVConnectorBaseType:
@@ -27,7 +27,7 @@ def has_kv_transfer_group() -> bool:
return _KV_CONNECTOR_AGENT is not None
def is_v1_kv_transfer_group(connector: Optional[KVConnectorBaseType] = None) -> bool:
def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> bool:
"""Check if the KV connector is the v1 connector.
If the argument is None, it will check the global KV connector
+43 -44
View File
@@ -28,11 +28,12 @@ import gc
import pickle
import weakref
from collections import namedtuple
from collections.abc import Callable
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from datetime import timedelta
from multiprocessing import shared_memory
from typing import Any, Callable, Optional, Union
from typing import Any, Optional
from unittest.mock import patch
import torch
@@ -65,7 +66,7 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
def _split_tensor_dict(
tensor_dict: dict[str, Union[torch.Tensor, Any]],
tensor_dict: dict[str, torch.Tensor | Any],
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
"""Split the tensor dictionary into two parts:
1. A list of (key, value) pairs. If the value is a tensor, it is replaced
@@ -300,17 +301,17 @@ class GroupCoordinator:
cpu_group: ProcessGroup # group for CPU communication
device_group: ProcessGroup # group for device communication
# device communicator (if use_device_communicator=True)
device_communicator: Optional[DeviceCommunicatorBase]
mq_broadcaster: Optional[Any] # shared memory broadcaster
device_communicator: DeviceCommunicatorBase | None
mq_broadcaster: Any | None # shared memory broadcaster
def __init__(
self,
group_ranks: list[list[int]],
local_rank: int,
torch_distributed_backend: Union[str, Backend],
torch_distributed_backend: str | Backend,
use_device_communicator: bool, # whether to use device communicator
use_message_queue_broadcaster: bool = False,
group_name: Optional[str] = None,
group_name: str | None = None,
):
group_name = group_name or "anonymous"
self.unique_name = _get_unique_name(group_name)
@@ -368,7 +369,7 @@ class GroupCoordinator:
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
self.mq_broadcaster: Optional[MessageQueue] = None
self.mq_broadcaster: MessageQueue | None = None
if use_message_queue_broadcaster and self.world_size > 1:
self.mq_broadcaster = MessageQueue.create_from_process_group(
self.cpu_group, 1 << 22, 6
@@ -419,9 +420,7 @@ class GroupCoordinator:
return self.ranks[(rank_in_group - 1) % world_size]
@contextmanager
def graph_capture(
self, graph_capture_context: Optional[GraphCaptureContext] = None
):
def graph_capture(self, graph_capture_context: GraphCaptureContext | None = None):
if graph_capture_context is None:
stream = torch.cuda.Stream()
graph_capture_context = GraphCaptureContext(stream)
@@ -502,9 +501,9 @@ class GroupCoordinator:
def all_gatherv(
self,
input_: Union[torch.Tensor, list[torch.Tensor]],
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: Optional[list[int]] = None,
sizes: list[int] | None = None,
):
if self.device_communicator is None:
raise ValueError("No device communicator found")
@@ -527,7 +526,7 @@ class GroupCoordinator:
return self._reduce_scatter_out_place(input_, dim)
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: Optional[list[int]] = None
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
) -> torch.Tensor:
if self.device_communicator is None:
raise ValueError("No device communicator found")
@@ -540,7 +539,7 @@ class GroupCoordinator:
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
all the ranks.
@@ -569,7 +568,7 @@ class GroupCoordinator:
)
return input_
def broadcast_object(self, obj: Optional[Any] = None, src: int = 0):
def broadcast_object(self, obj: Any | None = None, src: int = 0):
"""Broadcast the input object.
NOTE: `src` is the local rank of the source rank.
"""
@@ -594,7 +593,7 @@ class GroupCoordinator:
return recv[0]
def broadcast_object_list(
self, obj_list: list[Any], src: int = 0, group: Optional[ProcessGroup] = None
self, obj_list: list[Any], src: int = 0, group: ProcessGroup | None = None
):
"""Broadcast the input object list.
NOTE: `src` is the local rank of the source rank.
@@ -675,11 +674,11 @@ class GroupCoordinator:
def broadcast_tensor_dict(
self,
tensor_dict: Optional[dict[str, Union[torch.Tensor, Any]]] = None,
tensor_dict: dict[str, torch.Tensor | Any] | None = None,
src: int = 0,
group: Optional[ProcessGroup] = None,
metadata_group: Optional[ProcessGroup] = None,
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
group: ProcessGroup | None = None,
metadata_group: ProcessGroup | None = None,
) -> dict[str, torch.Tensor | Any] | None:
"""Broadcast the input tensor dictionary.
NOTE: `src` is the local rank of the source rank.
"""
@@ -757,11 +756,11 @@ class GroupCoordinator:
def send_tensor_dict(
self,
tensor_dict: dict[str, Union[torch.Tensor, Any]],
dst: Optional[int] = None,
tensor_dict: dict[str, torch.Tensor | Any],
dst: int | None = None,
all_gather_group: Optional["GroupCoordinator"] = None,
all_gather_tensors: Optional[dict[str, bool]] = None,
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
all_gather_tensors: dict[str, bool] | None = None,
) -> dict[str, torch.Tensor | Any] | None:
"""Send the input tensor dictionary.
NOTE: `dst` is the local rank of the source rank.
@@ -845,10 +844,10 @@ class GroupCoordinator:
def recv_tensor_dict(
self,
src: Optional[int] = None,
src: int | None = None,
all_gather_group: Optional["GroupCoordinator"] = None,
all_gather_tensors: Optional[dict[str, bool]] = None,
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
all_gather_tensors: dict[str, bool] | None = None,
) -> dict[str, torch.Tensor | Any] | None:
"""Recv the input tensor dictionary.
NOTE: `src` is the local rank of the source rank.
@@ -943,7 +942,7 @@ class GroupCoordinator:
"""
torch.distributed.barrier(group=self.cpu_group)
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if self.device_communicator is None:
@@ -951,7 +950,7 @@ class GroupCoordinator:
self.device_communicator.send(tensor, dst)
def recv(
self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None
self, size: torch.Size, dtype: torch.dtype, src: int | None = None
) -> torch.Tensor:
"""Receives a tensor from the source rank."""
"""NOTE: `src` is the local rank of the source rank."""
@@ -997,8 +996,8 @@ class GroupCoordinator:
return hidden_states
_WORLD: Optional[GroupCoordinator] = None
_NODE_COUNT: Optional[int] = None
_WORLD: GroupCoordinator | None = None
_NODE_COUNT: int | None = None
def get_world_group() -> GroupCoordinator:
@@ -1023,7 +1022,7 @@ def init_model_parallel_group(
local_rank: int,
backend: str,
use_message_queue_broadcaster: bool = False,
group_name: Optional[str] = None,
group_name: str | None = None,
) -> GroupCoordinator:
return GroupCoordinator(
group_ranks=group_ranks,
@@ -1035,7 +1034,7 @@ def init_model_parallel_group(
)
_TP: Optional[GroupCoordinator] = None
_TP: GroupCoordinator | None = None
def get_tp_group() -> GroupCoordinator:
@@ -1052,7 +1051,7 @@ def get_tensor_model_parallel_group():
return get_tp_group()
_DCP: Optional[GroupCoordinator] = None
_DCP: GroupCoordinator | None = None
def get_dcp_group() -> GroupCoordinator:
@@ -1063,9 +1062,9 @@ def get_dcp_group() -> GroupCoordinator:
# kept for backward compatibility
get_context_model_parallel_group = get_dcp_group
_PP: Optional[GroupCoordinator] = None
_PP: GroupCoordinator | None = None
_DP: Optional[GroupCoordinator] = None
_DP: GroupCoordinator | None = None
def get_dp_group() -> GroupCoordinator:
@@ -1073,7 +1072,7 @@ def get_dp_group() -> GroupCoordinator:
return _DP
_EP: Optional[GroupCoordinator] = None
_EP: GroupCoordinator | None = None
def get_ep_group() -> GroupCoordinator:
@@ -1131,7 +1130,7 @@ def init_distributed_environment(
distributed_init_method: str = "env://",
local_rank: int = -1,
backend: str = "nccl",
timeout: Optional[timedelta] = None,
timeout: timedelta | None = None,
):
logger.debug(
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
@@ -1208,8 +1207,8 @@ def init_distributed_environment(
def initialize_model_parallel(
tensor_model_parallel_size: int = 1,
pipeline_model_parallel_size: int = 1,
decode_context_model_parallel_size: Optional[int] = 1,
backend: Optional[str] = None,
decode_context_model_parallel_size: int | None = 1,
backend: str | None = None,
) -> None:
"""
Initialize model parallel groups.
@@ -1338,8 +1337,8 @@ def initialize_model_parallel(
def ensure_model_parallel_initialized(
tensor_model_parallel_size: int,
pipeline_model_parallel_size: int,
decode_context_model_parallel_size: Optional[int] = 1,
backend: Optional[str] = None,
decode_context_model_parallel_size: int | None = 1,
backend: str | None = None,
) -> None:
"""Helper to initialize model parallel groups if they are not initialized,
or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
@@ -1504,7 +1503,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
def in_the_same_node_as(
pg: Union[ProcessGroup, StatelessProcessGroup], source_rank: int = 0
pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0
) -> list[bool]:
"""
This is a collective operation that returns if each rank is in the same node
@@ -1624,7 +1623,7 @@ def is_global_first_rank() -> bool:
return True
def _node_count(pg: Union[ProcessGroup, StatelessProcessGroup]) -> int:
def _node_count(pg: ProcessGroup | StatelessProcessGroup) -> int:
"""
Returns the total number of nodes in the process group.

Some files were not shown because too many files have changed in this diff Show More