Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+103 -105
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Literal, Optional, Union
from typing import TYPE_CHECKING, Literal
import torch
@@ -37,7 +37,7 @@ def paged_attention_v1(
seq_lens: torch.Tensor,
block_size: int,
max_seq_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
@@ -84,7 +84,7 @@ def paged_attention_v2(
seq_lens: torch.Tensor,
block_size: int,
max_seq_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
@@ -132,14 +132,14 @@ def paged_attention_rocm(
scale: float,
block_tables: torch.Tensor,
seq_lens: torch.Tensor,
query_start_loc: Optional[torch.Tensor],
query_start_loc: torch.Tensor | None,
block_size: int,
max_seq_len: int,
alibi_slopes: Optional[torch.Tensor],
alibi_slopes: torch.Tensor | None,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
fp8_out_scale: Optional[torch.Tensor] = None,
fp8_out_scale: torch.Tensor | None = None,
mfma_type: str = "fp8" if envs.VLLM_ROCM_FP8_MFMA_PAGE_ATTN else "f16",
) -> None:
torch.ops._rocm_C.paged_attention(
@@ -186,7 +186,7 @@ def merge_attn_states(
prefix_lse: torch.Tensor,
suffix_output: torch.Tensor,
suffix_lse: torch.Tensor,
output_lse: Optional[torch.Tensor] = None,
output_lse: torch.Tensor | None = None,
) -> None:
torch.ops._C.merge_attn_states(
output, output_lse, prefix_output, prefix_lse, suffix_output, suffix_lse
@@ -314,7 +314,7 @@ def convert_vertical_slash_indexes_mergehead(
def rotary_embedding(
positions: torch.Tensor,
query: torch.Tensor,
key: Optional[torch.Tensor],
key: torch.Tensor | None,
head_size: int,
cos_sin_cache: torch.Tensor,
is_neox: bool,
@@ -408,8 +408,8 @@ def rms_norm_dynamic_per_token_quant(
weight: torch.Tensor,
epsilon: float,
quant_dtype: torch.dtype,
scale_ub: Optional[torch.Tensor] = None,
residual: Optional[torch.Tensor] = None,
scale_ub: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
output = torch.empty_like(input, dtype=quant_dtype)
scales = torch.empty(
@@ -527,14 +527,14 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
@register_fake("_C::gptq_marlin_gemm")
def _gptq_marlin_gemm_fake(
a: torch.Tensor,
c: Optional[torch.Tensor],
c: torch.Tensor | None,
b_q_weight: torch.Tensor,
b_bias: Optional[torch.Tensor],
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
global_scale: Optional[torch.Tensor],
b_zeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
global_scale: torch.Tensor | None,
b_zeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
b_q_type_id: int,
size_m: torch.SymInt,
@@ -582,13 +582,13 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
# b_q Should be the tensor returned by machete_prepack_B
b_q: torch.Tensor,
b_type: ScalarType,
out_type: Optional[torch.dtype] = None,
b_group_scales: Optional[torch.Tensor] = None,
b_group_zeros: Optional[torch.Tensor] = None,
b_group_size: Optional[int] = None,
b_channel_scales: Optional[torch.Tensor] = None,
a_token_scales: Optional[torch.Tensor] = None,
schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
b_group_scales: torch.Tensor | None = None,
b_group_zeros: torch.Tensor | None = None,
b_group_size: int | None = None,
b_channel_scales: torch.Tensor | None = None,
a_token_scales: torch.Tensor | None = None,
schedule: str | None = None,
) -> torch.Tensor:
m = a.size(0)
n = b_q.size(1)
@@ -599,7 +599,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
b_q_weight: torch.Tensor,
a_type: torch.dtype,
b_type: ScalarType,
group_scales_type: Optional[torch.dtype],
group_scales_type: torch.dtype | None,
) -> torch.Tensor:
return torch.empty_like(b_q_weight, memory_format=torch.contiguous_format)
@@ -612,8 +612,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
b_group_size: int,
b_channel_scales: torch.Tensor,
a_token_scales: torch.Tensor,
out_type: Optional[torch.dtype] = None,
maybe_schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
maybe_schedule: str | None = None,
) -> torch.Tensor:
m = a.size(0)
n = b_q.size(1)
@@ -636,7 +636,7 @@ if hasattr(torch.ops._C, "allspark_w8a16_gemm"):
a: torch.Tensor,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
n: torch.SymInt,
group_size: torch.SymInt,
sm_count: torch.SymInt,
@@ -657,7 +657,7 @@ if hasattr(torch.ops._C, "ggml_dequantize"):
quant_type: int,
m: torch.SymInt,
n: torch.SymInt,
dtype: Optional[torch.dtype] = None,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
return torch.empty((m, n), dtype=torch.float16, device=W.device)
@@ -760,7 +760,7 @@ def cutlass_scaled_mm(
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out_dtype: torch.dtype,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
`cutlass_scaled_mm` implements a fused version of
@@ -812,8 +812,8 @@ def cutlass_scaled_mm_azp(
scale_b: torch.Tensor,
out_dtype: torch.dtype,
azp_adj: torch.Tensor,
azp: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
azp: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
:param azp_adj: In the per-tensor case, this should include the azp.
@@ -890,7 +890,7 @@ def cutlass_scaled_sparse_mm(
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out_dtype: torch.dtype,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Performs a scaled sparse matrix multiplication using Cutlass.
@@ -940,7 +940,7 @@ def get_cutlass_moe_mm_data(
num_experts: int,
n: int,
k: int,
blockscale_offsets: Optional[torch.Tensor] = None,
blockscale_offsets: torch.Tensor | None = None,
):
"""
Prepare data necessary to perform CUTLASS grouped matrix multiplications
@@ -986,7 +986,7 @@ def get_cutlass_moe_mm_problem_sizes(
num_experts: int,
n: int,
k: int,
blockscale_offsets: Optional[torch.Tensor] = None,
blockscale_offsets: torch.Tensor | None = None,
):
"""
Compute only the per-expert problem sizes needed by the two grouped matrix
@@ -1195,14 +1195,14 @@ def awq_marlin_moe_repack(
def gptq_marlin_gemm(
a: torch.Tensor,
c: Optional[torch.Tensor],
c: torch.Tensor | None,
b_q_weight: torch.Tensor,
b_bias: Optional[torch.Tensor],
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
global_scale: Optional[torch.Tensor],
b_zeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
global_scale: torch.Tensor | None,
b_zeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
b_q_type: ScalarType,
size_m: int,
@@ -1239,11 +1239,11 @@ def gptq_marlin_gemm(
def machete_supported_schedules(
a_type: torch.dtype,
b_type: ScalarType,
group_scales_type: Optional[torch.dtype],
group_zeros_type: Optional[torch.dtype] = None,
channel_scales_type: Optional[torch.dtype] = None,
token_scales_type: Optional[torch.dtype] = None,
out_type: Optional[torch.dtype] = None,
group_scales_type: torch.dtype | None,
group_zeros_type: torch.dtype | None = None,
channel_scales_type: torch.dtype | None = None,
token_scales_type: torch.dtype | None = None,
out_type: torch.dtype | None = None,
) -> list[str]:
return torch.ops._C.machete_supported_schedules(
a_type,
@@ -1261,13 +1261,13 @@ def machete_mm(
# b_q Should be the tensor returned by machete_prepack_B
b_q: torch.Tensor,
b_type: ScalarType,
out_type: Optional[torch.dtype] = None,
b_group_scales: Optional[torch.Tensor] = None,
b_group_zeros: Optional[torch.Tensor] = None,
b_group_size: Optional[int] = None,
b_channel_scales: Optional[torch.Tensor] = None,
a_token_scales: Optional[torch.Tensor] = None,
schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
b_group_scales: torch.Tensor | None = None,
b_group_zeros: torch.Tensor | None = None,
b_group_size: int | None = None,
b_channel_scales: torch.Tensor | None = None,
a_token_scales: torch.Tensor | None = None,
schedule: str | None = None,
) -> torch.Tensor:
return torch.ops._C.machete_mm(
a,
@@ -1287,7 +1287,7 @@ def machete_prepack_B(
b_q_weight: torch.Tensor,
a_type: torch.dtype,
b_type: ScalarType,
group_scales_type: Optional[torch.dtype],
group_scales_type: torch.dtype | None,
) -> torch.Tensor:
return torch.ops._C.machete_prepack_B(
b_q_weight, a_type, b_type.id, group_scales_type
@@ -1303,8 +1303,8 @@ def cutlass_w4a8_mm(
b_group_size: int,
b_channel_scales: torch.Tensor,
a_token_scales: torch.Tensor,
out_type: Optional[torch.dtype] = None,
maybe_schedule: Optional[str] = None,
out_type: torch.dtype | None = None,
maybe_schedule: str | None = None,
) -> torch.Tensor:
return torch.ops._C.cutlass_w4a8_mm(
a,
@@ -1458,11 +1458,11 @@ def scaled_fp4_experts_quant(
# fp8
def scaled_fp8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
num_token_padding: Optional[int] = None,
scale_ub: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
num_token_padding: int | None = None,
scale_ub: torch.Tensor | None = None,
use_per_token_if_dynamic: bool = False,
output: Optional[torch.Tensor] = None,
output: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Quantize input tensor to FP8 and return quantized tensor and scale.
@@ -1489,7 +1489,7 @@ def scaled_fp8_quant(
"""
# This code assumes batch_dim and num_tokens are flattened
assert input.ndim == 2
shape: Union[tuple[int, int], torch.Size] = input.shape
shape: tuple[int, int] | torch.Size = input.shape
# For ROCm on MI300, the output fp8 dtype is torch.float_e3m3fnuz
out_dtype: torch.dtype = current_platform.fp8_dtype()
if num_token_padding:
@@ -1520,7 +1520,7 @@ def scaled_fp8_quant(
def allspark_repack_weight(
qweight: torch.Tensor,
scale: torch.Tensor,
zero_point: Optional[torch.Tensor] = None,
zero_point: torch.Tensor | None = None,
has_zp: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
@@ -1576,7 +1576,7 @@ def allspark_w8a16_gemm(
a: torch.Tensor,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
n: int,
group_size: int,
sm_count: int,
@@ -1603,10 +1603,10 @@ def allspark_w8a16_gemm(
# int8
def scaled_int8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
azp: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
azp: torch.Tensor | None = None,
symmetric: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
@@ -1643,7 +1643,7 @@ def scaled_int8_quant(
# gguf
def ggml_dequantize(
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: Optional[torch.dtype]
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None
) -> torch.Tensor:
return torch.ops._C.ggml_dequantize(W, quant_type, m, n, dtype)
@@ -1713,13 +1713,13 @@ def selective_scan_fwd(
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D_: Optional[torch.Tensor],
z_: Optional[torch.Tensor],
delta_bias_: Optional[torch.Tensor],
D_: torch.Tensor | None,
z_: torch.Tensor | None,
delta_bias_: torch.Tensor | None,
delta_softplus: bool,
query_start_loc: Optional[torch.Tensor],
cache_indices: Optional[torch.Tensor],
has_initial_state: Optional[torch.Tensor],
query_start_loc: torch.Tensor | None,
cache_indices: torch.Tensor | None,
has_initial_state: torch.Tensor | None,
ssm_states: torch.Tensor,
pad_slot_id: int,
):
@@ -1794,8 +1794,8 @@ def moe_wna16_gemm(
output: torch.Tensor,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
topk_weights: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
topk_weights: torch.Tensor | None,
sorted_token_ids: torch.Tensor,
experts_ids: torch.Tensor,
num_tokens_post_pad: torch.Tensor,
@@ -1864,14 +1864,14 @@ def grouped_topk(
def moe_wna16_marlin_gemm(
input: torch.Tensor,
output: Optional[torch.Tensor],
output: torch.Tensor | None,
b_qweight: torch.Tensor,
b_bias: Optional[torch.Tensor],
b_bias: torch.Tensor | None,
b_scales: torch.Tensor,
global_scale: Optional[torch.Tensor],
b_qzeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
global_scale: torch.Tensor | None,
b_qzeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
@@ -1950,12 +1950,12 @@ if hasattr(torch.ops, "_moe_C") and hasattr(torch.ops._moe_C, "marlin_gemm_moe")
@register_fake("_moe_C::moe_wna16_marlin_gemm")
def moe_wna16_marlin_gemm_fake(
input: torch.Tensor,
output: Optional[torch.Tensor],
output: torch.Tensor | None,
b_qweight: torch.Tensor,
b_scales: torch.Tensor,
b_qzeros: Optional[torch.Tensor],
g_idx: Optional[torch.Tensor],
perm: Optional[torch.Tensor],
b_qzeros: torch.Tensor | None,
g_idx: torch.Tensor | None,
perm: torch.Tensor | None,
workspace: torch.Tensor,
sorted_token_ids: torch.Tensor,
expert_ids: torch.Tensor,
@@ -2068,7 +2068,7 @@ def gather_and_maybe_dequant_cache(
batch_size: int,
kv_cache_dtype: str,
scale: torch.Tensor,
seq_starts: Optional[torch.Tensor] = None,
seq_starts: torch.Tensor | None = None,
) -> None:
torch.ops._C_cache_ops.gather_and_maybe_dequant_cache(
src_cache,
@@ -2088,7 +2088,7 @@ def cp_gather_cache(
block_table: torch.Tensor,
cu_seq_lens: torch.Tensor,
batch_size: int,
seq_starts: Optional[torch.Tensor] = None,
seq_starts: torch.Tensor | None = None,
) -> None:
torch.ops._C_cache_ops.cp_gather_cache(
src_cache, dst, block_table, cu_seq_lens, batch_size, seq_starts
@@ -2187,9 +2187,7 @@ def free_shared_buffer(ptr: int) -> None:
# quick all reduce
def init_custom_qr(
rank: int, world_size: int, qr_max_size: Optional[int] = None
) -> int:
def init_custom_qr(rank: int, world_size: int, qr_max_size: int | None = None) -> int:
return torch.ops._C_custom_ar.init_custom_qr(rank, world_size, qr_max_size)
@@ -2247,7 +2245,7 @@ def flash_mla_with_kvcache(
head_dim_v: int,
tile_scheduler_metadata: torch.Tensor,
num_splits: torch.Tensor,
softmax_scale: Optional[float] = None,
softmax_scale: float | None = None,
causal: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
@@ -2324,7 +2322,7 @@ if hasattr(torch.ops._C, "weight_packed_linear"):
def weight_packed_linear_fake(
mat1: torch.Tensor,
mat2: torch.Tensor,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
is_vnni: bool,
) -> torch.Tensor:
return torch.empty(
@@ -2344,11 +2342,11 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
inplace: bool,
use_int8_w8a8: bool,
use_fp8_w8a16: bool,
w1_scale: Optional[torch.Tensor],
w2_scale: Optional[torch.Tensor],
block_size: Optional[list[int]],
a1_scale: Optional[torch.Tensor],
a2_scale: Optional[torch.Tensor],
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
block_size: list[int] | None,
a1_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
is_vnni: bool,
) -> torch.Tensor:
return torch.empty_like(hidden_states)
@@ -2361,7 +2359,7 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
mat1: torch.Tensor,
mat2: torch.Tensor,
scales2: torch.Tensor,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
out_dtype: torch.dtype,
is_vnni: bool,
) -> torch.Tensor:
@@ -2372,7 +2370,7 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
class CPUDNNLGEMMHandler:
def __init__(self) -> None:
self.handler: Optional[int] = None
self.handler: int | None = None
self.n = -1
self.k = -1
@@ -2403,7 +2401,7 @@ def create_onednn_mm(
def onednn_mm(
dnnl_handler: CPUDNNLGEMMHandler,
x: torch.Tensor,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
) -> torch.Tensor:
output = torch.empty((*x.shape[0:-1], dnnl_handler.n), dtype=x.dtype)
torch.ops._C.onednn_mm(
@@ -2431,8 +2429,8 @@ def create_onednn_scaled_mm(
def onednn_scaled_int8_quant(
input: torch.Tensor,
scale: Optional[torch.Tensor] = None,
azp: Optional[torch.Tensor] = None,
scale: torch.Tensor | None = None,
azp: torch.Tensor | None = None,
symmetric: bool = True,
):
"""
@@ -2471,10 +2469,10 @@ def onednn_scaled_mm(
dnnl_handler: CPUDNNLGEMMHandler,
x: torch.Tensor,
output: torch.Tensor,
input_scale: Optional[torch.Tensor],
input_zp: Optional[torch.Tensor],
input_zp_adj: Optional[torch.Tensor],
bias: Optional[torch.Tensor],
input_scale: torch.Tensor | None,
input_zp: torch.Tensor | None,
input_zp_adj: torch.Tensor | None,
bias: torch.Tensor | None,
) -> torch.Tensor:
torch.ops._C.onednn_scaled_mm(
output, x, input_scale, input_zp, input_zp_adj, bias, dnnl_handler.handler