Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+103
-105
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Literal, Optional, Union
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
import torch
|
||||
|
||||
@@ -37,7 +37,7 @@ def paged_attention_v1(
|
||||
seq_lens: torch.Tensor,
|
||||
block_size: int,
|
||||
max_seq_len: int,
|
||||
alibi_slopes: Optional[torch.Tensor],
|
||||
alibi_slopes: torch.Tensor | None,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
@@ -84,7 +84,7 @@ def paged_attention_v2(
|
||||
seq_lens: torch.Tensor,
|
||||
block_size: int,
|
||||
max_seq_len: int,
|
||||
alibi_slopes: Optional[torch.Tensor],
|
||||
alibi_slopes: torch.Tensor | None,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
@@ -132,14 +132,14 @@ def paged_attention_rocm(
|
||||
scale: float,
|
||||
block_tables: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
query_start_loc: Optional[torch.Tensor],
|
||||
query_start_loc: torch.Tensor | None,
|
||||
block_size: int,
|
||||
max_seq_len: int,
|
||||
alibi_slopes: Optional[torch.Tensor],
|
||||
alibi_slopes: torch.Tensor | None,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
fp8_out_scale: Optional[torch.Tensor] = None,
|
||||
fp8_out_scale: torch.Tensor | None = None,
|
||||
mfma_type: str = "fp8" if envs.VLLM_ROCM_FP8_MFMA_PAGE_ATTN else "f16",
|
||||
) -> None:
|
||||
torch.ops._rocm_C.paged_attention(
|
||||
@@ -186,7 +186,7 @@ def merge_attn_states(
|
||||
prefix_lse: torch.Tensor,
|
||||
suffix_output: torch.Tensor,
|
||||
suffix_lse: torch.Tensor,
|
||||
output_lse: Optional[torch.Tensor] = None,
|
||||
output_lse: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
torch.ops._C.merge_attn_states(
|
||||
output, output_lse, prefix_output, prefix_lse, suffix_output, suffix_lse
|
||||
@@ -314,7 +314,7 @@ def convert_vertical_slash_indexes_mergehead(
|
||||
def rotary_embedding(
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: Optional[torch.Tensor],
|
||||
key: torch.Tensor | None,
|
||||
head_size: int,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
@@ -408,8 +408,8 @@ def rms_norm_dynamic_per_token_quant(
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
quant_dtype: torch.dtype,
|
||||
scale_ub: Optional[torch.Tensor] = None,
|
||||
residual: Optional[torch.Tensor] = None,
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
output = torch.empty_like(input, dtype=quant_dtype)
|
||||
scales = torch.empty(
|
||||
@@ -527,14 +527,14 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
|
||||
@register_fake("_C::gptq_marlin_gemm")
|
||||
def _gptq_marlin_gemm_fake(
|
||||
a: torch.Tensor,
|
||||
c: Optional[torch.Tensor],
|
||||
c: torch.Tensor | None,
|
||||
b_q_weight: torch.Tensor,
|
||||
b_bias: Optional[torch.Tensor],
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
global_scale: Optional[torch.Tensor],
|
||||
b_zeros: Optional[torch.Tensor],
|
||||
g_idx: Optional[torch.Tensor],
|
||||
perm: Optional[torch.Tensor],
|
||||
global_scale: torch.Tensor | None,
|
||||
b_zeros: torch.Tensor | None,
|
||||
g_idx: torch.Tensor | None,
|
||||
perm: torch.Tensor | None,
|
||||
workspace: torch.Tensor,
|
||||
b_q_type_id: int,
|
||||
size_m: torch.SymInt,
|
||||
@@ -582,13 +582,13 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
|
||||
# b_q Should be the tensor returned by machete_prepack_B
|
||||
b_q: torch.Tensor,
|
||||
b_type: ScalarType,
|
||||
out_type: Optional[torch.dtype] = None,
|
||||
b_group_scales: Optional[torch.Tensor] = None,
|
||||
b_group_zeros: Optional[torch.Tensor] = None,
|
||||
b_group_size: Optional[int] = None,
|
||||
b_channel_scales: Optional[torch.Tensor] = None,
|
||||
a_token_scales: Optional[torch.Tensor] = None,
|
||||
schedule: Optional[str] = None,
|
||||
out_type: torch.dtype | None = None,
|
||||
b_group_scales: torch.Tensor | None = None,
|
||||
b_group_zeros: torch.Tensor | None = None,
|
||||
b_group_size: int | None = None,
|
||||
b_channel_scales: torch.Tensor | None = None,
|
||||
a_token_scales: torch.Tensor | None = None,
|
||||
schedule: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
m = a.size(0)
|
||||
n = b_q.size(1)
|
||||
@@ -599,7 +599,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
|
||||
b_q_weight: torch.Tensor,
|
||||
a_type: torch.dtype,
|
||||
b_type: ScalarType,
|
||||
group_scales_type: Optional[torch.dtype],
|
||||
group_scales_type: torch.dtype | None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(b_q_weight, memory_format=torch.contiguous_format)
|
||||
|
||||
@@ -612,8 +612,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
|
||||
b_group_size: int,
|
||||
b_channel_scales: torch.Tensor,
|
||||
a_token_scales: torch.Tensor,
|
||||
out_type: Optional[torch.dtype] = None,
|
||||
maybe_schedule: Optional[str] = None,
|
||||
out_type: torch.dtype | None = None,
|
||||
maybe_schedule: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
m = a.size(0)
|
||||
n = b_q.size(1)
|
||||
@@ -636,7 +636,7 @@ if hasattr(torch.ops._C, "allspark_w8a16_gemm"):
|
||||
a: torch.Tensor,
|
||||
b_qweight: torch.Tensor,
|
||||
b_scales: torch.Tensor,
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
b_qzeros: torch.Tensor | None,
|
||||
n: torch.SymInt,
|
||||
group_size: torch.SymInt,
|
||||
sm_count: torch.SymInt,
|
||||
@@ -657,7 +657,7 @@ if hasattr(torch.ops._C, "ggml_dequantize"):
|
||||
quant_type: int,
|
||||
m: torch.SymInt,
|
||||
n: torch.SymInt,
|
||||
dtype: Optional[torch.dtype] = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((m, n), dtype=torch.float16, device=W.device)
|
||||
|
||||
@@ -760,7 +760,7 @@ def cutlass_scaled_mm(
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
`cutlass_scaled_mm` implements a fused version of
|
||||
@@ -812,8 +812,8 @@ def cutlass_scaled_mm_azp(
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
azp_adj: torch.Tensor,
|
||||
azp: Optional[torch.Tensor] = None,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
azp: torch.Tensor | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
:param azp_adj: In the per-tensor case, this should include the azp.
|
||||
@@ -890,7 +890,7 @@ def cutlass_scaled_sparse_mm(
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Performs a scaled sparse matrix multiplication using Cutlass.
|
||||
@@ -940,7 +940,7 @@ def get_cutlass_moe_mm_data(
|
||||
num_experts: int,
|
||||
n: int,
|
||||
k: int,
|
||||
blockscale_offsets: Optional[torch.Tensor] = None,
|
||||
blockscale_offsets: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Prepare data necessary to perform CUTLASS grouped matrix multiplications
|
||||
@@ -986,7 +986,7 @@ def get_cutlass_moe_mm_problem_sizes(
|
||||
num_experts: int,
|
||||
n: int,
|
||||
k: int,
|
||||
blockscale_offsets: Optional[torch.Tensor] = None,
|
||||
blockscale_offsets: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Compute only the per-expert problem sizes needed by the two grouped matrix
|
||||
@@ -1195,14 +1195,14 @@ def awq_marlin_moe_repack(
|
||||
|
||||
def gptq_marlin_gemm(
|
||||
a: torch.Tensor,
|
||||
c: Optional[torch.Tensor],
|
||||
c: torch.Tensor | None,
|
||||
b_q_weight: torch.Tensor,
|
||||
b_bias: Optional[torch.Tensor],
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
global_scale: Optional[torch.Tensor],
|
||||
b_zeros: Optional[torch.Tensor],
|
||||
g_idx: Optional[torch.Tensor],
|
||||
perm: Optional[torch.Tensor],
|
||||
global_scale: torch.Tensor | None,
|
||||
b_zeros: torch.Tensor | None,
|
||||
g_idx: torch.Tensor | None,
|
||||
perm: torch.Tensor | None,
|
||||
workspace: torch.Tensor,
|
||||
b_q_type: ScalarType,
|
||||
size_m: int,
|
||||
@@ -1239,11 +1239,11 @@ def gptq_marlin_gemm(
|
||||
def machete_supported_schedules(
|
||||
a_type: torch.dtype,
|
||||
b_type: ScalarType,
|
||||
group_scales_type: Optional[torch.dtype],
|
||||
group_zeros_type: Optional[torch.dtype] = None,
|
||||
channel_scales_type: Optional[torch.dtype] = None,
|
||||
token_scales_type: Optional[torch.dtype] = None,
|
||||
out_type: Optional[torch.dtype] = None,
|
||||
group_scales_type: torch.dtype | None,
|
||||
group_zeros_type: torch.dtype | None = None,
|
||||
channel_scales_type: torch.dtype | None = None,
|
||||
token_scales_type: torch.dtype | None = None,
|
||||
out_type: torch.dtype | None = None,
|
||||
) -> list[str]:
|
||||
return torch.ops._C.machete_supported_schedules(
|
||||
a_type,
|
||||
@@ -1261,13 +1261,13 @@ def machete_mm(
|
||||
# b_q Should be the tensor returned by machete_prepack_B
|
||||
b_q: torch.Tensor,
|
||||
b_type: ScalarType,
|
||||
out_type: Optional[torch.dtype] = None,
|
||||
b_group_scales: Optional[torch.Tensor] = None,
|
||||
b_group_zeros: Optional[torch.Tensor] = None,
|
||||
b_group_size: Optional[int] = None,
|
||||
b_channel_scales: Optional[torch.Tensor] = None,
|
||||
a_token_scales: Optional[torch.Tensor] = None,
|
||||
schedule: Optional[str] = None,
|
||||
out_type: torch.dtype | None = None,
|
||||
b_group_scales: torch.Tensor | None = None,
|
||||
b_group_zeros: torch.Tensor | None = None,
|
||||
b_group_size: int | None = None,
|
||||
b_channel_scales: torch.Tensor | None = None,
|
||||
a_token_scales: torch.Tensor | None = None,
|
||||
schedule: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.machete_mm(
|
||||
a,
|
||||
@@ -1287,7 +1287,7 @@ def machete_prepack_B(
|
||||
b_q_weight: torch.Tensor,
|
||||
a_type: torch.dtype,
|
||||
b_type: ScalarType,
|
||||
group_scales_type: Optional[torch.dtype],
|
||||
group_scales_type: torch.dtype | None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.machete_prepack_B(
|
||||
b_q_weight, a_type, b_type.id, group_scales_type
|
||||
@@ -1303,8 +1303,8 @@ def cutlass_w4a8_mm(
|
||||
b_group_size: int,
|
||||
b_channel_scales: torch.Tensor,
|
||||
a_token_scales: torch.Tensor,
|
||||
out_type: Optional[torch.dtype] = None,
|
||||
maybe_schedule: Optional[str] = None,
|
||||
out_type: torch.dtype | None = None,
|
||||
maybe_schedule: str | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.cutlass_w4a8_mm(
|
||||
a,
|
||||
@@ -1458,11 +1458,11 @@ def scaled_fp4_experts_quant(
|
||||
# fp8
|
||||
def scaled_fp8_quant(
|
||||
input: torch.Tensor,
|
||||
scale: Optional[torch.Tensor] = None,
|
||||
num_token_padding: Optional[int] = None,
|
||||
scale_ub: Optional[torch.Tensor] = None,
|
||||
scale: torch.Tensor | None = None,
|
||||
num_token_padding: int | None = None,
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
use_per_token_if_dynamic: bool = False,
|
||||
output: Optional[torch.Tensor] = None,
|
||||
output: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP8 and return quantized tensor and scale.
|
||||
@@ -1489,7 +1489,7 @@ def scaled_fp8_quant(
|
||||
"""
|
||||
# This code assumes batch_dim and num_tokens are flattened
|
||||
assert input.ndim == 2
|
||||
shape: Union[tuple[int, int], torch.Size] = input.shape
|
||||
shape: tuple[int, int] | torch.Size = input.shape
|
||||
# For ROCm on MI300, the output fp8 dtype is torch.float_e3m3fnuz
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
if num_token_padding:
|
||||
@@ -1520,7 +1520,7 @@ def scaled_fp8_quant(
|
||||
def allspark_repack_weight(
|
||||
qweight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
zero_point: Optional[torch.Tensor] = None,
|
||||
zero_point: torch.Tensor | None = None,
|
||||
has_zp: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
@@ -1576,7 +1576,7 @@ def allspark_w8a16_gemm(
|
||||
a: torch.Tensor,
|
||||
b_qweight: torch.Tensor,
|
||||
b_scales: torch.Tensor,
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
b_qzeros: torch.Tensor | None,
|
||||
n: int,
|
||||
group_size: int,
|
||||
sm_count: int,
|
||||
@@ -1603,10 +1603,10 @@ def allspark_w8a16_gemm(
|
||||
# int8
|
||||
def scaled_int8_quant(
|
||||
input: torch.Tensor,
|
||||
scale: Optional[torch.Tensor] = None,
|
||||
azp: Optional[torch.Tensor] = None,
|
||||
scale: torch.Tensor | None = None,
|
||||
azp: torch.Tensor | None = None,
|
||||
symmetric: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
Quantize the input tensor to int8 and return the quantized tensor and scale, and maybe azp.
|
||||
|
||||
@@ -1643,7 +1643,7 @@ def scaled_int8_quant(
|
||||
|
||||
# gguf
|
||||
def ggml_dequantize(
|
||||
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: Optional[torch.dtype]
|
||||
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_dequantize(W, quant_type, m, n, dtype)
|
||||
|
||||
@@ -1713,13 +1713,13 @@ def selective_scan_fwd(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D_: Optional[torch.Tensor],
|
||||
z_: Optional[torch.Tensor],
|
||||
delta_bias_: Optional[torch.Tensor],
|
||||
D_: torch.Tensor | None,
|
||||
z_: torch.Tensor | None,
|
||||
delta_bias_: torch.Tensor | None,
|
||||
delta_softplus: bool,
|
||||
query_start_loc: Optional[torch.Tensor],
|
||||
cache_indices: Optional[torch.Tensor],
|
||||
has_initial_state: Optional[torch.Tensor],
|
||||
query_start_loc: torch.Tensor | None,
|
||||
cache_indices: torch.Tensor | None,
|
||||
has_initial_state: torch.Tensor | None,
|
||||
ssm_states: torch.Tensor,
|
||||
pad_slot_id: int,
|
||||
):
|
||||
@@ -1794,8 +1794,8 @@ def moe_wna16_gemm(
|
||||
output: torch.Tensor,
|
||||
b_qweight: torch.Tensor,
|
||||
b_scales: torch.Tensor,
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
topk_weights: Optional[torch.Tensor],
|
||||
b_qzeros: torch.Tensor | None,
|
||||
topk_weights: torch.Tensor | None,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
experts_ids: torch.Tensor,
|
||||
num_tokens_post_pad: torch.Tensor,
|
||||
@@ -1864,14 +1864,14 @@ def grouped_topk(
|
||||
|
||||
def moe_wna16_marlin_gemm(
|
||||
input: torch.Tensor,
|
||||
output: Optional[torch.Tensor],
|
||||
output: torch.Tensor | None,
|
||||
b_qweight: torch.Tensor,
|
||||
b_bias: Optional[torch.Tensor],
|
||||
b_bias: torch.Tensor | None,
|
||||
b_scales: torch.Tensor,
|
||||
global_scale: Optional[torch.Tensor],
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
g_idx: Optional[torch.Tensor],
|
||||
perm: Optional[torch.Tensor],
|
||||
global_scale: torch.Tensor | None,
|
||||
b_qzeros: torch.Tensor | None,
|
||||
g_idx: torch.Tensor | None,
|
||||
perm: torch.Tensor | None,
|
||||
workspace: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
@@ -1950,12 +1950,12 @@ if hasattr(torch.ops, "_moe_C") and hasattr(torch.ops._moe_C, "marlin_gemm_moe")
|
||||
@register_fake("_moe_C::moe_wna16_marlin_gemm")
|
||||
def moe_wna16_marlin_gemm_fake(
|
||||
input: torch.Tensor,
|
||||
output: Optional[torch.Tensor],
|
||||
output: torch.Tensor | None,
|
||||
b_qweight: torch.Tensor,
|
||||
b_scales: torch.Tensor,
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
g_idx: Optional[torch.Tensor],
|
||||
perm: Optional[torch.Tensor],
|
||||
b_qzeros: torch.Tensor | None,
|
||||
g_idx: torch.Tensor | None,
|
||||
perm: torch.Tensor | None,
|
||||
workspace: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
@@ -2068,7 +2068,7 @@ def gather_and_maybe_dequant_cache(
|
||||
batch_size: int,
|
||||
kv_cache_dtype: str,
|
||||
scale: torch.Tensor,
|
||||
seq_starts: Optional[torch.Tensor] = None,
|
||||
seq_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.gather_and_maybe_dequant_cache(
|
||||
src_cache,
|
||||
@@ -2088,7 +2088,7 @@ def cp_gather_cache(
|
||||
block_table: torch.Tensor,
|
||||
cu_seq_lens: torch.Tensor,
|
||||
batch_size: int,
|
||||
seq_starts: Optional[torch.Tensor] = None,
|
||||
seq_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.cp_gather_cache(
|
||||
src_cache, dst, block_table, cu_seq_lens, batch_size, seq_starts
|
||||
@@ -2187,9 +2187,7 @@ def free_shared_buffer(ptr: int) -> None:
|
||||
|
||||
|
||||
# quick all reduce
|
||||
def init_custom_qr(
|
||||
rank: int, world_size: int, qr_max_size: Optional[int] = None
|
||||
) -> int:
|
||||
def init_custom_qr(rank: int, world_size: int, qr_max_size: int | None = None) -> int:
|
||||
return torch.ops._C_custom_ar.init_custom_qr(rank, world_size, qr_max_size)
|
||||
|
||||
|
||||
@@ -2247,7 +2245,7 @@ def flash_mla_with_kvcache(
|
||||
head_dim_v: int,
|
||||
tile_scheduler_metadata: torch.Tensor,
|
||||
num_splits: torch.Tensor,
|
||||
softmax_scale: Optional[float] = None,
|
||||
softmax_scale: float | None = None,
|
||||
causal: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
@@ -2324,7 +2322,7 @@ if hasattr(torch.ops._C, "weight_packed_linear"):
|
||||
def weight_packed_linear_fake(
|
||||
mat1: torch.Tensor,
|
||||
mat2: torch.Tensor,
|
||||
bias: Optional[torch.Tensor],
|
||||
bias: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
@@ -2344,11 +2342,11 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
inplace: bool,
|
||||
use_int8_w8a8: bool,
|
||||
use_fp8_w8a16: bool,
|
||||
w1_scale: Optional[torch.Tensor],
|
||||
w2_scale: Optional[torch.Tensor],
|
||||
block_size: Optional[list[int]],
|
||||
a1_scale: Optional[torch.Tensor],
|
||||
a2_scale: Optional[torch.Tensor],
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
block_size: list[int] | None,
|
||||
a1_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
@@ -2361,7 +2359,7 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
|
||||
mat1: torch.Tensor,
|
||||
mat2: torch.Tensor,
|
||||
scales2: torch.Tensor,
|
||||
bias: Optional[torch.Tensor],
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
@@ -2372,7 +2370,7 @@ if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
|
||||
|
||||
class CPUDNNLGEMMHandler:
|
||||
def __init__(self) -> None:
|
||||
self.handler: Optional[int] = None
|
||||
self.handler: int | None = None
|
||||
self.n = -1
|
||||
self.k = -1
|
||||
|
||||
@@ -2403,7 +2401,7 @@ def create_onednn_mm(
|
||||
def onednn_mm(
|
||||
dnnl_handler: CPUDNNLGEMMHandler,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor],
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty((*x.shape[0:-1], dnnl_handler.n), dtype=x.dtype)
|
||||
torch.ops._C.onednn_mm(
|
||||
@@ -2431,8 +2429,8 @@ def create_onednn_scaled_mm(
|
||||
|
||||
def onednn_scaled_int8_quant(
|
||||
input: torch.Tensor,
|
||||
scale: Optional[torch.Tensor] = None,
|
||||
azp: Optional[torch.Tensor] = None,
|
||||
scale: torch.Tensor | None = None,
|
||||
azp: torch.Tensor | None = None,
|
||||
symmetric: bool = True,
|
||||
):
|
||||
"""
|
||||
@@ -2471,10 +2469,10 @@ def onednn_scaled_mm(
|
||||
dnnl_handler: CPUDNNLGEMMHandler,
|
||||
x: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
input_scale: Optional[torch.Tensor],
|
||||
input_zp: Optional[torch.Tensor],
|
||||
input_zp_adj: Optional[torch.Tensor],
|
||||
bias: Optional[torch.Tensor],
|
||||
input_scale: torch.Tensor | None,
|
||||
input_zp: torch.Tensor | None,
|
||||
input_zp_adj: torch.Tensor | None,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
torch.ops._C.onednn_scaled_mm(
|
||||
output, x, input_scale, input_zp, input_zp_adj, bias, dnnl_handler.handler
|
||||
|
||||
Reference in New Issue
Block a user