Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
from packaging import version
@@ -29,7 +28,7 @@ BITBLAS_SUPPORTED_SYM = [False, True]
# Determines the supported quantization types for BitBLAS based on the
# device's capability and whether zero-point (zp) is used.
def query_bitblas_supported_quant_types(
has_zp: bool, device_capability: Optional[int] = None
has_zp: bool, device_capability: int | None = None
):
if device_capability is None:
capability_tuple = current_platform.get_device_capability()
@@ -52,10 +51,10 @@ def query_bitblas_supported_quant_types(
def _check_bitblas_supported(
quant_type: ScalarType,
group_size: Optional[int],
group_size: int | None,
has_zp: bool,
device_capability: Optional[int] = None,
) -> tuple[bool, Optional[str]]:
device_capability: int | None = None,
) -> tuple[bool, str | None]:
if device_capability is None:
capability_tuple = current_platform.get_device_capability()
device_capability = (
@@ -99,7 +98,7 @@ def check_bitblas_supported(
quant_type: ScalarType,
group_size: int,
has_zp: bool = False,
device_capability: Optional[int] = None,
device_capability: int | None = None,
) -> bool:
cond, _ = _check_bitblas_supported(
quant_type, group_size, has_zp, device_capability
@@ -156,7 +155,7 @@ def check_bitblas_supports_shape(
input_size_per_partition: int,
input_size: int,
group_size: int,
) -> tuple[bool, Optional[str]]:
) -> tuple[bool, str | None]:
try:
verify_bitblas_supports_shape(
output_size_per_partition, input_size_per_partition, input_size, group_size
@@ -2,8 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
from __future__ import annotations
import torch
import vllm.envs as envs
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Optional
import torch
@@ -101,10 +100,10 @@ def apply_flashinfer_per_tensor_scale_fp8(
layer: torch.nn.Module,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
routing_bias: Optional[torch.Tensor],
routing_bias: torch.Tensor | None,
top_k: int,
num_expert_group: Optional[int],
topk_group: Optional[int],
num_expert_group: int | None,
topk_group: int | None,
global_num_experts: int,
apply_router_weight_on_input: bool,
) -> torch.Tensor:
@@ -186,7 +185,7 @@ def register_moe_scaling_factors(layer: torch.nn.Module) -> None:
def build_flashinfer_fp8_cutlass_moe_prepare_finalize(
moe: Optional[FusedMoEConfig],
moe: FusedMoEConfig | None,
) -> mk.FusedMoEPrepareAndFinalize:
"""Create a FlashInfer CUTLASS fused-MoE prepare finalize kernel"""
use_dp = moe.moe_parallel_config.dp_size > 1 if moe is not None else False
@@ -194,9 +193,9 @@ def build_flashinfer_fp8_cutlass_moe_prepare_finalize(
def select_cutlass_fp8_gemm_impl(
moe: Optional[FusedMoEConfig],
moe: FusedMoEConfig | None,
quant_config: FusedMoEQuantConfig,
out_dtype: Optional[torch.dtype] = None,
out_dtype: torch.dtype | None = None,
) -> mk.FusedMoEPermuteExpertsUnpermute:
"""Return a GEMM *experts* implementation for fused-MoE layers"""
@@ -225,7 +224,7 @@ def flashinfer_cutlass_moe_fp8(
inplace: bool = False,
activation: str = "silu",
global_num_experts: int = -1,
expert_map: Optional[torch.Tensor] = None,
expert_map: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
) -> torch.Tensor:
quant_config = layer.quant_method.get_fused_moe_quant_config(layer)
@@ -5,8 +5,8 @@
import functools
import json
import os
from collections.abc import Sequence
from typing import Any, Callable, Optional, Union
from collections.abc import Callable, Sequence
from typing import Any
import torch
@@ -39,7 +39,7 @@ from vllm.utils.deep_gemm import (
logger = init_logger(__name__)
def is_fp8(x: Union[torch.dtype, torch.Tensor]) -> bool:
def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
if isinstance(x, torch.Tensor):
x = x.dtype
return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
@@ -54,7 +54,7 @@ def cutlass_scaled_mm(
Bs: torch.Tensor,
block_size: list[int],
output_dtype: torch.dtype = torch.float16,
is_hopper: Optional[bool] = None,
is_hopper: bool | None = None,
) -> torch.Tensor:
if is_hopper is None:
is_hopper = current_platform.is_device_capability(90)
@@ -279,8 +279,8 @@ class W8A8BlockFp8LinearOp:
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
input_scale: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
input_scale: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
assert input_scale is None
# View input as 2D matrix for fp8 methods
@@ -394,7 +394,7 @@ class W8A8BlockFp8LinearOp:
],
torch.Tensor,
],
Optional[QuantFP8],
QuantFP8 | None,
]:
if use_cutlass:
return self._run_cutlass, (
@@ -418,7 +418,7 @@ class W8A8BlockFp8LinearOp:
def input_to_float8(
x: torch.Tensor, dtype: Optional[torch.dtype] = None
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values "
"with tensor-wise quantization."""
@@ -568,10 +568,10 @@ def per_token_group_quant_fp8(
x: torch.Tensor,
group_size: int,
eps: float = 1e-10,
dtype: Optional[torch.dtype] = None,
dtype: torch.dtype | None = None,
column_major_scales: bool = False,
out_q: Optional[torch.Tensor] = None,
use_ue8m0: Optional[bool] = None,
out_q: torch.Tensor | None = None,
use_ue8m0: bool | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Function to perform per-token-group quantization on an input tensor `x`.
It converts the tensor values into signed float8 values and returns the
@@ -754,7 +754,7 @@ def _w8a8_triton_block_scaled_mm(
@functools.lru_cache
def get_w8a8_block_fp8_configs(
N: int, K: int, block_n: int, block_k: int
) -> Optional[dict[int, Any]]:
) -> dict[int, Any] | None:
"""
Return optimized configurations for the w8a8 block fp8 kernel.
The return value will be a dictionary that maps an irregular grid of
@@ -1012,7 +1012,7 @@ def validate_fp8_block_shape(
def create_fp8_weight_parameter(
output_size_per_partition: int,
input_size_per_partition: int,
weight_loader: Optional[Callable],
weight_loader: Callable | None,
) -> torch.nn.Parameter:
"""Create FP8 weight parameter."""
from vllm.model_executor.parameter import ModelWeightParameter
@@ -1033,8 +1033,8 @@ def create_fp8_scale_parameter(
parameter_type: torch.nn.Parameter,
output_partition_sizes: list[int],
input_size_per_partition: int,
block_size: Optional[list[int]],
weight_loader: Optional[Callable],
block_size: list[int] | None,
weight_loader: Callable | None,
) -> torch.nn.Parameter:
"""Create scale parameter based on quantization strategy."""
if parameter_type == ChannelQuantScaleParameter:
@@ -1070,7 +1070,7 @@ def create_fp8_scale_parameter(
def create_fp8_input_scale(
output_partition_sizes: list[int], weight_loader: Optional[Callable]
output_partition_sizes: list[int], weight_loader: Callable | None
) -> torch.nn.Parameter:
"""Create input scale parameter for static activation quantization."""
from vllm.model_executor.parameter import PerTensorScaleParameter
@@ -1087,8 +1087,8 @@ def process_fp8_weight_tensor_strategy(
weight: torch.Tensor,
weight_scale: torch.Tensor,
logical_widths: list[int],
input_scale: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
input_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Process weights for tensor-wise quantization strategy."""
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
@@ -1114,8 +1114,8 @@ def process_fp8_weight_tensor_strategy(
def process_fp8_weight_channel_strategy(
weight: torch.Tensor,
weight_scale: torch.Tensor,
input_scale: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
input_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Process weights for channel-wise quantization strategy."""
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
@@ -4,7 +4,7 @@ from collections.abc import Mapping
from copy import deepcopy
from fractions import Fraction
from types import MappingProxyType
from typing import TYPE_CHECKING, Optional, Union
from typing import TYPE_CHECKING
import regex as re
import torch
@@ -25,7 +25,7 @@ else:
# Match dynamic rules with module name (prefix) and override quantize
# config if module (prefix) matches a rule
def override_config(config: Union[GPTQConfig, GPTQMarlinConfig], prefix: str):
def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
weight_bits = get_dynamic_override(config, prefix, "bits", config.weight_bits)
if isinstance(weight_bits, int):
config.weight_bits = weight_bits
@@ -60,11 +60,11 @@ def override_config(config: Union[GPTQConfig, GPTQMarlinConfig], prefix: str):
def get_dynamic_override(
config: Union[GPTQConfig, GPTQMarlinConfig],
config: GPTQConfig | GPTQMarlinConfig,
layer_name: str,
key: Optional[str] = None,
default_value: Union[int, bool, None] = None,
) -> Union[dict, int, bool, None]:
key: str | None = None,
default_value: int | bool | None = None,
) -> dict | int | bool | None:
for pattern, pattern_dict in config.dynamic.items():
# Negative match: matched modules are excluded from quantized init
if pattern.startswith("-:"):
@@ -126,7 +126,7 @@ def is_layer_gptq_quantized(
def get_linear_quant_method(
config: Union[GPTQConfig, GPTQMarlinConfig],
config: GPTQConfig | GPTQMarlinConfig,
layer: torch.nn.Module,
prefix: str,
linear_method_cls: type,
@@ -6,7 +6,7 @@ import functools
import json
import logging
import os
from typing import Any, Optional
from typing import Any
import torch
@@ -21,8 +21,8 @@ def apply_w8a8_block_int8_linear(
weight: torch.Tensor,
block_size: list[int],
weight_scale: torch.Tensor,
input_scale: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
input_scale: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
assert input_scale is None
# View input as 2D matrix for fp8 methods
@@ -359,7 +359,7 @@ def _w8a8_block_int8_matmul(
@functools.lru_cache
def get_w8a8_block_int8_configs(
N: int, K: int, block_n: int, block_k: int
) -> Optional[dict[int, Any]]:
) -> dict[int, Any] | None:
"""
Return optimized configurations for the w8a8 block fp8 kernel.
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Union
import torch
@@ -21,7 +20,7 @@ def update_tensor_inplace(dst: torch.Tensor, src: torch.Tensor):
# Newly generated tensors need to replace existing tensors that are
# already registered as parameters by vLLM (and won't be freed)
def replace_parameter(
mod: torch.nn.Module, name: str, new: Union[torch.Tensor, torch.nn.Parameter]
mod: torch.nn.Module, name: str, new: torch.Tensor | torch.nn.Parameter
) -> None:
old = getattr(mod, name)
if (
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
@@ -41,7 +40,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
def check_machete_supports_shape(
in_features: int, out_featrues: int
) -> tuple[bool, Optional[str]]:
) -> tuple[bool, str | None]:
if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
return (
False,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import numpy
import torch
@@ -34,9 +33,9 @@ USE_FP32_REDUCE_DEFAULT = True
# without runtime zero-point. We support common cases, i.e. AWQ and GPTQ.
# TODO: we may want to move this into the C++ so its closer to the actual impl
def query_marlin_supported_quant_types(
has_zp: Optional[bool] = None,
has_zp: bool | None = None,
include_fp_type: bool = True,
device_capability: Optional[int] = None,
device_capability: int | None = None,
):
if device_capability is None:
capability_tuple = current_platform.get_device_capability()
@@ -72,10 +71,10 @@ def query_marlin_supported_quant_types(
def _check_marlin_supported(
quant_type: ScalarType,
group_size: Optional[int],
group_size: int | None,
has_zp: bool,
device_capability: Optional[int] = None,
) -> tuple[bool, Optional[str]]:
device_capability: int | None = None,
) -> tuple[bool, str | None]:
if device_capability is None:
capability_tuple = current_platform.get_device_capability()
device_capability = (
@@ -109,7 +108,7 @@ def check_marlin_supported(
quant_type: ScalarType,
group_size: int,
has_zp: bool = False,
device_capability: Optional[int] = None,
device_capability: int | None = None,
) -> bool:
cond, _ = _check_marlin_supported(quant_type, group_size, has_zp, device_capability)
return cond
@@ -164,7 +163,7 @@ def check_marlin_supports_shape(
input_size_per_partition: int,
input_size: int,
group_size: int,
) -> tuple[bool, Optional[str]]:
) -> tuple[bool, str | None]:
try:
verify_marlin_supports_shape(
output_size_per_partition, input_size_per_partition, input_size, group_size
@@ -445,7 +444,7 @@ def apply_gptq_marlin_linear(
output_size_per_partition: int,
input_size_per_partition: int,
is_k_full: bool,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
) -> torch.Tensor:
reshaped_x = input.reshape(-1, input.shape[-1])
@@ -494,7 +493,7 @@ def apply_awq_marlin_linear(
quant_type: ScalarType,
output_size_per_partition: int,
input_size_per_partition: int,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
) -> torch.Tensor:
reshaped_x = input.reshape(-1, input.shape[-1])
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
@@ -95,11 +94,11 @@ def apply_fp4_marlin_linear(
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
weight_scale_2: Optional[torch.Tensor],
weight_scale_2: torch.Tensor | None,
workspace: torch.Tensor,
size_n: int,
size_k: int,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
) -> torch.Tensor:
# For GPUs that lack FP4 hardware support, we can leverage the
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
@@ -45,7 +44,7 @@ def apply_fp8_marlin_linear(
workspace: torch.Tensor,
size_n: int,
size_k: int,
bias: Optional[torch.Tensor],
bias: torch.Tensor | None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
) -> torch.Tensor:
# For GPUs that lack FP8 hardware support, we can leverage the
@@ -2,8 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions used for tests and benchmarks"""
from typing import Optional
import numpy as np
import torch
@@ -100,7 +98,7 @@ def marlin_quantize(
quant_type: ScalarType,
group_size: int,
act_order: bool,
test_perm: Optional[torch.Tensor] = None,
test_perm: torch.Tensor | None = None,
):
size_k, size_n = w.shape
num_bits = quant_type.size_bits
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Callable, Optional
from collections.abc import Callable
from typing import Any
import torch
@@ -71,17 +72,17 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps):
def _can_support_mxfp4(
use_grouped_topk: bool = False,
topk_group: Optional[int] = None,
num_expert_group: Optional[int] = None,
expert_map: Optional[torch.Tensor] = None,
custom_routing_function: Optional[Callable] = None,
e_score_correction_bias: Optional[torch.Tensor] = None,
topk_group: int | None = None,
num_expert_group: int | None = None,
expert_map: torch.Tensor | None = None,
custom_routing_function: Callable | None = None,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
scoring_func: str = "softmax",
activation: str = "swigluoai",
expert_load_view: Optional[torch.Tensor] = None,
logical_to_physical_map: Optional[torch.Tensor] = None,
logical_replica_count: Optional[torch.Tensor] = None,
expert_load_view: torch.Tensor | None = None,
logical_to_physical_map: torch.Tensor | None = None,
logical_replica_count: torch.Tensor | None = None,
):
return not (
use_grouped_topk
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Union
from vllm.logger import init_logger
@@ -28,9 +27,7 @@ class OCP_MX_Scheme(str, Enum):
w_mxfp6_e2m3_a_mxfp6_e2m3 = "w_mxfp6_e2m3_a_mxfp6_e2m3"
@classmethod
def from_quant_dtype(
cls, input_dtype: Union[str, None], weight_dtype: Union[str, None]
):
def from_quant_dtype(cls, input_dtype: str | None, weight_dtype: str | None):
if input_dtype not in OCP_MX_DTYPES or weight_dtype not in OCP_MX_DTYPES:
return None
elif input_dtype == "mxfp4" and weight_dtype == "mxfp4":
@@ -43,8 +43,8 @@ _require_petit = _import_petit_kernel
def _check_petit_nvfp4_supported(
quant_method: str, group_size: Optional[int]
) -> tuple[bool, Optional[str]]:
quant_method: str, group_size: int | None
) -> tuple[bool, str | None]:
if quant_method != "NVFP4":
return (
False,
@@ -62,7 +62,7 @@ def _check_petit_nvfp4_supported(
return (True, None)
def verify_petit_nvfp4_supported(quant_method: str, group_size: Optional[int]) -> None:
def verify_petit_nvfp4_supported(quant_method: str, group_size: int | None) -> None:
supported, error_msg = _check_petit_nvfp4_supported(quant_method, group_size)
if not supported:
assert error_msg is not None
@@ -98,7 +98,7 @@ def apply_petit_nvfp4_linear(
weight_scale_2: torch.Tensor,
size_n: int,
size_k: int,
bias: Optional[torch.Tensor] = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
# Trigger (or get) the import here as well.
petit_kernel = _import_petit_kernel()
@@ -5,7 +5,7 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import ClassVar, NamedTuple, Optional
from typing import ClassVar, NamedTuple
import numpy
import torch
@@ -91,7 +91,7 @@ class QuantKey:
dtype: torch.dtype
scale: ScaleDesc
scale2: Optional[ScaleDesc] = None
scale2: ScaleDesc | None = None
symmetric: bool = True
def __str__(self):
@@ -205,7 +205,7 @@ def scaled_quantize(
def scaled_dequantize(
x_q: torch.Tensor,
x_s: torch.Tensor,
group_shape: Optional[GroupShape] = None,
group_shape: GroupShape | None = None,
out_dtype: torch.dtype = torch.float32,
) -> tuple[torch.Tensor, torch.Tensor]:
if group_shape is not None:
@@ -336,7 +336,7 @@ def permute_rows(
q_w: torch.Tensor,
w_ref: torch.Tensor,
group_size: int,
test_perm: Optional[torch.Tensor] = None,
test_perm: torch.Tensor | None = None,
):
assert q_w.shape == w_ref.shape
@@ -365,7 +365,7 @@ def permute_rows(
def quantize_weights(
w: torch.Tensor,
quant_type: ScalarType,
group_size: Optional[int],
group_size: int | None,
zero_points: bool = False,
ref_zero_points_after_scales: bool = False,
):
@@ -466,7 +466,7 @@ def gptq_quantize_weights(
quant_type: ScalarType,
group_size: int,
act_order: bool,
test_perm: Optional[torch.Tensor] = None,
test_perm: torch.Tensor | None = None,
):
size_k, _ = w.shape
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Callable, Optional, Union
from collections.abc import Callable
import torch
from packaging import version
@@ -75,7 +75,7 @@ CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported()
def per_tensor_dequantize(
tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor]
tensor: torch.Tensor, inv_scale: float | torch.Tensor
) -> torch.Tensor:
fake_qweight = tensor.to(torch.float16)
dq_weight = fake_qweight * inv_scale
@@ -399,7 +399,7 @@ class Fp8LinearOp:
self,
act_quant_static: bool,
act_quant_group_shape: GroupShape = GroupShape.PER_TENSOR,
pad_output: Optional[bool] = None,
pad_output: bool | None = None,
):
if current_platform.is_rocm():
self.preferred_backend = "rocm"
@@ -437,10 +437,10 @@ class Fp8LinearOp:
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: Optional[torch.dtype] = None,
input_scale: Optional[torch.Tensor] = None,
input_scale_ub: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
out_dtype: torch.dtype | None = None,
input_scale: torch.Tensor | None = None,
input_scale_ub: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
# ops.scaled_fp8_quant supports both dynamic and static quant.
# If dynamic, layer.input_scale is None and x_scale computed from x.
@@ -486,8 +486,8 @@ class Fp8LinearOp:
def normalize_e4m3fn_to_e4m3fnuz(
weight: torch.Tensor,
weight_scale: torch.Tensor,
input_scale: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
input_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
assert weight.dtype == torch.float8_e4m3fn
# The bits pattern 10000000(-128) represents zero in e4m3fn
# but NaN in e4m3fnuz. So here we set it to 0.