Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -63,6 +63,7 @@ def is_shared_expert_quant_fse_compatible(
|
||||
|
||||
Returns:
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
|
||||
"""
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
@@ -303,13 +304,11 @@ def is_equal_or_regex_match(
|
||||
check_contains: bool = False,
|
||||
use_fnmatch: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
"""Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
If use_fnmatch is set, supports shell-style patterns in target.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
|
||||
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path"""
|
||||
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path."""
|
||||
|
||||
import functools
|
||||
|
||||
@@ -37,8 +37,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
|
||||
gemm2_weights: torch.Tensor,
|
||||
gemm2_scales: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Prepare MxInt4 weights for TRT-LLM kernel.
|
||||
"""Prepare MxInt4 weights for TRT-LLM kernel.
|
||||
|
||||
Input:
|
||||
gemm1_weights: [num_experts, 2*intermediate_size, hidden_size//8] int32
|
||||
@@ -51,6 +50,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
|
||||
Returns:
|
||||
Dict with keys 'gemm1_weights', 'gemm1_scales', 'gemm2_weights',
|
||||
'gemm2_scales' containing shuffled/packed tensors ready for kernel
|
||||
|
||||
"""
|
||||
from flashinfer import block_scale_interleave
|
||||
from flashinfer.fused_moe import (
|
||||
@@ -192,8 +192,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
routing_method_type: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
"""Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
|
||||
Args:
|
||||
x: Input hidden states. dtype: bfloat16
|
||||
@@ -209,11 +208,14 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
ep_rank: Expert parallelism rank (default: 0)
|
||||
num_expert_group: Number of expert groups (default: None -> 0)
|
||||
topk_group: Top-k within groups (default: None -> 0)
|
||||
routing_replay_out: Optional buffer receiving the routing decisions
|
||||
so they can be replayed later
|
||||
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
||||
routing_method_type: FlashInfer RoutingMethodType enum value
|
||||
|
||||
Returns:
|
||||
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
||||
|
||||
"""
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
@@ -59,7 +59,7 @@ def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
|
||||
def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
|
||||
gemm1_weights: torch.Tensor, gemm2_weights: torch.Tensor, is_gated_activation: bool
|
||||
):
|
||||
"""Shuffle weights for FI TRT-LLM Format"""
|
||||
"""Shuffle weights for FI TRT-LLM Format."""
|
||||
from flashinfer import reorder_rows_for_gated_act_gemm, shuffle_matrix_a
|
||||
|
||||
epilogue_tile_m = 128
|
||||
@@ -210,7 +210,6 @@ def align_fp4_moe_weights_for_fi(
|
||||
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
|
||||
gate/up and down projection weights along the intermediate dim.
|
||||
"""
|
||||
|
||||
# Current local intermediate size (per partition) is the K dimension of
|
||||
# the down projection.
|
||||
num_experts, hidden_size, intermediate = w2.shape
|
||||
@@ -316,7 +315,6 @@ def align_moe_weights_for_fi(
|
||||
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
|
||||
gate/up and down projection weights along the intermediate dim.
|
||||
"""
|
||||
|
||||
# Current local intermediate size (per partition) is the K dimension of
|
||||
# the down projection.
|
||||
num_experts, hidden_size, intermediate = w2.shape
|
||||
@@ -453,8 +451,7 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
w2_input_scale: torch.Tensor | None,
|
||||
is_trtllm: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Convert Fp8 MoE weights to flashinfer kernel format
|
||||
"""Convert Fp8 MoE weights to flashinfer kernel format.
|
||||
|
||||
Note that for trtllm we update the model state dict
|
||||
with the scale format needed for these kernels.
|
||||
@@ -462,7 +459,6 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
Note that for per-tensor, we update the layer's
|
||||
intermediate size if the weights needed padding.
|
||||
"""
|
||||
|
||||
assert hasattr(layer.moe_config, "is_act_and_mul")
|
||||
block_quant = (
|
||||
hasattr(layer, "weight_block_size") and layer.weight_block_size is not None
|
||||
|
||||
@@ -69,7 +69,8 @@ def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values "
|
||||
"with tensor-wise quantization."""
|
||||
"with tensor-wise quantization.
|
||||
"""
|
||||
dtype = current_platform.fp8_dtype() if dtype is None else dtype
|
||||
finfo = torch.finfo(dtype)
|
||||
min_val, max_val = x.aminmax()
|
||||
@@ -347,12 +348,10 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
|
||||
"""
|
||||
Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
||||
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
||||
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
|
||||
the outputs tensors at the right positions.
|
||||
"""
|
||||
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
N_2 = N // 2
|
||||
@@ -421,8 +420,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
):
|
||||
"""
|
||||
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
||||
"""
|
||||
GROUP_SIZE = group_size
|
||||
@@ -564,18 +562,23 @@ def per_token_group_quant_fp8(
|
||||
"""Function to perform per-token-group quantization on an input tensor `x`.
|
||||
It converts the tensor values into signed float8 values and returns the
|
||||
quantized tensor along with the scaling factor used for quantization.
|
||||
|
||||
Args:
|
||||
x: The input tensor with ndim >= 2.
|
||||
group_size: The group size used for quantization.
|
||||
eps: The minimum to avoid dividing zero.
|
||||
dtype: The dtype of output tensor. Note that only `torch.float8_e4m3fn`
|
||||
is supported for now.
|
||||
is supported for now.
|
||||
column_major_scales: Outputs scales in column major.
|
||||
tma_aligned_scales: Outputs scales in TMA-aligned layout.
|
||||
out_q: Optional output tensor. If not provided, function will create.
|
||||
use_ue8m0: If True, round each scale down to a power of two (UE8M0
|
||||
format). None defers to the platform default.
|
||||
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
|
||||
scaling factor.
|
||||
|
||||
"""
|
||||
if use_ue8m0 is None:
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
@@ -695,6 +698,7 @@ def per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x_s_packed: Int32 tensor with logical shape
|
||||
[mn, ceil(num_groups_per_row / 4)], laid out with
|
||||
TMA-aligned stride along the packed-K dimension
|
||||
|
||||
"""
|
||||
if use_ue8m0 is None:
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
@@ -789,7 +793,6 @@ def _w8a8_triton_block_scaled_mm(
|
||||
product) on input tensors `A` and `B` with block-wise quantization, and
|
||||
store the result in output tensor `C`.
|
||||
"""
|
||||
|
||||
pid = tl.program_id(axis=0)
|
||||
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
@@ -842,14 +845,12 @@ def _w8a8_triton_block_scaled_mm(
|
||||
def get_w8a8_block_fp8_configs(
|
||||
N: int, K: int, block_n: int, block_k: int
|
||||
) -> dict[int, Any] | None:
|
||||
"""
|
||||
Return optimized configurations for the w8a8 block fp8 kernel.
|
||||
"""Return optimized configurations for the w8a8 block fp8 kernel.
|
||||
The return value will be a dictionary that maps an irregular grid of
|
||||
batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the
|
||||
kernel on a given batch size bs, the closest batch size in the grid should
|
||||
be picked and the associated configuration chosen to invoke the kernel.
|
||||
"""
|
||||
|
||||
# First look up if an optimized configuration is available in the configs
|
||||
# directory
|
||||
device_name = get_device_name_as_file_name()
|
||||
@@ -889,18 +890,20 @@ def w8a8_triton_block_scaled_mm(
|
||||
quantization.
|
||||
It takes two input tensors `A` and `B` with scales `As` and `Bs`.
|
||||
The output is returned in the specified `output_dtype`.
|
||||
|
||||
Args:
|
||||
A: The input tensor, e.g., activation.
|
||||
B: The input tensor, e.g., weight.
|
||||
As: The per-token-group quantization scale for `A`.
|
||||
Bs: The per-block quantization scale for `B`.
|
||||
block_size: The block size for per-block quantization. It should
|
||||
be 2-dim, e.g., [128, 128].
|
||||
output_dytpe: The dtype of the returned tensor.
|
||||
be 2-dim, e.g., [128, 128].
|
||||
output_dtype: The dtype of the returned tensor.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: The result of matmul.
|
||||
"""
|
||||
|
||||
"""
|
||||
_on_gfx1250 = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
@@ -1022,6 +1025,7 @@ def requant_weight_ue8m0_inplace(
|
||||
with shape `(..., M // block_size[0], K // block_size[1])`.
|
||||
block_size: 2-element iterable `[block_m, block_k]` describing the
|
||||
block quantisation granularity.
|
||||
|
||||
"""
|
||||
if weight.numel() == 0:
|
||||
return
|
||||
@@ -1191,7 +1195,8 @@ def prepare_fp8_moe_layer_for_deepgemm(
|
||||
|
||||
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Pad the weight tensor. This is an optimization on ROCm platform, which
|
||||
can benefit from tensors located far enough from one another in memory"""
|
||||
can benefit from tensors located far enough from one another in memory
|
||||
"""
|
||||
if (
|
||||
envs.VLLM_ROCM_FP8_PADDING
|
||||
and current_platform.is_rocm()
|
||||
@@ -1486,7 +1491,6 @@ def process_fp8_input_tensor_strategy_moe(
|
||||
enable_eplb: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Process moe input scales for tensor-wise quantization strategy."""
|
||||
|
||||
if not all_close_1d(w13_input_scale) or not all_close_1d(w2_input_scale):
|
||||
logger.info_once(
|
||||
"Found input_scales that are not equal for "
|
||||
|
||||
@@ -88,8 +88,7 @@ if has_humming():
|
||||
|
||||
|
||||
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
|
||||
"""
|
||||
Map humming group sizes to QuantKey GroupShape.
|
||||
"""Map humming group sizes to QuantKey GroupShape.
|
||||
|
||||
group_size: elements per group along K (col); 0 means full dimension.
|
||||
group_size_n: elements per group along N (row); 0 means 1 (per-row).
|
||||
@@ -298,7 +297,8 @@ def _humming_input_schema_to_quant_key(
|
||||
schema: "HummingInputSchema",
|
||||
) -> QuantKey | None:
|
||||
"""Convert a HummingInputSchema to a QuantKey. Returns None if
|
||||
the schema represents unquantized (bf16/fp16) inputs."""
|
||||
the schema represents unquantized (bf16/fp16) inputs.
|
||||
"""
|
||||
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
@@ -427,7 +427,8 @@ def convert_linear_layer_to_humming_standard(
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
|
||||
"""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
@@ -691,11 +692,9 @@ def select_humming_moe_experts(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> type[mk.FusedMoEExperts] | None:
|
||||
"""
|
||||
Select the primary Humming MoE Experts class
|
||||
"""Select the primary Humming MoE Experts class
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
if not has_humming():
|
||||
return None
|
||||
|
||||
@@ -809,14 +808,14 @@ def _replace_layer_parameters(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
preserve_bias: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Replace layer parameters for a sublayer with new tensors.
|
||||
"""Replace layer parameters for a sublayer with new tensors.
|
||||
|
||||
Args:
|
||||
layer: The RoutedExperts layer
|
||||
sublayer_name: Name of the sublayer (e.g., "w13", "w2")
|
||||
tensors: Dict of parameter name to tensor
|
||||
preserve_bias: If True, don't delete bias parameters
|
||||
|
||||
"""
|
||||
# Delete old parameters
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
@@ -843,11 +842,11 @@ def _convert_sublayer_to_humming(
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
) -> tuple[Any, Any]:
|
||||
"""
|
||||
Convert a sublayer's weights from checkpoint format to Humming format.
|
||||
"""Convert a sublayer's weights from checkpoint format to Humming format.
|
||||
|
||||
Returns:
|
||||
Tuple of (converted_weight_schema, converted_input_schema)
|
||||
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
@@ -931,8 +930,7 @@ def _process_single_sublayer(
|
||||
param_dtype: torch.dtype,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> tuple[Any, Any, "LayerConfig"]:
|
||||
"""
|
||||
Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
|
||||
This combines the common logic from convert_to_humming_moe_kernel_format
|
||||
for processing a single sublayer.
|
||||
@@ -951,6 +949,7 @@ def _process_single_sublayer(
|
||||
|
||||
Returns:
|
||||
Tuple of the final weight schema, input schema, and Humming layer config.
|
||||
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
@@ -1005,8 +1004,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
input_schema: Any | None = None,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> dict[str, "LayerConfig"]:
|
||||
"""
|
||||
Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
"""Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
|
||||
This function processes weights for each sublayer (w13, w2) by:
|
||||
1. Converting from checkpoint format to humming format if needed
|
||||
@@ -1032,8 +1030,8 @@ def convert_to_humming_moe_kernel_format(
|
||||
- Modifies layer parameters in place
|
||||
- Sets layer.weight_schemas and layer.input_schemas
|
||||
- Sets layer.humming_configs for quant config construction
|
||||
"""
|
||||
|
||||
"""
|
||||
# Build schemas from quant_config if not provided
|
||||
has_bias = layer.moe_config.has_bias
|
||||
num_experts = layer.moe_config.num_local_experts
|
||||
|
||||
@@ -178,6 +178,7 @@ def per_token_group_quant_int8(
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
|
||||
scaling factor for quantization.
|
||||
|
||||
"""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` cannot be divisible by `group_size`"
|
||||
|
||||
@@ -17,8 +17,7 @@ def query_machete_supported_quant_types(zero_points: bool) -> list[ScalarType]:
|
||||
|
||||
|
||||
def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
"""
|
||||
Queries the supported group sizes for Machete based on the activation type.
|
||||
"""Queries the supported group sizes for Machete based on the activation type.
|
||||
|
||||
Args:
|
||||
act_type: The activation data type (torch.float16, torch.bfloat16).
|
||||
@@ -27,6 +26,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
A list of supported group sizes. The group size must
|
||||
be divisible by `TileShapeK = 128 * 8 // num_bits(act_type)`.
|
||||
-1 indicates per-channel quantization.
|
||||
|
||||
"""
|
||||
if act_type in [torch.float16, torch.bfloat16]:
|
||||
return [-1, 64, 128]
|
||||
|
||||
@@ -245,7 +245,8 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
|
||||
|
||||
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
|
||||
"""Recover the (size_n, size_k) a Marlin weight was repacked with
|
||||
(including any tile padding) from its packed shape."""
|
||||
(including any tile padding) from its packed shape.
|
||||
"""
|
||||
pack_factor = 32 // num_bits
|
||||
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
|
||||
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
|
||||
@@ -256,7 +257,8 @@ def marlin_pad_qweight(
|
||||
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
|
||||
gptq_marlin_repack."""
|
||||
gptq_marlin_repack.
|
||||
"""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return qweight
|
||||
pack_factor = size_k // qweight.size(0)
|
||||
@@ -274,7 +276,8 @@ def marlin_pad_scales(
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad weight scales (num_groups, size_n); call before
|
||||
marlin_permute_scales and pass the padded extents to it."""
|
||||
marlin_permute_scales and pass the padded extents to it.
|
||||
"""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return scales
|
||||
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
|
||||
@@ -397,8 +400,7 @@ def check_moe_marlin_supports_layer(
|
||||
|
||||
|
||||
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
|
||||
"""
|
||||
Given Marlin packed weight matrices w1_packed, and w2_packed,
|
||||
"""Given Marlin packed weight matrices w1_packed, and w2_packed,
|
||||
return the MoE intermediate size N
|
||||
"""
|
||||
marlin_tile_size = 16
|
||||
|
||||
@@ -41,8 +41,8 @@ def _nvfp4_compute_scale_factor(
|
||||
) -> float:
|
||||
"""Compute the power-of-2 scale_factor needed so that all non-zero
|
||||
values in marlin_scales * 2^7 are >= 2 after rescaling.
|
||||
Returns a Python float (power of 2, >= 1.0)."""
|
||||
|
||||
Returns a Python float (power of 2, >= 1.0).
|
||||
"""
|
||||
# Since half has a smaller dynamic range compared to bfloat16,
|
||||
# no rescaling is applied here if active dtype is half.
|
||||
if a_dtype is not None and a_dtype == torch.half:
|
||||
@@ -76,9 +76,12 @@ def nvfp4_marlin_process_scales(
|
||||
given value is used directly. The caller is responsible for
|
||||
dividing ``global_scale`` by the returned ``scale_factor`` to
|
||||
preserve numerical correctness.
|
||||
a_dtype: dtype of the activations the scales will be used with. None
|
||||
keeps the dtype of ``marlin_scales``.
|
||||
|
||||
Returns:
|
||||
A tuple of (processed_scales, scale_factor).
|
||||
|
||||
"""
|
||||
if not (marlin_scales >= 0).all():
|
||||
logger.warning_once(
|
||||
@@ -373,7 +376,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w13(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
|
||||
tensor to padded_N rows."""
|
||||
tensor to padded_N rows.
|
||||
"""
|
||||
if padded_N == N:
|
||||
return x
|
||||
x = x.view(E, num_shards, N, x.size(-1))
|
||||
@@ -382,7 +386,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
|
||||
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
|
||||
tensor."""
|
||||
tensor.
|
||||
"""
|
||||
if padded_N == N:
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
|
||||
|
||||
@@ -217,7 +217,8 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
|
||||
"""
|
||||
if padded_n == n:
|
||||
return x
|
||||
e = x.size(0)
|
||||
@@ -241,14 +242,12 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Shuffle weights and scales into marlin format.
|
||||
"""Shuffle weights and scales into marlin format.
|
||||
|
||||
Note that this function has the side effect of adding a `workspace`
|
||||
attribute to the layer. This `workspace` does not need to be
|
||||
registered as a Parameter as it is not used during weight reloading.
|
||||
"""
|
||||
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP8 computation but "
|
||||
"FP8 quantization is being used. Weight-only FP8 compression will "
|
||||
@@ -372,9 +371,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
def pack_fp8_to_int32(
|
||||
fp8_tensor: torch.Tensor, size_k_first: bool = True
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Repack FP8 weights to gptq format (packed int32 elements)
|
||||
"""
|
||||
"""Repack FP8 weights to gptq format (packed int32 elements)."""
|
||||
assert fp8_tensor.dtype == torch.float8_e4m3fn
|
||||
assert fp8_tensor.ndim == 2
|
||||
|
||||
@@ -524,6 +521,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
|
||||
Returns:
|
||||
(w13, w2, w13_scale, w2_scale) in Marlin format.
|
||||
|
||||
"""
|
||||
group_size = 32
|
||||
e = w13.shape[0]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions used for tests and benchmarks"""
|
||||
"""Utility functions used for tests and benchmarks."""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
@@ -28,7 +28,7 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
|
||||
"""Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel."""
|
||||
assert has_triton_kernels()
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
from triton_kernels.numerics import InFlexData
|
||||
@@ -374,8 +374,7 @@ def downcast_to_mxfp(
|
||||
BLOCK_OUT_DIM: int = 128,
|
||||
BLOCK_QUANT_DIM: int = 32,
|
||||
):
|
||||
"""
|
||||
Convert the src weights to MXFP4. The src weight is quantized along the
|
||||
"""Convert the src weights to MXFP4. The src weight is quantized along the
|
||||
axis dimension into packed e2m1 values (torch.uint8, two values per
|
||||
byte), so the size of that dimension in the output is half of the
|
||||
logical (unpacked) size.
|
||||
|
||||
@@ -657,8 +657,7 @@ def ref_nvfp4_quant(x, global_scale, block_size):
|
||||
def ref_nvfp4_quant_dequant(
|
||||
x: torch.Tensor, global_scale: torch.Tensor, block_size: int
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
NVFP4 quantize-dequantize operation.
|
||||
"""NVFP4 quantize-dequantize operation.
|
||||
|
||||
`global_scale` is expected to have a single element.
|
||||
"""
|
||||
|
||||
@@ -11,18 +11,15 @@ from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Pad and block-interleave the FP4 block-scales so that they match the data
|
||||
"""Pad and block-interleave the FP4 block-scales so that they match the data
|
||||
layout expected by the CUTLASS / FlashInfer kernels.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
scale: torch.Tensor
|
||||
Args:
|
||||
scale: The FP4 block-scale tensor to swizzle.
|
||||
|
||||
Returns:
|
||||
The swizzled tensor with the same logical shape as `scale`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
torch.Tensor
|
||||
The swizzled tensor with the same logical shape as *scale*.
|
||||
"""
|
||||
assert scale.dtype == torch.float8_e4m3fn, (
|
||||
"swizzle_blockscale expects the input tensor to be in "
|
||||
@@ -65,8 +62,7 @@ def pad_nvfp4_weight_for_cutlass(
|
||||
weight: torch.Tensor,
|
||||
alignment: int = 32,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""
|
||||
Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
|
||||
"""Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
|
||||
the alignment constraints required by CUTLASS / FlashInfer FP4 kernels.
|
||||
|
||||
CUTLASS FP4 kernel requires both K and N matrix dimensions to be divisible
|
||||
@@ -99,8 +95,7 @@ def pad_nvfp4_weight_for_cutlass(
|
||||
|
||||
|
||||
def nvfp4_weight_padding_bytes(layer: torch.nn.Module) -> int:
|
||||
"""
|
||||
Compute K-dim padding (packed bytes) of an NVFP4 weight,
|
||||
"""Compute K-dim padding (packed bytes) of an NVFP4 weight,
|
||||
Aligned with `pad_nvfp4_weight_for_cutlass` above.
|
||||
"""
|
||||
return layer.weight.shape[1] - layer.input_size_per_partition // 2
|
||||
@@ -110,8 +105,7 @@ def pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4: torch.Tensor,
|
||||
weights_padding_bytes: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pad packed FP4 activations to match the K-dimension padding applied to weights.
|
||||
"""Pad packed FP4 activations to match the K-dimension padding applied to weights.
|
||||
The padding is in bytes (tensor dimension), not FP4 elements.
|
||||
"""
|
||||
if weights_padding_bytes > 0:
|
||||
@@ -123,9 +117,7 @@ def slice_nvfp4_output(
|
||||
out: torch.Tensor,
|
||||
output_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Slice the output tensor to remove padding in N dimension if weight was padded.
|
||||
"""
|
||||
"""Slice the output tensor to remove padding in N dimension if weight was padded."""
|
||||
if out.shape[-1] != output_size:
|
||||
return out[..., :output_size].contiguous()
|
||||
return out
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This file is used for /tests and /benchmarks"""
|
||||
"""This file is used for /tests and /benchmarks."""
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
@@ -109,8 +109,7 @@ class _GroupShape(NamedTuple):
|
||||
|
||||
|
||||
class GroupShape(_GroupShape):
|
||||
"""
|
||||
This class describes the quantization group shape.
|
||||
"""This class describes the quantization group shape.
|
||||
It includes static members for common shapes (per-tensor, per-token).
|
||||
"""
|
||||
|
||||
@@ -139,8 +138,7 @@ GroupShape.PER_CHANNEL = GroupShape(-1, 1)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScaleDesc:
|
||||
"""
|
||||
Class for describing a single quantization scaling factor.
|
||||
"""Class for describing a single quantization scaling factor.
|
||||
dtype: data type of the scale
|
||||
static: static scale if True, dynamic if False
|
||||
group_shape: group shape of the scale
|
||||
@@ -166,8 +164,7 @@ class ScaleDesc:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuantKey:
|
||||
"""
|
||||
Class for identifying the type of quantization.
|
||||
"""Class for identifying the type of quantization.
|
||||
dtype: quantized data type
|
||||
scale: scale descriptor
|
||||
scale2: second-level scale descriptor
|
||||
@@ -378,8 +375,7 @@ def prep_scale_for_group_broadcast(
|
||||
x: torch.Tensor,
|
||||
group_shape: GroupShape | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Prepare the input quantization scale for group broadcasting.
|
||||
"""Prepare the input quantization scale for group broadcasting.
|
||||
|
||||
Args:
|
||||
scale: The scale tensor (scalar or 1D).
|
||||
@@ -388,6 +384,7 @@ def prep_scale_for_group_broadcast(
|
||||
|
||||
Returns:
|
||||
scale reshaped for correct broadcasting.
|
||||
|
||||
"""
|
||||
if scale.numel() == 1:
|
||||
# For per-tensor quant, keep the scale as a scalar (not reshaped to (1, 1)).
|
||||
@@ -432,13 +429,13 @@ def scaled_quantize(
|
||||
quant_dtype: torch.dtype,
|
||||
compute_dtype: torch.dtype | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Args:
|
||||
x: Input tensor to quantize
|
||||
group_shape: Shape of quantization groups
|
||||
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
|
||||
compute_dtype: Optional dtype for intermediate computations.
|
||||
If None, uses input dtype. Use torch.float32 for higher precision.
|
||||
"""Args:
|
||||
x: Input tensor to quantize
|
||||
group_shape: Shape of quantization groups
|
||||
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
|
||||
compute_dtype: Optional dtype for intermediate computations.
|
||||
If None, uses input dtype. Use torch.float32 for higher precision.
|
||||
|
||||
"""
|
||||
group_shape = _normalize_quant_group_shape(x, group_shape)
|
||||
assert quant_dtype.is_floating_point, (
|
||||
@@ -506,7 +503,7 @@ def get_attribute_fallback(obj, attributes: list[str]):
|
||||
def get_and_maybe_dequant_weights(
|
||||
layer: "LinearBase", out_dtype: torch.dtype = torch.float32
|
||||
):
|
||||
"""Return layer's unquantized weights in [out, in] layout"""
|
||||
"""Return layer's unquantized weights in [out, in] layout."""
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
@@ -924,8 +921,7 @@ def awq_pack(
|
||||
def convert_bf16_scales_to_fp8(
|
||||
quant_fp8: Callable, scales: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
|
||||
"""Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
|
||||
expected by W4A8 GEMM kernels.
|
||||
"""
|
||||
assert scales.is_contiguous(), (
|
||||
@@ -949,9 +945,7 @@ def convert_bf16_scales_to_fp8(
|
||||
|
||||
|
||||
def convert_packed_uint4b8_to_signed_int4_inplace(t: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Convert int4b8 (packed to int32) to signed int4
|
||||
"""
|
||||
"""Convert int4b8 (packed to int32) to signed int4."""
|
||||
assert t.is_cuda, "tensor must be on gpu"
|
||||
assert t.dtype == torch.int32, f"expected int32 packed weights but got {t.dtype}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user