Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -63,6 +63,7 @@ def is_shared_expert_quant_fse_compatible(
Returns:
A compatibility flag and, when incompatible, the reason.
"""
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
@@ -303,13 +304,11 @@ def is_equal_or_regex_match(
check_contains: bool = False,
use_fnmatch: bool = False,
) -> bool:
"""
Checks whether a value is exactly equal or a regex match for target
"""Checks whether a value is exactly equal or a regex match for target
if target starts with 're:'. If check_contains is set to True,
additionally checks if the target string is contained within the value.
If use_fnmatch is set, supports shell-style patterns in target.
"""
if target.startswith("re:"):
pattern = target[3:]
if re.match(pattern, value):
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path."""
from typing import TYPE_CHECKING
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path"""
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path."""
import functools
@@ -37,8 +37,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
gemm2_weights: torch.Tensor,
gemm2_scales: torch.Tensor,
) -> dict[str, torch.Tensor]:
"""
Prepare MxInt4 weights for TRT-LLM kernel.
"""Prepare MxInt4 weights for TRT-LLM kernel.
Input:
gemm1_weights: [num_experts, 2*intermediate_size, hidden_size//8] int32
@@ -51,6 +50,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
Returns:
Dict with keys 'gemm1_weights', 'gemm1_scales', 'gemm2_weights',
'gemm2_scales' containing shuffled/packed tensors ready for kernel
"""
from flashinfer import block_scale_interleave
from flashinfer.fused_moe import (
@@ -192,8 +192,7 @@ def flashinfer_trtllm_mxint4_moe(
routing_method_type: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
"""Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
Args:
x: Input hidden states. dtype: bfloat16
@@ -209,11 +208,14 @@ def flashinfer_trtllm_mxint4_moe(
ep_rank: Expert parallelism rank (default: 0)
num_expert_group: Number of expert groups (default: None -> 0)
topk_group: Top-k within groups (default: None -> 0)
routing_replay_out: Optional buffer receiving the routing decisions
so they can be replayed later
e_score_correction_bias: Optional routing bias. dtype: bfloat16
routing_method_type: FlashInfer RoutingMethodType enum value
Returns:
Output tensor from MoE layer. dtype: same as x (bfloat16)
"""
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
@@ -59,7 +59,7 @@ def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
gemm1_weights: torch.Tensor, gemm2_weights: torch.Tensor, is_gated_activation: bool
):
"""Shuffle weights for FI TRT-LLM Format"""
"""Shuffle weights for FI TRT-LLM Format."""
from flashinfer import reorder_rows_for_gated_act_gemm, shuffle_matrix_a
epilogue_tile_m = 128
@@ -210,7 +210,6 @@ def align_fp4_moe_weights_for_fi(
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
gate/up and down projection weights along the intermediate dim.
"""
# Current local intermediate size (per partition) is the K dimension of
# the down projection.
num_experts, hidden_size, intermediate = w2.shape
@@ -316,7 +315,6 @@ def align_moe_weights_for_fi(
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
gate/up and down projection weights along the intermediate dim.
"""
# Current local intermediate size (per partition) is the K dimension of
# the down projection.
num_experts, hidden_size, intermediate = w2.shape
@@ -453,8 +451,7 @@ def prepare_fp8_moe_layer_for_fi(
w2_input_scale: torch.Tensor | None,
is_trtllm: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Convert Fp8 MoE weights to flashinfer kernel format
"""Convert Fp8 MoE weights to flashinfer kernel format.
Note that for trtllm we update the model state dict
with the scale format needed for these kernels.
@@ -462,7 +459,6 @@ def prepare_fp8_moe_layer_for_fi(
Note that for per-tensor, we update the layer's
intermediate size if the weights needed padding.
"""
assert hasattr(layer.moe_config, "is_act_and_mul")
block_quant = (
hasattr(layer, "weight_block_size") and layer.weight_block_size is not None
@@ -69,7 +69,8 @@ def input_to_float8(
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values "
"with tensor-wise quantization."""
"with tensor-wise quantization.
"""
dtype = current_platform.fp8_dtype() if dtype is None else dtype
finfo = torch.finfo(dtype)
min_val, max_val = x.aminmax()
@@ -347,12 +348,10 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
BLOCK_N: tl.constexpr,
):
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
"""
Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
the outputs tensors at the right positions.
"""
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
N_2 = N // 2
@@ -421,8 +420,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
alpha: float = 1.0,
beta: float = 0.0,
):
"""
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
"""
GROUP_SIZE = group_size
@@ -564,18 +562,23 @@ def per_token_group_quant_fp8(
"""Function to perform per-token-group quantization on an input tensor `x`.
It converts the tensor values into signed float8 values and returns the
quantized tensor along with the scaling factor used for quantization.
Args:
x: The input tensor with ndim >= 2.
group_size: The group size used for quantization.
eps: The minimum to avoid dividing zero.
dtype: The dtype of output tensor. Note that only `torch.float8_e4m3fn`
is supported for now.
is supported for now.
column_major_scales: Outputs scales in column major.
tma_aligned_scales: Outputs scales in TMA-aligned layout.
out_q: Optional output tensor. If not provided, function will create.
use_ue8m0: If True, round each scale down to a power of two (UE8M0
format). None defers to the platform default.
Returns:
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
scaling factor.
"""
if use_ue8m0 is None:
use_ue8m0 = is_deep_gemm_e8m0_used()
@@ -695,6 +698,7 @@ def per_token_group_quant_fp8_packed_for_deepgemm(
x_s_packed: Int32 tensor with logical shape
[mn, ceil(num_groups_per_row / 4)], laid out with
TMA-aligned stride along the packed-K dimension
"""
if use_ue8m0 is None:
use_ue8m0 = is_deep_gemm_e8m0_used()
@@ -789,7 +793,6 @@ def _w8a8_triton_block_scaled_mm(
product) on input tensors `A` and `B` with block-wise quantization, and
store the result in output tensor `C`.
"""
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
@@ -842,14 +845,12 @@ def _w8a8_triton_block_scaled_mm(
def get_w8a8_block_fp8_configs(
N: int, K: int, block_n: int, block_k: int
) -> dict[int, Any] | None:
"""
Return optimized configurations for the w8a8 block fp8 kernel.
"""Return optimized configurations for the w8a8 block fp8 kernel.
The return value will be a dictionary that maps an irregular grid of
batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the
kernel on a given batch size bs, the closest batch size in the grid should
be picked and the associated configuration chosen to invoke the kernel.
"""
# First look up if an optimized configuration is available in the configs
# directory
device_name = get_device_name_as_file_name()
@@ -889,18 +890,20 @@ def w8a8_triton_block_scaled_mm(
quantization.
It takes two input tensors `A` and `B` with scales `As` and `Bs`.
The output is returned in the specified `output_dtype`.
Args:
A: The input tensor, e.g., activation.
B: The input tensor, e.g., weight.
As: The per-token-group quantization scale for `A`.
Bs: The per-block quantization scale for `B`.
block_size: The block size for per-block quantization. It should
be 2-dim, e.g., [128, 128].
output_dytpe: The dtype of the returned tensor.
be 2-dim, e.g., [128, 128].
output_dtype: The dtype of the returned tensor.
Returns:
torch.Tensor: The result of matmul.
"""
"""
_on_gfx1250 = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx1250
@@ -1022,6 +1025,7 @@ def requant_weight_ue8m0_inplace(
with shape `(..., M // block_size[0], K // block_size[1])`.
block_size: 2-element iterable `[block_m, block_k]` describing the
block quantisation granularity.
"""
if weight.numel() == 0:
return
@@ -1191,7 +1195,8 @@ def prepare_fp8_moe_layer_for_deepgemm(
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
"""Pad the weight tensor. This is an optimization on ROCm platform, which
can benefit from tensors located far enough from one another in memory"""
can benefit from tensors located far enough from one another in memory
"""
if (
envs.VLLM_ROCM_FP8_PADDING
and current_platform.is_rocm()
@@ -1486,7 +1491,6 @@ def process_fp8_input_tensor_strategy_moe(
enable_eplb: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Process moe input scales for tensor-wise quantization strategy."""
if not all_close_1d(w13_input_scale) or not all_close_1d(w2_input_scale):
logger.info_once(
"Found input_scales that are not equal for "
@@ -88,8 +88,7 @@ if has_humming():
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
"""
Map humming group sizes to QuantKey GroupShape.
"""Map humming group sizes to QuantKey GroupShape.
group_size: elements per group along K (col); 0 means full dimension.
group_size_n: elements per group along N (row); 0 means 1 (per-row).
@@ -298,7 +297,8 @@ def _humming_input_schema_to_quant_key(
schema: "HummingInputSchema",
) -> QuantKey | None:
"""Convert a HummingInputSchema to a QuantKey. Returns None if
the schema represents unquantized (bf16/fp16) inputs."""
the schema represents unquantized (bf16/fp16) inputs.
"""
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
@@ -427,7 +427,8 @@ def convert_linear_layer_to_humming_standard(
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
"""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
@@ -691,11 +692,9 @@ def select_humming_moe_experts(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> type[mk.FusedMoEExperts] | None:
"""
Select the primary Humming MoE Experts class
"""Select the primary Humming MoE Experts class
Note: Shape-specific fallbacks may still occur at runtime.
"""
if not has_humming():
return None
@@ -809,14 +808,14 @@ def _replace_layer_parameters(
tensors: dict[str, torch.Tensor],
preserve_bias: bool = False,
) -> None:
"""
Replace layer parameters for a sublayer with new tensors.
"""Replace layer parameters for a sublayer with new tensors.
Args:
layer: The RoutedExperts layer
sublayer_name: Name of the sublayer (e.g., "w13", "w2")
tensors: Dict of parameter name to tensor
preserve_bias: If True, don't delete bias parameters
"""
# Delete old parameters
for name, _ in list(layer.named_parameters()):
@@ -843,11 +842,11 @@ def _convert_sublayer_to_humming(
num_experts: int,
param_dtype: torch.dtype,
) -> tuple[Any, Any]:
"""
Convert a sublayer's weights from checkpoint format to Humming format.
"""Convert a sublayer's weights from checkpoint format to Humming format.
Returns:
Tuple of (converted_weight_schema, converted_input_schema)
"""
from vllm.utils.humming import HummingWeightSchema
@@ -931,8 +930,7 @@ def _process_single_sublayer(
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
) -> tuple[Any, Any, "LayerConfig"]:
"""
Process a single sublayer: convert, optionally requant, prepare, and transform.
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
This combines the common logic from convert_to_humming_moe_kernel_format
for processing a single sublayer.
@@ -951,6 +949,7 @@ def _process_single_sublayer(
Returns:
Tuple of the final weight schema, input schema, and Humming layer config.
"""
from vllm.utils.humming import HummingWeightSchema
@@ -1005,8 +1004,7 @@ def convert_to_humming_moe_kernel_format(
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
) -> dict[str, "LayerConfig"]:
"""
Convert MoE weights from checkpoint format to Humming kernel format.
"""Convert MoE weights from checkpoint format to Humming kernel format.
This function processes weights for each sublayer (w13, w2) by:
1. Converting from checkpoint format to humming format if needed
@@ -1032,8 +1030,8 @@ def convert_to_humming_moe_kernel_format(
- Modifies layer parameters in place
- Sets layer.weight_schemas and layer.input_schemas
- Sets layer.humming_configs for quant config construction
"""
"""
# Build schemas from quant_config if not provided
has_bias = layer.moe_config.has_bias
num_experts = layer.moe_config.num_local_experts
@@ -178,6 +178,7 @@ def per_token_group_quant_int8(
Returns:
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
scaling factor for quantization.
"""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` cannot be divisible by `group_size`"
@@ -17,8 +17,7 @@ def query_machete_supported_quant_types(zero_points: bool) -> list[ScalarType]:
def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
"""
Queries the supported group sizes for Machete based on the activation type.
"""Queries the supported group sizes for Machete based on the activation type.
Args:
act_type: The activation data type (torch.float16, torch.bfloat16).
@@ -27,6 +26,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
A list of supported group sizes. The group size must
be divisible by `TileShapeK = 128 * 8 // num_bits(act_type)`.
-1 indicates per-channel quantization.
"""
if act_type in [torch.float16, torch.bfloat16]:
return [-1, 64, 128]
@@ -245,7 +245,8 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
"""Recover the (size_n, size_k) a Marlin weight was repacked with
(including any tile padding) from its packed shape."""
(including any tile padding) from its packed shape.
"""
pack_factor = 32 // num_bits
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
@@ -256,7 +257,8 @@ def marlin_pad_qweight(
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
) -> torch.Tensor:
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
gptq_marlin_repack."""
gptq_marlin_repack.
"""
if (padded_n, padded_k) == (size_n, size_k):
return qweight
pack_factor = size_k // qweight.size(0)
@@ -274,7 +276,8 @@ def marlin_pad_scales(
group_size: int,
) -> torch.Tensor:
"""Zero-pad weight scales (num_groups, size_n); call before
marlin_permute_scales and pass the padded extents to it."""
marlin_permute_scales and pass the padded extents to it.
"""
if (padded_n, padded_k) == (size_n, size_k):
return scales
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
@@ -397,8 +400,7 @@ def check_moe_marlin_supports_layer(
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
"""
Given Marlin packed weight matrices w1_packed, and w2_packed,
"""Given Marlin packed weight matrices w1_packed, and w2_packed,
return the MoE intermediate size N
"""
marlin_tile_size = 16
@@ -41,8 +41,8 @@ def _nvfp4_compute_scale_factor(
) -> float:
"""Compute the power-of-2 scale_factor needed so that all non-zero
values in marlin_scales * 2^7 are >= 2 after rescaling.
Returns a Python float (power of 2, >= 1.0)."""
Returns a Python float (power of 2, >= 1.0).
"""
# Since half has a smaller dynamic range compared to bfloat16,
# no rescaling is applied here if active dtype is half.
if a_dtype is not None and a_dtype == torch.half:
@@ -76,9 +76,12 @@ def nvfp4_marlin_process_scales(
given value is used directly. The caller is responsible for
dividing ``global_scale`` by the returned ``scale_factor`` to
preserve numerical correctness.
a_dtype: dtype of the activations the scales will be used with. None
keeps the dtype of ``marlin_scales``.
Returns:
A tuple of (processed_scales, scale_factor).
"""
if not (marlin_scales >= 0).all():
logger.warning_once(
@@ -373,7 +376,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w13(x: torch.Tensor) -> torch.Tensor:
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
tensor to padded_N rows."""
tensor to padded_N rows.
"""
if padded_N == N:
return x
x = x.view(E, num_shards, N, x.size(-1))
@@ -382,7 +386,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
tensor."""
tensor.
"""
if padded_N == N:
return x
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
@@ -217,7 +217,8 @@ def prepare_fp8_layer_for_marlin(
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
"""
if padded_n == n:
return x
e = x.size(0)
@@ -241,14 +242,12 @@ def prepare_fp8_moe_layer_for_marlin(
w13_weight_scale: torch.Tensor,
w2_weight_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Shuffle weights and scales into marlin format.
"""Shuffle weights and scales into marlin format.
Note that this function has the side effect of adding a `workspace`
attribute to the layer. This `workspace` does not need to be
registered as a Parameter as it is not used during weight reloading.
"""
logger.warning_once(
"Your GPU does not have native support for FP8 computation but "
"FP8 quantization is being used. Weight-only FP8 compression will "
@@ -372,9 +371,7 @@ def prepare_fp8_moe_layer_for_marlin(
def pack_fp8_to_int32(
fp8_tensor: torch.Tensor, size_k_first: bool = True
) -> torch.Tensor:
"""
Repack FP8 weights to gptq format (packed int32 elements)
"""
"""Repack FP8 weights to gptq format (packed int32 elements)."""
assert fp8_tensor.dtype == torch.float8_e4m3fn
assert fp8_tensor.ndim == 2
@@ -524,6 +521,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
Returns:
(w13, w2, w13_scale, w2_scale) in Marlin format.
"""
group_size = 32
e = w13.shape[0]
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions used for tests and benchmarks"""
"""Utility functions used for tests and benchmarks."""
import numpy as np
import torch
@@ -28,7 +28,7 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
"""Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel."""
assert has_triton_kernels()
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
from triton_kernels.numerics import InFlexData
@@ -374,8 +374,7 @@ def downcast_to_mxfp(
BLOCK_OUT_DIM: int = 128,
BLOCK_QUANT_DIM: int = 32,
):
"""
Convert the src weights to MXFP4. The src weight is quantized along the
"""Convert the src weights to MXFP4. The src weight is quantized along the
axis dimension into packed e2m1 values (torch.uint8, two values per
byte), so the size of that dimension in the output is half of the
logical (unpacked) size.
@@ -657,8 +657,7 @@ def ref_nvfp4_quant(x, global_scale, block_size):
def ref_nvfp4_quant_dequant(
x: torch.Tensor, global_scale: torch.Tensor, block_size: int
) -> torch.Tensor:
"""
NVFP4 quantize-dequantize operation.
"""NVFP4 quantize-dequantize operation.
`global_scale` is expected to have a single element.
"""
@@ -11,18 +11,15 @@ from vllm.utils.math_utils import round_up
def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
"""
Pad and block-interleave the FP4 block-scales so that they match the data
"""Pad and block-interleave the FP4 block-scales so that they match the data
layout expected by the CUTLASS / FlashInfer kernels.
Parameters
----------
scale: torch.Tensor
Args:
scale: The FP4 block-scale tensor to swizzle.
Returns:
The swizzled tensor with the same logical shape as `scale`.
Returns
-------
torch.Tensor
The swizzled tensor with the same logical shape as *scale*.
"""
assert scale.dtype == torch.float8_e4m3fn, (
"swizzle_blockscale expects the input tensor to be in "
@@ -65,8 +62,7 @@ def pad_nvfp4_weight_for_cutlass(
weight: torch.Tensor,
alignment: int = 32,
) -> tuple[torch.Tensor, int]:
"""
Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
"""Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
the alignment constraints required by CUTLASS / FlashInfer FP4 kernels.
CUTLASS FP4 kernel requires both K and N matrix dimensions to be divisible
@@ -99,8 +95,7 @@ def pad_nvfp4_weight_for_cutlass(
def nvfp4_weight_padding_bytes(layer: torch.nn.Module) -> int:
"""
Compute K-dim padding (packed bytes) of an NVFP4 weight,
"""Compute K-dim padding (packed bytes) of an NVFP4 weight,
Aligned with `pad_nvfp4_weight_for_cutlass` above.
"""
return layer.weight.shape[1] - layer.input_size_per_partition // 2
@@ -110,8 +105,7 @@ def pad_nvfp4_activation_for_cutlass(
x_fp4: torch.Tensor,
weights_padding_bytes: int,
) -> torch.Tensor:
"""
Pad packed FP4 activations to match the K-dimension padding applied to weights.
"""Pad packed FP4 activations to match the K-dimension padding applied to weights.
The padding is in bytes (tensor dimension), not FP4 elements.
"""
if weights_padding_bytes > 0:
@@ -123,9 +117,7 @@ def slice_nvfp4_output(
out: torch.Tensor,
output_size: int,
) -> torch.Tensor:
"""
Slice the output tensor to remove padding in N dimension if weight was padded.
"""
"""Slice the output tensor to remove padding in N dimension if weight was padded."""
if out.shape[-1] != output_size:
return out[..., :output_size].contiguous()
return out
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file is used for /tests and /benchmarks"""
"""This file is used for /tests and /benchmarks."""
from collections.abc import Callable, Mapping
from dataclasses import dataclass
@@ -109,8 +109,7 @@ class _GroupShape(NamedTuple):
class GroupShape(_GroupShape):
"""
This class describes the quantization group shape.
"""This class describes the quantization group shape.
It includes static members for common shapes (per-tensor, per-token).
"""
@@ -139,8 +138,7 @@ GroupShape.PER_CHANNEL = GroupShape(-1, 1)
@dataclass(frozen=True)
class ScaleDesc:
"""
Class for describing a single quantization scaling factor.
"""Class for describing a single quantization scaling factor.
dtype: data type of the scale
static: static scale if True, dynamic if False
group_shape: group shape of the scale
@@ -166,8 +164,7 @@ class ScaleDesc:
@dataclass(frozen=True)
class QuantKey:
"""
Class for identifying the type of quantization.
"""Class for identifying the type of quantization.
dtype: quantized data type
scale: scale descriptor
scale2: second-level scale descriptor
@@ -378,8 +375,7 @@ def prep_scale_for_group_broadcast(
x: torch.Tensor,
group_shape: GroupShape | None,
) -> torch.Tensor:
"""
Prepare the input quantization scale for group broadcasting.
"""Prepare the input quantization scale for group broadcasting.
Args:
scale: The scale tensor (scalar or 1D).
@@ -388,6 +384,7 @@ def prep_scale_for_group_broadcast(
Returns:
scale reshaped for correct broadcasting.
"""
if scale.numel() == 1:
# For per-tensor quant, keep the scale as a scalar (not reshaped to (1, 1)).
@@ -432,13 +429,13 @@ def scaled_quantize(
quant_dtype: torch.dtype,
compute_dtype: torch.dtype | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Args:
x: Input tensor to quantize
group_shape: Shape of quantization groups
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
compute_dtype: Optional dtype for intermediate computations.
If None, uses input dtype. Use torch.float32 for higher precision.
"""Args:
x: Input tensor to quantize
group_shape: Shape of quantization groups
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
compute_dtype: Optional dtype for intermediate computations.
If None, uses input dtype. Use torch.float32 for higher precision.
"""
group_shape = _normalize_quant_group_shape(x, group_shape)
assert quant_dtype.is_floating_point, (
@@ -506,7 +503,7 @@ def get_attribute_fallback(obj, attributes: list[str]):
def get_and_maybe_dequant_weights(
layer: "LinearBase", out_dtype: torch.dtype = torch.float32
):
"""Return layer's unquantized weights in [out, in] layout"""
"""Return layer's unquantized weights in [out, in] layout."""
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
@@ -924,8 +921,7 @@ def awq_pack(
def convert_bf16_scales_to_fp8(
quant_fp8: Callable, scales: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
"""Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
expected by W4A8 GEMM kernels.
"""
assert scales.is_contiguous(), (
@@ -949,9 +945,7 @@ def convert_bf16_scales_to_fp8(
def convert_packed_uint4b8_to_signed_int4_inplace(t: torch.Tensor) -> torch.Tensor:
"""
Convert int4b8 (packed to int32) to signed int4
"""
"""Convert int4b8 (packed to int32) to signed int4."""
assert t.is_cuda, "tensor must be on gpu"
assert t.dtype == torch.int32, f"expected int32 packed weights but got {t.dtype}"