[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
+1 -2
View File
@@ -804,8 +804,7 @@ _ACTIVATION_REGISTRY = LazyDict(
def _get_gelu_pytorch_tanh() -> nn.Module:
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
and fast GELU for ARM.
"""
and fast GELU for ARM."""
if current_platform.is_rocm():
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
logger.warning_once(
@@ -1568,8 +1568,7 @@ class MLACommonBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack an inline fp32 scale pair after the
latent data (single-sided: ``head_size_v == 0`` for MLA).
"""
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
spec = super().customize_spec(spec)
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
@@ -55,8 +55,7 @@ _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.
"""
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
@@ -170,8 +170,7 @@ def _fused_embed_eh_norm_kernel(
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
cat. Requires the full table on-rank (replicated embedding).
"""
cat. Requires the full table on-rank (replicated embedding)."""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
@@ -56,8 +56,7 @@ class MoEActivation(Enum):
@property
def custom_op_name(self) -> str:
"""Maps to the CustomOp name of activations
in vllm/model_executor/layers/activation.py.
"""
in vllm/model_executor/layers/activation.py."""
return _CUSTOM_OP_NAMES[self]
def without_mul(self) -> "MoEActivation":
@@ -36,8 +36,7 @@ def is_aiter_mxfp8_moe_available() -> bool:
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
closed (returns False -> triton dot_scaled), which is always safe.
"""
closed (returns False -> triton dot_scaled), which is always safe."""
if not (current_platform.is_rocm() and current_platform.supports_mx()):
return False
try:
@@ -192,8 +192,7 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
"""Zero-pad the per-partition MoE intermediate dim of both weights and
the expert bias, see `_padded_intermediate_size`.
"""
the expert bias, see `_padded_intermediate_size`."""
intermediate_size = self.moe_config.intermediate_size_per_partition
padded_size = self._padded_intermediate_size(self.moe_config)
if padded_size == intermediate_size:
@@ -953,8 +952,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
modular MoE experts.
"""
modular MoE experts."""
def __init__(
self,
@@ -1287,8 +1285,7 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
quantization, dispatched through zentorch.
"""
quantization, dispatched through zentorch."""
def __init__(
self,
@@ -34,8 +34,7 @@ from vllm.triton_utils import tl, triton
def _select_cfg(M, N, K, block_m):
"""Pick the launch config from host constants only (M=num_valid_tokens, N, K,
block_m) — graph-capture safe (no GPU-scalar branch).
"""
block_m) — graph-capture safe (no GPU-scalar branch)."""
# Per-regime winners (measured, isolated cuda-event A/B on gfx950, GPU 3):
# BLOCK_K=256 (fewer K-iters + bigger MX scale-load coalesced with the dot),
# num_stages=2 (software-pipeline overlaps the E8M0 scale-load with the scaled
@@ -55,28 +55,24 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
@abstractmethod
def backend_enum_cls(self) -> type[BackendT]:
"""Return the concrete `Enum` class enumerating this oracle's
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).
"""
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
@abstractmethod
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
"""Return platform-appropriate backends in priority order for
this `moe_config`.
"""
this `moe_config`."""
@abstractmethod
def backend_to_kernel_cls(
self, backend: BackendT
) -> list[type[mk.FusedMoEExperts]]:
"""Map a backend enum value to its concrete `FusedMoEExperts`
subclasses, in selection priority order.
"""
subclasses, in selection priority order."""
@abstractmethod
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
"""Map a user-facing `MoEBackend` (from the runner config) to
this oracle's enum.
"""
this oracle's enum."""
@abstractmethod
def select_backend(
@@ -108,8 +104,7 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
combinator) for the chosen backend.
"""
combinator) for the chosen backend."""
def convert_to_kernel_format(
self,
@@ -451,8 +451,7 @@ def _humming_fp8_weight_schema(
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
tensors (scale dtype/shape, block size), not the producing quant method.
"""
tensors (scale dtype/shape, block size), not the producing quant method."""
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
@@ -224,8 +224,7 @@ def _humming_int8_weight_schema(
weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming compressed-tensors int8 schema from the canonical
on-device tensors; humming does the signed-int8 -> native conversion.
"""
on-device tensors; humming does the signed-int8 -> native conversion."""
config: dict[str, Any] = {
"quant_method": "compressed-tensors",
"format": "int-quantized",
@@ -512,8 +512,7 @@ def _process_weights_flashinfer(
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
tensor along its last dim, from ``unit`` to ``padded_unit`` columns.
"""
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
if padded_unit == unit:
return x
e, rows, _ = x.shape
@@ -1081,8 +1080,7 @@ def _humming_wna16_weight_schema(
quant_config: QuantizationConfig | QuantizationArgs | None,
) -> dict[str, Any]:
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
config rather than the running kernel.
"""
config rather than the running kernel."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
@@ -331,8 +331,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
"""Available backends in priority order, BF16-act variant before
activation-quantized variant within each vendor family.
"""
activation-quantized variant within each vendor family."""
_AVAILABLE_BACKENDS = [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
@@ -411,8 +410,7 @@ def _resolve_activation_key(
model_activation_key: QuantKey | None,
) -> QuantKey | None:
"""Combine the model-supplied activation key with the user override.
Raises on conflict (both set and disagreeing).
"""
Raises on conflict (both set and disagreeing)."""
user_override = _user_moe_activation_override()
if user_override is None:
return model_activation_key
@@ -462,8 +460,7 @@ def _filter_by_activation(
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land.
"""
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
if requested_activation_key is not None:
return [
b
@@ -71,8 +71,7 @@ class ZeroExpertRouter(BaseRouter):
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute routing with full bias, compute zero expert output,
mask zero expert IDs.
"""
mask zero expert IDs."""
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=router_logits,
+3 -6
View File
@@ -163,8 +163,7 @@ class SparseIndexerTopk(torch.nn.Module):
) -> str:
"""Resolve the decode top-k implementation from the configured
backend ("auto" = the pre-existing chain, or a validated explicit
value).
"""
value)."""
if self._backend == "auto":
return self._resolve_auto(logits, topk_tokens, num_rows)
@@ -211,8 +210,7 @@ class SparseIndexerTopk(torch.nn.Module):
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The pre-existing priority chain: cooperative -> persistent ->
per_row. deep_select/flashinfer/torch are opt-in only.
"""
per_row. deep_select/flashinfer/torch are opt-in only."""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
if self._is_cuda and topk_tokens in (512, 1024, 2048):
@@ -274,8 +272,7 @@ class SparseIndexerTopk(torch.nn.Module):
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
+1 -2
View File
@@ -158,8 +158,7 @@ class LinearMethodBase(QuantizeMethodBase):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply the weights in layer to the input tensor.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
raise NotImplementedError
@@ -861,8 +861,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""ROCm forward using AITER Triton fused projection+attention when
available, otherwise falling back to the generic CUDA path.
"""
available, otherwise falling back to the generic CUDA path."""
if GDN_AITER_TRITON_AVAILABLE:
num_tokens = hidden_states.size(0)
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
@@ -261,8 +261,7 @@ class MambaStateShapeCalculator:
@classmethod
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
"""Compute the increase in group numbers to account for
replication in order to accompany the head shards.
"""
replication in order to accompany the head shards."""
# in the case ngoups % tp_size == 0, this will be zero
if ngroups % tp_size == 0:
return 0
@@ -59,8 +59,7 @@ def get_ssm_device_name() -> str:
def _canonical_cache_dtype(cache_dtype: str) -> str:
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
configs because the kernel only sees bit width when accessing state.
"""
configs because the kernel only sees bit width when accessing state."""
return "float16" if cache_dtype == "bfloat16" else cache_dtype
@@ -161,8 +160,7 @@ _ssm_config_override: tuple[int, int] | None = None
@contextmanager
def override_ssm_config(config: tuple[int, int]):
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
the context. Used by the tuning benchmark to time specific configs.
"""
the context. Used by the tuning benchmark to time specific configs."""
global _ssm_config_override
prev = _ssm_config_override
_ssm_config_override = config
@@ -210,8 +210,7 @@ class LamportWorkspace:
@property
def workspace(self) -> torch.Tensor:
"""Device tensor (int64) that can be passed to the kernel
as ``void** workspace``.
"""
as ``void** workspace``."""
return self._workspace
# ------------------------------------------------------------------
@@ -46,24 +46,21 @@ class QuantizeMethodBase(ABC):
):
"""Create weights for a layer.
The weights will be set as attributes of the layer.
"""
The weights will be set as attributes of the layer."""
raise NotImplementedError
@abstractmethod
def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
"""Apply the weights in layer to the input tensor.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
raise NotImplementedError
# Not required functions
def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
"""Gather embeddings in the layer based on indices in the input tensor.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
raise NotImplementedError
# Not required functions
@@ -76,8 +73,7 @@ class QuantizeMethodBase(ABC):
Quantization methods that need special weight handling (e.g. repacked
weights) override this.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
layer.weight = embed_tokens.weight
return layer
@@ -665,8 +665,7 @@ class CompressedTensorsConfig(QuantizationConfig):
) -> bool:
"""Weight N-bit INT (pack-quantized for sub-byte, int-quantized for 8-bit)
with static per-tensor INT8 input/output activation quant, applied as a float
fake-quant around a weight-only matmul.
"""
fake-quant around a weight-only matmul."""
is_int_pack_format = format in (
CompressionFormat.pack_quantized.value,
CompressionFormat.int_quantized.value,
@@ -702,8 +701,7 @@ class CompressedTensorsConfig(QuantizationConfig):
format: str | None,
) -> bool:
"""Weight N-bit INT with symmetric dynamic INT activation quant
via Humming kernel.
"""
via Humming kernel."""
if input_quant is None:
return False
is_pack_format = format == CompressionFormat.pack_quantized.value
@@ -37,8 +37,7 @@ def _dequant_gather_kernel(
BLOCK: tl.constexpr,
):
"""Gather embedding rows by token id, unpack int32-packed INT weights, and
dequantize to ``out`` dtype in one pass (no int8 intermediate).
"""
dequantize to ``out`` dtype in one pass (no int8 intermediate)."""
row = tl.program_id(0)
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
col_mask = col < hidden
@@ -211,8 +211,7 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
def _pack_int_quantized_weight(self, layer: torch.nn.Module) -> None:
"""Normalize an int-quantized (plain int8) weight to the canonical
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect.
"""
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect."""
weight = layer.weight
out_features, in_features = weight.shape
packed = pack_to_int32(weight.data.contiguous(), self.num_bits)
@@ -86,8 +86,7 @@ class INCConfigParser:
def is_explicitly_configured(name: str) -> bool:
"""Return True if *name* has an explicit entry in extra_config,
either via exact key match or via a regex pattern key.
"""
either via exact key match or via a regex pattern key."""
if not self._config.extra_config:
return False
if name in self._config.extra_config:
@@ -48,8 +48,7 @@ class INCScheme(ABC):
layer_config: "INCLayerConfig",
) -> "FusedMoEMethodBase | None":
"""Optional. Override if this scheme supports MoE.
Default raises NotImplementedError.
"""
Default raises NotImplementedError."""
raise NotImplementedError(
f"{type(self).__name__} does not support MoE layers. "
f"Layer config: {layer_config}"
@@ -1988,8 +1988,7 @@ class KNvfp4Static(QuantKeyScheme):
class KNvfp4Dynamic(QuantKeyScheme):
"""NVFP4 activation scheme (W4A4). Has a static global input scale on disk;
the per-group scale is computed at runtime inside the kernel.
"""
the per-group scale is computed at runtime inside the kernel."""
key = kNvfp4Dynamic
@@ -2026,8 +2025,7 @@ class KNvfp4Dynamic(QuantKeyScheme):
class KFp8StaticTensor(QuantKeyScheme):
"""Plain per-tensor static FP8 — bivalent: serves BOTH the weight slot and
the activation slot (W8A8). One key in both QuantSpec slots.
"""
the activation slot (W8A8). One key in both QuantSpec slots."""
key = kFp8StaticTensorSym
@@ -2097,8 +2095,7 @@ class KFp8StaticTensor(QuantKeyScheme):
class KFp8StaticChannel(QuantKeyScheme):
"""Per-channel static FP8 weight (the 'PcPt' weight). Weight-role only —
there is no static per-channel *activation* today.
"""
there is no static per-channel *activation* today."""
key = kFp8StaticTokenSym
@@ -2139,8 +2136,7 @@ class KFp8StaticChannel(QuantKeyScheme):
class KFp8Block128(QuantKeyScheme):
"""128x128 block-static FP8 weight ('PbWo'). Weight-role only. ModelOpt
exports the scale 4-D [out_blk,1,in_blk,1]; process squeezes to 2-D.
No transpose (block kernel keeps [out,in]).
"""
No transpose (block kernel keeps [out,in])."""
key = kFp8Static128BlockSym
@@ -2196,8 +2192,7 @@ class KFp8Block128(QuantKeyScheme):
class KMxfp8Static(QuantKeyScheme):
"""MXFP8 weight: fp8-e4m3 values + per-32-block e8m0 (uint8) scale.
Weight-role only. process is validate-only plus an idempotency guard.
"""
Weight-role only. process is validate-only plus an idempotency guard."""
key = kMxfp8Static
@@ -2272,8 +2267,7 @@ class KDynamicNoParam(QuantKeyScheme):
"""Dynamic activation with no stored scale (W8A8): quantized at runtime in
the kernel. NOT the same as activation=None (weight-only) — init_fp8 needs a
non-None activation key. Activation-role only. Serves the fp8 per-token, fp8
per-block, and mxfp8 dynamic activation keys.
"""
per-block, and mxfp8 dynamic activation keys."""
def create_weights(self, layer, role, ctx, shapes, wl) -> None:
if role is not ACT:
@@ -79,8 +79,7 @@ class Mxfp4Config(QuantizationConfig):
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
"""MoE method for RoutedExperts. Subclasses override to pick a
checkpoint-specific kernel family.
"""
checkpoint-specific kernel family."""
return Mxfp4MoEMethod(moe)
def get_quant_method(
@@ -139,8 +139,7 @@ def _find_matching_targets(
class OnlineQuantizationConfig(QuantizationConfig):
"""Model-level config for online quantization (quantize fp16/bf16 weights
during model loading, without requiring a pre-quantized checkpoint).
"""
during model loading, without requiring a pre-quantized checkpoint)."""
def __init__(
self,
@@ -118,8 +118,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time.
"""
weights onto meta device and materializes them just-in-time."""
uses_meta_device: bool = True
@@ -163,8 +162,7 @@ class OnlineLinearBase(LinearMethodBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
"""
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
def __init__(self):
super().__init__()
@@ -265,8 +263,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading.
"""
Loads fp16/bf16 weights and quantizes them per-block during loading."""
def __init__(self):
super().__init__()
@@ -440,8 +437,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
"""Shared base for online FP8 MoE methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time.
"""
weights onto meta device and materializes them just-in-time."""
# Declared here for mypy; actual values are set in __init__.
fp8_backend: "Fp8MoeBackend"
@@ -562,8 +558,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online tensorwise FP8 MoE quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
"""
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
def __init__(
self,
@@ -620,8 +615,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online blockwise FP8 MoE quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading.
"""
Loads fp16/bf16 weights and quantizes them per-block during loading."""
def __init__(
self,
@@ -69,8 +69,7 @@ def input_to_float8(
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values "
"with tensor-wise quantization.
"""
"with tensor-wise quantization."""
dtype = current_platform.fp8_dtype() if dtype is None else dtype
finfo = torch.finfo(dtype)
min_val, max_val = x.aminmax()
@@ -1195,8 +1194,7 @@ def prepare_fp8_moe_layer_for_deepgemm(
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
"""Pad the weight tensor. This is an optimization on ROCm platform, which
can benefit from tensors located far enough from one another in memory
"""
can benefit from tensors located far enough from one another in memory"""
if (
envs.VLLM_ROCM_FP8_PADDING
and current_platform.is_rocm()
@@ -297,8 +297,7 @@ def _humming_input_schema_to_quant_key(
schema: "HummingInputSchema",
) -> QuantKey | None:
"""Convert a HummingInputSchema to a QuantKey. Returns None if
the schema represents unquantized (bf16/fp16) inputs.
"""
the schema represents unquantized (bf16/fp16) inputs."""
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
@@ -427,8 +426,7 @@ def convert_linear_layer_to_humming_standard(
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
"""
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
@@ -245,8 +245,7 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
"""Recover the (size_n, size_k) a Marlin weight was repacked with
(including any tile padding) from its packed shape.
"""
(including any tile padding) from its packed shape."""
pack_factor = 32 // num_bits
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
@@ -257,8 +256,7 @@ def marlin_pad_qweight(
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
) -> torch.Tensor:
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
gptq_marlin_repack.
"""
gptq_marlin_repack."""
if (padded_n, padded_k) == (size_n, size_k):
return qweight
pack_factor = size_k // qweight.size(0)
@@ -276,8 +274,7 @@ def marlin_pad_scales(
group_size: int,
) -> torch.Tensor:
"""Zero-pad weight scales (num_groups, size_n); call before
marlin_permute_scales and pass the padded extents to it.
"""
marlin_permute_scales and pass the padded extents to it."""
if (padded_n, padded_k) == (size_n, size_k):
return scales
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
@@ -41,8 +41,7 @@ def _nvfp4_compute_scale_factor(
) -> float:
"""Compute the power-of-2 scale_factor needed so that all non-zero
values in marlin_scales * 2^7 are >= 2 after rescaling.
Returns a Python float (power of 2, >= 1.0).
"""
Returns a Python float (power of 2, >= 1.0)."""
# Since half has a smaller dynamic range compared to bfloat16,
# no rescaling is applied here if active dtype is half.
if a_dtype is not None and a_dtype == torch.half:
@@ -376,8 +375,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w13(x: torch.Tensor) -> torch.Tensor:
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
tensor to padded_N rows.
"""
tensor to padded_N rows."""
if padded_N == N:
return x
x = x.view(E, num_shards, N, x.size(-1))
@@ -386,8 +384,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
tensor.
"""
tensor."""
if padded_N == N:
return x
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
@@ -217,8 +217,7 @@ def prepare_fp8_layer_for_marlin(
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
"""
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
if padded_n == n:
return x
e = x.size(0)
@@ -9,8 +9,7 @@ from .base import RotaryEmbedding
class NTKScalingRotaryEmbedding(RotaryEmbedding):
"""RotaryEmbedding extended with fixed and mixed NTK scaling.
https://kexue.fm/archives/9706
"""
https://kexue.fm/archives/9706"""
def __init__(
self,
@@ -285,8 +285,7 @@ def _gather_workspace_shapes(
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
"""
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
if use_fp4_cache:
return (
((total_seq_lens, head_dim // 2), torch.uint8),
@@ -304,8 +303,7 @@ def kv_cache_as_quant_view(
use_fp4_cache: bool,
) -> torch.Tensor:
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
DeepGEMM, from the 3D indexer kv-cache allocation.
"""
DeepGEMM, from the 3D indexer kv-cache allocation."""
if use_fp4_cache:
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
num_blocks, block_size, _ = kv_cache.shape
@@ -41,8 +41,7 @@ def _prefill_k_workspaces(
total_seq_lens: int, head_dim: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""The packed MXFP4 K-gather workspace ``(values, scales)``, shared with
the dense indexer layers of the same model.
"""
the dense indexer layers of the same model."""
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, current_platform.fp8_dtype(), use_fp4_cache=True
)
@@ -120,8 +119,7 @@ class SparseMQAIndexer(nn.Module):
def _reserve_workspaces(self, device: torch.device) -> None:
"""Profiling run: claim the K-gather workspace and the peak sparse
logits allocation so the memory estimate covers them.
"""
logits allocation so the memory estimate covers them."""
_prefill_k_workspaces(self.max_total_seq_len, self.head_dim)
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
torch.empty(max_logits_bytes, dtype=torch.uint8, device=device)
@@ -376,8 +376,7 @@ class VocabParallelEmbedding(PluggableLayer):
) -> VocabParallelEmbeddingShardIndices:
"""Get start and end indices for vocab parallel embedding, following the
layout outlined in the class docstring, based on the given tp_rank and
tp_size.
"""
tp_size."""
num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded
padded_org_vocab_start_index, padded_org_vocab_end_index = (
vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size)