[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
+1 -2
View File
@@ -135,8 +135,7 @@ class CustomOp(nn.Module):
def forward_native(self, *args, **kwargs):
"""PyTorch-native implementation; OOT plugins override via
:meth:`forward_oot`.
"""
:meth:`forward_oot`."""
raise NotImplementedError
def forward_oot(self, *args, **kwargs):
@@ -10,8 +10,7 @@ from vllm.model_executor.hw_agnostic.custom_op import CustomOp
@CustomOp.register("rms_norm")
class RMSNorm(CustomOp):
"""``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses
``residual += x`` then RMSNorm and returns ``(normalized, residual)``.
"""
``residual += x`` then RMSNorm and returns ``(normalized, residual)``."""
def __init__(
self,
@@ -881,8 +881,7 @@ def choose_mp_linear_kernel(
def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8LinearKernel:
"""Select and instantiate the best MXFP8 linear kernel for the
current platform.
"""
current platform."""
config = Mxfp8LinearLayerConfig(bmm_batch_size=bmm_batch_size)
platform = current_platform._enum
@@ -934,8 +933,7 @@ def init_mxfp4_linear_kernel(
activation_quant_key: QuantKey | None = None,
) -> MxFp4LinearKernel:
"""Select and instantiate the best MXFP4 linear kernel for the
current platform.
"""
current platform."""
config = MxFp4LinearLayerConfig(
activation_quant_key=activation_quant_key,
)
@@ -982,8 +980,7 @@ def init_mxfp6_linear_kernel(
activation_quant_key: QuantKey | None = None,
) -> MxFp6LinearKernel:
"""Select and instantiate the best MXFP6 linear kernel for the
current platform.
"""
current platform."""
config = MxFp6LinearLayerConfig(
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
@@ -1071,8 +1068,7 @@ def init_wfp8_a16_linear_kernel(
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
"""Select and instantiate the best NVFP4 linear kernel for the
current platform.
"""
current platform."""
config = NvFp4LinearLayerConfig()
a16_kernels = (
FlashInferCuteDslNvFp4W4A16LinearKernel,
@@ -177,8 +177,7 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
def input_quant_key(self) -> QuantKey | None:
"""This kernel supports dynamic quantization of the input. By
convention, pre-quantized blockscales must use the swizzled layout.
"""
convention, pre-quantized blockscales must use the swizzled layout."""
return kNvfp4Dynamic
@classmethod
@@ -41,8 +41,7 @@ class FP8BlockParams(FP8Params):
@property
def block_scale_attr(self) -> str:
"""Fp8LinearMethod registers the block scale as ``weight_scale_inv``,
compressed-tensors as ``weight_scale``.
"""
compressed-tensors as ``weight_scale``."""
return (
self.WEIGHT_SCALE
if self.weight_scale_inv is None
@@ -174,8 +174,7 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
def input_quant_key(self) -> QuantKey | None:
"""Only static per-tensor activation quantization is supported for external
quantization.
"""
quantization."""
if self.config.activation_quant_key == kFp8StaticTensorSym:
return kFp8StaticTensorSym
return None
@@ -185,8 +184,7 @@ class CutlassFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
x: torch.Tensor, dim: int, alignment: int, value: float = 0.0
) -> torch.Tensor:
"""Pad tensor ``x`` along ``dim`` to the next multiple of
``alignment``.
"""
``alignment``."""
remainder = x.shape[dim] % alignment
if remainder == 0:
return x
+1 -2
View File
@@ -804,8 +804,7 @@ _ACTIVATION_REGISTRY = LazyDict(
def _get_gelu_pytorch_tanh() -> nn.Module:
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
and fast GELU for ARM.
"""
and fast GELU for ARM."""
if current_platform.is_rocm():
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
logger.warning_once(
@@ -1568,8 +1568,7 @@ class MLACommonBackend(AttentionBackend):
@classmethod
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
"""Per-token-head modes pack an inline fp32 scale pair after the
latent data (single-sided: ``head_size_v == 0`` for MLA).
"""
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
spec = super().customize_spec(spec)
mode = spec.kv_quant_mode
if spec.state_content_bytes is not None or not mode.is_per_token_head:
@@ -55,8 +55,7 @@ _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.
"""
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
@@ -170,8 +170,7 @@ def _fused_embed_eh_norm_kernel(
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
cat. Requires the full table on-rank (replicated embedding).
"""
cat. Requires the full table on-rank (replicated embedding)."""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
@@ -56,8 +56,7 @@ class MoEActivation(Enum):
@property
def custom_op_name(self) -> str:
"""Maps to the CustomOp name of activations
in vllm/model_executor/layers/activation.py.
"""
in vllm/model_executor/layers/activation.py."""
return _CUSTOM_OP_NAMES[self]
def without_mul(self) -> "MoEActivation":
@@ -36,8 +36,7 @@ def is_aiter_mxfp8_moe_available() -> bool:
nightly lacking #3811 would wrongly select FlyDSL instead of falling back to
the native Triton dot_scaled path. #3811 added no probe-able public symbol,
so detect the ``minimax_m3_mxfp8`` tuned config it shipped. Every check fails
closed (returns False -> triton dot_scaled), which is always safe.
"""
closed (returns False -> triton dot_scaled), which is always safe."""
if not (current_platform.is_rocm() and current_platform.supports_mx()):
return False
try:
@@ -192,8 +192,7 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
"""Zero-pad the per-partition MoE intermediate dim of both weights and
the expert bias, see `_padded_intermediate_size`.
"""
the expert bias, see `_padded_intermediate_size`."""
intermediate_size = self.moe_config.intermediate_size_per_partition
padded_size = self._padded_intermediate_size(self.moe_config)
if padded_size == intermediate_size:
@@ -953,8 +952,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
modular MoE experts.
"""
modular MoE experts."""
def __init__(
self,
@@ -1287,8 +1285,7 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
quantization, dispatched through zentorch.
"""
quantization, dispatched through zentorch."""
def __init__(
self,
@@ -34,8 +34,7 @@ from vllm.triton_utils import tl, triton
def _select_cfg(M, N, K, block_m):
"""Pick the launch config from host constants only (M=num_valid_tokens, N, K,
block_m) — graph-capture safe (no GPU-scalar branch).
"""
block_m) — graph-capture safe (no GPU-scalar branch)."""
# Per-regime winners (measured, isolated cuda-event A/B on gfx950, GPU 3):
# BLOCK_K=256 (fewer K-iters + bigger MX scale-load coalesced with the dot),
# num_stages=2 (software-pipeline overlaps the E8M0 scale-load with the scaled
@@ -55,28 +55,24 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
@abstractmethod
def backend_enum_cls(self) -> type[BackendT]:
"""Return the concrete `Enum` class enumerating this oracle's
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).
"""
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
@abstractmethod
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
"""Return platform-appropriate backends in priority order for
this `moe_config`.
"""
this `moe_config`."""
@abstractmethod
def backend_to_kernel_cls(
self, backend: BackendT
) -> list[type[mk.FusedMoEExperts]]:
"""Map a backend enum value to its concrete `FusedMoEExperts`
subclasses, in selection priority order.
"""
subclasses, in selection priority order."""
@abstractmethod
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
"""Map a user-facing `MoEBackend` (from the runner config) to
this oracle's enum.
"""
this oracle's enum."""
@abstractmethod
def select_backend(
@@ -108,8 +104,7 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
combinator) for the chosen backend.
"""
combinator) for the chosen backend."""
def convert_to_kernel_format(
self,
@@ -451,8 +451,7 @@ def _humming_fp8_weight_schema(
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
tensors (scale dtype/shape, block size), not the producing quant method.
"""
tensors (scale dtype/shape, block size), not the producing quant method."""
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
@@ -224,8 +224,7 @@ def _humming_int8_weight_schema(
weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming compressed-tensors int8 schema from the canonical
on-device tensors; humming does the signed-int8 -> native conversion.
"""
on-device tensors; humming does the signed-int8 -> native conversion."""
config: dict[str, Any] = {
"quant_method": "compressed-tensors",
"format": "int-quantized",
@@ -512,8 +512,7 @@ def _process_weights_flashinfer(
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
tensor along its last dim, from ``unit`` to ``padded_unit`` columns.
"""
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
if padded_unit == unit:
return x
e, rows, _ = x.shape
@@ -1081,8 +1080,7 @@ def _humming_wna16_weight_schema(
quant_config: QuantizationConfig | QuantizationArgs | None,
) -> dict[str, Any]:
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
config rather than the running kernel.
"""
config rather than the running kernel."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
@@ -331,8 +331,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
"""Available backends in priority order, BF16-act variant before
activation-quantized variant within each vendor family.
"""
activation-quantized variant within each vendor family."""
_AVAILABLE_BACKENDS = [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
@@ -411,8 +410,7 @@ def _resolve_activation_key(
model_activation_key: QuantKey | None,
) -> QuantKey | None:
"""Combine the model-supplied activation key with the user override.
Raises on conflict (both set and disagreeing).
"""
Raises on conflict (both set and disagreeing)."""
user_override = _user_moe_activation_override()
if user_override is None:
return model_activation_key
@@ -462,8 +460,7 @@ def _filter_by_activation(
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land.
"""
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
if requested_activation_key is not None:
return [
b
@@ -71,8 +71,7 @@ class ZeroExpertRouter(BaseRouter):
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute routing with full bias, compute zero expert output,
mask zero expert IDs.
"""
mask zero expert IDs."""
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=router_logits,
+3 -6
View File
@@ -163,8 +163,7 @@ class SparseIndexerTopk(torch.nn.Module):
) -> str:
"""Resolve the decode top-k implementation from the configured
backend ("auto" = the pre-existing chain, or a validated explicit
value).
"""
value)."""
if self._backend == "auto":
return self._resolve_auto(logits, topk_tokens, num_rows)
@@ -211,8 +210,7 @@ class SparseIndexerTopk(torch.nn.Module):
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The pre-existing priority chain: cooperative -> persistent ->
per_row. deep_select/flashinfer/torch are opt-in only.
"""
per_row. deep_select/flashinfer/torch are opt-in only."""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
if self._is_cuda and topk_tokens in (512, 1024, 2048):
@@ -274,8 +272,7 @@ class SparseIndexerTopk(torch.nn.Module):
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
+1 -2
View File
@@ -158,8 +158,7 @@ class LinearMethodBase(QuantizeMethodBase):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply the weights in layer to the input tensor.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
raise NotImplementedError
@@ -861,8 +861,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""ROCm forward using AITER Triton fused projection+attention when
available, otherwise falling back to the generic CUDA path.
"""
available, otherwise falling back to the generic CUDA path."""
if GDN_AITER_TRITON_AVAILABLE:
num_tokens = hidden_states.size(0)
projected_states_qkvz, _ = self.in_proj_qkvz(hidden_states)
@@ -261,8 +261,7 @@ class MambaStateShapeCalculator:
@classmethod
def extra_groups_for_head_shards(cls, ngroups: int, tp_size: int):
"""Compute the increase in group numbers to account for
replication in order to accompany the head shards.
"""
replication in order to accompany the head shards."""
# in the case ngoups % tp_size == 0, this will be zero
if ngroups % tp_size == 0:
return 0
@@ -59,8 +59,7 @@ def get_ssm_device_name() -> str:
def _canonical_cache_dtype(cache_dtype: str) -> str:
"""Canonical key for config lookup. bf16 and fp16 share the same tuned
configs because the kernel only sees bit width when accessing state.
"""
configs because the kernel only sees bit width when accessing state."""
return "float16" if cache_dtype == "bfloat16" else cache_dtype
@@ -161,8 +160,7 @@ _ssm_config_override: tuple[int, int] | None = None
@contextmanager
def override_ssm_config(config: tuple[int, int]):
"""Pin ``try_get_optimal_ssm_config`` to ``config`` for the duration of
the context. Used by the tuning benchmark to time specific configs.
"""
the context. Used by the tuning benchmark to time specific configs."""
global _ssm_config_override
prev = _ssm_config_override
_ssm_config_override = config
@@ -210,8 +210,7 @@ class LamportWorkspace:
@property
def workspace(self) -> torch.Tensor:
"""Device tensor (int64) that can be passed to the kernel
as ``void** workspace``.
"""
as ``void** workspace``."""
return self._workspace
# ------------------------------------------------------------------
@@ -46,24 +46,21 @@ class QuantizeMethodBase(ABC):
):
"""Create weights for a layer.
The weights will be set as attributes of the layer.
"""
The weights will be set as attributes of the layer."""
raise NotImplementedError
@abstractmethod
def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
"""Apply the weights in layer to the input tensor.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
raise NotImplementedError
# Not required functions
def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
"""Gather embeddings in the layer based on indices in the input tensor.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
raise NotImplementedError
# Not required functions
@@ -76,8 +73,7 @@ class QuantizeMethodBase(ABC):
Quantization methods that need special weight handling (e.g. repacked
weights) override this.
Expects create_weights to have been called before on the layer.
"""
Expects create_weights to have been called before on the layer."""
layer.weight = embed_tokens.weight
return layer
@@ -665,8 +665,7 @@ class CompressedTensorsConfig(QuantizationConfig):
) -> bool:
"""Weight N-bit INT (pack-quantized for sub-byte, int-quantized for 8-bit)
with static per-tensor INT8 input/output activation quant, applied as a float
fake-quant around a weight-only matmul.
"""
fake-quant around a weight-only matmul."""
is_int_pack_format = format in (
CompressionFormat.pack_quantized.value,
CompressionFormat.int_quantized.value,
@@ -702,8 +701,7 @@ class CompressedTensorsConfig(QuantizationConfig):
format: str | None,
) -> bool:
"""Weight N-bit INT with symmetric dynamic INT activation quant
via Humming kernel.
"""
via Humming kernel."""
if input_quant is None:
return False
is_pack_format = format == CompressionFormat.pack_quantized.value
@@ -37,8 +37,7 @@ def _dequant_gather_kernel(
BLOCK: tl.constexpr,
):
"""Gather embedding rows by token id, unpack int32-packed INT weights, and
dequantize to ``out`` dtype in one pass (no int8 intermediate).
"""
dequantize to ``out`` dtype in one pass (no int8 intermediate)."""
row = tl.program_id(0)
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
col_mask = col < hidden
@@ -211,8 +211,7 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
def _pack_int_quantized_weight(self, layer: torch.nn.Module) -> None:
"""Normalize an int-quantized (plain int8) weight to the canonical
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect.
"""
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect."""
weight = layer.weight
out_features, in_features = weight.shape
packed = pack_to_int32(weight.data.contiguous(), self.num_bits)
@@ -86,8 +86,7 @@ class INCConfigParser:
def is_explicitly_configured(name: str) -> bool:
"""Return True if *name* has an explicit entry in extra_config,
either via exact key match or via a regex pattern key.
"""
either via exact key match or via a regex pattern key."""
if not self._config.extra_config:
return False
if name in self._config.extra_config:
@@ -48,8 +48,7 @@ class INCScheme(ABC):
layer_config: "INCLayerConfig",
) -> "FusedMoEMethodBase | None":
"""Optional. Override if this scheme supports MoE.
Default raises NotImplementedError.
"""
Default raises NotImplementedError."""
raise NotImplementedError(
f"{type(self).__name__} does not support MoE layers. "
f"Layer config: {layer_config}"
@@ -1988,8 +1988,7 @@ class KNvfp4Static(QuantKeyScheme):
class KNvfp4Dynamic(QuantKeyScheme):
"""NVFP4 activation scheme (W4A4). Has a static global input scale on disk;
the per-group scale is computed at runtime inside the kernel.
"""
the per-group scale is computed at runtime inside the kernel."""
key = kNvfp4Dynamic
@@ -2026,8 +2025,7 @@ class KNvfp4Dynamic(QuantKeyScheme):
class KFp8StaticTensor(QuantKeyScheme):
"""Plain per-tensor static FP8 — bivalent: serves BOTH the weight slot and
the activation slot (W8A8). One key in both QuantSpec slots.
"""
the activation slot (W8A8). One key in both QuantSpec slots."""
key = kFp8StaticTensorSym
@@ -2097,8 +2095,7 @@ class KFp8StaticTensor(QuantKeyScheme):
class KFp8StaticChannel(QuantKeyScheme):
"""Per-channel static FP8 weight (the 'PcPt' weight). Weight-role only —
there is no static per-channel *activation* today.
"""
there is no static per-channel *activation* today."""
key = kFp8StaticTokenSym
@@ -2139,8 +2136,7 @@ class KFp8StaticChannel(QuantKeyScheme):
class KFp8Block128(QuantKeyScheme):
"""128x128 block-static FP8 weight ('PbWo'). Weight-role only. ModelOpt
exports the scale 4-D [out_blk,1,in_blk,1]; process squeezes to 2-D.
No transpose (block kernel keeps [out,in]).
"""
No transpose (block kernel keeps [out,in])."""
key = kFp8Static128BlockSym
@@ -2196,8 +2192,7 @@ class KFp8Block128(QuantKeyScheme):
class KMxfp8Static(QuantKeyScheme):
"""MXFP8 weight: fp8-e4m3 values + per-32-block e8m0 (uint8) scale.
Weight-role only. process is validate-only plus an idempotency guard.
"""
Weight-role only. process is validate-only plus an idempotency guard."""
key = kMxfp8Static
@@ -2272,8 +2267,7 @@ class KDynamicNoParam(QuantKeyScheme):
"""Dynamic activation with no stored scale (W8A8): quantized at runtime in
the kernel. NOT the same as activation=None (weight-only) — init_fp8 needs a
non-None activation key. Activation-role only. Serves the fp8 per-token, fp8
per-block, and mxfp8 dynamic activation keys.
"""
per-block, and mxfp8 dynamic activation keys."""
def create_weights(self, layer, role, ctx, shapes, wl) -> None:
if role is not ACT:
@@ -79,8 +79,7 @@ class Mxfp4Config(QuantizationConfig):
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
"""MoE method for RoutedExperts. Subclasses override to pick a
checkpoint-specific kernel family.
"""
checkpoint-specific kernel family."""
return Mxfp4MoEMethod(moe)
def get_quant_method(
@@ -139,8 +139,7 @@ def _find_matching_targets(
class OnlineQuantizationConfig(QuantizationConfig):
"""Model-level config for online quantization (quantize fp16/bf16 weights
during model loading, without requiring a pre-quantized checkpoint).
"""
during model loading, without requiring a pre-quantized checkpoint)."""
def __init__(
self,
@@ -118,8 +118,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time.
"""
weights onto meta device and materializes them just-in-time."""
uses_meta_device: bool = True
@@ -163,8 +162,7 @@ class OnlineLinearBase(LinearMethodBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
"""
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
def __init__(self):
super().__init__()
@@ -265,8 +263,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading.
"""
Loads fp16/bf16 weights and quantizes them per-block during loading."""
def __init__(self):
super().__init__()
@@ -440,8 +437,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
"""Shared base for online FP8 MoE methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time.
"""
weights onto meta device and materializes them just-in-time."""
# Declared here for mypy; actual values are set in __init__.
fp8_backend: "Fp8MoeBackend"
@@ -562,8 +558,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online tensorwise FP8 MoE quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
"""
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
def __init__(
self,
@@ -620,8 +615,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online blockwise FP8 MoE quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading.
"""
Loads fp16/bf16 weights and quantizes them per-block during loading."""
def __init__(
self,
@@ -69,8 +69,7 @@ def input_to_float8(
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values "
"with tensor-wise quantization.
"""
"with tensor-wise quantization."""
dtype = current_platform.fp8_dtype() if dtype is None else dtype
finfo = torch.finfo(dtype)
min_val, max_val = x.aminmax()
@@ -1195,8 +1194,7 @@ def prepare_fp8_moe_layer_for_deepgemm(
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
"""Pad the weight tensor. This is an optimization on ROCm platform, which
can benefit from tensors located far enough from one another in memory
"""
can benefit from tensors located far enough from one another in memory"""
if (
envs.VLLM_ROCM_FP8_PADDING
and current_platform.is_rocm()
@@ -297,8 +297,7 @@ def _humming_input_schema_to_quant_key(
schema: "HummingInputSchema",
) -> QuantKey | None:
"""Convert a HummingInputSchema to a QuantKey. Returns None if
the schema represents unquantized (bf16/fp16) inputs.
"""
the schema represents unquantized (bf16/fp16) inputs."""
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
@@ -427,8 +426,7 @@ def convert_linear_layer_to_humming_standard(
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
"""
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
@@ -245,8 +245,7 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
"""Recover the (size_n, size_k) a Marlin weight was repacked with
(including any tile padding) from its packed shape.
"""
(including any tile padding) from its packed shape."""
pack_factor = 32 // num_bits
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
@@ -257,8 +256,7 @@ def marlin_pad_qweight(
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
) -> torch.Tensor:
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
gptq_marlin_repack.
"""
gptq_marlin_repack."""
if (padded_n, padded_k) == (size_n, size_k):
return qweight
pack_factor = size_k // qweight.size(0)
@@ -276,8 +274,7 @@ def marlin_pad_scales(
group_size: int,
) -> torch.Tensor:
"""Zero-pad weight scales (num_groups, size_n); call before
marlin_permute_scales and pass the padded extents to it.
"""
marlin_permute_scales and pass the padded extents to it."""
if (padded_n, padded_k) == (size_n, size_k):
return scales
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
@@ -41,8 +41,7 @@ def _nvfp4_compute_scale_factor(
) -> float:
"""Compute the power-of-2 scale_factor needed so that all non-zero
values in marlin_scales * 2^7 are >= 2 after rescaling.
Returns a Python float (power of 2, >= 1.0).
"""
Returns a Python float (power of 2, >= 1.0)."""
# Since half has a smaller dynamic range compared to bfloat16,
# no rescaling is applied here if active dtype is half.
if a_dtype is not None and a_dtype == torch.half:
@@ -376,8 +375,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w13(x: torch.Tensor) -> torch.Tensor:
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
tensor to padded_N rows.
"""
tensor to padded_N rows."""
if padded_N == N:
return x
x = x.view(E, num_shards, N, x.size(-1))
@@ -386,8 +384,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
tensor.
"""
tensor."""
if padded_N == N:
return x
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
@@ -217,8 +217,7 @@ def prepare_fp8_layer_for_marlin(
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
"""
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
if padded_n == n:
return x
e = x.size(0)
@@ -9,8 +9,7 @@ from .base import RotaryEmbedding
class NTKScalingRotaryEmbedding(RotaryEmbedding):
"""RotaryEmbedding extended with fixed and mixed NTK scaling.
https://kexue.fm/archives/9706
"""
https://kexue.fm/archives/9706"""
def __init__(
self,
@@ -285,8 +285,7 @@ def _gather_workspace_shapes(
"""Return ((values_shape, values_dtype), (scales_shape, scales_dtype)) for
the K-gather workspace. FP8 path: (T, head_dim) fp8 + (T, 4) uint8 fp32
scales. MXFP4 path: (T, head_dim // 2) uint8 packed mxfp4 +
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales.
"""
(T, head_dim // MXFP4_BLOCK_SIZE) uint8 ue8m0 scales."""
if use_fp4_cache:
return (
((total_seq_lens, head_dim // 2), torch.uint8),
@@ -304,8 +303,7 @@ def kv_cache_as_quant_view(
use_fp4_cache: bool,
) -> torch.Tensor:
"""4D ``[num_blocks, block_size, 1, head_width]`` view expected by
DeepGEMM, from the 3D indexer kv-cache allocation.
"""
DeepGEMM, from the 3D indexer kv-cache allocation."""
if use_fp4_cache:
assert kv_cache.ndim == 3 and kv_cache.dtype == torch.uint8
num_blocks, block_size, _ = kv_cache.shape
@@ -41,8 +41,7 @@ def _prefill_k_workspaces(
total_seq_lens: int, head_dim: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""The packed MXFP4 K-gather workspace ``(values, scales)``, shared with
the dense indexer layers of the same model.
"""
the dense indexer layers of the same model."""
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, current_platform.fp8_dtype(), use_fp4_cache=True
)
@@ -120,8 +119,7 @@ class SparseMQAIndexer(nn.Module):
def _reserve_workspaces(self, device: torch.device) -> None:
"""Profiling run: claim the K-gather workspace and the peak sparse
logits allocation so the memory estimate covers them.
"""
logits allocation so the memory estimate covers them."""
_prefill_k_workspaces(self.max_total_seq_len, self.head_dim)
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
torch.empty(max_logits_bytes, dtype=torch.uint8, device=device)
@@ -376,8 +376,7 @@ class VocabParallelEmbedding(PluggableLayer):
) -> VocabParallelEmbeddingShardIndices:
"""Get start and end indices for vocab parallel embedding, following the
layout outlined in the class docstring, based on the given tp_rank and
tp_size.
"""
tp_size."""
num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded
padded_org_vocab_start_index, padded_org_vocab_end_index = (
vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size)
@@ -36,8 +36,7 @@ class BaseModelLoader(ABC):
@abstractmethod
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
"""Load weights into a model. This standalone API allows
inplace weights loading for an already-initialized model
"""
inplace weights loading for an already-initialized model"""
raise NotImplementedError
def create_model(
@@ -135,8 +135,7 @@ class DefaultModelLoader(BaseModelLoader):
) -> tuple[str, list[str], bool]:
"""Prepare weights for the model.
If the model is not local, it will be downloaded.
"""
If the model is not local, it will be downloaded."""
model_name_or_path = (
maybe_download_from_modelscope(model_name_or_path, revision)
or model_name_or_path
@@ -23,8 +23,7 @@ def parse_expert_id(weight_name: str) -> int | None:
Returns ``None`` for dense weights (attention, layernorm, embedding),
shared experts, and 3D fused-expert tensors where all experts are stored
in a single tensor without a numeric expert id in the name.
"""
in a single tensor without a numeric expert id in the name."""
m = _EXPERT_ID_RE.search(weight_name)
return int(m.group(1)) if m else None
@@ -71,8 +70,7 @@ def should_skip_weight(
local_expert_ids: set[int] | None,
) -> bool:
"""Return ``True`` if *weight_name* is an expert weight that does not
belong to the local rank and should be skipped during loading.
"""
belong to the local rank and should be skipped during loading."""
if local_expert_ids is None:
return False
eid = parse_expert_id(weight_name)
@@ -308,8 +308,7 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
Assumes dtype/shapes of attention tensors do not change during
processing, since we use .data.copy_() to preserve kernel tensor
references.
"""
references."""
quant_method = getattr(layer, "quant_method", None)
if quant_method is not None:
# Re-create scale Parameters with sentinel values so unloaded scales
@@ -388,8 +387,7 @@ def _get_weight_loader(tensor: torch.Tensor):
def _copy_and_restore_kernel_tensors(layer: torch.nn.Module, info: LayerReloadingInfo):
"""Copy processed values into original kernel tensor storage and restore
kernel tensor references on the layer. Preserves cudagraph references.
"""
kernel tensor references on the layer. Preserves cudagraph references."""
assert info.kernel_tensors is not None
parameters, buffers = info.kernel_tensors
non_persistent = info.kernel_non_persistent_buffers
@@ -81,8 +81,7 @@ class RunaiModelStreamerLoader(BaseModelLoader):
) -> list[str]:
"""Prepare weights for the model.
If the model is not local, it will be downloaded.
"""
If the model is not local, it will be downloaded."""
is_object_storage_path = is_runai_obj_uri(model_name_or_path)
is_local = os.path.isdir(model_name_or_path)
safetensors_pattern = "*.safetensors"
@@ -497,8 +497,7 @@ def _check_tensors_on_meta_device(model: nn.Module) -> None:
def _resize_lora_embeddings(model: nn.Module):
"""Modify LoRA embedding layers to use bigger tensors
to allow for adapter added tokens.
"""
to allow for adapter added tokens."""
for child in model.modules():
if (
isinstance(child, VocabParallelEmbedding)
@@ -105,8 +105,7 @@ class TensorizerLoader(BaseModelLoader):
Expects a vLLM-tensorized model. See the
examples/features/tensorize_vllm_model.py example script
for serializing vLLM models.
"""
for serializing vLLM models."""
if is_vllm_tensorized(self.tensorizer_config):
tensorizer_config = self._patch_tensorizer_config(model_config)
deserialize_tensorizer_model(model, tensorizer_config)
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""IPC model loader: maps post-quantized weights from a local weight cache
daemon via CUDA IPC instead of loading from disk.
"""
daemon via CUDA IPC instead of loading from disk."""
import dataclasses
import socket
@@ -78,8 +78,7 @@ def maybe_retie_word_embeddings(model: nn.Module, model_config: ModelConfig) ->
Checkpoints produced by quantization or fine-tuning tooling often keep a redundant
copy of the tied `lm_head`. Sharing the storage again reclaims the memory it would
otherwise cost.
"""
otherwise cost."""
if not model_config.word_embeddings_untied_by_checkpoint:
return
if (untied := _get_untied_lm_head(model)) is None:
@@ -144,8 +144,7 @@ def atomic_writer(
def _natural_sort_key(filepath: str) -> list:
"""Natural sort key for filenames with numeric components, such as
model-00001-of-00005.safetensors -> ['model-', 1, '-of-', 5, '.safetensors']
"""
model-00001-of-00005.safetensors -> ['model-', 1, '-of-', 5, '.safetensors']"""
return [
int(s) if s.isdigit() else s
for s in re.split(r"(\d+)", os.path.basename(filepath))
@@ -162,8 +161,7 @@ def maybe_download_from_modelscope(
"""Download model from ModelScope hub if VLLM_USE_MODELSCOPE is True.
Returns the path to the downloaded model, or None if the model is not
downloaded from ModelScope.
"""
downloaded from ModelScope."""
if envs.VLLM_USE_MODELSCOPE:
# download model from ModelScope hub,
# lazy import so that modelscope is not required for normal use.
@@ -1148,8 +1146,7 @@ def instanttensor_weights_iterator(
use_tqdm_on_load: bool,
) -> Generator[tuple[str, torch.Tensor], None, None]:
"""Iterate over the weights in the model safetensor files
using instanttensor library.
"""
using instanttensor library."""
try:
import instanttensor
except ImportError as e:
+5 -10
View File
@@ -47,8 +47,7 @@ from .utils import (
class ArceeMLP(nn.Module):
"""Feed-forward layer for Arcee using ReLU^2 activation
(no gating as in LLaMA).
"""
(no gating as in LLaMA)."""
def __init__(
self,
@@ -96,8 +95,7 @@ class ArceeMLP(nn.Module):
class ArceeDecoderLayer(nn.Module):
"""Transformer decoder block for Arcee, with self-attention and
ReLU^2 MLP.
"""
ReLU^2 MLP."""
def __init__(
self,
@@ -177,8 +175,7 @@ class ArceeDecoderLayer(nn.Module):
@support_torch_compile
class ArceeModel(nn.Module, EagleModelMixin):
"""The transformer model backbone for Arcee (embedding layer + stacked
decoder blocks + final norm).
"""
decoder blocks + final norm)."""
def __init__(
self,
@@ -280,8 +277,7 @@ class ArceeForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
"""Arcee Model for causal language modeling, integrated with vLLM
runtime.
"""
runtime."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
@@ -362,8 +358,7 @@ class ArceeForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights into the model (delegates to inner model and handles
tied embeddings).
"""
tied embeddings)."""
loader = AutoWeightsLoader(self)
# AutoWeightLoader handles weight name remapping, including fusing
# separate q_proj, k_proj, v_proj into qkv_proj
+1 -2
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Minimal implementation of BlipVisionModel intended to be only used
within a vision language model.
"""
within a vision language model."""
from collections.abc import Iterable
+1 -2
View File
@@ -96,8 +96,7 @@ class RMSNorm(nn.Module):
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
"""Returns (norm_class, eps). Uses RMSNorm when config.rms_norm_eps is set,
otherwise falls back to LayerNorm with config.layer_norm_eps.
"""
otherwise falls back to LayerNorm with config.layer_norm_eps."""
rms_eps = getattr(config, "rms_norm_eps", None)
if rms_eps is not None:
return RMSNorm, rms_eps
+1 -2
View File
@@ -667,8 +667,7 @@ class ConvSubsampling(nn.Module):
repeat_num: int = 1,
) -> torch.Tensor:
"""Calculates the output length of a Tensor passed
through a convolution or max pooling layer
"""
through a convolution or max pooling layer"""
add_pad: float = all_paddings - kernel_size
one: float = 1.0
for i in range(repeat_num):
+1 -2
View File
@@ -363,8 +363,7 @@ class ImageEncoderViT(nn.Module):
class Block(nn.Module):
"""Transformer blocks with support of window attention and residual propagation
blocks
"""
blocks"""
def __init__(
self,
+1 -2
View File
@@ -139,8 +139,7 @@ class NoRepeatNGramLogitsProcessor:
class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
"""Example of overriding the wrapper class `__init__()` in order to utilize
info about the device type
"""
info about the device type"""
@classmethod
def validate_params(cls, params: SamplingParams):
@@ -511,8 +511,7 @@ def _compiled_sample_step(
accept/renoise → convergence, all as vectorized PyTorch ops.
Returns the temperature-scaled logits ``[num_decode, CL, vocab]`` so the
caller can compute logprobs outside the compiled region.
"""
caller can compute logprobs outside the compiled region."""
num_decode = decode_slots.shape[0]
device = decode_slots.device
+1 -2
View File
@@ -814,8 +814,7 @@ class Gemma3nCrossDecoder(nn.Module):
def _kv_sharing_weights_mapper(config: Gemma3nTextConfig) -> WeightsMapper:
"""KV-shared layers only have q_proj, so qkv_proj packing applies to the
other layers. Original checkpoints still ship K/V tensors for the shared
layers (fine-tuned ones omit them); those are dropped.
"""
layers (fine-tuned ones omit them); those are dropped."""
first_kv_shared_layer_idx = config.num_hidden_layers - config.num_kv_shared_layers
return WeightsMapper(
orig_to_new_substr={
+1 -2
View File
@@ -499,8 +499,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
class Gemma3nMultimodalEmbedder(nn.Module):
"""Embeds token ids or soft tokens for multimodal content into language
model space.
"""
model space."""
def __init__(
self,
+1 -2
View File
@@ -25,8 +25,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
compatible with HuggingFace weights.
"""
compatible with HuggingFace weights."""
import math
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
+1 -2
View File
@@ -22,8 +22,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
compatible with HuggingFace weights.
"""
compatible with HuggingFace weights."""
from collections.abc import Iterable
from itertools import islice
+1 -2
View File
@@ -22,8 +22,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 MTP
model compatible with HuggingFace weights.
"""
model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
+1 -2
View File
@@ -462,8 +462,7 @@ class GraniteSpeechConformerConvModule(nn.Module):
class GraniteSpeechConformerBlock(nn.Module):
"""Conformer block, consisting largely of linear layers,
attention, and convolutional layers.
"""
attention, and convolutional layers."""
def __init__(self, config: PretrainedConfig, prefix: str = ""):
super().__init__()
+5 -10
View File
@@ -892,8 +892,7 @@ def has_inner_state(
@runtime_checkable
class IsAttentionFree(Protocol):
"""The interface required for all models like Mamba that lack attention,
but do have state whose size is constant wrt the number of tokens.
"""
but do have state whose size is constant wrt the number of tokens."""
is_attention_free: ClassVar[Literal[True]] = True
"""
@@ -921,8 +920,7 @@ def is_attention_free(
class IsHybrid(Protocol):
"""The interface required for all models like Jamba that have both
attention and mamba blocks, indicates that
hf_config has 'layers_block_type'
"""
hf_config has 'layers_block_type'"""
is_hybrid: ClassVar[Literal[True]] = True
"""
@@ -1338,8 +1336,7 @@ class SupportsTranscription(Protocol):
) -> "PromptType":
"""Get the prompt for the ASR model.
The model has control over the construction, as long as it
returns a valid PromptType.
"""
returns a valid PromptType."""
...
@classmethod
@@ -1631,8 +1628,7 @@ class EagleModelMixin:
@runtime_checkable
class SupportsEagle(SupportsEagleBase, Protocol):
"""The interface required for models that support
EAGLE-1 and EAGLE-2 speculative decoding.
"""
EAGLE-1 and EAGLE-2 speculative decoding."""
supports_eagle: ClassVar[Literal[True]] = True
"""
@@ -1662,8 +1658,7 @@ def supports_eagle(
@runtime_checkable
class SupportsEagle3(SupportsEagleBase, Protocol):
"""The interface required for models that support
EAGLE-3 speculative decoding.
"""
EAGLE-3 speculative decoding."""
supports_eagle3: ClassVar[Literal[True]] = True
"""
+1 -2
View File
@@ -345,8 +345,7 @@ def apply_rotary_pos_emb_flashatt(
class KeyeSiglipAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You
Need' paper.
"""
Need' paper."""
def __init__(
self,
+1 -2
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Implementation of Siglip2VisionModel intended to be only used
within a vision language model.
"""
within a vision language model."""
from collections.abc import Iterable
+1 -2
View File
@@ -51,8 +51,7 @@ class Medusa(nn.Module):
truncated_vocab_size < vocab_size. To use this technique, one has to find
the top-k most frequent tokens in target dataset and add that as a tensor
in the draft checkpoint (using key token_map). Also, the draft config
needs to have truncated_vocab_size (=k) as an attribute.
"""
needs to have truncated_vocab_size (=k) as an attribute."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
speculative_config = vllm_config.speculative_config
+1 -2
View File
@@ -858,8 +858,7 @@ class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
class MiniCPMV4_6DownsampleMLP(nn.Module):
"""Match HF (transformers v5.7+) parameter naming: pre_norm/linear_1/
act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2)).
"""
act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2))."""
def __init__(
self,
+1 -2
View File
@@ -1090,8 +1090,7 @@ class Llama4ForConditionalGeneration(
def _rename_weight_for_modelopt_checkpoint(self, name: str) -> str:
"""Rename weights from ModelOpt llama4 fp8 checkpoints to vLLM
format.
"""
format."""
if name.startswith("model.") or name.startswith("language_model.model."):
renamed = (
name.replace("model.", "language_model.model.", 1)
+3 -6
View File
@@ -114,8 +114,7 @@ logger = init_logger(__name__)
def _text_config(config):
"""MuseGlimmer checkpoints may nest the text config under ``text_config``
(multimodal ``MuseGlimmerConfig``) or expose it directly
(``MuseGlimmerTextConfig``).
"""
(``MuseGlimmerTextConfig``)."""
return getattr(config, "text_config", config)
@@ -489,8 +488,7 @@ class MuseGlimmerMultiModalProcessor(
def _muse_glimmer_use_qk_norm(config) -> bool:
"""Whether QK-norm is applied. MuseGlimmer ALWAYS applies QK-norm; the modular HF
``text_config`` schema simply omits ``use_qk_norm`` (reads as ``None``).
Treat a missing/None flag as True — only an explicit ``False`` disables it.
"""
Treat a missing/None flag as True — only an explicit ``False`` disables it."""
val = getattr(config, "use_qk_norm", None)
return True if val is None else bool(val)
@@ -498,8 +496,7 @@ def _muse_glimmer_use_qk_norm(config) -> bool:
def _muse_glimmer_use_attn_output_gate(config) -> bool:
"""Whether the per-head sigmoid attention output gate is applied. MuseGlimmer ALWAYS
applies it; the modular HF ``text_config`` omits ``use_attn_output_gate``
(reads as ``None``). Missing/None -> True; only explicit ``False`` disables.
"""
(reads as ``None``). Missing/None -> True; only explicit ``False`` disables."""
val = getattr(config, "use_attn_output_gate", None)
return True if val is None else bool(val)
@@ -1188,8 +1188,7 @@ class NemotronH_Nano_VL_V2(
self, video_input: NanoNemotronVLVideoPixelInputs
) -> tuple[torch.Tensor, ...]:
"""Process video input and create final embeddings with video content
and indicator tokens.
"""
and indicator tokens."""
T = self.video_temporal_patch_size
if T > 1:
+1 -2
View File
@@ -1462,8 +1462,7 @@ class NemoConvSubsampling(torch.nn.Module):
def conv_split_by_channel(self, x: Tensor) -> Tensor:
"""For dw convs, tries to split input by time, run conv and concat
results
"""
results"""
x = self.conv[0](x) # full conv2D
x = self.conv[1](x) # activation
+4 -8
View File
@@ -69,8 +69,7 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
"""Whether the draft needs a non-causal-capable backend, resolved from config
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build).
"""
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build)."""
return not all(
_dflash_layer_causal(config, i) for i in range(config.num_hidden_layers)
)
@@ -155,8 +154,7 @@ class DFlashQwen3Attention(nn.Module):
Context KVs are pre-inserted into the KV cache before the forward pass.
This layer handles only query tokens via standard attention.
Adapted from Qwen3Attention.
"""
Adapted from Qwen3Attention."""
def __init__(
self,
@@ -250,8 +248,7 @@ class DFlashQwen3Attention(nn.Module):
"""DFlash attention assumes that the KV cache is already populated
with the context K/V from the target model's hidden states. This forward op
computes attention for the query tokens only.
See also: precompute_and_store_context_kv
"""
See also: precompute_and_store_context_kv"""
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
@@ -743,8 +740,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
def get_draft_attn_causal(self) -> list[bool]:
"""Per-layer attention causality, aligned with
get_draft_kv_cache_layer_names.
"""
get_draft_kv_cache_layer_names."""
return [layer.self_attn.causal for layer in self.model.layers]
def compute_logits(
+1 -2
View File
@@ -1321,8 +1321,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
"""Transformers>=5.10 processors override `replace_video_token`
to expand only the bare video token, keeping the prompt's outer
`<|vision_start|>`/`<|vision_end|>` markers.
"""
`<|vision_start|>`/`<|vision_end|>` markers."""
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl
+2 -4
View File
@@ -599,8 +599,7 @@ class RadioInternVisionModel(nn.Module):
self, imgs_sizes: list[tuple[int, int]], device: torch.device
) -> MaskMetadata:
"""Build mask metadata from image pixel sizes. Adds num_skip to each
sequence length (cls/register tokens) to match patch generator output.
"""
sequence length (cls/register tokens) to match patch generator output."""
patch_size = self.patch_generator.patch_size
num_skip = self.patch_generator.num_skip
@@ -613,8 +612,7 @@ class RadioInternVisionModel(nn.Module):
) -> MaskMetadata:
"""Build mask metadata from actual sequence lengths (already including
cls/register tokens, i.e. patch_count + num_skip per item).
Use inter_image_mask_metadata() when you only have imgs_sizes.
"""
Use inter_image_mask_metadata() when you only have imgs_sizes."""
assert len(seq_lens) > 0
cu_seqlens = torch.tensor(
list(accumulate(seq_lens, initial=0)), dtype=torch.int32, device=device
+1 -2
View File
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Implementation of SiglipVisionModel intended to be only used
within a vision language model.
"""
within a vision language model."""
from collections.abc import Iterable
+1 -2
View File
@@ -20,8 +20,7 @@
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
"""Inference-only StableLM (https://github.com/Stability-AI/StableLM)
model compatible with HuggingFace weights.
"""
model compatible with HuggingFace weights."""
from collections.abc import Iterable
from itertools import islice
@@ -207,8 +207,7 @@ class Base(
def _mark_model_components(self, vllm_config: "VllmConfig"):
"""Mark language model and tower submodules as `self.model` is created.
Nothing to do in `Base`, `MultiModalMixin` will override.
"""
Nothing to do in `Base`, `MultiModalMixin` will override."""
yield
@cached_property
@@ -281,8 +280,7 @@ class Base(
def _decorate_for_torch_compile(self):
"""Decorate the model's decoder class to indicate to vLLM that it
supports torch compile if `can_enable_torch_compile` is True.
"""
supports torch compile if `can_enable_torch_compile` is True."""
self._decorate_cls_for_torch_compile(
cls=self._pre_trained_model_classes.decoder,
# Applied to a PreTrainedModel so the batch dimension will exist
@@ -55,20 +55,17 @@ class BaseFuser(ABC):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built).
"""
module to install in its place (mutated in place, or freshly built)."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
(none unless it stacks weights).
"""
(none unless it stacks weights)."""
return {}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`packed_modules_mapping` entries this fuser contributes (none unless
it stacks weights).
"""
it stacks weights)."""
return {}
@@ -122,8 +119,7 @@ class RewriteFuser(BaseFuser):
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward.
"""
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@@ -132,8 +128,7 @@ class RewriteFuser(BaseFuser):
@dataclass
class StackedFuser(RewriteFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it.
"""
rewrites the forward to call it."""
merged_name: ClassVar[str]
"""Attribute name of the merged module created by `update_attrs`."""
@@ -152,16 +147,14 @@ class StackedFuser(RewriteFuser):
def shards(self) -> list[tuple[str, ShardId]]:
"""Each projection's original name and its shard id in the merged module.
Source for both `orig_to_new_stacked` and `packed_modules_mapping`.
"""
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
so only this exact layer is remapped, never a same-named projection
elsewhere (e.g. an unfused MoE expert's `gate_proj`).
"""
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
merged = maybe_prefix(prefix, self.merged_name)
return {
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
@@ -170,8 +163,7 @@ class StackedFuser(RewriteFuser):
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`{merged_name: [projection names]}` so quantization can unpack the
fused layer into its per-shard configs.
"""
fused layer into its per-shard configs."""
return {self.merged_name: [name for name, _ in self.shards]}
def _unguarded_calls(
@@ -182,8 +174,7 @@ class StackedFuser(RewriteFuser):
`update_attrs` deletes the projections it merges, so any reference to one beyond
its call site must be a guard on its existence; folding those to their constant
value keeps the rewritten forward off a name that no longer exists. A reference
that is not such a guard raises, so fusion is skipped.
"""
that is not such a guard raises, so fusion is skipped."""
calls = []
for name in names:
call, refs = self_call_and_refs(funcdef, name)
@@ -232,8 +232,7 @@ class MLAFuser(StackedFuser):
def update_forward(self, module: nn.Module) -> None:
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
attention interface unexpanded.
"""
attention interface unexpanded."""
funcdef, fn = recover_forward(type(module))
if self.has_q_lora:
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
@@ -80,8 +80,7 @@ def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it.
"""
even when the config does not name it."""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
@@ -126,8 +125,7 @@ class SharedExpertMLP(nn.Module):
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
"""Standard MoE block forward.
Routing and any shared experts are handled inside `self.experts: MoERunner`.
"""
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
orig_shape = hidden_states.shape
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
num_tokens = hidden_states.shape[0]
@@ -154,8 +152,7 @@ class MoEBlockFuser:
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in.
"""
Returns the scoring function and the dtype the router computes in."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
@@ -283,8 +280,7 @@ class MoEBlockFuser:
self, moe_block: nn.Module, prefix: str
) -> SharedExpertMLP | None:
"""Build the HF shared expert (and its optional gate)
as a `SharedExpertMLP` for vLLM's fused MoE.
"""
as a `SharedExpertMLP` for vLLM's fused MoE."""
if self.shared_name is None:
return None
shared_experts = getattr(moe_block, self.shared_name)
@@ -43,8 +43,7 @@ def _reference_weight(module: nn.Module) -> torch.Tensor | None:
"""A weight whose trailing dim is the module's hidden size.
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype.
"""
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
for child in module.modules():
if isinstance(child, nn.Linear):
return child.weight
@@ -59,8 +58,7 @@ class _MetaProxy(fx.Proxy):
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
executing each op on the meta values, so PyTorch's meta kernels are the
single source of shape inference — no per-op rules.
"""
single source of shape inference — no per-op rules."""
meta: object = _UNKNOWN
@@ -84,8 +82,7 @@ class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
`Tracer.proxy`, so the meta value must be grafted on here too.
"""
`Tracer.proxy`, so the meta value must be grafted on here too."""
def __init__(self, root: fx.Proxy, attr: str):
super().__init__(root, attr)
@@ -221,8 +218,7 @@ def _leaf_attention_interfaces():
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
`vllm_attention_function` needs runtime context so it is untraceable.
Every interface returns `(attn_output, attn_weights)`.
"""
Every interface returns `(attn_output, attn_weights)`."""
from transformers.modeling_utils import AttentionInterface
original = AttentionInterface.get_interface
@@ -300,8 +296,7 @@ def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
def forward_input_count(cls: type[nn.Module]) -> int:
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
any `*args`/`**kwargs`. Read from the signature, so it is independent of
whether the trace completes (unlike counting placeholders).
"""
whether the trace completes (unlike counting placeholders)."""
params = list(forward_parameters(cls).values())
if not params:
return 1 # uninspectable: assume a single input and let matching decide
@@ -378,8 +373,7 @@ def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
In these positions the object's identity never escapes, so a reference that
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
yield an operand, so the module could escape (`x and self.<name>`).
"""
yield an operand, so the module could escape (`x and self.<name>`)."""
for node in ast.walk(funcdef):
if (
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
@@ -412,8 +406,7 @@ def bypass_existence_guard(
`not self.<name>`): the reference itself to `True`.
Any other surviving reference escapes the projection's value, which no longer
exists after fusion, so refuse rather than change semantics.
"""
exists after fusion, so refuse rather than change semantics."""
for node in ast.walk(funcdef):
# `self.<name> is (not) None`, either operand order.
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
@@ -474,8 +467,7 @@ def _base_name(node: ast.expr) -> str | None:
def _rebound_names(region: list[ast.stmt]) -> set[str]:
"""Names rebound outright within `region` (`x = ...`, `del x`).
These are `Name` nodes in a `Store`/`Del` context.
"""
These are `Name` nodes in a `Store`/`Del` context."""
return {
node.id
for stmt in region
@@ -489,8 +481,7 @@ def _inplace_target(node: ast.AST) -> str | None:
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
plain `Name` store (see `_rebound_names`).
"""
plain `Name` store (see `_rebound_names`)."""
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
node.ctx, (ast.Store, ast.Del)
):
@@ -711,8 +702,7 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
Non-linear submodules are transparent too (e.g. the dropout GPT-style
attentions apply after their output projection). Never walks through a leaf
call (e.g. an attention interface): its inputs are what attention consumes,
not what produced the value.
"""
not what produced the value."""
stack = [node]
seen: set[fx.Node] = set()
while stack:
@@ -735,8 +725,7 @@ def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
Never walks through a leaf call (e.g. an attention interface): what crosses
it is consumed by the attention computation, not projected.
"""
it is consumed by the attention computation, not projected."""
queue = list(node.users)
seen: set[fx.Node] = set()
while queue:
@@ -22,8 +22,7 @@ _VLLM_PKG = "vllm.model_executor.layers"
def _resolve(module: str, name: str):
"""Return `name` from the hw-agnostic `module` when enabled and available,
else from vLLM. Logs which source was used.
"""
else from vLLM. Logs which source was used."""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
@@ -75,8 +75,7 @@ class TransformersMoERunner(MoERunner):
) -> torch.Tensor:
"""In Transformers `experts.forward` will have this signature.
We discard any extra kwargs because we cannot use them here.
"""
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
@@ -368,8 +367,7 @@ class MoEMixin(MixtureOfExperts):
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier.
"""
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
@@ -424,8 +424,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Empty, because `apply` writes the placeholder ranges itself rather than
deriving them from updates.
"""
deriving them from updates."""
return []
def _get_mm_token_ids(self, modality: str) -> list[int]:
@@ -459,8 +458,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
num_audios: int,
) -> dict[str, list[PlaceholderRange]]:
"""Take each contiguous run of the audio token as one item's placeholder,
and record how many tokens the run holds.
"""
and record how many tokens the run holds."""
audio_token_ids = self._get_mm_token_ids("audio")
prompt_tensor = torch.tensor(prompt_ids)
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
@@ -518,8 +516,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
"""Split the positions the processor marks as image into one placeholder per
item, sized by the token count it reports for each image.
"""
item, sized by the token count it reports for each image."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if mm_token_type_ids is None:
raise ValueError(
@@ -572,8 +569,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
timing_ctx: TimingContext,
) -> MultiModalInput:
"""Process the prompt and every multi-modal item in one HF processor call,
then read the placeholder ranges out of the token ids it returns.
"""
then read the placeholder ranges out of the token ids it returns."""
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
@@ -711,8 +707,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Replace each modality's placeholder token with the token ids that item's
replacement text encodes to, marking which of them hold embeddings.
"""
replacement text encodes to, marking which of them hold embeddings."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
@@ -907,8 +902,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
self, model: "PreTrainedModel"
) -> dict[str, type["PreTrainedModel"]]:
"""Modalities whose encoder cannot be told apart from the model itself are
omitted, as are those `get_encoder` rejects.
"""
omitted, as are those `get_encoder` rejects."""
encoder_classes: dict[str, type[PreTrainedModel]] = {}
for modality in _MODALITY_TO_TOKEN_TYPE_ID:
try:
@@ -1027,8 +1021,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
def get_language_model(self) -> torch.nn.Module:
"""Transformers modeling backend multimodal classes do not contain a separate
vLLM language model class. Therefore, in order to return a language model vLLM
class, we use a wrapper to give `self` the same interface as a text model.
"""
class, we use a wrapper to give `self` the same interface as a text model."""
# Exclude self and object
bases = self.__class__.mro()[1:-1]
# Keep only classes defined in `vllm.model_executor.models.transformers`
@@ -1182,8 +1175,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
) -> dict[str, Any]:
"""Narrow the entries of `kwargs` that hold one row per item down to the item
at `index`. Length is all there is to match on, so an unrelated entry of the
same length is narrowed too.
"""
same length is narrowed too."""
return {
key: value[index : index + 1]
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
@@ -201,8 +201,7 @@ def attrsetter(attr: str) -> Callable[[object, object], None]:
class _UninitializedEmbedding(nn.Embedding):
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
call to `super().__init__` does not invoke `nn.Embedding.__init__`.
"""
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
def __init__(self):
pass
@@ -210,8 +209,7 @@ class _UninitializedEmbedding(nn.Embedding):
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.
"""
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
@lru_cache
+2 -4
View File
@@ -47,8 +47,7 @@ ShardId: TypeAlias = str | int | tuple[int, ...]
class WeightsMapper:
"""Maps the name of each weight if they match the following patterns.
If a key maps to a value of `None`, the corresponding weight is ignored.
"""
If a key maps to a value of `None`, the corresponding weight is ignored."""
orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
@@ -172,8 +171,7 @@ class WeightsMapper:
rather than being rewritten to the stacked vLLM name (`qkv_proj`). Mappings to
`None` are dropped because "do not load this weight" is meaningless to such a
consumer, and applying it would silently shrink a quantization config's ignore
list or make LoRA name parsing fail.
"""
list or make LoRA name parsing fail."""
remove_none = lambda d: {k: v for k, v in d.items() if v is not None}
return replace(
self,
+1 -2
View File
@@ -116,8 +116,7 @@ def get_multimodal_config() -> MultiModalConfig | None:
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
config context is active (e.g., during unit tests) or when the current
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
stubs used in tests).
"""
stubs used in tests)."""
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None or vllm_config.model_config is None:
return None
+1 -2
View File
@@ -174,8 +174,7 @@ def get_moe_expert_mapping(
It will be retrieved from the first module that has a `get_expert_mapping` method.
If the model manually implements `get_expert_mapping`, it will be used.
Otherwise, it will use the first RoutedExperts layer.
"""
Otherwise, it will use the first RoutedExperts layer."""
for _, module in model.named_modules():
get_mapping = getattr(module, "get_expert_mapping", None)
if get_mapping is not None:
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
def qwen4_exp_qsa_triton_warmup(worker: "Worker") -> None:
"""Warm every reachable QSA specialization: indexer decode-query-length
profiles plus the sparse attention split-K/merge configs.
"""
profiles plus the sparse attention split-K/merge configs."""
qsa_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.indexer_qsa")
attn_module = sys.modules.get("vllm.models.qwen4_exp.nvidia.qsa")
if qsa_module is None or attn_module is None: