[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -47,8 +47,7 @@ from .utils import (
|
||||
|
||||
class ArceeMLP(nn.Module):
|
||||
"""Feed-forward layer for Arcee using ReLU^2 activation
|
||||
(no gating as in LLaMA).
|
||||
"""
|
||||
(no gating as in LLaMA)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -96,8 +95,7 @@ class ArceeMLP(nn.Module):
|
||||
|
||||
class ArceeDecoderLayer(nn.Module):
|
||||
"""Transformer decoder block for Arcee, with self-attention and
|
||||
ReLU^2 MLP.
|
||||
"""
|
||||
ReLU^2 MLP."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -177,8 +175,7 @@ class ArceeDecoderLayer(nn.Module):
|
||||
@support_torch_compile
|
||||
class ArceeModel(nn.Module, EagleModelMixin):
|
||||
"""The transformer model backbone for Arcee (embedding layer + stacked
|
||||
decoder blocks + final norm).
|
||||
"""
|
||||
decoder blocks + final norm)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -280,8 +277,7 @@ class ArceeForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
"""Arcee Model for causal language modeling, integrated with vLLM
|
||||
runtime.
|
||||
"""
|
||||
runtime."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
@@ -362,8 +358,7 @@ class ArceeForCausalLM(
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights into the model (delegates to inner model and handles
|
||||
tied embeddings).
|
||||
"""
|
||||
tied embeddings)."""
|
||||
loader = AutoWeightsLoader(self)
|
||||
# AutoWeightLoader handles weight name remapping, including fusing
|
||||
# separate q_proj, k_proj, v_proj into qkv_proj
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Minimal implementation of BlipVisionModel intended to be only used
|
||||
within a vision language model.
|
||||
"""
|
||||
within a vision language model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
@@ -96,8 +96,7 @@ class RMSNorm(nn.Module):
|
||||
|
||||
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
|
||||
"""Returns (norm_class, eps). Uses RMSNorm when config.rms_norm_eps is set,
|
||||
otherwise falls back to LayerNorm with config.layer_norm_eps.
|
||||
"""
|
||||
otherwise falls back to LayerNorm with config.layer_norm_eps."""
|
||||
rms_eps = getattr(config, "rms_norm_eps", None)
|
||||
if rms_eps is not None:
|
||||
return RMSNorm, rms_eps
|
||||
|
||||
@@ -667,8 +667,7 @@ class ConvSubsampling(nn.Module):
|
||||
repeat_num: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Calculates the output length of a Tensor passed
|
||||
through a convolution or max pooling layer
|
||||
"""
|
||||
through a convolution or max pooling layer"""
|
||||
add_pad: float = all_paddings - kernel_size
|
||||
one: float = 1.0
|
||||
for i in range(repeat_num):
|
||||
|
||||
@@ -363,8 +363,7 @@ class ImageEncoderViT(nn.Module):
|
||||
|
||||
class Block(nn.Module):
|
||||
"""Transformer blocks with support of window attention and residual propagation
|
||||
blocks
|
||||
"""
|
||||
blocks"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -139,8 +139,7 @@ class NoRepeatNGramLogitsProcessor:
|
||||
|
||||
class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
"""Example of overriding the wrapper class `__init__()` in order to utilize
|
||||
info about the device type
|
||||
"""
|
||||
info about the device type"""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
|
||||
@@ -511,8 +511,7 @@ def _compiled_sample_step(
|
||||
accept/renoise → convergence, all as vectorized PyTorch ops.
|
||||
|
||||
Returns the temperature-scaled logits ``[num_decode, CL, vocab]`` so the
|
||||
caller can compute logprobs outside the compiled region.
|
||||
"""
|
||||
caller can compute logprobs outside the compiled region."""
|
||||
num_decode = decode_slots.shape[0]
|
||||
device = decode_slots.device
|
||||
|
||||
|
||||
@@ -814,8 +814,7 @@ class Gemma3nCrossDecoder(nn.Module):
|
||||
def _kv_sharing_weights_mapper(config: Gemma3nTextConfig) -> WeightsMapper:
|
||||
"""KV-shared layers only have q_proj, so qkv_proj packing applies to the
|
||||
other layers. Original checkpoints still ship K/V tensors for the shared
|
||||
layers (fine-tuned ones omit them); those are dropped.
|
||||
"""
|
||||
layers (fine-tuned ones omit them); those are dropped."""
|
||||
first_kv_shared_layer_idx = config.num_hidden_layers - config.num_kv_shared_layers
|
||||
return WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
|
||||
@@ -499,8 +499,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
|
||||
class Gemma3nMultimodalEmbedder(nn.Module):
|
||||
"""Embeds token ids or soft tokens for multimodal content into language
|
||||
model space.
|
||||
"""
|
||||
model space."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -25,8 +25,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
|
||||
compatible with HuggingFace weights.
|
||||
"""
|
||||
compatible with HuggingFace weights."""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
|
||||
@@ -22,8 +22,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
|
||||
compatible with HuggingFace weights.
|
||||
"""
|
||||
compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
@@ -22,8 +22,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 MTP
|
||||
model compatible with HuggingFace weights.
|
||||
"""
|
||||
model compatible with HuggingFace weights."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
@@ -462,8 +462,7 @@ class GraniteSpeechConformerConvModule(nn.Module):
|
||||
|
||||
class GraniteSpeechConformerBlock(nn.Module):
|
||||
"""Conformer block, consisting largely of linear layers,
|
||||
attention, and convolutional layers.
|
||||
"""
|
||||
attention, and convolutional layers."""
|
||||
|
||||
def __init__(self, config: PretrainedConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -892,8 +892,7 @@ def has_inner_state(
|
||||
@runtime_checkable
|
||||
class IsAttentionFree(Protocol):
|
||||
"""The interface required for all models like Mamba that lack attention,
|
||||
but do have state whose size is constant wrt the number of tokens.
|
||||
"""
|
||||
but do have state whose size is constant wrt the number of tokens."""
|
||||
|
||||
is_attention_free: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -921,8 +920,7 @@ def is_attention_free(
|
||||
class IsHybrid(Protocol):
|
||||
"""The interface required for all models like Jamba that have both
|
||||
attention and mamba blocks, indicates that
|
||||
hf_config has 'layers_block_type'
|
||||
"""
|
||||
hf_config has 'layers_block_type'"""
|
||||
|
||||
is_hybrid: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -1338,8 +1336,7 @@ class SupportsTranscription(Protocol):
|
||||
) -> "PromptType":
|
||||
"""Get the prompt for the ASR model.
|
||||
The model has control over the construction, as long as it
|
||||
returns a valid PromptType.
|
||||
"""
|
||||
returns a valid PromptType."""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
@@ -1631,8 +1628,7 @@ class EagleModelMixin:
|
||||
@runtime_checkable
|
||||
class SupportsEagle(SupportsEagleBase, Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE-1 and EAGLE-2 speculative decoding.
|
||||
"""
|
||||
EAGLE-1 and EAGLE-2 speculative decoding."""
|
||||
|
||||
supports_eagle: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -1662,8 +1658,7 @@ def supports_eagle(
|
||||
@runtime_checkable
|
||||
class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE-3 speculative decoding.
|
||||
"""
|
||||
EAGLE-3 speculative decoding."""
|
||||
|
||||
supports_eagle3: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
|
||||
@@ -345,8 +345,7 @@ def apply_rotary_pos_emb_flashatt(
|
||||
|
||||
class KeyeSiglipAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You
|
||||
Need' paper.
|
||||
"""
|
||||
Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Implementation of Siglip2VisionModel intended to be only used
|
||||
within a vision language model.
|
||||
"""
|
||||
within a vision language model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
@@ -51,8 +51,7 @@ class Medusa(nn.Module):
|
||||
truncated_vocab_size < vocab_size. To use this technique, one has to find
|
||||
the top-k most frequent tokens in target dataset and add that as a tensor
|
||||
in the draft checkpoint (using key token_map). Also, the draft config
|
||||
needs to have truncated_vocab_size (=k) as an attribute.
|
||||
"""
|
||||
needs to have truncated_vocab_size (=k) as an attribute."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
|
||||
@@ -858,8 +858,7 @@ class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
|
||||
|
||||
class MiniCPMV4_6DownsampleMLP(nn.Module):
|
||||
"""Match HF (transformers v5.7+) parameter naming: pre_norm/linear_1/
|
||||
act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2)).
|
||||
"""
|
||||
act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2))."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1090,8 +1090,7 @@ class Llama4ForConditionalGeneration(
|
||||
|
||||
def _rename_weight_for_modelopt_checkpoint(self, name: str) -> str:
|
||||
"""Rename weights from ModelOpt llama4 fp8 checkpoints to vLLM
|
||||
format.
|
||||
"""
|
||||
format."""
|
||||
if name.startswith("model.") or name.startswith("language_model.model."):
|
||||
renamed = (
|
||||
name.replace("model.", "language_model.model.", 1)
|
||||
|
||||
@@ -114,8 +114,7 @@ logger = init_logger(__name__)
|
||||
def _text_config(config):
|
||||
"""MuseGlimmer checkpoints may nest the text config under ``text_config``
|
||||
(multimodal ``MuseGlimmerConfig``) or expose it directly
|
||||
(``MuseGlimmerTextConfig``).
|
||||
"""
|
||||
(``MuseGlimmerTextConfig``)."""
|
||||
return getattr(config, "text_config", config)
|
||||
|
||||
|
||||
@@ -489,8 +488,7 @@ class MuseGlimmerMultiModalProcessor(
|
||||
def _muse_glimmer_use_qk_norm(config) -> bool:
|
||||
"""Whether QK-norm is applied. MuseGlimmer ALWAYS applies QK-norm; the modular HF
|
||||
``text_config`` schema simply omits ``use_qk_norm`` (reads as ``None``).
|
||||
Treat a missing/None flag as True — only an explicit ``False`` disables it.
|
||||
"""
|
||||
Treat a missing/None flag as True — only an explicit ``False`` disables it."""
|
||||
val = getattr(config, "use_qk_norm", None)
|
||||
return True if val is None else bool(val)
|
||||
|
||||
@@ -498,8 +496,7 @@ def _muse_glimmer_use_qk_norm(config) -> bool:
|
||||
def _muse_glimmer_use_attn_output_gate(config) -> bool:
|
||||
"""Whether the per-head sigmoid attention output gate is applied. MuseGlimmer ALWAYS
|
||||
applies it; the modular HF ``text_config`` omits ``use_attn_output_gate``
|
||||
(reads as ``None``). Missing/None -> True; only explicit ``False`` disables.
|
||||
"""
|
||||
(reads as ``None``). Missing/None -> True; only explicit ``False`` disables."""
|
||||
val = getattr(config, "use_attn_output_gate", None)
|
||||
return True if val is None else bool(val)
|
||||
|
||||
|
||||
@@ -1188,8 +1188,7 @@ class NemotronH_Nano_VL_V2(
|
||||
self, video_input: NanoNemotronVLVideoPixelInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Process video input and create final embeddings with video content
|
||||
and indicator tokens.
|
||||
"""
|
||||
and indicator tokens."""
|
||||
T = self.video_temporal_patch_size
|
||||
|
||||
if T > 1:
|
||||
|
||||
@@ -1462,8 +1462,7 @@ class NemoConvSubsampling(torch.nn.Module):
|
||||
|
||||
def conv_split_by_channel(self, x: Tensor) -> Tensor:
|
||||
"""For dw convs, tries to split input by time, run conv and concat
|
||||
results
|
||||
"""
|
||||
results"""
|
||||
x = self.conv[0](x) # full conv2D
|
||||
x = self.conv[1](x) # activation
|
||||
|
||||
|
||||
@@ -69,8 +69,7 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
|
||||
|
||||
def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||
"""Whether the draft needs a non-causal-capable backend, resolved from config
|
||||
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build).
|
||||
"""
|
||||
(config mirror of the model's ``get_draft_attn_causal``, usable pre-build)."""
|
||||
return not all(
|
||||
_dflash_layer_causal(config, i) for i in range(config.num_hidden_layers)
|
||||
)
|
||||
@@ -155,8 +154,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
|
||||
Context KVs are pre-inserted into the KV cache before the forward pass.
|
||||
This layer handles only query tokens via standard attention.
|
||||
Adapted from Qwen3Attention.
|
||||
"""
|
||||
Adapted from Qwen3Attention."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -250,8 +248,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
"""DFlash attention assumes that the KV cache is already populated
|
||||
with the context K/V from the target model's hidden states. This forward op
|
||||
computes attention for the query tokens only.
|
||||
See also: precompute_and_store_context_kv
|
||||
"""
|
||||
See also: precompute_and_store_context_kv"""
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
@@ -743,8 +740,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
|
||||
def get_draft_attn_causal(self) -> list[bool]:
|
||||
"""Per-layer attention causality, aligned with
|
||||
get_draft_kv_cache_layer_names.
|
||||
"""
|
||||
get_draft_kv_cache_layer_names."""
|
||||
return [layer.self_attn.causal for layer in self.model.layers]
|
||||
|
||||
def compute_logits(
|
||||
|
||||
@@ -1321,8 +1321,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
||||
"""Transformers>=5.10 processors override `replace_video_token`
|
||||
to expand only the bare video token, keeping the prompt's outer
|
||||
`<|vision_start|>`/`<|vision_end|>` markers.
|
||||
"""
|
||||
`<|vision_start|>`/`<|vision_end|>` markers."""
|
||||
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
|
||||
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
||||
return proc_impl is not None and proc_impl is not mixin_impl
|
||||
|
||||
@@ -599,8 +599,7 @@ class RadioInternVisionModel(nn.Module):
|
||||
self, imgs_sizes: list[tuple[int, int]], device: torch.device
|
||||
) -> MaskMetadata:
|
||||
"""Build mask metadata from image pixel sizes. Adds num_skip to each
|
||||
sequence length (cls/register tokens) to match patch generator output.
|
||||
"""
|
||||
sequence length (cls/register tokens) to match patch generator output."""
|
||||
patch_size = self.patch_generator.patch_size
|
||||
num_skip = self.patch_generator.num_skip
|
||||
|
||||
@@ -613,8 +612,7 @@ class RadioInternVisionModel(nn.Module):
|
||||
) -> MaskMetadata:
|
||||
"""Build mask metadata from actual sequence lengths (already including
|
||||
cls/register tokens, i.e. patch_count + num_skip per item).
|
||||
Use inter_image_mask_metadata() when you only have imgs_sizes.
|
||||
"""
|
||||
Use inter_image_mask_metadata() when you only have imgs_sizes."""
|
||||
assert len(seq_lens) > 0
|
||||
cu_seqlens = torch.tensor(
|
||||
list(accumulate(seq_lens, initial=0)), dtype=torch.int32, device=device
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Implementation of SiglipVisionModel intended to be only used
|
||||
within a vision language model.
|
||||
"""
|
||||
within a vision language model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
@@ -20,8 +20,7 @@
|
||||
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
|
||||
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
|
||||
"""Inference-only StableLM (https://github.com/Stability-AI/StableLM)
|
||||
model compatible with HuggingFace weights.
|
||||
"""
|
||||
model compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
@@ -207,8 +207,7 @@ class Base(
|
||||
def _mark_model_components(self, vllm_config: "VllmConfig"):
|
||||
"""Mark language model and tower submodules as `self.model` is created.
|
||||
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override.
|
||||
"""
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override."""
|
||||
yield
|
||||
|
||||
@cached_property
|
||||
@@ -281,8 +280,7 @@ class Base(
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
"""Decorate the model's decoder class to indicate to vLLM that it
|
||||
supports torch compile if `can_enable_torch_compile` is True.
|
||||
"""
|
||||
supports torch compile if `can_enable_torch_compile` is True."""
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._pre_trained_model_classes.decoder,
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
|
||||
@@ -55,20 +55,17 @@ class BaseFuser(ABC):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built).
|
||||
"""
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
|
||||
(none unless it stacks weights).
|
||||
"""
|
||||
(none unless it stacks weights)."""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`packed_modules_mapping` entries this fuser contributes (none unless
|
||||
it stacks weights).
|
||||
"""
|
||||
it stacks weights)."""
|
||||
return {}
|
||||
|
||||
|
||||
@@ -122,8 +119,7 @@ class RewriteFuser(BaseFuser):
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward.
|
||||
"""
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
@@ -132,8 +128,7 @@ class RewriteFuser(BaseFuser):
|
||||
@dataclass
|
||||
class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it.
|
||||
"""
|
||||
rewrites the forward to call it."""
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
@@ -152,16 +147,14 @@ class StackedFuser(RewriteFuser):
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""Each projection's original name and its shard id in the merged module.
|
||||
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`.
|
||||
"""
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
|
||||
|
||||
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
|
||||
so only this exact layer is remapped, never a same-named projection
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`).
|
||||
"""
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
|
||||
merged = maybe_prefix(prefix, self.merged_name)
|
||||
return {
|
||||
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
|
||||
@@ -170,8 +163,7 @@ class StackedFuser(RewriteFuser):
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs.
|
||||
"""
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _unguarded_calls(
|
||||
@@ -182,8 +174,7 @@ class StackedFuser(RewriteFuser):
|
||||
`update_attrs` deletes the projections it merges, so any reference to one beyond
|
||||
its call site must be a guard on its existence; folding those to their constant
|
||||
value keeps the rewritten forward off a name that no longer exists. A reference
|
||||
that is not such a guard raises, so fusion is skipped.
|
||||
"""
|
||||
that is not such a guard raises, so fusion is skipped."""
|
||||
calls = []
|
||||
for name in names:
|
||||
call, refs = self_call_and_refs(funcdef, name)
|
||||
|
||||
@@ -232,8 +232,7 @@ class MLAFuser(StackedFuser):
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
|
||||
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
|
||||
attention interface unexpanded.
|
||||
"""
|
||||
attention interface unexpanded."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
if self.has_q_lora:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
|
||||
@@ -80,8 +80,7 @@ def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it.
|
||||
"""
|
||||
even when the config does not name it."""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
@@ -126,8 +125,7 @@ class SharedExpertMLP(nn.Module):
|
||||
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Standard MoE block forward.
|
||||
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`.
|
||||
"""
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
|
||||
num_tokens = hidden_states.shape[0]
|
||||
@@ -154,8 +152,7 @@ class MoEBlockFuser:
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in.
|
||||
"""
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
@@ -283,8 +280,7 @@ class MoEBlockFuser:
|
||||
self, moe_block: nn.Module, prefix: str
|
||||
) -> SharedExpertMLP | None:
|
||||
"""Build the HF shared expert (and its optional gate)
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE.
|
||||
"""
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE."""
|
||||
if self.shared_name is None:
|
||||
return None
|
||||
shared_experts = getattr(moe_block, self.shared_name)
|
||||
|
||||
@@ -43,8 +43,7 @@ def _reference_weight(module: nn.Module) -> torch.Tensor | None:
|
||||
"""A weight whose trailing dim is the module's hidden size.
|
||||
|
||||
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype.
|
||||
"""
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
|
||||
for child in module.modules():
|
||||
if isinstance(child, nn.Linear):
|
||||
return child.weight
|
||||
@@ -59,8 +58,7 @@ class _MetaProxy(fx.Proxy):
|
||||
|
||||
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
|
||||
executing each op on the meta values, so PyTorch's meta kernels are the
|
||||
single source of shape inference — no per-op rules.
|
||||
"""
|
||||
single source of shape inference — no per-op rules."""
|
||||
|
||||
meta: object = _UNKNOWN
|
||||
|
||||
@@ -84,8 +82,7 @@ class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
|
||||
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
|
||||
|
||||
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too.
|
||||
"""
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too."""
|
||||
|
||||
def __init__(self, root: fx.Proxy, attr: str):
|
||||
super().__init__(root, attr)
|
||||
@@ -221,8 +218,7 @@ def _leaf_attention_interfaces():
|
||||
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
|
||||
|
||||
`vllm_attention_function` needs runtime context so it is untraceable.
|
||||
Every interface returns `(attn_output, attn_weights)`.
|
||||
"""
|
||||
Every interface returns `(attn_output, attn_weights)`."""
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
original = AttentionInterface.get_interface
|
||||
@@ -300,8 +296,7 @@ def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
|
||||
def forward_input_count(cls: type[nn.Module]) -> int:
|
||||
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
|
||||
any `*args`/`**kwargs`. Read from the signature, so it is independent of
|
||||
whether the trace completes (unlike counting placeholders).
|
||||
"""
|
||||
whether the trace completes (unlike counting placeholders)."""
|
||||
params = list(forward_parameters(cls).values())
|
||||
if not params:
|
||||
return 1 # uninspectable: assume a single input and let matching decide
|
||||
@@ -378,8 +373,7 @@ def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
|
||||
|
||||
In these positions the object's identity never escapes, so a reference that
|
||||
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
|
||||
yield an operand, so the module could escape (`x and self.<name>`).
|
||||
"""
|
||||
yield an operand, so the module could escape (`x and self.<name>`)."""
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
|
||||
@@ -412,8 +406,7 @@ def bypass_existence_guard(
|
||||
`not self.<name>`): the reference itself to `True`.
|
||||
|
||||
Any other surviving reference escapes the projection's value, which no longer
|
||||
exists after fusion, so refuse rather than change semantics.
|
||||
"""
|
||||
exists after fusion, so refuse rather than change semantics."""
|
||||
for node in ast.walk(funcdef):
|
||||
# `self.<name> is (not) None`, either operand order.
|
||||
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
|
||||
@@ -474,8 +467,7 @@ def _base_name(node: ast.expr) -> str | None:
|
||||
def _rebound_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names rebound outright within `region` (`x = ...`, `del x`).
|
||||
|
||||
These are `Name` nodes in a `Store`/`Del` context.
|
||||
"""
|
||||
These are `Name` nodes in a `Store`/`Del` context."""
|
||||
return {
|
||||
node.id
|
||||
for stmt in region
|
||||
@@ -489,8 +481,7 @@ def _inplace_target(node: ast.AST) -> str | None:
|
||||
|
||||
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
|
||||
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
|
||||
plain `Name` store (see `_rebound_names`).
|
||||
"""
|
||||
plain `Name` store (see `_rebound_names`)."""
|
||||
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
|
||||
node.ctx, (ast.Store, ast.Del)
|
||||
):
|
||||
@@ -711,8 +702,7 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
Non-linear submodules are transparent too (e.g. the dropout GPT-style
|
||||
attentions apply after their output projection). Never walks through a leaf
|
||||
call (e.g. an attention interface): its inputs are what attention consumes,
|
||||
not what produced the value.
|
||||
"""
|
||||
not what produced the value."""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
@@ -735,8 +725,7 @@ def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): what crosses
|
||||
it is consumed by the attention computation, not projected.
|
||||
"""
|
||||
it is consumed by the attention computation, not projected."""
|
||||
queue = list(node.users)
|
||||
seen: set[fx.Node] = set()
|
||||
while queue:
|
||||
|
||||
@@ -22,8 +22,7 @@ _VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used.
|
||||
"""
|
||||
else from vLLM. Logs which source was used."""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
|
||||
@@ -75,8 +75,7 @@ class TransformersMoERunner(MoERunner):
|
||||
) -> torch.Tensor:
|
||||
"""In Transformers `experts.forward` will have this signature.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here.
|
||||
"""
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
@@ -368,8 +367,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier.
|
||||
"""
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
|
||||
@@ -424,8 +424,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||
deriving them from updates.
|
||||
"""
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
@@ -459,8 +458,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
num_audios: int,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||
and record how many tokens the run holds.
|
||||
"""
|
||||
and record how many tokens the run holds."""
|
||||
audio_token_ids = self._get_mm_token_ids("audio")
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||
@@ -518,8 +516,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Split the positions the processor marks as image into one placeholder per
|
||||
item, sized by the token count it reports for each image.
|
||||
"""
|
||||
item, sized by the token count it reports for each image."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if mm_token_type_ids is None:
|
||||
raise ValueError(
|
||||
@@ -572,8 +569,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||
then read the placeholder ranges out of the token ids it returns.
|
||||
"""
|
||||
then read the placeholder ranges out of the token ids it returns."""
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
@@ -711,8 +707,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Replace each modality's placeholder token with the token ids that item's
|
||||
replacement text encodes to, marking which of them hold embeddings.
|
||||
"""
|
||||
replacement text encodes to, marking which of them hold embeddings."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
@@ -907,8 +902,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
self, model: "PreTrainedModel"
|
||||
) -> dict[str, type["PreTrainedModel"]]:
|
||||
"""Modalities whose encoder cannot be told apart from the model itself are
|
||||
omitted, as are those `get_encoder` rejects.
|
||||
"""
|
||||
omitted, as are those `get_encoder` rejects."""
|
||||
encoder_classes: dict[str, type[PreTrainedModel]] = {}
|
||||
for modality in _MODALITY_TO_TOKEN_TYPE_ID:
|
||||
try:
|
||||
@@ -1027,8 +1021,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
"""Transformers modeling backend multimodal classes do not contain a separate
|
||||
vLLM language model class. Therefore, in order to return a language model vLLM
|
||||
class, we use a wrapper to give `self` the same interface as a text model.
|
||||
"""
|
||||
class, we use a wrapper to give `self` the same interface as a text model."""
|
||||
# Exclude self and object
|
||||
bases = self.__class__.mro()[1:-1]
|
||||
# Keep only classes defined in `vllm.model_executor.models.transformers`
|
||||
@@ -1182,8 +1175,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
) -> dict[str, Any]:
|
||||
"""Narrow the entries of `kwargs` that hold one row per item down to the item
|
||||
at `index`. Length is all there is to match on, so an unrelated entry of the
|
||||
same length is narrowed too.
|
||||
"""
|
||||
same length is narrowed too."""
|
||||
return {
|
||||
key: value[index : index + 1]
|
||||
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
|
||||
|
||||
@@ -201,8 +201,7 @@ def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
|
||||
class _UninitializedEmbedding(nn.Embedding):
|
||||
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`.
|
||||
"""
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
@@ -210,8 +209,7 @@ class _UninitializedEmbedding(nn.Embedding):
|
||||
|
||||
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
|
||||
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.
|
||||
"""
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -47,8 +47,7 @@ ShardId: TypeAlias = str | int | tuple[int, ...]
|
||||
class WeightsMapper:
|
||||
"""Maps the name of each weight if they match the following patterns.
|
||||
|
||||
If a key maps to a value of `None`, the corresponding weight is ignored.
|
||||
"""
|
||||
If a key maps to a value of `None`, the corresponding weight is ignored."""
|
||||
|
||||
orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)
|
||||
orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)
|
||||
@@ -172,8 +171,7 @@ class WeightsMapper:
|
||||
rather than being rewritten to the stacked vLLM name (`qkv_proj`). Mappings to
|
||||
`None` are dropped because "do not load this weight" is meaningless to such a
|
||||
consumer, and applying it would silently shrink a quantization config's ignore
|
||||
list or make LoRA name parsing fail.
|
||||
"""
|
||||
list or make LoRA name parsing fail."""
|
||||
remove_none = lambda d: {k: v for k, v in d.items() if v is not None}
|
||||
return replace(
|
||||
self,
|
||||
|
||||
@@ -116,8 +116,7 @@ def get_multimodal_config() -> MultiModalConfig | None:
|
||||
"""Return the current ``MultiModalConfig``, or ``None`` when no engine
|
||||
config context is active (e.g., during unit tests) or when the current
|
||||
``model_config`` does not carry a ``multimodal_config`` (e.g., minimal
|
||||
stubs used in tests).
|
||||
"""
|
||||
stubs used in tests)."""
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None or vllm_config.model_config is None:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user