Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -14,7 +14,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Wrapper around `transformers` models"""
"""Wrapper around `transformers` models."""
from typing import TYPE_CHECKING
+11 -16
View File
@@ -193,8 +193,7 @@ class Base(
)
def _patch_config(self):
"""
Patch the config to ensure that the model is created correctly:
"""Patch the config to ensure that the model is created correctly:
- Sets the attention implementation to "vllm" so the attention instances from
`_create_attention_instances` are used
@@ -208,7 +207,8 @@ class Base(
def _mark_model_components(self, vllm_config: "VllmConfig"):
"""Mark language model and tower submodules as `self.model` is created.
Nothing to do in `Base`, `MultiModalMixin` will override."""
Nothing to do in `Base`, `MultiModalMixin` will override.
"""
yield
@cached_property
@@ -253,8 +253,7 @@ class Base(
enable_if: Callable[["VllmConfig"], bool],
is_encoder: bool,
):
"""
Decorate `cls` to indicate to vLLM that it supports torch compile.
"""Decorate `cls` to indicate to vLLM that it supports torch compile.
Args:
cls: The PreTrainedModel class to decorate.
@@ -265,6 +264,7 @@ class Base(
enable_if: A function which takes in the vLLM config and returns whether
torch compile should be enabled for this class.
is_encoder: Whether the class being decorated is an encoder.
"""
logger.debug(
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
@@ -281,7 +281,8 @@ class Base(
def _decorate_for_torch_compile(self):
"""Decorate the model's decoder class to indicate to vLLM that it
supports torch compile if `can_enable_torch_compile` is True."""
supports torch compile if `can_enable_torch_compile` is True.
"""
self._decorate_cls_for_torch_compile(
cls=self._pre_trained_model_classes.decoder,
# Applied to a PreTrainedModel so the batch dimension will exist
@@ -295,8 +296,7 @@ class Base(
)
def _create_hf_to_vllm_mapper(self):
"""
Create a WeightsMapper to map checkpoint weight names to module qualnames.
"""Create a WeightsMapper to map checkpoint weight names to module qualnames.
This handles:
@@ -347,18 +347,14 @@ class Base(
self._maybe_apply_model_mapping()
def _get_tie_word_embeddings(self):
"""
Check if the model has tied word embeddings.
"""
"""Check if the model has tied word embeddings."""
# Models created with Transformers v4 and v5 will store this in different places
tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False)
tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False)
return tie_word_embeddings_v4 or tie_word_embeddings_v5
def pipeline_parallel(self):
"""
Apply the model's pipeline parallelization plan.
"""
"""Apply the model's pipeline parallelization plan."""
if self.pp_group.world_size <= 1:
return
@@ -697,8 +693,7 @@ class Base(
return Attention
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
"""
If a `parameter` is on the `meta` device, then its parent
"""If a `parameter` is on the `meta` device, then its parent
`module` is the original module created by:
```python
@@ -65,7 +65,6 @@ class CausalMixin(VllmModelForTextGeneration):
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
lm_head_bias = set()
def auto_load_lm_head_bias(weights):
@@ -55,17 +55,20 @@ class BaseFuser(ABC):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built)."""
module to install in its place (mutated in place, or freshly built).
"""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
(none unless it stacks weights)."""
(none unless it stacks weights).
"""
return {}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`packed_modules_mapping` entries this fuser contributes (none unless
it stacks weights)."""
it stacks weights).
"""
return {}
@@ -119,7 +122,8 @@ class RewriteFuser(BaseFuser):
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
Builds the merged submodule and binds the compiled forward.
"""
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@@ -128,7 +132,8 @@ class RewriteFuser(BaseFuser):
@dataclass
class StackedFuser(RewriteFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it."""
rewrites the forward to call it.
"""
merged_name: ClassVar[str]
"""Attribute name of the merged module created by `update_attrs`."""
@@ -147,14 +152,16 @@ class StackedFuser(RewriteFuser):
def shards(self) -> list[tuple[str, ShardId]]:
"""Each projection's original name and its shard id in the merged module.
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
Source for both `orig_to_new_stacked` and `packed_modules_mapping`.
"""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
so only this exact layer is remapped, never a same-named projection
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
elsewhere (e.g. an unfused MoE expert's `gate_proj`).
"""
merged = maybe_prefix(prefix, self.merged_name)
return {
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
@@ -163,7 +170,8 @@ class StackedFuser(RewriteFuser):
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`{merged_name: [projection names]}` so quantization can unpack the
fused layer into its per-shard configs."""
fused layer into its per-shard configs.
"""
return {self.merged_name: [name for name, _ in self.shards]}
def _unguarded_calls(
@@ -174,7 +182,8 @@ class StackedFuser(RewriteFuser):
`update_attrs` deletes the projections it merges, so any reference to one beyond
its call site must be a guard on its existence; folding those to their constant
value keeps the rewritten forward off a name that no longer exists. A reference
that is not such a guard raises, so fusion is skipped."""
that is not such a guard raises, so fusion is skipped.
"""
calls = []
for name in names:
call, refs = self_call_and_refs(funcdef, name)
@@ -232,7 +232,8 @@ class MLAFuser(StackedFuser):
def update_forward(self, module: nn.Module) -> None:
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
attention interface unexpanded."""
attention interface unexpanded.
"""
funcdef, fn = recover_forward(type(module))
if self.has_q_lora:
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
@@ -80,7 +80,8 @@ def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it."""
even when the config does not name it.
"""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
@@ -125,7 +126,8 @@ class SharedExpertMLP(nn.Module):
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
"""Standard MoE block forward.
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
Routing and any shared experts are handled inside `self.experts: MoERunner`.
"""
orig_shape = hidden_states.shape
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
num_tokens = hidden_states.shape[0]
@@ -152,7 +154,8 @@ class MoEBlockFuser:
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
Returns the scoring function and the dtype the router computes in.
"""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
@@ -280,7 +283,8 @@ class MoEBlockFuser:
self, moe_block: nn.Module, prefix: str
) -> SharedExpertMLP | None:
"""Build the HF shared expert (and its optional gate)
as a `SharedExpertMLP` for vLLM's fused MoE."""
as a `SharedExpertMLP` for vLLM's fused MoE.
"""
if self.shared_name is None:
return None
shared_experts = getattr(moe_block, self.shared_name)
@@ -51,7 +51,7 @@ def _is_squared(node: object, x: fx.Node) -> bool:
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
"""eps from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
"""Eps from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
add = peel(rsqrt.args[0])
if not is_op(add, "add"):
return None
@@ -43,7 +43,8 @@ def _reference_weight(module: nn.Module) -> torch.Tensor | None:
"""A weight whose trailing dim is the module's hidden size.
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype.
"""
for child in module.modules():
if isinstance(child, nn.Linear):
return child.weight
@@ -58,7 +59,8 @@ class _MetaProxy(fx.Proxy):
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
executing each op on the meta values, so PyTorch's meta kernels are the
single source of shape inference — no per-op rules."""
single source of shape inference — no per-op rules.
"""
meta: object = _UNKNOWN
@@ -82,7 +84,8 @@ class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
`Tracer.proxy`, so the meta value must be grafted on here too."""
`Tracer.proxy`, so the meta value must be grafted on here too.
"""
def __init__(self, root: fx.Proxy, attr: str):
super().__init__(root, attr)
@@ -218,7 +221,8 @@ def _leaf_attention_interfaces():
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
`vllm_attention_function` needs runtime context so it is untraceable.
Every interface returns `(attn_output, attn_weights)`."""
Every interface returns `(attn_output, attn_weights)`.
"""
from transformers.modeling_utils import AttentionInterface
original = AttentionInterface.get_interface
@@ -296,7 +300,8 @@ def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
def forward_input_count(cls: type[nn.Module]) -> int:
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
any `*args`/`**kwargs`. Read from the signature, so it is independent of
whether the trace completes (unlike counting placeholders)."""
whether the trace completes (unlike counting placeholders).
"""
params = list(forward_parameters(cls).values())
if not params:
return 1 # uninspectable: assume a single input and let matching decide
@@ -373,7 +378,8 @@ def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
In these positions the object's identity never escapes, so a reference that
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
yield an operand, so the module could escape (`x and self.<name>`)."""
yield an operand, so the module could escape (`x and self.<name>`).
"""
for node in ast.walk(funcdef):
if (
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
@@ -406,7 +412,8 @@ def bypass_existence_guard(
`not self.<name>`): the reference itself to `True`.
Any other surviving reference escapes the projection's value, which no longer
exists after fusion, so refuse rather than change semantics."""
exists after fusion, so refuse rather than change semantics.
"""
for node in ast.walk(funcdef):
# `self.<name> is (not) None`, either operand order.
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
@@ -467,7 +474,8 @@ def _base_name(node: ast.expr) -> str | None:
def _rebound_names(region: list[ast.stmt]) -> set[str]:
"""Names rebound outright within `region` (`x = ...`, `del x`).
These are `Name` nodes in a `Store`/`Del` context."""
These are `Name` nodes in a `Store`/`Del` context.
"""
return {
node.id
for stmt in region
@@ -481,7 +489,8 @@ def _inplace_target(node: ast.AST) -> str | None:
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
plain `Name` store (see `_rebound_names`)."""
plain `Name` store (see `_rebound_names`).
"""
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
node.ctx, (ast.Store, ast.Del)
):
@@ -702,7 +711,8 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
Non-linear submodules are transparent too (e.g. the dropout GPT-style
attentions apply after their output projection). Never walks through a leaf
call (e.g. an attention interface): its inputs are what attention consumes,
not what produced the value."""
not what produced the value.
"""
stack = [node]
seen: set[fx.Node] = set()
while stack:
@@ -725,7 +735,8 @@ def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
Never walks through a leaf call (e.g. an attention interface): what crosses
it is consumed by the attention computation, not projected."""
it is consumed by the attention computation, not projected.
"""
queue = list(node.users)
seen: set[fx.Node] = set()
while queue:
@@ -787,9 +798,7 @@ def is_method(node: object, name: str) -> bool:
def is_op(node: object, name: str) -> bool:
"""
Is node `torch.<name>()`, `F.<name>()`, `operator.<name>()`, or `Tensor.<name>()`.
"""
"""Is node `<mod>.<name>()` for torch, F, operator, or Tensor."""
return any(
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
) or (hasattr(torch.Tensor, name) and is_method(node, name))
@@ -22,7 +22,8 @@ _VLLM_PKG = "vllm.model_executor.layers"
def _resolve(module: str, name: str):
"""Return `name` from the hw-agnostic `module` when enabled and available,
else from vLLM. Logs which source was used."""
else from vLLM. Logs which source was used.
"""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
@@ -75,7 +75,8 @@ class TransformersMoERunner(MoERunner):
) -> torch.Tensor:
"""In Transformers `experts.forward` will have this signature.
We discard any extra kwargs because we cannot use them here."""
We discard any extra kwargs because we cannot use them here.
"""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
@@ -367,7 +368,8 @@ class MoEMixin(MixtureOfExperts):
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
`topk_ids` we stored in the layer earlier.
"""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
@@ -224,7 +224,8 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Rename the parser's `audios` key to the `audio` argument HF audio
processors take."""
processors take.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
@@ -432,7 +433,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Empty, because `apply` writes the placeholder ranges itself rather than
deriving them from updates."""
deriving them from updates.
"""
return []
def _get_mm_token_ids(self, modality: str) -> list[int]:
@@ -466,7 +468,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
num_audios: int,
) -> dict[str, list[PlaceholderRange]]:
"""Take each contiguous run of the audio token as one item's placeholder,
and record how many tokens the run holds."""
and record how many tokens the run holds.
"""
audio_token_ids = self._get_mm_token_ids("audio")
prompt_tensor = torch.tensor(prompt_ids)
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
@@ -524,7 +527,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
"""Split the positions the processor marks as image into one placeholder per
item, sized by the token count it reports for each image."""
item, sized by the token count it reports for each image.
"""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if mm_token_type_ids is None:
raise ValueError(
@@ -577,7 +581,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
timing_ctx: TimingContext,
) -> MultiModalInput:
"""Process the prompt and every multi-modal item in one HF processor call,
then read the placeholder ranges out of the token ids it returns."""
then read the placeholder ranges out of the token ids it returns.
"""
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
@@ -705,7 +710,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Replace each modality's placeholder token with the token ids that item's
replacement text encodes to, marking which of them hold embeddings."""
replacement text encodes to, marking which of them hold embeddings.
"""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
@@ -892,7 +898,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
self, model: "PreTrainedModel"
) -> dict[str, type["PreTrainedModel"]]:
"""Modalities whose encoder cannot be told apart from the model itself are
omitted, as are those `get_encoder` rejects."""
omitted, as are those `get_encoder` rejects.
"""
encoder_classes: dict[str, type[PreTrainedModel]] = {}
for modality in _MODALITY_TO_TOKEN_TYPE_ID:
try:
@@ -953,9 +960,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
yield
def _decorate_for_torch_compile(self):
"""
Decorate the model's decoder and encoder classes to indicate to vLLM that they
support torch compile if `can_enable_torch_compile` and
"""Decorate the model's decoder and encoder classes to indicate to vLLM
that they support torch compile if `can_enable_torch_compile` and
`should_torch_compile_mm_encoder` are True respectively.
"""
super()._decorate_for_torch_compile()
@@ -1012,8 +1018,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
def get_language_model(self) -> torch.nn.Module:
"""Transformers modeling backend multimodal classes do not contain a separate
vLLM language model class. Therefore, in order to return a language model vLLM
class, we use a wrapper to give `self` the same interface as a text model."""
class, we use a wrapper to give `self` the same interface as a text model.
"""
# Exclude self and object
bases = self.__class__.mro()[1:-1]
# Keep only classes defined in `vllm.model_executor.models.transformers`
@@ -1031,9 +1037,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
return LanguageModel(self)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
"""Get the module prefix in multimodal models"""
for name in ("language_model", "text_model"):
if getattr(self.model, name, None) is not None:
return MultiModelKeys.from_string_field(language_model=f"model.{name}")
@@ -1169,7 +1173,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
) -> dict[str, Any]:
"""Narrow the entries of `kwargs` that hold one row per item down to the item
at `index`. Length is all there is to match on, so an unrelated entry of the
same length is narrowed too."""
same length is narrowed too.
"""
return {
key: value[index : index + 1]
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
@@ -84,8 +84,7 @@ class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling):
self.init_parameters(self.classifier, dtype=self.model_config.head_dtype)
class ClassifierWithReshape(self.classifier.__class__):
"""
Token extraction has already been applied in `pooler.pooling`.
"""Token extraction has already been applied in `pooler.pooling`.
Add dim to match expected input shape of `classifier.forward`.
"""
@@ -49,16 +49,15 @@ logger = init_logger(__name__)
# Copied from `accelerate`
@contextmanager
def init_on_device_without_buffers(device: torch.device):
"""
A context manager under which models are initialized with all
"""A context manager under which models are initialized with all
parameters on the specified device. However buffers are not
initialized on specified device.
Args:
device (`torch.device`):
Device to initialize all parameters on.
"""
"""
old_register_parameter = nn.Module.register_parameter
def register_empty_parameter(module, name, param):
@@ -114,17 +113,18 @@ def replace_linear_class(
*,
prefix: str = "",
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
"""
Replace nn.Linear with one of vLLM's tensor parallel linear classes.
"""Replace nn.Linear with one of vLLM's tensor parallel linear classes.
Args:
linear: `nn.Linear` to be replaced.
style: Tensor parallel style of the new linear, e.g. "colwise".
quant_config: Quantization config for the new linear.
prefix: Module prefix of the new linear, used for quantization lookup.
Returns:
The new linear.
"""
"""
if not isinstance(style, str):
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
@@ -156,9 +156,11 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
Args:
conv: `nn.Conv2d` or `nn.Conv3d` to be replaced.
Returns:
The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported,
returns the original conv module.
"""
# vLLM does not handle non-zero padding modes
if conv.padding_mode != "zeros":
@@ -199,7 +201,8 @@ def attrsetter(attr: str) -> Callable[[object, object], None]:
class _UninitializedEmbedding(nn.Embedding):
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
call to `super().__init__` does not invoke `nn.Embedding.__init__`.
"""
def __init__(self):
pass
@@ -207,7 +210,8 @@ class _UninitializedEmbedding(nn.Embedding):
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.
"""
@lru_cache
@@ -217,8 +221,10 @@ def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbe
Args:
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
maps to the same class.
Returns:
The new class, to assign to `__class__` of an instance of `cls`.
"""
return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
@@ -235,8 +241,10 @@ def replace_embedding_class(
embedding: The `nn.Embedding` holding a vocab table.
quant_config: Quantization config for the new embedding.
prefix: Qualname of `embedding`, used to look up its quantization method.
Returns:
The module to install in `embedding`'s place.
"""
# Tied tables are reached more than once, so don't rebase an already new embedding
if isinstance(embedding, VocabParallelEmbedding):
@@ -320,8 +328,7 @@ def get_feature_request_tip(
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
"""
Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
Defaults to `True` but is disabled in the following situations: