Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -14,7 +14,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Wrapper around `transformers` models"""
|
||||
"""Wrapper around `transformers` models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
||||
@@ -193,8 +193,7 @@ class Base(
|
||||
)
|
||||
|
||||
def _patch_config(self):
|
||||
"""
|
||||
Patch the config to ensure that the model is created correctly:
|
||||
"""Patch the config to ensure that the model is created correctly:
|
||||
|
||||
- Sets the attention implementation to "vllm" so the attention instances from
|
||||
`_create_attention_instances` are used
|
||||
@@ -208,7 +207,8 @@ class Base(
|
||||
def _mark_model_components(self, vllm_config: "VllmConfig"):
|
||||
"""Mark language model and tower submodules as `self.model` is created.
|
||||
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override."""
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override.
|
||||
"""
|
||||
yield
|
||||
|
||||
@cached_property
|
||||
@@ -253,8 +253,7 @@ class Base(
|
||||
enable_if: Callable[["VllmConfig"], bool],
|
||||
is_encoder: bool,
|
||||
):
|
||||
"""
|
||||
Decorate `cls` to indicate to vLLM that it supports torch compile.
|
||||
"""Decorate `cls` to indicate to vLLM that it supports torch compile.
|
||||
|
||||
Args:
|
||||
cls: The PreTrainedModel class to decorate.
|
||||
@@ -265,6 +264,7 @@ class Base(
|
||||
enable_if: A function which takes in the vLLM config and returns whether
|
||||
torch compile should be enabled for this class.
|
||||
is_encoder: Whether the class being decorated is an encoder.
|
||||
|
||||
"""
|
||||
logger.debug(
|
||||
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
|
||||
@@ -281,7 +281,8 @@ class Base(
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
"""Decorate the model's decoder class to indicate to vLLM that it
|
||||
supports torch compile if `can_enable_torch_compile` is True."""
|
||||
supports torch compile if `can_enable_torch_compile` is True.
|
||||
"""
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._pre_trained_model_classes.decoder,
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
@@ -295,8 +296,7 @@ class Base(
|
||||
)
|
||||
|
||||
def _create_hf_to_vllm_mapper(self):
|
||||
"""
|
||||
Create a WeightsMapper to map checkpoint weight names to module qualnames.
|
||||
"""Create a WeightsMapper to map checkpoint weight names to module qualnames.
|
||||
|
||||
This handles:
|
||||
|
||||
@@ -347,18 +347,14 @@ class Base(
|
||||
self._maybe_apply_model_mapping()
|
||||
|
||||
def _get_tie_word_embeddings(self):
|
||||
"""
|
||||
Check if the model has tied word embeddings.
|
||||
"""
|
||||
"""Check if the model has tied word embeddings."""
|
||||
# Models created with Transformers v4 and v5 will store this in different places
|
||||
tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False)
|
||||
tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False)
|
||||
return tie_word_embeddings_v4 or tie_word_embeddings_v5
|
||||
|
||||
def pipeline_parallel(self):
|
||||
"""
|
||||
Apply the model's pipeline parallelization plan.
|
||||
"""
|
||||
"""Apply the model's pipeline parallelization plan."""
|
||||
if self.pp_group.world_size <= 1:
|
||||
return
|
||||
|
||||
@@ -697,8 +693,7 @@ class Base(
|
||||
return Attention
|
||||
|
||||
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
|
||||
"""
|
||||
If a `parameter` is on the `meta` device, then its parent
|
||||
"""If a `parameter` is on the `meta` device, then its parent
|
||||
`module` is the original module created by:
|
||||
|
||||
```python
|
||||
|
||||
@@ -65,7 +65,6 @@ class CausalMixin(VllmModelForTextGeneration):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
|
||||
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
|
||||
|
||||
lm_head_bias = set()
|
||||
|
||||
def auto_load_lm_head_bias(weights):
|
||||
|
||||
@@ -55,17 +55,20 @@ class BaseFuser(ABC):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
module to install in its place (mutated in place, or freshly built).
|
||||
"""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
|
||||
(none unless it stacks weights)."""
|
||||
(none unless it stacks weights).
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`packed_modules_mapping` entries this fuser contributes (none unless
|
||||
it stacks weights)."""
|
||||
it stacks weights).
|
||||
"""
|
||||
return {}
|
||||
|
||||
|
||||
@@ -119,7 +122,8 @@ class RewriteFuser(BaseFuser):
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
Builds the merged submodule and binds the compiled forward.
|
||||
"""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
@@ -128,7 +132,8 @@ class RewriteFuser(BaseFuser):
|
||||
@dataclass
|
||||
class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it."""
|
||||
rewrites the forward to call it.
|
||||
"""
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
@@ -147,14 +152,16 @@ class StackedFuser(RewriteFuser):
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""Each projection's original name and its shard id in the merged module.
|
||||
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`.
|
||||
"""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
|
||||
|
||||
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
|
||||
so only this exact layer is remapped, never a same-named projection
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`).
|
||||
"""
|
||||
merged = maybe_prefix(prefix, self.merged_name)
|
||||
return {
|
||||
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
|
||||
@@ -163,7 +170,8 @@ class StackedFuser(RewriteFuser):
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs."""
|
||||
fused layer into its per-shard configs.
|
||||
"""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _unguarded_calls(
|
||||
@@ -174,7 +182,8 @@ class StackedFuser(RewriteFuser):
|
||||
`update_attrs` deletes the projections it merges, so any reference to one beyond
|
||||
its call site must be a guard on its existence; folding those to their constant
|
||||
value keeps the rewritten forward off a name that no longer exists. A reference
|
||||
that is not such a guard raises, so fusion is skipped."""
|
||||
that is not such a guard raises, so fusion is skipped.
|
||||
"""
|
||||
calls = []
|
||||
for name in names:
|
||||
call, refs = self_call_and_refs(funcdef, name)
|
||||
|
||||
@@ -232,7 +232,8 @@ class MLAFuser(StackedFuser):
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
|
||||
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
|
||||
attention interface unexpanded."""
|
||||
attention interface unexpanded.
|
||||
"""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
if self.has_q_lora:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
|
||||
@@ -80,7 +80,8 @@ def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it."""
|
||||
even when the config does not name it.
|
||||
"""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
@@ -125,7 +126,8 @@ class SharedExpertMLP(nn.Module):
|
||||
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Standard MoE block forward.
|
||||
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`.
|
||||
"""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
|
||||
num_tokens = hidden_states.shape[0]
|
||||
@@ -152,7 +154,8 @@ class MoEBlockFuser:
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
Returns the scoring function and the dtype the router computes in.
|
||||
"""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
@@ -280,7 +283,8 @@ class MoEBlockFuser:
|
||||
self, moe_block: nn.Module, prefix: str
|
||||
) -> SharedExpertMLP | None:
|
||||
"""Build the HF shared expert (and its optional gate)
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE."""
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE.
|
||||
"""
|
||||
if self.shared_name is None:
|
||||
return None
|
||||
shared_experts = getattr(moe_block, self.shared_name)
|
||||
|
||||
@@ -51,7 +51,7 @@ def _is_squared(node: object, x: fx.Node) -> bool:
|
||||
|
||||
|
||||
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
|
||||
"""eps from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
|
||||
"""Eps from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
|
||||
add = peel(rsqrt.args[0])
|
||||
if not is_op(add, "add"):
|
||||
return None
|
||||
|
||||
@@ -43,7 +43,8 @@ def _reference_weight(module: nn.Module) -> torch.Tensor | None:
|
||||
"""A weight whose trailing dim is the module's hidden size.
|
||||
|
||||
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype.
|
||||
"""
|
||||
for child in module.modules():
|
||||
if isinstance(child, nn.Linear):
|
||||
return child.weight
|
||||
@@ -58,7 +59,8 @@ class _MetaProxy(fx.Proxy):
|
||||
|
||||
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
|
||||
executing each op on the meta values, so PyTorch's meta kernels are the
|
||||
single source of shape inference — no per-op rules."""
|
||||
single source of shape inference — no per-op rules.
|
||||
"""
|
||||
|
||||
meta: object = _UNKNOWN
|
||||
|
||||
@@ -82,7 +84,8 @@ class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
|
||||
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
|
||||
|
||||
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too."""
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too.
|
||||
"""
|
||||
|
||||
def __init__(self, root: fx.Proxy, attr: str):
|
||||
super().__init__(root, attr)
|
||||
@@ -218,7 +221,8 @@ def _leaf_attention_interfaces():
|
||||
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
|
||||
|
||||
`vllm_attention_function` needs runtime context so it is untraceable.
|
||||
Every interface returns `(attn_output, attn_weights)`."""
|
||||
Every interface returns `(attn_output, attn_weights)`.
|
||||
"""
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
original = AttentionInterface.get_interface
|
||||
@@ -296,7 +300,8 @@ def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
|
||||
def forward_input_count(cls: type[nn.Module]) -> int:
|
||||
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
|
||||
any `*args`/`**kwargs`. Read from the signature, so it is independent of
|
||||
whether the trace completes (unlike counting placeholders)."""
|
||||
whether the trace completes (unlike counting placeholders).
|
||||
"""
|
||||
params = list(forward_parameters(cls).values())
|
||||
if not params:
|
||||
return 1 # uninspectable: assume a single input and let matching decide
|
||||
@@ -373,7 +378,8 @@ def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
|
||||
|
||||
In these positions the object's identity never escapes, so a reference that
|
||||
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
|
||||
yield an operand, so the module could escape (`x and self.<name>`)."""
|
||||
yield an operand, so the module could escape (`x and self.<name>`).
|
||||
"""
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
|
||||
@@ -406,7 +412,8 @@ def bypass_existence_guard(
|
||||
`not self.<name>`): the reference itself to `True`.
|
||||
|
||||
Any other surviving reference escapes the projection's value, which no longer
|
||||
exists after fusion, so refuse rather than change semantics."""
|
||||
exists after fusion, so refuse rather than change semantics.
|
||||
"""
|
||||
for node in ast.walk(funcdef):
|
||||
# `self.<name> is (not) None`, either operand order.
|
||||
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
|
||||
@@ -467,7 +474,8 @@ def _base_name(node: ast.expr) -> str | None:
|
||||
def _rebound_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names rebound outright within `region` (`x = ...`, `del x`).
|
||||
|
||||
These are `Name` nodes in a `Store`/`Del` context."""
|
||||
These are `Name` nodes in a `Store`/`Del` context.
|
||||
"""
|
||||
return {
|
||||
node.id
|
||||
for stmt in region
|
||||
@@ -481,7 +489,8 @@ def _inplace_target(node: ast.AST) -> str | None:
|
||||
|
||||
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
|
||||
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
|
||||
plain `Name` store (see `_rebound_names`)."""
|
||||
plain `Name` store (see `_rebound_names`).
|
||||
"""
|
||||
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
|
||||
node.ctx, (ast.Store, ast.Del)
|
||||
):
|
||||
@@ -702,7 +711,8 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
Non-linear submodules are transparent too (e.g. the dropout GPT-style
|
||||
attentions apply after their output projection). Never walks through a leaf
|
||||
call (e.g. an attention interface): its inputs are what attention consumes,
|
||||
not what produced the value."""
|
||||
not what produced the value.
|
||||
"""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
@@ -725,7 +735,8 @@ def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): what crosses
|
||||
it is consumed by the attention computation, not projected."""
|
||||
it is consumed by the attention computation, not projected.
|
||||
"""
|
||||
queue = list(node.users)
|
||||
seen: set[fx.Node] = set()
|
||||
while queue:
|
||||
@@ -787,9 +798,7 @@ def is_method(node: object, name: str) -> bool:
|
||||
|
||||
|
||||
def is_op(node: object, name: str) -> bool:
|
||||
"""
|
||||
Is node `torch.<name>()`, `F.<name>()`, `operator.<name>()`, or `Tensor.<name>()`.
|
||||
"""
|
||||
"""Is node `<mod>.<name>()` for torch, F, operator, or Tensor."""
|
||||
return any(
|
||||
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
|
||||
) or (hasattr(torch.Tensor, name) and is_method(node, name))
|
||||
|
||||
@@ -22,7 +22,8 @@ _VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used."""
|
||||
else from vLLM. Logs which source was used.
|
||||
"""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
|
||||
@@ -75,7 +75,8 @@ class TransformersMoERunner(MoERunner):
|
||||
) -> torch.Tensor:
|
||||
"""In Transformers `experts.forward` will have this signature.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
We discard any extra kwargs because we cannot use them here.
|
||||
"""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
@@ -367,7 +368,8 @@ class MoEMixin(MixtureOfExperts):
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
`topk_ids` we stored in the layer earlier.
|
||||
"""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
|
||||
@@ -224,7 +224,8 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Rename the parser's `audios` key to the `audio` argument HF audio
|
||||
processors take."""
|
||||
processors take.
|
||||
"""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
@@ -432,7 +433,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||
deriving them from updates."""
|
||||
deriving them from updates.
|
||||
"""
|
||||
return []
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
@@ -466,7 +468,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
num_audios: int,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||
and record how many tokens the run holds."""
|
||||
and record how many tokens the run holds.
|
||||
"""
|
||||
audio_token_ids = self._get_mm_token_ids("audio")
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||
@@ -524,7 +527,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Split the positions the processor marks as image into one placeholder per
|
||||
item, sized by the token count it reports for each image."""
|
||||
item, sized by the token count it reports for each image.
|
||||
"""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if mm_token_type_ids is None:
|
||||
raise ValueError(
|
||||
@@ -577,7 +581,8 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||
then read the placeholder ranges out of the token ids it returns."""
|
||||
then read the placeholder ranges out of the token ids it returns.
|
||||
"""
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
@@ -705,7 +710,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Replace each modality's placeholder token with the token ids that item's
|
||||
replacement text encodes to, marking which of them hold embeddings."""
|
||||
replacement text encodes to, marking which of them hold embeddings.
|
||||
"""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
|
||||
@@ -892,7 +898,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
self, model: "PreTrainedModel"
|
||||
) -> dict[str, type["PreTrainedModel"]]:
|
||||
"""Modalities whose encoder cannot be told apart from the model itself are
|
||||
omitted, as are those `get_encoder` rejects."""
|
||||
omitted, as are those `get_encoder` rejects.
|
||||
"""
|
||||
encoder_classes: dict[str, type[PreTrainedModel]] = {}
|
||||
for modality in _MODALITY_TO_TOKEN_TYPE_ID:
|
||||
try:
|
||||
@@ -953,9 +960,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
yield
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
"""
|
||||
Decorate the model's decoder and encoder classes to indicate to vLLM that they
|
||||
support torch compile if `can_enable_torch_compile` and
|
||||
"""Decorate the model's decoder and encoder classes to indicate to vLLM
|
||||
that they support torch compile if `can_enable_torch_compile` and
|
||||
`should_torch_compile_mm_encoder` are True respectively.
|
||||
"""
|
||||
super()._decorate_for_torch_compile()
|
||||
@@ -1012,8 +1018,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
"""Transformers modeling backend multimodal classes do not contain a separate
|
||||
vLLM language model class. Therefore, in order to return a language model vLLM
|
||||
class, we use a wrapper to give `self` the same interface as a text model."""
|
||||
|
||||
class, we use a wrapper to give `self` the same interface as a text model.
|
||||
"""
|
||||
# Exclude self and object
|
||||
bases = self.__class__.mro()[1:-1]
|
||||
# Keep only classes defined in `vllm.model_executor.models.transformers`
|
||||
@@ -1031,9 +1037,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
return LanguageModel(self)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models"""
|
||||
for name in ("language_model", "text_model"):
|
||||
if getattr(self.model, name, None) is not None:
|
||||
return MultiModelKeys.from_string_field(language_model=f"model.{name}")
|
||||
@@ -1169,7 +1173,8 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
) -> dict[str, Any]:
|
||||
"""Narrow the entries of `kwargs` that hold one row per item down to the item
|
||||
at `index`. Length is all there is to match on, so an unrelated entry of the
|
||||
same length is narrowed too."""
|
||||
same length is narrowed too.
|
||||
"""
|
||||
return {
|
||||
key: value[index : index + 1]
|
||||
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
|
||||
|
||||
@@ -84,8 +84,7 @@ class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling):
|
||||
self.init_parameters(self.classifier, dtype=self.model_config.head_dtype)
|
||||
|
||||
class ClassifierWithReshape(self.classifier.__class__):
|
||||
"""
|
||||
Token extraction has already been applied in `pooler.pooling`.
|
||||
"""Token extraction has already been applied in `pooler.pooling`.
|
||||
Add dim to match expected input shape of `classifier.forward`.
|
||||
"""
|
||||
|
||||
|
||||
@@ -49,16 +49,15 @@ logger = init_logger(__name__)
|
||||
# Copied from `accelerate`
|
||||
@contextmanager
|
||||
def init_on_device_without_buffers(device: torch.device):
|
||||
"""
|
||||
A context manager under which models are initialized with all
|
||||
"""A context manager under which models are initialized with all
|
||||
parameters on the specified device. However buffers are not
|
||||
initialized on specified device.
|
||||
|
||||
Args:
|
||||
device (`torch.device`):
|
||||
Device to initialize all parameters on.
|
||||
"""
|
||||
|
||||
"""
|
||||
old_register_parameter = nn.Module.register_parameter
|
||||
|
||||
def register_empty_parameter(module, name, param):
|
||||
@@ -114,17 +113,18 @@ def replace_linear_class(
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
|
||||
"""
|
||||
Replace nn.Linear with one of vLLM's tensor parallel linear classes.
|
||||
"""Replace nn.Linear with one of vLLM's tensor parallel linear classes.
|
||||
|
||||
Args:
|
||||
linear: `nn.Linear` to be replaced.
|
||||
style: Tensor parallel style of the new linear, e.g. "colwise".
|
||||
quant_config: Quantization config for the new linear.
|
||||
prefix: Module prefix of the new linear, used for quantization lookup.
|
||||
|
||||
Returns:
|
||||
The new linear.
|
||||
"""
|
||||
|
||||
"""
|
||||
if not isinstance(style, str):
|
||||
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
|
||||
|
||||
@@ -156,9 +156,11 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
|
||||
|
||||
Args:
|
||||
conv: `nn.Conv2d` or `nn.Conv3d` to be replaced.
|
||||
|
||||
Returns:
|
||||
The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported,
|
||||
returns the original conv module.
|
||||
|
||||
"""
|
||||
# vLLM does not handle non-zero padding modes
|
||||
if conv.padding_mode != "zeros":
|
||||
@@ -199,7 +201,8 @@ def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
|
||||
class _UninitializedEmbedding(nn.Embedding):
|
||||
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
@@ -207,7 +210,8 @@ class _UninitializedEmbedding(nn.Embedding):
|
||||
|
||||
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
|
||||
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.
|
||||
"""
|
||||
|
||||
|
||||
@lru_cache
|
||||
@@ -217,8 +221,10 @@ def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbe
|
||||
Args:
|
||||
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
|
||||
maps to the same class.
|
||||
|
||||
Returns:
|
||||
The new class, to assign to `__class__` of an instance of `cls`.
|
||||
|
||||
"""
|
||||
return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
|
||||
|
||||
@@ -235,8 +241,10 @@ def replace_embedding_class(
|
||||
embedding: The `nn.Embedding` holding a vocab table.
|
||||
quant_config: Quantization config for the new embedding.
|
||||
prefix: Qualname of `embedding`, used to look up its quantization method.
|
||||
|
||||
Returns:
|
||||
The module to install in `embedding`'s place.
|
||||
|
||||
"""
|
||||
# Tied tables are reached more than once, so don't rebase an already new embedding
|
||||
if isinstance(embedding, VocabParallelEmbedding):
|
||||
@@ -320,8 +328,7 @@ def get_feature_request_tip(
|
||||
|
||||
|
||||
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
|
||||
"""
|
||||
Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
|
||||
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
|
||||
|
||||
Defaults to `True` but is disabled in the following situations:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user