[Mypy] Fix mypy typing for Transformers models (#54320)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-17 12:11:31 -04:00
committed by GitHub
co-authored by Harry Mellor
parent b78fbab4be
commit e960ead0aa
16 changed files with 224 additions and 145 deletions
@@ -5,6 +5,7 @@
import ast
import inspect
import textwrap
from collections.abc import Callable
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
@@ -36,16 +37,16 @@ def _is_interface_lookup(node: ast.expr | None) -> bool:
@cache
def interface_call(cls: type[nn.Module]) -> ast.Call | None:
"""The attention interface call in `cls.forward`, if it makes exactly one."""
def interface_call(forward: Callable) -> ast.Call | None:
"""The attention interface call in `forward`, if it makes exactly one."""
try:
source = inspect.getsource(inspect.unwrap(cls.forward))
source = inspect.getsource(inspect.unwrap(forward))
tree = ast.parse(textwrap.dedent(source))
except (AttributeError, OSError, SyntaxError, TypeError):
return None
# Find the names of the local variables that read from the interface lookup
names = set()
names: set[str] = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
continue
@@ -97,7 +98,7 @@ class AttentionFuser(BaseFuser):
def match(
cls, graph: fx.Graph | None, module: nn.Module
) -> "AttentionFuser | None":
if (call := interface_call(type(module))) is None:
if (call := interface_call(type(module).forward)) is None:
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
@@ -120,6 +121,7 @@ class AttentionFuser(BaseFuser):
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
if (sinks := self.sinks(module)) is not None:
assert isinstance(self.s_aux_expr, ast.Attribute)
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
device = vllm_config.device_config.device
data = torch.empty(size, dtype=sinks.dtype, device=device)
@@ -130,11 +130,14 @@ class StackedFuser(RewriteFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it."""
merged_name: ClassVar[str]
"""Attribute name of the merged module created by `update_attrs`."""
merged_cls_name: ClassVar[str]
"""Name of the vLLM class the merged projection becomes (for logging)."""
@property
@abstractmethod
def merged_name(self) -> str:
"""Attribute name of the merged module created by `update_attrs`."""
def info(self, name: str) -> str:
sources = " + ".join(shard for shard, _ in self.shards)
return (
@@ -24,6 +24,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
Style,
log_replacement,
replace_linear_class,
)
@@ -128,30 +129,26 @@ class MLAFuser(StackedFuser):
merged_name: ClassVar[str] = "fused_qkv_a_proj"
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
@property
def has_q_lora(self) -> bool:
return self.q_a_proj_name is not None
def info(self, name: str) -> str:
info_str = (
f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)"
)
if self.has_q_lora:
if self.q_a_proj_name is not None:
info_str += "; " + super().info(name).removeprefix("Fused: ")
return info_str
@property
def shards(self) -> list[tuple[str, ShardId]]:
"""`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection."""
if self.has_q_lora:
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
return []
if self.q_a_proj_name is None:
return []
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
if self.has_q_lora:
return super().packed_modules_mapping
return {}
if self.q_a_proj_name is None:
return {}
return super().packed_modules_mapping
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None":
@@ -234,10 +231,10 @@ class MLAFuser(StackedFuser):
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
attention interface unexpanded."""
funcdef, fn = recover_forward(type(module))
if self.has_q_lora:
if (q_a_proj_name := self.q_a_proj_name) is not None:
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
# The fused call is inserted at the top-level statement preceding both.
names = [self.q_a_proj_name, self.kv_a_proj_name]
names = [q_a_proj_name, self.kv_a_proj_name]
calls = self._unguarded_calls(funcdef, names)
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
raise ValueError("down-projections read different inputs")
@@ -258,7 +255,7 @@ class MLAFuser(StackedFuser):
def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"):
quant_config = vllm_config.quant_config
def replace_linear_by_name(name: str, style: str):
def replace_linear_by_name(name: str, style: Style):
linear = module.get_submodule(name)
replacement = replace_linear_class(
linear, style, quant_config, prefix=maybe_prefix(prefix, name)
@@ -266,8 +263,10 @@ class MLAFuser(StackedFuser):
setattr(module, name, replacement)
log_replacement(maybe_prefix(prefix, name), linear, replacement)
if self.has_q_lora:
q_a = module.get_submodule(self.q_a_proj_name)
if (q_a_proj_name := self.q_a_proj_name) is not None:
# `match` sets the q-LoRA names together, or none of them
assert self.q_b_proj_name is not None
q_a = module.get_submodule(q_a_proj_name)
kv_a = module.get_submodule(self.kv_a_proj_name)
merged = MergedColumnParallelLinear(
input_size=q_a.in_features,
@@ -280,7 +279,7 @@ class MLAFuser(StackedFuser):
)
logger.debug(
"%s: %s, %s: %s -> %s: %s",
self.q_a_proj_name,
q_a_proj_name,
q_a,
self.kv_a_proj_name,
kv_a,
@@ -289,10 +288,11 @@ class MLAFuser(StackedFuser):
)
setattr(module, self.merged_name, merged)
# The rewritten forward calls the merged projection instead.
delattr(module, self.q_a_proj_name)
delattr(module, q_a_proj_name)
delattr(module, self.kv_a_proj_name)
replace_linear_by_name(self.q_b_proj_name, "colwise")
else:
assert self.q_proj_name is not None
replace_linear_by_name(self.kv_a_proj_name, "replicate")
replace_linear_by_name(self.q_proj_name, "colwise")
@@ -216,7 +216,7 @@ class RMSNormFuser(BaseFuser):
finally:
for name, value in candidates.items():
setattr(module, name, value)
if (name := markers.get(marked)) is not None:
if marked is not None and (name := markers.get(marked)) is not None:
return name, None
logger.debug_once(
"%s does not hold its eps (%s) in an attribute. Every instance in this "
@@ -251,18 +251,18 @@ class RMSNormFuser(BaseFuser):
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
weight = getattr(module, "weight", None)
has_weight = weight is not None
hidden_size = weight.size(0) if has_weight else 0
hidden_size = weight.size(0) if weight is not None else 0
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
if not isinstance(eps, (int, float)):
# If eps was not detected, match torch behaviour.
dtype = weight.dtype if has_weight else vllm_config.model_config.dtype
config_dtype = vllm_config.model_config.dtype
dtype = weight.dtype if weight is not None else config_dtype
eps = torch.finfo(dtype).eps
if self.zero_centered:
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
return TPAwareRMSNorm(
hidden_size=hidden_size,
eps=eps,
has_weight=has_weight,
dtype=weight.dtype if has_weight else None,
has_weight=weight is not None,
dtype=weight.dtype if weight is not None else None,
)