[Mypy] Fix mypy typing for Transformers models (#54320)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
b78fbab4be
commit
e960ead0aa
@@ -5,6 +5,7 @@
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
@@ -36,16 +37,16 @@ def _is_interface_lookup(node: ast.expr | None) -> bool:
|
||||
|
||||
|
||||
@cache
|
||||
def interface_call(cls: type[nn.Module]) -> ast.Call | None:
|
||||
"""The attention interface call in `cls.forward`, if it makes exactly one."""
|
||||
def interface_call(forward: Callable) -> ast.Call | None:
|
||||
"""The attention interface call in `forward`, if it makes exactly one."""
|
||||
try:
|
||||
source = inspect.getsource(inspect.unwrap(cls.forward))
|
||||
source = inspect.getsource(inspect.unwrap(forward))
|
||||
tree = ast.parse(textwrap.dedent(source))
|
||||
except (AttributeError, OSError, SyntaxError, TypeError):
|
||||
return None
|
||||
|
||||
# Find the names of the local variables that read from the interface lookup
|
||||
names = set()
|
||||
names: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
|
||||
continue
|
||||
@@ -97,7 +98,7 @@ class AttentionFuser(BaseFuser):
|
||||
def match(
|
||||
cls, graph: fx.Graph | None, module: nn.Module
|
||||
) -> "AttentionFuser | None":
|
||||
if (call := interface_call(type(module))) is None:
|
||||
if (call := interface_call(type(module).forward)) is None:
|
||||
return None
|
||||
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
|
||||
scale_expr = scaling[0] if len(scaling) == 1 else None
|
||||
@@ -120,6 +121,7 @@ class AttentionFuser(BaseFuser):
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
if (sinks := self.sinks(module)) is not None:
|
||||
assert isinstance(self.s_aux_expr, ast.Attribute)
|
||||
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
|
||||
device = vllm_config.device_config.device
|
||||
data = torch.empty(size, dtype=sinks.dtype, device=device)
|
||||
|
||||
@@ -130,11 +130,14 @@ class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it."""
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
merged_cls_name: ClassVar[str]
|
||||
"""Name of the vLLM class the merged projection becomes (for logging)."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def merged_name(self) -> str:
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
sources = " + ".join(shard for shard, _ in self.shards)
|
||||
return (
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
Style,
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
@@ -128,30 +129,26 @@ class MLAFuser(StackedFuser):
|
||||
merged_name: ClassVar[str] = "fused_qkv_a_proj"
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
@property
|
||||
def has_q_lora(self) -> bool:
|
||||
return self.q_a_proj_name is not None
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
info_str = (
|
||||
f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)"
|
||||
)
|
||||
if self.has_q_lora:
|
||||
if self.q_a_proj_name is not None:
|
||||
info_str += "; " + super().info(name).removeprefix("Fused: ")
|
||||
return info_str
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection."""
|
||||
if self.has_q_lora:
|
||||
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
|
||||
return []
|
||||
if self.q_a_proj_name is None:
|
||||
return []
|
||||
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
if self.has_q_lora:
|
||||
return super().packed_modules_mapping
|
||||
return {}
|
||||
if self.q_a_proj_name is None:
|
||||
return {}
|
||||
return super().packed_modules_mapping
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None":
|
||||
@@ -234,10 +231,10 @@ class MLAFuser(StackedFuser):
|
||||
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
|
||||
attention interface unexpanded."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
if self.has_q_lora:
|
||||
if (q_a_proj_name := self.q_a_proj_name) is not None:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
# The fused call is inserted at the top-level statement preceding both.
|
||||
names = [self.q_a_proj_name, self.kv_a_proj_name]
|
||||
names = [q_a_proj_name, self.kv_a_proj_name]
|
||||
calls = self._unguarded_calls(funcdef, names)
|
||||
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
|
||||
raise ValueError("down-projections read different inputs")
|
||||
@@ -258,7 +255,7 @@ class MLAFuser(StackedFuser):
|
||||
def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
def replace_linear_by_name(name: str, style: str):
|
||||
def replace_linear_by_name(name: str, style: Style):
|
||||
linear = module.get_submodule(name)
|
||||
replacement = replace_linear_class(
|
||||
linear, style, quant_config, prefix=maybe_prefix(prefix, name)
|
||||
@@ -266,8 +263,10 @@ class MLAFuser(StackedFuser):
|
||||
setattr(module, name, replacement)
|
||||
log_replacement(maybe_prefix(prefix, name), linear, replacement)
|
||||
|
||||
if self.has_q_lora:
|
||||
q_a = module.get_submodule(self.q_a_proj_name)
|
||||
if (q_a_proj_name := self.q_a_proj_name) is not None:
|
||||
# `match` sets the q-LoRA names together, or none of them
|
||||
assert self.q_b_proj_name is not None
|
||||
q_a = module.get_submodule(q_a_proj_name)
|
||||
kv_a = module.get_submodule(self.kv_a_proj_name)
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=q_a.in_features,
|
||||
@@ -280,7 +279,7 @@ class MLAFuser(StackedFuser):
|
||||
)
|
||||
logger.debug(
|
||||
"%s: %s, %s: %s -> %s: %s",
|
||||
self.q_a_proj_name,
|
||||
q_a_proj_name,
|
||||
q_a,
|
||||
self.kv_a_proj_name,
|
||||
kv_a,
|
||||
@@ -289,10 +288,11 @@ class MLAFuser(StackedFuser):
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
# The rewritten forward calls the merged projection instead.
|
||||
delattr(module, self.q_a_proj_name)
|
||||
delattr(module, q_a_proj_name)
|
||||
delattr(module, self.kv_a_proj_name)
|
||||
replace_linear_by_name(self.q_b_proj_name, "colwise")
|
||||
else:
|
||||
assert self.q_proj_name is not None
|
||||
replace_linear_by_name(self.kv_a_proj_name, "replicate")
|
||||
replace_linear_by_name(self.q_proj_name, "colwise")
|
||||
|
||||
|
||||
@@ -216,7 +216,7 @@ class RMSNormFuser(BaseFuser):
|
||||
finally:
|
||||
for name, value in candidates.items():
|
||||
setattr(module, name, value)
|
||||
if (name := markers.get(marked)) is not None:
|
||||
if marked is not None and (name := markers.get(marked)) is not None:
|
||||
return name, None
|
||||
logger.debug_once(
|
||||
"%s does not hold its eps (%s) in an attribute. Every instance in this "
|
||||
@@ -251,18 +251,18 @@ class RMSNormFuser(BaseFuser):
|
||||
) -> nn.Module:
|
||||
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
|
||||
weight = getattr(module, "weight", None)
|
||||
has_weight = weight is not None
|
||||
hidden_size = weight.size(0) if has_weight else 0
|
||||
hidden_size = weight.size(0) if weight is not None else 0
|
||||
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
|
||||
if not isinstance(eps, (int, float)):
|
||||
# If eps was not detected, match torch behaviour.
|
||||
dtype = weight.dtype if has_weight else vllm_config.model_config.dtype
|
||||
config_dtype = vllm_config.model_config.dtype
|
||||
dtype = weight.dtype if weight is not None else config_dtype
|
||||
eps = torch.finfo(dtype).eps
|
||||
if self.zero_centered:
|
||||
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
|
||||
return TPAwareRMSNorm(
|
||||
hidden_size=hidden_size,
|
||||
eps=eps,
|
||||
has_weight=has_weight,
|
||||
dtype=weight.dtype if has_weight else None,
|
||||
has_weight=weight is not None,
|
||||
dtype=weight.dtype if weight is not None else None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user