[Mypy] Fix mypy typing for Transformers models (#54320)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-17 12:11:31 -04:00
committed by GitHub
co-authored by Harry Mellor
parent b78fbab4be
commit e960ead0aa
16 changed files with 224 additions and 145 deletions
+29 -15
View File
@@ -35,7 +35,7 @@ from transformers.conversion_mapping import (
get_model_conversion_mapping,
)
from vllm.compilation.decorators import support_torch_compile
from vllm.compilation.decorators import DynamicArgDims, support_torch_compile
from vllm.config.utils import getattr_iter
from vllm.distributed import get_pp_group, get_tp_group
from vllm.distributed.utils import get_pp_indices
@@ -76,6 +76,7 @@ from vllm.model_executor.models.transformers.utils import (
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
ShardId,
WeightsMapper,
make_empty_intermediate_tensors_factory,
maybe_prefix,
@@ -110,10 +111,13 @@ class Base(
SupportsEagle,
SupportsEagle3,
):
embedding_modules = ["embed_tokens"] # TODO transformers will have a util to get it
hf_to_vllm_mapper: WeightsMapper
# TODO transformers will have a util to get these
embedding_modules = {"embed_tokens": "input_embeddings"}
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
super().__init__()
nn.Module.__init__(self)
logger.info("Using Transformers modeling backend.")
self.vllm_config = vllm_config
@@ -248,7 +252,7 @@ class Base(
def _decorate_cls_for_torch_compile(
self,
cls: type["PreTrainedModel"],
dynamic_arg_dims: dict[str, int] | None,
dynamic_arg_dims: DynamicArgDims | None,
enable_if: Callable[["VllmConfig"], bool],
is_encoder: bool,
):
@@ -284,7 +288,7 @@ class Base(
self._decorate_cls_for_torch_compile(
cls=self._pre_trained_model_classes.decoder,
# Applied to a PreTrainedModel so the batch dimension will exist
dynamic_arg_dims=dict[str, int](
dynamic_arg_dims=DynamicArgDims(
input_ids=1, # shape: [1, seq_len]
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
@@ -303,9 +307,8 @@ class Base(
- Checkpoints saved with no base model prefix
- Any quantization config specific mappings
"""
self.hf_to_vllm_mapper = WeightsMapper()
orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming
orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex
orig_to_new_renaming: list[WeightRenaming] = []
orig_to_new_regex: dict[re.Pattern, str | None] = {}
for mapping in get_model_conversion_mapping(self.model):
# Handle weights which have been renamed in Transformers
@@ -341,6 +344,11 @@ class Base(
nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)")
orig_to_new_regex[nested_lm_head_pattern] = r"\2"
self.hf_to_vllm_mapper = WeightsMapper(
orig_to_new_renaming=orig_to_new_renaming,
orig_to_new_regex=orig_to_new_regex,
)
# Apply mapping to quantization config if needed
self._maybe_apply_model_mapping()
@@ -371,8 +379,8 @@ class Base(
"modeling backend will infer the split from the layers of %s in order "
"of declaration and keep parameter-free modules on every rank. This "
"may fail if the model's structure is non-standard. %s",
type(self.model),
type(module),
type(self.model).__name__,
type(module).__name__,
tip,
)
@@ -470,7 +478,7 @@ class Base(
"backend will shard the model the best it can during graph fusion and "
"replicate the rest. This may be suboptimal or fail if the model does "
"not fuse cleanly. %s",
type(self.model),
type(self.model).__name__,
tip,
)
@@ -481,6 +489,8 @@ class Base(
vocab_embeddings = self._vocab_embeddings()
orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {}
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
"""Register a fused layer's mappings just before it is built."""
self.fusers.setdefault(prefix, []).append(fuser)
@@ -498,8 +508,7 @@ class Base(
)
self.attention_fusers[index] = (prefix, fuser)
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix))
packed_modules_mapping = fuser.packed_modules_mapping
self.packed_modules_mapping.update(packed_modules_mapping)
@@ -564,6 +573,8 @@ class Base(
_recursive_replace(self.model, prefix="model")
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
def _create_attention_instances(self):
"""Create `Attention` instances to inform KV cache allocation."""
text_config = self.text_config
@@ -589,8 +600,11 @@ class Base(
for i in range(start, end):
if i not in self.attention_fusers:
in_range = layer_types and i < len(layer_types)
layer = f"{i} ({layer_types[i]})" if in_range else str(i)
layer = (
f"{i} ({layer_types[i]})"
if layer_types and i < len(layer_types)
else str(i)
)
raise ValueError(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."