[Mypy] Fix mypy typing for Transformers models (#54320)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
b78fbab4be
commit
e960ead0aa
@@ -35,7 +35,7 @@ from transformers.conversion_mapping import (
|
||||
get_model_conversion_mapping,
|
||||
)
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.compilation.decorators import DynamicArgDims, support_torch_compile
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_pp_group, get_tp_group
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
@@ -76,6 +76,7 @@ from vllm.model_executor.models.transformers.utils import (
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
ShardId,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
@@ -110,10 +111,13 @@ class Base(
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
):
|
||||
embedding_modules = ["embed_tokens"] # TODO transformers will have a util to get it
|
||||
hf_to_vllm_mapper: WeightsMapper
|
||||
|
||||
# TODO transformers will have a util to get these
|
||||
embedding_modules = {"embed_tokens": "input_embeddings"}
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
super().__init__()
|
||||
nn.Module.__init__(self)
|
||||
logger.info("Using Transformers modeling backend.")
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
@@ -248,7 +252,7 @@ class Base(
|
||||
def _decorate_cls_for_torch_compile(
|
||||
self,
|
||||
cls: type["PreTrainedModel"],
|
||||
dynamic_arg_dims: dict[str, int] | None,
|
||||
dynamic_arg_dims: DynamicArgDims | None,
|
||||
enable_if: Callable[["VllmConfig"], bool],
|
||||
is_encoder: bool,
|
||||
):
|
||||
@@ -284,7 +288,7 @@ class Base(
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._pre_trained_model_classes.decoder,
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
dynamic_arg_dims=dict[str, int](
|
||||
dynamic_arg_dims=DynamicArgDims(
|
||||
input_ids=1, # shape: [1, seq_len]
|
||||
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
|
||||
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
|
||||
@@ -303,9 +307,8 @@ class Base(
|
||||
- Checkpoints saved with no base model prefix
|
||||
- Any quantization config specific mappings
|
||||
"""
|
||||
self.hf_to_vllm_mapper = WeightsMapper()
|
||||
orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming
|
||||
orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex
|
||||
orig_to_new_renaming: list[WeightRenaming] = []
|
||||
orig_to_new_regex: dict[re.Pattern, str | None] = {}
|
||||
|
||||
for mapping in get_model_conversion_mapping(self.model):
|
||||
# Handle weights which have been renamed in Transformers
|
||||
@@ -341,6 +344,11 @@ class Base(
|
||||
nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)")
|
||||
orig_to_new_regex[nested_lm_head_pattern] = r"\2"
|
||||
|
||||
self.hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_renaming=orig_to_new_renaming,
|
||||
orig_to_new_regex=orig_to_new_regex,
|
||||
)
|
||||
|
||||
# Apply mapping to quantization config if needed
|
||||
self._maybe_apply_model_mapping()
|
||||
|
||||
@@ -371,8 +379,8 @@ class Base(
|
||||
"modeling backend will infer the split from the layers of %s in order "
|
||||
"of declaration and keep parameter-free modules on every rank. This "
|
||||
"may fail if the model's structure is non-standard. %s",
|
||||
type(self.model),
|
||||
type(module),
|
||||
type(self.model).__name__,
|
||||
type(module).__name__,
|
||||
tip,
|
||||
)
|
||||
|
||||
@@ -470,7 +478,7 @@ class Base(
|
||||
"backend will shard the model the best it can during graph fusion and "
|
||||
"replicate the rest. This may be suboptimal or fail if the model does "
|
||||
"not fuse cleanly. %s",
|
||||
type(self.model),
|
||||
type(self.model).__name__,
|
||||
tip,
|
||||
)
|
||||
|
||||
@@ -481,6 +489,8 @@ class Base(
|
||||
|
||||
vocab_embeddings = self._vocab_embeddings()
|
||||
|
||||
orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {}
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
self.fusers.setdefault(prefix, []).append(fuser)
|
||||
@@ -498,8 +508,7 @@ class Base(
|
||||
)
|
||||
self.attention_fusers[index] = (prefix, fuser)
|
||||
|
||||
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
|
||||
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
|
||||
orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix))
|
||||
|
||||
packed_modules_mapping = fuser.packed_modules_mapping
|
||||
self.packed_modules_mapping.update(packed_modules_mapping)
|
||||
@@ -564,6 +573,8 @@ class Base(
|
||||
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
||||
|
||||
def _create_attention_instances(self):
|
||||
"""Create `Attention` instances to inform KV cache allocation."""
|
||||
text_config = self.text_config
|
||||
@@ -589,8 +600,11 @@ class Base(
|
||||
|
||||
for i in range(start, end):
|
||||
if i not in self.attention_fusers:
|
||||
in_range = layer_types and i < len(layer_types)
|
||||
layer = f"{i} ({layer_types[i]})" if in_range else str(i)
|
||||
layer = (
|
||||
f"{i} ({layer_types[i]})"
|
||||
if layer_types and i < len(layer_types)
|
||||
else str(i)
|
||||
)
|
||||
raise ValueError(
|
||||
f"Layer {layer} does not dispatch through the Transformers "
|
||||
"attention interface and vLLM has no other way to handle it."
|
||||
|
||||
Reference in New Issue
Block a user