Reduce AutoWeightsLoader kwargs (#53106)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -45,11 +45,7 @@ from vllm.v1.attention.backend import AttentionType
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .llama import LlamaMLP as Glm4MLP
|
||||
from .llama import LlamaModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
class Glm4Attention(nn.Module):
|
||||
@@ -269,6 +265,16 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
# Drop the speculative (MTP) layers, which are loaded by the draft
|
||||
# model instead. They are appended after the main decoder layers.
|
||||
num_nextn_layers = getattr(config, "num_nextn_predict_layers", 0)
|
||||
self.hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
f"model.layers.{config.num_hidden_layers + i}.": None
|
||||
for i in range(num_nextn_layers)
|
||||
}
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
@@ -292,12 +298,5 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Skip the speculative (MTP) layers, which are loaded by the
|
||||
# draft model instead.
|
||||
num_nextn_layers = getattr(self.config, "num_nextn_predict_layers", 0)
|
||||
skip_prefixes = [
|
||||
f"model.layers.{self.config.num_hidden_layers + i}."
|
||||
for i in range(num_nextn_layers)
|
||||
]
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
return loader.load_weights(weights)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
Reference in New Issue
Block a user