[Migration] Migrate GGUF quantization support to plugin (#39612)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-12 12:02:21 -07:00
committed by GitHub
parent d6fd7ce8da
commit 6635279d8a
57 changed files with 72 additions and 9048 deletions
@@ -6,7 +6,7 @@ from dataclasses import dataclass
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter, UninitializedParameter
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.distributed import (
@@ -77,6 +77,12 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor:
return F.embedding(input_, layer.weight)
def tie_weights(
self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding"
):
layer.weight = embed_tokens.weight
return layer
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
"""Pad the vocab size to the given value."""
@@ -425,17 +431,6 @@ class VocabParallelEmbedding(PluggableLayer):
output_dim = getattr(param, "output_dim", None)
packed_dim = getattr(param, "packed_dim", None)
# If the parameter is a gguf weight, then load it directly.
if getattr(param, "is_gguf_weight_type", None):
param.data.copy_(loaded_weight)
param.weight_type = loaded_weight.item()
return
elif isinstance(param, UninitializedParameter):
shape = list(loaded_weight.shape)
if output_dim is not None:
shape[output_dim] = self.num_embeddings_per_partition
param.materialize(tuple(shape), dtype=loaded_weight.dtype)
# If parameter does not have output dim, then it should
# be copied onto all gpus (e.g. g_idx for act_order gptq).
if output_dim is None:
@@ -562,12 +557,7 @@ class ParallelLMHead(VocabParallelEmbedding):
def tie_weights(self, embed_tokens: VocabParallelEmbedding):
"""Tie the weights with word embeddings."""
# GGUF quantized embed_tokens.
if self.quant_config and self.quant_config.get_name() == "gguf":
return embed_tokens
else:
self.weight = embed_tokens.weight
return self
return self.quant_method.tie_weights(self, embed_tokens)
def forward(self, input_):
del input_