[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
parent
e1726d9142
commit
c9bfa574ce
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
|
||||
|
||||
class OnlineRequantizationMixin:
|
||||
"""Shared lifecycle for converting serialized weights before quantization."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.requantization_source: QuantizeMethodBase | None = None
|
||||
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Configure serialized-weight conversion before online quantization."""
|
||||
self.requantization_source = source_method
|
||||
self.uses_meta_device = False
|
||||
|
||||
def create_requantization_source_weights(
|
||||
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
|
||||
) -> bool:
|
||||
"""Delegate weight creation to the source and track its parameters."""
|
||||
if self.requantization_source is None:
|
||||
return False
|
||||
|
||||
existing_parameter_names = set(layer._parameters)
|
||||
self.requantization_source.create_weights(
|
||||
layer, *weight_args, **extra_weight_attrs
|
||||
)
|
||||
self.requantization_source_parameters = {
|
||||
name: parameter
|
||||
for name, parameter in layer._parameters.items()
|
||||
if name not in existing_parameter_names and parameter is not None
|
||||
}
|
||||
return True
|
||||
|
||||
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Release serialized parameters after successful requantization."""
|
||||
for name, source_parameter in self.requantization_source_parameters.items():
|
||||
if layer._parameters.get(name) is source_parameter:
|
||||
delattr(layer, name)
|
||||
self.requantization_source_parameters.clear()
|
||||
Reference in New Issue
Block a user