[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-14 15:49:37 +08:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang
parent e1726d9142
commit c9bfa574ce
16 changed files with 1353 additions and 148 deletions
@@ -34,10 +34,12 @@ from vllm.model_executor.layers.fusion.quant_activation import (
from vllm.model_executor.layers.linear import (
LinearMethodBase,
)
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.online.requantization import (
OnlineRequantizationMixin,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
@@ -118,7 +120,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class OnlineLinearBase(LinearMethodBase):
class OnlineLinearBase(OnlineRequantizationMixin, LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -126,15 +128,17 @@ class OnlineLinearBase(LinearMethodBase):
activation_quant_key: QuantKey | None
def __init__(self):
super().__init__()
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
self.requantization_source: QuantizeMethodBase | None = None
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Configure serialized-weight conversion before online quantization."""
self.requantization_source = source_method
self.uses_meta_device = False
def get_weight_for_quantization(self, layer: torch.nn.Module) -> torch.Tensor:
if self.requantization_source is None:
return layer.weight
weight = self.requantization_source.dequantize_weight(layer)
if not isinstance(weight, torch.Tensor):
raise TypeError("Linear requantization requires a single weight tensor.")
return weight
def create_weights(
self,
@@ -146,22 +150,15 @@ class OnlineLinearBase(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if self.requantization_source is not None:
existing_parameter_names = set(layer._parameters)
self.requantization_source.create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
self.requantization_source_parameters = {
name: parameter
for name, parameter in layer._parameters.items()
if name not in existing_parameter_names and parameter is not None
}
if self.create_requantization_source_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
):
return
output_size_per_partition = sum(output_partition_sizes)
@@ -187,19 +184,6 @@ class OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
"""Release checkpoint parameters after successful requantization."""
for name, source_parameter in self.requantization_source_parameters.items():
if layer._parameters.get(name) is source_parameter:
delattr(layer, name)
self.requantization_source_parameters.clear()
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
"""Return checkpoint weights materialized for online quantization."""
if self.requantization_source is None:
return layer.weight
return self.requantization_source.dequantize_weight(layer)
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
@@ -268,6 +252,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
self.fp8_linear.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -368,6 +353,7 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
del layer.weight_scale
self.fp8_linear.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -624,27 +610,29 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
# If checkpoint is fp16, quantize in place.
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
layer.w13_input_scale = None
layer.w2_input_scale = None
moe_tp_size = self.moe.tp_size
w13_amax = weight_amax(layer.w13_weight.flatten(1), dim=-1)
w13_amax = weight_amax(w13_weight.flatten(1), dim=-1)
w13_amax = amax_for_moe_weight_quant(w13_amax, moe_tp_size)
w13_scale = _fp8_scale(w13_amax)
w2_amax = weight_amax(layer.w2_weight.flatten(1), dim=-1)
w2_amax = weight_amax(w2_weight.flatten(1), dim=-1)
w2_amax = amax_for_moe_weight_quant(w2_amax, moe_tp_size)
w2_scale = _fp8_scale(w2_amax)
for expert in range(layer.local_num_experts):
w13[expert, :, :], _ = ops.scaled_fp8_quant(
layer.w13_weight[expert, :, :], scale=w13_scale[expert]
w13_weight[expert, :, :], scale=w13_scale[expert]
)
w2[expert, :, :], _ = ops.scaled_fp8_quant(
layer.w2_weight[expert, :, :], scale=w2_scale[expert]
w2_weight[expert, :, :], scale=w2_scale[expert]
)
# Shuffle weights to runtime format and setup kernel.
@@ -657,6 +645,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
w13_input_scale=layer.w13_input_scale,
w2_input_scale=layer.w2_input_scale,
)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -700,11 +689,12 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._zero_padding(layer)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
self._zero_padding(layer, w13_weight, w2_weight)
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
block_size = self.weight_block_size
assert block_size is not None
@@ -713,8 +703,8 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
# Create block-shaped scales (computed here rather than in
# create_weights because online quant doesn't need them until now).
num_experts = layer.local_num_experts
_, w13_out, w13_in = layer.w13_weight.shape
_, w2_out, w2_in = layer.w2_weight.shape
_, w13_out, w13_in = w13_weight.shape
_, w2_out, w2_in = w2_weight.shape
w13_scale = torch.ones(
num_experts,
@@ -733,18 +723,22 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
for expert in range(num_experts):
w13[expert], w13_scale[expert] = per_block_cast_to_fp8(
layer.w13_weight[expert],
w13_weight[expert],
block_size=block_size,
use_ue8m0=False,
)
w2[expert], w2_scale[expert] = per_block_cast_to_fp8(
layer.w2_weight[expert],
w2_weight[expert],
block_size=block_size,
use_ue8m0=False,
)
layer.weight_block_size = block_size
# Activation quantization is dynamic (including in re-quantization)
layer.w13_input_scale = None
layer.w2_input_scale = None
# Shuffle weights to runtime format and setup kernel.
self._setup_kernel(
layer,
@@ -752,9 +746,10 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
w2,
w13_scale,
w2_scale,
layer.w13_input_scale,
layer.w2_input_scale,
w13_input_scale=None,
w2_input_scale=None,
)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -806,32 +801,32 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
# Scale's leading dim is taken from the fp8 weight tensor by
# construction, so it cannot drift from the weight's expert count
# under EP / padded MoE.
n_w13 = layer.w13_weight.shape[1]
n_w13 = w13_weight.shape[1]
w13_scale = torch.ones(
w13.shape[0], n_w13, 1, device=w13.device, dtype=torch.float32
)
layer.w13_input_scale = None
layer.w2_input_scale = None
w2_amax = weight_amax(layer.w2_weight, dim=-1, keepdim=True)
w2_amax = weight_amax(w2_weight, dim=-1, keepdim=True)
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
w2_scale = _fp8_channel_scale(w2_amax)
for expert in range(layer.local_num_experts):
w13[expert], w13_scale[expert] = ops.scaled_fp8_quant(
layer.w13_weight[expert],
w13_weight[expert],
scale=None,
use_per_token_if_dynamic=True,
)
w2[expert] = _fp8_quant_per_channel(
layer.w2_weight[expert], w2_scale[expert]
)
w2[expert] = _fp8_quant_per_channel(w2_weight[expert], w2_scale[expert])
self._setup_kernel(
layer,
@@ -842,5 +837,6 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
w13_input_scale=None,
w2_input_scale=None,
)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True