[Refactor] Remove Fp8OnlineLinearMethod as scheduled (#45463)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d53f4593ce
commit
3f53e2138f
@@ -80,7 +80,6 @@ from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
@@ -188,7 +187,11 @@ class Fp8Config(QuantizationConfig):
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
if not self.is_checkpoint_fp8_serialized:
|
||||
online_method = Fp8OnlineLinearMethod(self)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
|
||||
online_method = Fp8PerTensorOnlineLinearMethod()
|
||||
online_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
return online_method
|
||||
else:
|
||||
@@ -489,81 +492,6 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
# TODO(future PR): remove this class in favor of
|
||||
# online/fp8.py::Fp8PerTensorOnlineLinearMethod
|
||||
class Fp8OnlineLinearMethod(Fp8LinearMethod):
|
||||
"""Online version of Fp8LinearMethod which loads a full precision checkpoint
|
||||
and quantizes weights during loading."""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
device="meta", # materialized and processed during loading
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
# TODO(future): support block_quant in online quant path
|
||||
assert not self.block_quant
|
||||
|
||||
layer.input_scale = None
|
||||
qweight, weight_scale = ops.scaled_fp8_quant(layer.weight, scale=None)
|
||||
|
||||
# Update layer with new values. Canonicalize to (K, N) for the kernel.
|
||||
replace_parameter(layer, "weight", qweight.t().data)
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
|
||||
# Only Marlin kernels support `marlin_input_dtype`; guard to avoid
|
||||
# AttributeError if backend selection changes.
|
||||
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
"""MoE method for FP8.
|
||||
Supports loading FP8 checkpoints with static weight scale and
|
||||
|
||||
@@ -109,6 +109,10 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.block_quant = False
|
||||
self.use_deep_gemm = False
|
||||
self.use_marlin = False
|
||||
self.marlin_input_dtype = None
|
||||
self.weight_quant_key = kFp8StaticTensorSym
|
||||
# Use per-token quantization for better perf if dynamic and cutlass
|
||||
if cutlass_fp8_supported():
|
||||
@@ -144,6 +148,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
@@ -156,6 +161,8 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
replace_parameter(layer, "weight", qweight.t().data)
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
|
||||
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
|
||||
@@ -401,6 +401,9 @@ def get_and_maybe_dequant_weights(
|
||||
"""Return layer's unquantized weights in [out, in] layout"""
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
|
||||
# LoRA linear wrappers store quantization metadata on `base_layer`.
|
||||
# Unwrap here so callers can pass either a raw linear layer or its LoRA
|
||||
@@ -418,7 +421,9 @@ def get_and_maybe_dequant_weights(
|
||||
|
||||
# Simple Fp8 case: rescale with tensor or block weight scales
|
||||
if (
|
||||
isinstance(layer.quant_method, Fp8LinearMethod)
|
||||
isinstance(
|
||||
layer.quant_method, (Fp8LinearMethod, Fp8PerTensorOnlineLinearMethod)
|
||||
)
|
||||
and not layer.quant_method.use_marlin
|
||||
# DeepGEMM transforms the scales using `transform_sf_into_required_layout` into
|
||||
# a layout that is not compatible with `scaled_dequantize`.
|
||||
|
||||
Reference in New Issue
Block a user