diff --git a/docs/training/layerwise.md b/docs/training/layerwise.md index d304c4a842..9e7d187710 100644 --- a/docs/training/layerwise.md +++ b/docs/training/layerwise.md @@ -28,9 +28,9 @@ For more information on implementation, see [Low Level `layerwise` API](#low-lev Online quantization refers to when a user provides full precision weights and those weights are quantized on-the-fly as they are loaded into the model. The layerwise reloading system handles this by treating online quantization as a **processing** step, which is then handled in an online way both during first-time load and during reload. A typical online quantization method implementation should look like this: ```python -class Fp8OnlineLinearMethod(Fp8LinearMethod): - """Online version of Fp8LinearMethod which loads a full precision checkpoint - and quantizes weights during loading.""" +class Fp8PerTensorOnlineLinearMethod(LinearMethodBase): + """Online version of FP8 per-tensor quantization which loads a full + precision checkpoint and quantizes weights during loading.""" uses_meta_device: bool = True diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 7c92c3dc15..4c789059c7 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -21,6 +21,9 @@ from vllm.model_executor.layers.quantization.fp8 import ( Fp8LinearMethod, Fp8MoEMethod, ) +from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.platforms import current_platform @@ -164,7 +167,7 @@ def test_online_quantization( def check_model(model): fc1 = model.model.decoder.layers[0].fc1 - assert isinstance(fc1.quant_method, Fp8LinearMethod) + assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod) if kv_cache_dtype == "fp8": attn = model.model.decoder.layers[0].self_attn.attn assert isinstance(attn.quant_method, Fp8KVCacheMethod) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index fcf14d66cd..1c97e48771 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -80,7 +80,6 @@ from vllm.model_executor.model_loader.reload.layerwise import ( ) from vllm.model_executor.parameter import ( BlockQuantScaleParameter, - ModelWeightParameter, PerTensorScaleParameter, ) from vllm.model_executor.utils import replace_parameter, set_weight_attrs @@ -188,7 +187,11 @@ class Fp8Config(QuantizationConfig): ): return UnquantizedLinearMethod() if not self.is_checkpoint_fp8_serialized: - online_method = Fp8OnlineLinearMethod(self) + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, + ) + + online_method = Fp8PerTensorOnlineLinearMethod() online_method.marlin_input_dtype = get_marlin_input_dtype(prefix) return online_method else: @@ -489,81 +492,6 @@ class Fp8LinearMethod(LinearMethodBase): return self.fp8_linear.apply_weights(layer, x, bias) -# TODO(future PR): remove this class in favor of -# online/fp8.py::Fp8PerTensorOnlineLinearMethod -class Fp8OnlineLinearMethod(Fp8LinearMethod): - """Online version of Fp8LinearMethod which loads a full precision checkpoint - and quantizes weights during loading.""" - - uses_meta_device: bool = True - - def create_weights( - self, - layer: torch.nn.Module, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - output_size_per_partition = sum(output_partition_sizes) - weight_loader = extra_weight_attrs.get("weight_loader") - layer.logical_widths = output_partition_sizes - layer.input_size_per_partition = input_size_per_partition - layer.output_size_per_partition = output_size_per_partition - layer.orig_dtype = params_dtype - layer.weight_block_size = None - - weight = ModelWeightParameter( - data=torch.empty( - output_size_per_partition, - input_size_per_partition, - device="meta", # materialized and processed during loading - dtype=params_dtype, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) - layer.register_parameter("weight", weight) - - initialize_online_processing(layer) - - self.fp8_linear = init_fp8_linear_kernel( - activation_quant_key=self.activation_quant_key, - weight_quant_key=self.weight_quant_key, - weight_shape=layer.weight.shape, - input_dtype=self.input_dtype, - out_dtype=self.out_dtype, - module_name=self.__class__.__name__, - ) - self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel) - - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - if getattr(layer, "_already_called_process_weights_after_loading", False): - return - - # TODO(future): support block_quant in online quant path - assert not self.block_quant - - layer.input_scale = None - qweight, weight_scale = ops.scaled_fp8_quant(layer.weight, scale=None) - - # Update layer with new values. Canonicalize to (K, N) for the kernel. - replace_parameter(layer, "weight", qweight.t().data) - replace_parameter(layer, "weight_scale", weight_scale.data) - - if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"): - # Only Marlin kernels support `marlin_input_dtype`; guard to avoid - # AttributeError if backend selection changes. - self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype - self.fp8_linear.process_weights_after_loading(layer) - - # Prevent duplicate processing (e.g., during weight reload) - layer._already_called_process_weights_after_loading = True - - class Fp8MoEMethod(FusedMoEMethodBase): """MoE method for FP8. Supports loading FP8 checkpoints with static weight scale and diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 0490500bae..10847851cc 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -109,6 +109,10 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase): def __init__(self): super().__init__() + self.block_quant = False + self.use_deep_gemm = False + self.use_marlin = False + self.marlin_input_dtype = None self.weight_quant_key = kFp8StaticTensorSym # Use per-token quantization for better perf if dynamic and cutlass if cutlass_fp8_supported(): @@ -144,6 +148,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase): out_dtype=self.out_dtype, module_name=self.__class__.__name__, ) + self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel) def process_weights_after_loading(self, layer: Module) -> None: if getattr(layer, "_already_called_process_weights_after_loading", False): @@ -156,6 +161,8 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase): replace_parameter(layer, "weight", qweight.t().data) replace_parameter(layer, "weight_scale", weight_scale.data) + if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"): + self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype self.fp8_linear.process_weights_after_loading(layer) # Prevent duplicate processing (e.g., during weight reload) diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index f1639e3216..0c5cbae2a4 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -401,6 +401,9 @@ def get_and_maybe_dequant_weights( """Return layer's unquantized weights in [out, in] layout""" from vllm.model_executor.layers.linear import UnquantizedLinearMethod from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod + from vllm.model_executor.layers.quantization.online.fp8 import ( + Fp8PerTensorOnlineLinearMethod, + ) # LoRA linear wrappers store quantization metadata on `base_layer`. # Unwrap here so callers can pass either a raw linear layer or its LoRA @@ -418,7 +421,9 @@ def get_and_maybe_dequant_weights( # Simple Fp8 case: rescale with tensor or block weight scales if ( - isinstance(layer.quant_method, Fp8LinearMethod) + isinstance( + layer.quant_method, (Fp8LinearMethod, Fp8PerTensorOnlineLinearMethod) + ) and not layer.quant_method.use_marlin # DeepGEMM transforms the scales using `transform_sf_into_required_layout` into # a layout that is not compatible with `scaled_dequantize`.