diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index d3a4edd271..a81ea219f7 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -99,6 +99,35 @@ vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8 Combine with `--moe-backend` to pin a specific kernel family. +### Re-quantizing layers to a different precision + +An online target can re-quantize a checkpoint-quantized (prequantized) layer at load time. + +vLLM uses the checkpoint method to load and dequantize the checkpoint weight, and then applies the online quantization method. This requires the checkpoint +method to implement weight dequantization, unsupported source and target +combinations raise an error. + +Currently supported combinations are: + +| Checkpoint layer | Online target | Requirements | +| ---------------- | ------------- | ------------ | +| Serialized ModelOpt MXFP8 linear | any | Target-specific hardware requirements | + +For example, serialized ModelOpt MXFP8 linears can be converted to +per-channel-weight/per-token-activation FP8 at load time: + +```bash +vllm serve MiniMaxAI/MiniMax-M3-MXFP8 \ + --linear-backend auto \ + --quantization-config.linear fp8_per_channel +``` + +!!! info + + This is only available for linear layers at the moment. + + Checkpoint-exclusion policy is not enforced by online quantization/re-quantization. Only exclusion through `--quantization-config.targets`/`--quantization-config.ignore` is enforced. + ### Online quantization on unquantized layers from partially-quantized checkpoints Online quantization can be used on already quantized checkpoints independently of their original `quant_method` (`modelopt`, `compressed-tensors`, `quark`, etc.), for layers that are left unquantized in the original checkpoint. diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index b4dffbb375..f3b72707f3 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -58,6 +58,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso ) from vllm.model_executor.layers.quantization.modelopt import ( ModelOptFp8Config, + ModelOptLinearMethod, ModelOptMxFp8Config, ) from vllm.model_executor.layers.quantization.online.base import ( @@ -93,12 +94,14 @@ from vllm.model_executor.layers.quantization.quark.quark import ( ) from vllm.model_executor.layers.quantization.utils import quant_utils from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_SCALE_DTYPE, MXFP8_VALUE_DTYPE, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( amax_for_moe_weight_quant, amax_for_tp_weight_quant, kMxfp8Dynamic, + kMxfp8Static, weight_amax, ) from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -112,6 +115,7 @@ from vllm.model_executor.models.granitemoe import ( ) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe +from vllm.utils.torch_utils import set_default_torch_dtype if current_platform.is_rocm(): from vllm.platforms.rocm import on_gfx942, on_gfx950 @@ -307,6 +311,14 @@ def _fully_quantized_modelopt_config() -> ModelOptFp8Config: ) +def _fully_quantized_modelopt_mxfp8_config() -> ModelOptMxFp8Config: + return ModelOptMxFp8Config( + is_checkpoint_mxfp8_serialized=True, + kv_cache_quant_algo=None, + exclude_modules=["lm_head"], + ) + + def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig: return CompressedTensorsConfig( target_scheme_map={"RoutedExperts": {}}, @@ -334,10 +346,19 @@ def _write_minimal_llama_config( @pytest.mark.parametrize( - "checkpoint_config_factory,raises_conflict", + "checkpoint_config_factory,raises_requantization_error", [ pytest.param(_fully_quantized_quark_config, True, id="quark"), - pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"), + pytest.param( + _fully_quantized_modelopt_config, + True, + id="modelopt", + ), + pytest.param( + _fully_quantized_modelopt_mxfp8_config, + False, + id="modelopt_mxfp8", + ), pytest.param( _moe_only_compressed_tensors_config, False, @@ -345,9 +366,13 @@ def _write_minimal_llama_config( ), ], ) +@pytest.mark.skipif( + not (current_platform.is_cuda() or current_platform.is_rocm()), + reason="MXFP8 requantization requires CUDA or ROCm.", +) def test_online_prequantized_compatibility( checkpoint_config_factory, - raises_conflict: bool, + raises_requantization_error: bool, default_vllm_config, dist_init, ) -> None: @@ -370,35 +395,69 @@ def test_online_prequantized_compatibility( "disable_tp": True, } - if raises_conflict: - with pytest.raises(ValueError, match="pre-quantized layer"): - ColumnParallelLinear(**layer_kwargs) + layer = ColumnParallelLinear(**layer_kwargs) + assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod) + + if layer.weight.is_meta: + # Weight loading. + layer = layer.to_empty(device=DEVICE) + for parameter in layer.parameters(): + parameter.zero_() else: - layer = ColumnParallelLinear(**layer_kwargs) + layer = layer.to(device=DEVICE) + + if raises_requantization_error: + with pytest.raises( + NotImplementedError, + match="does not implement dequantize_weight|only supported for MXFP8", + ): + layer.quant_method.process_weights_after_loading(layer) + else: + layer.quant_method.process_weights_after_loading(layer) + + if isinstance(checkpoint_config, ModelOptMxFp8Config): assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod) + assert not layer.quant_method.uses_meta_device + assert isinstance( + layer.quant_method.requantization_source, ModelOptLinearMethod + ) + elif not raises_requantization_error: + assert layer.quant_method.uses_meta_device + assert layer.quant_method.requantization_source is None -def test_online_target_rejects_prequantized_layer( - default_vllm_config, dist_init +@pytest.mark.skipif( + not (current_platform.is_cuda() or current_platform.is_rocm()) + or not is_quant_method_supported("fp8"), + reason="Requires FP8 support on CUDA or ROCm.", +) +def test_online_block_fp8_requantization_removes_source_scales( + default_vllm_config, dist_init, workspace_init ) -> None: - """A targets match participates in checkpoint compatibility checks.""" - default_vllm_config.model_config = ModelConfig() - prefix = "model.layers.0.self_attn.o_proj" - quant_config = _fully_quantized_quark_config() + """Blockwise FP8 replaces MXFP8 scales with its own scale parameter.""" + default_vllm_config.model_config = ModelConfig(dtype="bfloat16") + quant_config = _fully_quantized_modelopt_mxfp8_config() quant_config.online_quantization_config = OnlineQuantizationConfig( - QuantizationConfigArgs(targets={prefix: "mxfp8"}) + QuantizationConfigArgs(linear="fp8_per_block") ) - - with pytest.raises(ValueError, match="pre-quantized layer"): - ColumnParallelLinear( - input_size=32, - output_size=32, + with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE): + layer = ColumnParallelLinear( + input_size=256, + output_size=256, bias=False, params_dtype=torch.bfloat16, quant_config=quant_config, - prefix=prefix, + prefix="model.layers.0.self_attn.o_proj", disable_tp=True, ) + with torch.no_grad(): + layer.weight.fill_(1) + layer.weight_scale.fill_(127) # E8M0 encoding of scale 1. + + layer.quant_method.process_weights_after_loading(layer) + + assert not hasattr(layer, "weight_scale") + assert layer.weight_scale_inv.numel() > 0 def test_online_ignore_keeps_checkpoint_quantization_linear( @@ -431,17 +490,25 @@ def test_online_ignore_keeps_checkpoint_quantization_linear( def test_online_quantization_rejects_prequantized_moe( - default_vllm_config, dist_init + default_vllm_config, dist_init, monkeypatch ) -> None: - """Online linear and MoE quantization reject a pre-quantized MoE layer.""" + """Reject pre-quantized MoE before constructing an online target backend.""" default_vllm_config.model_config = ModelConfig() prefix = "model.layers.0.mlp.experts" quant_config = _fully_quantized_quark_config() quant_config.online_quantization_config = OnlineQuantizationConfig( quant_config_args(linear="mxfp4", moe="mxfp4") ) + monkeypatch.setattr( + quant_config.online_quantization_config, + "get_quant_method", + lambda *args: pytest.fail("unsupported online backend must not be constructed"), + ) - with pytest.raises(ValueError, match="pre-quantized layer"): + with pytest.raises( + NotImplementedError, + match="Requantizing checkpoint-quantized MoE layers is not supported", + ): FusedMoEFactory( num_experts=4, top_k=2, @@ -591,7 +658,7 @@ def test_log_online_quantization_for_composable_config(monkeypatch) -> None: assert log_args == [ ( - "Quantized %d layers of types: %s", + "Quantizing %d layers of types: %s", 2, "; ".join(online_config.quantized_layer_summaries), ) @@ -836,6 +903,24 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp {}, id="partially_prequantized_checkpoint", ), + pytest.param( + PARTIALLY_PREQUANTIZED_MODEL_NAME, + None, + {"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}}, + Mxfp8OnlineLinearMethod, + CompressedTensorsMoEMethod, + {}, + id="partially_prequantized_checkpoint", + ), + pytest.param( + "mgoin/Qwen3-0.6B-MXFP8", + None, + {"linear": "fp8_per_channel"}, + Fp8PtpcOnlineLinearMethod, + None, + {}, + id="requantization_mxfp8_ptcp_fp8", + ), ], ) @pytest.mark.parametrize( @@ -888,6 +973,25 @@ def test_online_quantization( moe_backend=runner_kwargs.get("moe_backend", "auto"), ), } + if model_name == "mgoin/Qwen3-0.6B-MXFP8": + original_process = Fp8PtpcOnlineLinearMethod.process_weights_after_loading + + def assert_source_weights_released(method, layer) -> None: + source_parameters = dict(method.requantization_source_parameters) + assert set(source_parameters) == {"weight", "weight_scale"} + + original_process(method, layer) + + assert not method.requantization_source_parameters + assert layer.weight is not source_parameters["weight"] + assert layer.weight_scale is not source_parameters["weight_scale"] + assert layer.weight_scale.shape[-1] == 1 + + monkeypatch.setattr( + Fp8PtpcOnlineLinearMethod, + "process_weights_after_loading", + assert_source_weights_released, + ) if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: model, vllm_config = load_model_without_vllm_runner( @@ -943,11 +1047,17 @@ def test_online_quantization( if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: o_proj = model.model.layers[1].self_attn.o_proj moe = model.model.layers[0].mlp.experts + elif model_name == "mgoin/Qwen3-0.6B-MXFP8": + o_proj = model.model.layers[0].self_attn.o_proj + moe = None else: o_proj = model.model.layers[0].self_attn.o_proj moe = model.model.layers[0].block_sparse_moe.experts + assert isinstance(o_proj.quant_method, expected_linear_cls) - assert isinstance(moe._quant_method, expected_moe_cls) + + if moe is not None: + assert isinstance(moe._quant_method, expected_moe_cls) if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance( o_proj.quant_method.kernel, MarlinMxfp8LinearKernel @@ -1242,7 +1352,7 @@ def test_log_online_quantization(default_vllm_config, monkeypatch) -> None: log_online_quantization(default_vllm_config) assert logged_messages == [ - "Quantized 3 layers of types: mlp.down_proj: 2 (from linear: " + "Quantizing 3 layers of types: mlp.down_proj: 2 (from linear: " "fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: " "re:.*qkv_proj.*, mxfp4)" ] @@ -1365,6 +1475,48 @@ def test_online_linear_tp_weight_quant_matches_unsharded( assert torch.equal(tp_scale, full_scale) +@pytest.mark.skipif( + not is_quant_method_supported("fp8"), + reason="FP8 is not supported on this GPU type.", +) +def test_modelopt_mxfp8_requantized_ptpc_tp_shards_match_unsharded() -> None: + """MXFP8 source shards convert to slices of the unsharded PTPC weight.""" + torch.manual_seed(0) + serialized_weight = torch.randn(16, 64, device=DEVICE, dtype=torch.bfloat16).to( + MXFP8_VALUE_DTYPE + ) + serialized_scale = torch.randint( + 124, + 131, + (16, 2), + device=DEVICE, + dtype=MXFP8_SCALE_DTYPE, + ) + + source = ModelOptLinearMethod.__new__(ModelOptLinearMethod) + source.wkey = SimpleNamespace(key=kMxfp8Static) + + def dequantize(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + layer = SimpleNamespace(weight=weight, weight_scale=scale) + return source.dequantize_weight(layer) + + full_weight = dequantize(serialized_weight, serialized_scale) + full_scale = _fp8_channel_scale(weight_amax(full_weight, dim=-1, keepdim=True)) + full_qweight = _fp8_quant_per_channel(full_weight, full_scale) + + # Mimic the two K-dimension shards of a row-parallel layer. PTPC's global + # per-output-channel scale is the same on both ranks after its MAX reduce. + for rank in range(2): + start = rank * 32 + shard_weight = serialized_weight[:, start : start + 32].contiguous() + shard_scale = serialized_scale[:, rank : rank + 1].contiguous() + weight = dequantize(shard_weight, shard_scale) + qweight = _fp8_quant_per_channel(weight, full_scale) + + assert torch.equal(weight, full_weight[:, start : start + 32]) + assert torch.equal(qweight, full_qweight[:, start : start + 32]) + + def _quantize_moe(weight, scheme, moe_tp_size): if scheme == "nvfp4": return _quantize_moe_weight_to_nvfp4(weight, moe_tp_size) diff --git a/vllm/model_executor/layers/quantization/base_config.py b/vllm/model_executor/layers/quantization/base_config.py index 1b0457854e..6c1f934739 100644 --- a/vllm/model_executor/layers/quantization/base_config.py +++ b/vllm/model_executor/layers/quantization/base_config.py @@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC): """ return + def dequantize_weight(self, layer: nn.Module) -> torch.Tensor: + """Materialize a serialized quantized weight for requantization.""" + raise NotImplementedError( + f"The quantization method {type(self)} does not implement " + "dequantize_weight. Please open an issue." + ) + def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool: """Not all quant methods have embedding implemented, so we need to check that @@ -294,13 +301,16 @@ def resolve_quant_method( LinearBase, UnquantizedLinearMethod, ) + from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase base_quant_method = quant_config.get_quant_method(layer, prefix) if quant_config.online_quantization_config is None: + # No online configuration: retain the checkpoint method. return base_quant_method # Online quantization currently supports only LinearBase and RoutedExperts. # Embeddings and ParallelLMHead retain their checkpoint quantization method. if not isinstance(layer, (LinearBase, RoutedExperts)): + # Online quantization only supports linear and routed MoE layers. return base_quant_method quant_config.online_quantization_config.packed_modules_mapping = ( @@ -312,14 +322,34 @@ def resolve_quant_method( online_target = quant_config.online_quantization_config.resolve_quant_method_cls( layer, prefix ) + if checkpoint_is_quantized: - if online_target is not None: - raise ValueError( - f"Cannot apply requested online quantization {online_target[3]} to " - f"pre-quantized layer {prefix}: {base_quant_method} was already " - "selected by the checkpoint quantization config." + if online_target is None: + # The checkpoint quant method is applied as there is no online override. + return base_quant_method + + if isinstance(layer, RoutedExperts): + raise NotImplementedError( + "Requantizing checkpoint-quantized MoE layers is not supported." ) - return base_quant_method + + online_quant_method = quant_config.online_quantization_config.get_quant_method( + layer, prefix + ) + + assert base_quant_method is not None and not isinstance( + base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod) + ) + + assert isinstance(online_quant_method, OnlineLinearBase) + online_quant_method.set_requantization_source(base_quant_method) + + # The online method dequantizes the checkpoint method before requantizing. + return online_quant_method + if online_target is None: + # The layer is unquantized and no online target applies. return base_quant_method + + # Quantize an unquantized layer with the online method. return quant_config.online_quantization_config.get_quant_method(layer, prefix) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 272eb457bb..e33226cad3 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -76,6 +76,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( MXFP8_BLOCK_SIZE, MXFP8_SCALE_DTYPE, MXFP8_VALUE_DTYPE, + dequant_mxfp8_to_bf16, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( FP4_DTYPE, @@ -2572,6 +2573,17 @@ class ModelOptLinearMethod(LinearMethodBase): layer._nvfp4_group_size_for_gather = self.ctx.group_size self.kernel.process_weights_after_loading(layer) + def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor: + """Reconstruct serialized weights for online requantization.""" + if self.wkey.key is kMxfp8Static: + return dequant_mxfp8_to_bf16( + layer.weight.contiguous(), layer.weight_scale.contiguous() + ) + else: + raise NotImplementedError( + "ModelOpt weight dequantization is only supported for MXFP8." + ) + def apply(self, layer, x, bias=None): return self.fmt.apply( layer, diff --git a/vllm/model_executor/layers/quantization/online/base.py b/vllm/model_executor/layers/quantization/online/base.py index 8ed48b2b64..3f35b56a31 100644 --- a/vllm/model_executor/layers/quantization/online/base.py +++ b/vllm/model_executor/layers/quantization/online/base.py @@ -156,6 +156,7 @@ class OnlineQuantizationConfig(QuantizationConfig): self.args = args self.ignored_layers: list[str] = args.ignore self.quantized_layers: dict[str, tuple[str, str, str | None]] = {} + self.online_quantization_time = 0.0 @property def quantized_layer_summaries(self) -> list[str]: diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index ed018c81bf..23a36c8fbc 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.quant_activation import ( from vllm.model_executor.layers.linear import ( LinearMethodBase, ) +from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.layers.quantization.online.moe_base import ( OnlineMoEMethodBase, ) @@ -125,6 +126,13 @@ class OnlineLinearBase(LinearMethodBase): def __init__(self): self.out_dtype = torch.get_default_dtype() self.input_dtype = get_current_vllm_config().model_config.dtype + self.requantization_source: QuantizeMethodBase | None = None + self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {} + + def set_requantization_source(self, source_method: QuantizeMethodBase) -> None: + """Configure serialized-weight conversion before online quantization.""" + self.requantization_source = source_method + self.uses_meta_device = False def create_weights( self, @@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase): params_dtype: torch.dtype, **extra_weight_attrs, ): + if self.requantization_source is not None: + existing_parameter_names = set(layer._parameters) + self.requantization_source.create_weights( + layer, + input_size_per_partition, + output_partition_sizes, + input_size, + output_size, + params_dtype, + **extra_weight_attrs, + ) + self.requantization_source_parameters = { + name: parameter + for name, parameter in layer._parameters.items() + if name not in existing_parameter_names and parameter is not None + } + return + output_size_per_partition = sum(output_partition_sizes) weight_loader = extra_weight_attrs.get("weight_loader") layer.logical_widths = output_partition_sizes @@ -159,6 +185,19 @@ class OnlineLinearBase(LinearMethodBase): initialize_online_processing(layer) + def release_requantization_source_weights(self, layer: torch.nn.Module) -> None: + """Release checkpoint parameters after successful requantization.""" + for name, source_parameter in self.requantization_source_parameters.items(): + if layer._parameters.get(name) is source_parameter: + delattr(layer, name) + self.requantization_source_parameters.clear() + + def get_weight_for_quantization(self, layer: Module) -> torch.Tensor: + """Return checkpoint weights materialized for online quantization.""" + if self.requantization_source is None: + return layer.weight + return self.requantization_source.dequantize_weight(layer) + class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase): """Online tensorwise FP8 linear quantization. @@ -213,10 +252,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase): return layer.input_scale = None - amax = weight_amax(layer.weight).reshape(1) + weight = self.get_weight_for_quantization(layer) + amax = weight_amax(weight).reshape(1) amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer)) weight_scale = _fp8_scale(amax) - qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale) + qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale) # Update layer with new values. replace_parameter(layer, "weight", qweight.t().data) @@ -312,14 +352,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase): layer.input_scale = None block_size = self.weight_block_size + weight = self.get_weight_for_quantization(layer) qweight, weight_scale_inv = per_block_cast_to_fp8( - layer.weight, block_size=block_size, use_ue8m0=False + weight, block_size=block_size, use_ue8m0=False ) replace_parameter(layer, "weight", qweight.data) replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data) + if self.requantization_source is not None and hasattr(layer, "weight_scale"): + del layer.weight_scale + self.fp8_linear.process_weights_after_loading(layer) # Prevent duplicate processing (e.g., during weight reload) @@ -395,18 +439,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase): return layer.input_scale = None - amax = weight_amax(layer.weight, dim=-1, keepdim=True) + weight = self.get_weight_for_quantization(layer) + amax = weight_amax(weight, dim=-1, keepdim=True) amax = amax_for_tp_weight_quant( amax, _is_tp_sharded(layer, reduces_output_dim=False) ) weight_scale = _fp8_channel_scale(amax) - qweight = _fp8_quant_per_channel(layer.weight, weight_scale) + qweight = _fp8_quant_per_channel(weight, weight_scale) replace_parameter(layer, "weight", qweight.t()) replace_parameter(layer, "weight_scale", weight_scale) self.fp8_linear.process_weights_after_loading(layer) expose_input_quant_key(layer, self.fp8_linear) + self.release_requantization_source_weights(layer) layer._already_called_process_weights_after_loading = True diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index 6e21bc9092..e296eab373 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import ( SharedExperts, ) from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput +from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.model_loader.reload.layerwise import ( initialize_online_processing, ) @@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase): uses_meta_device: bool = True + def set_requantization_source(self, source_method: QuantizeMethodBase) -> None: + """Reject requantization from a checkpoint-quantized MoE method.""" + raise NotImplementedError( + "Requantizing checkpoint-quantized MoE layers is not supported." + ) + def create_weights( self, layer: torch.nn.Module, diff --git a/vllm/model_executor/layers/quantization/online/mxfp4.py b/vllm/model_executor/layers/quantization/online/mxfp4.py index 72914a9726..b949c9b7ff 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp4.py +++ b/vllm/model_executor/layers/quantization/online/mxfp4.py @@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase): if getattr(layer, "_already_called_process_weights_after_loading", False): return - weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous()) + weight = self.get_weight_for_quantization(layer) + weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous()) layer.input_scale = None replace_parameter(layer, "weight", weight_fp4.data) diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index 9d5814206c..9cddab85c1 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -75,7 +75,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase): if getattr(layer, "_already_called_process_weights_after_loading", False): return - weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous()) + weight = self.get_weight_for_quantization(layer) + weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous()) layer.input_scale = None replace_parameter(layer, "weight", weight_fp8.data) diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index 2b7cb72fff..4c774808bf 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -101,6 +101,7 @@ class BaseModelLoader(ABC): finalize_layerwise_processing(model, model_config) process_weights_after_loading(model, model_config, target_device) + log_online_quantization_time(vllm_config) return model.eval() @@ -115,8 +116,8 @@ def log_model_inspection(model: nn.Module) -> None: logger.info("vLLM model structure:\n%s", format_model_inspection(model)) -def log_online_quantization(vllm_config: VllmConfig) -> None: - """Log the online-quantized layer count and types, when applicable.""" +def _get_online_quantization_config(vllm_config: VllmConfig): + """Return the online quantization config when one is configured.""" from vllm.model_executor.layers.quantization.online.base import ( OnlineQuantizationConfig, ) @@ -128,15 +129,35 @@ def log_online_quantization(vllm_config: VllmConfig) -> None: if isinstance(online_quantization_config, OnlineQuantizationConfig): quant_config = online_quantization_config if not isinstance(quant_config, OnlineQuantizationConfig): + return None + return quant_config + + +def log_online_quantization(vllm_config: VllmConfig) -> None: + """Log the online-quantized layer count and types, when applicable.""" + quant_config = _get_online_quantization_config(vllm_config) + if quant_config is None: return logger.info( - "Quantized %d layers of types: %s", + "Quantizing %d layers of types: %s", len(quant_config.quantized_layers), "; ".join(quant_config.quantized_layer_summaries), ) +def log_online_quantization_time(vllm_config: VllmConfig) -> None: + """Log online quantization processing time, when applicable.""" + quant_config = _get_online_quantization_config(vllm_config) + if quant_config is None: + return + + logger.info( + "Online quantization as part of model loading took %.2f seconds", + quant_config.online_quantization_time, + ) + + def _has_online_quant(model: nn.Module): for module in model.modules(): quant_method = getattr(module, "quant_method", None) diff --git a/vllm/model_executor/model_loader/reload/layerwise.py b/vllm/model_executor/model_loader/reload/layerwise.py index 062bfa6090..df91522732 100644 --- a/vllm/model_executor/model_loader/reload/layerwise.py +++ b/vllm/model_executor/model_loader/reload/layerwise.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import inspect from collections.abc import Callable +from contextlib import nullcontext from functools import wraps from weakref import WeakKeyDictionary, WeakSet @@ -321,7 +322,21 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) - _get_weight_loader(param)(*args.args, **args.kwargs) if quant_method is not None: - quant_method.process_weights_after_loading(layer) + from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase + from vllm.model_executor.layers.quantization.online.moe_base import ( + OnlineMoEMethodBase, + ) + from vllm.model_executor.model_loader.utils import ( + online_quantization_timing_context, + ) + + timing_context = ( + online_quantization_timing_context(layer, quant_method) + if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase)) + else nullcontext() + ) + with timing_context: + quant_method.process_weights_after_loading(layer) _copy_and_restore_kernel_tensors(layer, info) @@ -356,7 +371,21 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo): # Process weights (quantization, repacking, etc.) quant_method = getattr(layer, "quant_method", None) if isinstance(quant_method, QuantizeMethodBase): - quant_method.process_weights_after_loading(layer) + from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase + from vllm.model_executor.layers.quantization.online.moe_base import ( + OnlineMoEMethodBase, + ) + from vllm.model_executor.model_loader.utils import ( + online_quantization_timing_context, + ) + + timing_context = ( + online_quantization_timing_context(layer, quant_method) + if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase)) + else nullcontext() + ) + with timing_context: + quant_method.process_weights_after_loading(layer) # Re-reconcile parameter TP state: process_weights_after_loading may # have re-created Parameters (stamped with the global rank), which would # otherwise break replicated (disable_tp) weights on a subsequent reload. diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 465703f759..c12b8e76c5 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -3,6 +3,7 @@ """Utilities for selecting and loading models.""" import inspect +import time import warnings from contextlib import contextmanager, nullcontext from typing import Any @@ -141,6 +142,11 @@ def process_weights_after_loading( """ # Reclaim memory when an explicit lm_head has been # loaded, but it is identical to the input embeddings. + from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase + from vllm.model_executor.layers.quantization.online.moe_base import ( + OnlineMoEMethodBase, + ) + maybe_retie_word_embeddings(model, model_config) for name, module in model.named_modules(): @@ -166,7 +172,12 @@ def process_weights_after_loading( if quant_method.requires_device_loading else nullcontext() ) - with loading_context: + timing_context = ( + online_quantization_timing_context(module, quant_method) + if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase)) + else nullcontext() + ) + with loading_context, timing_context: quant_method.process_weights_after_loading(module) # process_weights_after_loading may swap in freshly-created # Parameters (e.g. FP8 requantization), which are stamped with the @@ -245,6 +256,43 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device) p._vllm_is_uva_offloaded = True +@contextmanager +def online_quantization_timing_context( + layer: nn.Module, quant_method: QuantizeMethodBase +): + """Accumulate online quantization processing time for ``quant_method``.""" + from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + ) + from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase + from vllm.model_executor.layers.quantization.online.moe_base import ( + OnlineMoEMethodBase, + ) + + assert isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase)) + quant_config = layer.quant_config + assert quant_config is not None + online_quantization_config = ( + quant_config + if isinstance(quant_config, OnlineQuantizationConfig) + else quant_config.online_quantization_config + ) + # reload/layerwise.py's _layerwise_process calls this for experts_int8, + # whose online MoE method has no OnlineQuantizationConfig. + if online_quantization_config is None: + yield + return + assert isinstance(online_quantization_config, OnlineQuantizationConfig) + + start_time = time.perf_counter() + try: + yield + finally: + online_quantization_config.online_quantization_time += ( + time.perf_counter() - start_time + ) + + _MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]() """Caches the outputs of `_get_model_architecture`."""