From 90a951500699b1aa499f350b3352450cdcc13340 Mon Sep 17 00:00:00 2001 From: fxmarty-amd Date: Thu, 24 Sep 2026 21:48:14 +0200 Subject: [PATCH] [Cleanup] Remove online quantization support in `fp8.py` in favor of online shorthands (#53585) Signed-off-by: Felix Marty Signed-off-by: mgoin Co-authored-by: mgoin Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/intel_jobs/test-intel.yaml | 6 +- .../scripts/hardware_ci/run-intel-ci-test.sh | 6 +- .../quantization/llm_compressor/fp8.md | 4 +- .../passes/test_mla_attn_quant_fusion.py | 1 - tests/kernels/moe/test_moe_layer.py | 4 +- tests/kernels/moe/utils.py | 2 +- .../layers/test_fused_shared_expert.py | 3 +- .../model_loader/test_reload.py | 6 +- tests/models/qwen4_exp/test_ple.py | 12 +- tests/quantization/test_fp8.py | 196 ----------------- tests/quantization/test_online.py | 200 ++++++++++++++++-- .../test_speculators_correctness.py | 2 +- .../model_executor/layers/quantization/fp8.py | 39 ++-- .../model_loader/weight_utils.py | 14 +- vllm/model_executor/models/qianfan_ocr.py | 14 +- 15 files changed, 240 insertions(+), 269 deletions(-) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index e15b20ea40..ad1a03e9cf 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -38,7 +38,7 @@ steps: python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 && + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 && python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 && @@ -74,8 +74,8 @@ steps: python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --linear-backend torch --model Qwen/Qwen3-4B-Instruct-2507-FP8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 && - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 ' - label: "XPU V1 test" key: xpu-v1-test diff --git a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh index 905ce0a963..d71ee9cf62 100644 --- a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh @@ -17,7 +17,7 @@ case "${test_suite}" in python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 @@ -30,8 +30,8 @@ case "${test_suite}" in python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 ;; v1) cd tests diff --git a/docs/features/quantization/llm_compressor/fp8.md b/docs/features/quantization/llm_compressor/fp8.md index f823c089f4..f0d1b88264 100644 --- a/docs/features/quantization/llm_compressor/fp8.md +++ b/docs/features/quantization/llm_compressor/fp8.md @@ -143,14 +143,14 @@ If you encounter any issues or have feature requests, please open an issue on th ## Online Dynamic Quantization -Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8"` in the command line or setting `quantization="fp8"` in the LLM constructor. +Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8_per_tensor"` in the command line or setting `quantization="fp8_per_tensor"` in the LLM constructor. In this mode, all Linear modules (except for the final `lm_head`) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode. ```python from vllm import LLM -llm = LLM("facebook/opt-125m", quantization="fp8") +llm = LLM("facebook/opt-125m", quantization="fp8_per_tensor") # INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB result = llm.generate("Hello, my name is") print(result[0].outputs[0].text) diff --git a/tests/compile/passes/test_mla_attn_quant_fusion.py b/tests/compile/passes/test_mla_attn_quant_fusion.py index f1de3dde0c..56d2bc30cc 100644 --- a/tests/compile/passes/test_mla_attn_quant_fusion.py +++ b/tests/compile/passes/test_mla_attn_quant_fusion.py @@ -271,7 +271,6 @@ class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel): quant_key = kFp8Dynamic128Sym quant_config = Fp8Config( - is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128], ) diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index 5fe5ed1ab2..802d1028be 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -796,11 +796,11 @@ def make_quant_config( return None, QuantizedWeights(w13_weight=w1, w2_weight=w2) if quantization == "fp8": - return Fp8Config(True), _quantize_fp8_halves(w1, w2) + return Fp8Config(), _quantize_fp8_halves(w1, w2) if quantization == "fp8_blocked": block_shape = [128, 128] - return Fp8Config(True, weight_block_size=block_shape), _quantize_fp8_halves( + return Fp8Config(weight_block_size=block_shape), _quantize_fp8_halves( w1, w2, block_shape ) diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index d92e4d6129..36189e4dd7 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -641,7 +641,7 @@ def make_shared_experts_with_weights( if quant_dtype == torch.float8_e4m3fn: from vllm.model_executor.layers.quantization.fp8 import Fp8Config - quant_config = Fp8Config(True) + quant_config = Fp8Config() else: quant_config = None diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 08fb6818e3..42f134e6de 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -1227,7 +1227,6 @@ def test_non_quark_shared_expert_fse_is_incompatible() -> None: def _fp8_config(**kwargs: Any) -> Fp8Config: return Fp8Config( - is_checkpoint_fp8_serialized=True, activation_scheme="dynamic", weight_block_size=[128, 128], **kwargs, @@ -1248,7 +1247,7 @@ def test_block_fp8_shared_expert_fse_is_compatible() -> None: def test_per_tensor_fp8_shared_expert_fse_is_incompatible() -> None: """Per-tensor scales are 0-D, so the shared-expert chunker cannot slice them.""" compatible, reason = is_shared_expert_quant_fse_compatible( - Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"), + Fp8Config(activation_scheme="dynamic"), "model.layers.0.mlp.experts", "model.layers.0.mlp.shared_experts", ) diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index 155957160f..85ddea3946 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -1158,13 +1158,13 @@ def test_kv_scale_reload(vllm_runner): "Qwen/Qwen3-0.6B", "inference-optimization/Qwen3-0.6B-debug-multiply", "inference-optimization/Qwen3-0.6B-debug-add", - "fp8", + "fp8_per_tensor", ), pytest.param( "inference-optimization/DeepSeek-V3-debug-empty", "inference-optimization/DeepSeek-V3-debug-multiply", "inference-optimization/DeepSeek-V3-debug-add", - "fp8", + "fp8_per_tensor", marks=[pytest.mark.slow_test], ), pytest.param( @@ -1192,7 +1192,7 @@ def test_online_quantize_reload( if current_platform.device_count() < tp_size: pytest.skip(reason="Not enough GPU devices") - if quantization == "fp8" and _fp8_reload_unsupported(): + if quantization == "fp8_per_tensor" and _fp8_reload_unsupported(): pytest.skip(reason="Requires FP8 support") with vllm_runner( diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index e7d0e72d8a..3a418c4732 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -14,11 +14,13 @@ from torch.nn import functional as F import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module +from vllm.config.quantization import QuantizationConfigArgs from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.modelopt import ( ModelOptMixedPrecisionConfig, ModelOptNvFp4Config, ) +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig from vllm.models.qwen4_exp.common.ple import ( PLEShardOverlap, compute_ple_shard_overlap, @@ -396,7 +398,6 @@ def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None: def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None: prefix = "model.layers.1.ple.ple_embedding.ngram_embedding" quant_config = Fp8Config( - is_checkpoint_fp8_serialized=True, ignored_layers=[], weight_block_size=[128, 128], ) @@ -422,12 +423,11 @@ def test_ple_embedding_rejects_unsupported_quantization_configs() -> None: with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"): Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix) - dynamic_fp8_config = Fp8Config( - is_checkpoint_fp8_serialized=False, - ignored_layers=[], + online_fp8_config = OnlineQuantizationConfig( + QuantizationConfigArgs(linear="fp8_per_tensor") ) - with pytest.raises(NotImplementedError, match="serialized FP8"): - Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix) + with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"): + Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix) def test_ple_embedding_respects_modelopt_exclusion() -> None: diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 65e119b083..78f342a1d7 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -5,11 +5,9 @@ Run `pytest tests/quantization/test_fp8.py --forked`. """ -import logging from types import SimpleNamespace import pytest -import regex as re import torch from tests.quantization.utils import ( @@ -22,9 +20,6 @@ from vllm.config.cache import CacheConfig from vllm.config.kernel import KernelConfig from vllm.config.model import ModelConfig from vllm.forward_context import set_forward_context -from vllm.model_executor.kernels.linear.scaled_mm import ( - MarlinFP8ScaledMMLinearKernel, -) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention.attention import ( set_default_quant_scales, @@ -32,15 +27,11 @@ from vllm.model_executor.layers.attention.attention import ( from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.quantization.fp8 import ( Fp8Config, - Fp8KVCacheMethod, Fp8LinearMethod, Fp8MoEMethod, ) from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod -from vllm.model_executor.layers.quantization.online.fp8 import ( - Fp8PerTensorOnlineLinearMethod, -) from vllm.model_executor.layers.quantization.utils import flashinfer_utils from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( prepare_fp8_moe_layer_for_fi, @@ -531,187 +522,6 @@ def test_model_load_and_run( model(input_ids, positions, None) -@pytest.mark.skipif( - not is_quant_method_supported("fp8"), - reason="FP8 is not supported on this GPU type.", -) -@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) -@pytest.mark.parametrize( - "force_marlin", [True, False] if current_platform.is_cuda() else [False] -) -@pytest.mark.parametrize( - "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] -) -def test_online_quantization( - vllm_runner, - kv_cache_dtype: str, - force_marlin: bool, - use_rocm_aiter: bool, - monkeypatch, -) -> None: - if use_rocm_aiter: - monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") - - # `LLM.apply_model` requires pickling a function. - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - - kwargs = {} - if force_marlin: - kwargs["linear_backend"] = "marlin" - kwargs["moe_backend"] = "marlin" - - model_dtype = "auto" - if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90): - # FA3 requires BF16 output when the query input is FP8. - model_dtype = "bfloat16" - - with vllm_runner( - "facebook/opt-125m", - quantization="fp8", - dtype=model_dtype, - enforce_eager=True, - kv_cache_dtype=kv_cache_dtype, - **kwargs, - ) as llm: - - def check_model(model): - fc1 = model.model.decoder.layers[0].fc1 - assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod) - if kv_cache_dtype == "fp8": - attn = model.model.decoder.layers[0].self_attn.attn - assert isinstance(attn.quant_method, Fp8KVCacheMethod) - assert attn._k_scale == 1.0 - assert attn._v_scale == 1.0 - - if current_platform.is_cuda() or current_platform.is_xpu(): - if current_platform.supports_fp8() and not force_marlin: - # For GPUs with hardware support, we keep weights in fp8 - assert fc1.weight.dtype == torch.float8_e4m3fn - assert not isinstance( - fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel - ) - else: - # For GPUs without hardware support, we pack the fp8 weights - # for weight-only quantization using Marlin kernels - assert fc1.weight.dtype == torch.int32 - assert isinstance( - fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel - ) - elif current_platform.is_rocm(): - if current_platform.supports_fp8() and not force_marlin: - # For GPUs with hardware support, we keep weights in fp8 - assert fc1.weight.dtype == current_platform.fp8_dtype() - else: # unsupported ROCm platform - pytest.skip( - "Skip `test_load_fp16_model`. " - "It only runs on ROCm platform with FP8 compute." - " e.g. MI300X and above." - ) - else: # unsupported platform - pytest.skip( - "Skip `test_load_fp16_model`. " - "It only runs on CUDA and ROCm platform." - ) - - llm.apply_model(check_model) - - outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4) - print(outputs[0][1]) - - -@pytest.mark.skipif( - not is_quant_method_supported("fp8"), - reason="FP8 is not supported on this GPU type.", -) -def test_online_quant_peak_mem( - vllm_runner, - caplog_mp_spawn, - monkeypatch, -) -> None: - # Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because: - # 1. it covers both Linear and MoE paths - # 2. it is already used by other tests in CI, so adding it here - # does not increase disk space for CI runners - # I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base` - # which I think is the smallest MoE model in vLLM (2.5 GiB bf16, - # 1.3 GiB fp8), but could not as adding one more model makes CI - # run out of disk space. - model_name = "allenai/OLMoE-1B-7B-0125-Instruct" - - # Force spawn to ensure caplog_mp_spawn works consistently - # (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores) - monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") - - with ( - caplog_mp_spawn(logging.DEBUG) as log_holder, - vllm_runner( - model_name, - quantization="fp8", - enforce_eager=True, - ) as llm, - ): - outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4) - print(outputs[0][1]) - - log_text = log_holder.text - - # Parse memory usage from captured logs - model_memory_gib = None - peak_memory_gib = None - for line in log_text.splitlines(): - if model_memory_gib is None: - match = re.search(r"Model loading took ([\d.]+) GiB memory", line) - if match: - model_memory_gib = float(match.group(1)) - if peak_memory_gib is None: - match = re.search( - r"Peak GPU memory after loading weights: ([\d.]+) GiB", line - ) - if match: - peak_memory_gib = float(match.group(1)) - - assert model_memory_gib is not None, "Could not find model loading memory log" - assert peak_memory_gib is not None, "Could not find peak memory log" - print(f"GPU memory used after loading weights: {model_memory_gib} GiB") - print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB") - - # model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant - # uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB) - expected_model_memory_gib = 6.7 - - # for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06 - # GiB, which is 1.36x above model_memory_gib. A slightly higher number is - # expected as when we load and quantize weights in a streaming fashion we - # need to have individual weights in bf16 + fp8 alive at the same time. - expected_peak_memory_gib = expected_model_memory_gib * 1.4 - - assert model_memory_gib < expected_model_memory_gib, ( - f"{model_memory_gib=} higher than {expected_model_memory_gib}" - ) - assert peak_memory_gib < expected_peak_memory_gib, ( - f"{peak_memory_gib=} higher than {expected_peak_memory_gib}" - ) - - -@pytest.mark.skipif( - not is_quant_method_supported("fp8"), - reason="FP8 is not supported on this GPU type.", -) -def test_online_quant_load_format_dummy( - vllm_runner, - monkeypatch, - caplog, -) -> None: - with vllm_runner( - "ibm-granite/granite-3.0-1b-a400m-base", - quantization="fp8", - enforce_eager=True, - load_format="dummy", - ) as llm: - outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4) - print(outputs[0][1]) - - @pytest.mark.skipif( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", @@ -799,7 +609,6 @@ def test_scaled_fp8_quant(dtype) -> None: ) @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod]) # FP8 weight reloading does not support online quantization -@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False @pytest.mark.parametrize("weight_block_size", [None, [128, 128]]) # any postprocessing that is applied to the weights such as padding and repacking # (excluding device sharding) must also be applied to the reloaded weights @@ -809,7 +618,6 @@ def test_scaled_fp8_quant(dtype) -> None: def test_fp8_reloading( default_vllm_config, method_cls, - is_checkpoint_fp8_serialized, weight_block_size, use_marlin, dist_init, @@ -820,9 +628,6 @@ def test_fp8_reloading( # we set fp8_backend to None, which sidestepped the issue. monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0") - if is_checkpoint_fp8_serialized is False: - pytest.skip("FP8 weight reloading does not support online quantization") - if method_cls is Fp8MoEMethod and weight_block_size is None: pytest.skip( "FP8 Tensor weight reloading does not support fusing w13_weight_scale. " @@ -835,7 +640,6 @@ def test_fp8_reloading( layer_size = 128 if weight_block_size is not None else 1 with torch.device(f"{DEVICE_TYPE}:0"): config = Fp8Config( - is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, weight_block_size=weight_block_size, ) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 4fc908f484..743a10ecd1 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -20,6 +20,8 @@ from tests.quantization.utils import ( from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm._custom_ops import scaled_fp4_quant +from vllm.config.cache import CacheConfig +from vllm.config.kernel import KernelConfig from vllm.config.load import LoadConfig from vllm.config.model import ModelConfig from vllm.config.quantization import ( @@ -34,6 +36,9 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import ( from vllm.model_executor.kernels.linear.mxfp8.marlin import ( MarlinMxfp8LinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm import ( + MarlinFP8ScaledMMLinearKernel, +) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle @@ -125,6 +130,81 @@ PARTIALLY_PREQUANTIZED_MODEL_NAME = ( ) +@pytest.mark.parametrize( + "overrides", + [ + None, + {"ignore": ["model.layers.0.self_attn.o_proj"]}, + { + "linear": "fp8_per_block", + "ignore": ["model.layers.0.self_attn.o_proj"], + }, + ], + ids=["defaults", "ignore", "linear-and-ignore"], +) +def test_legacy_fp8_online_quantization_uses_per_tensor_shorthand( + tmp_path, caplog, disable_log_dedup, overrides +) -> None: + """The legacy alias preserves overrides and fills in per-tensor defaults.""" + _write_minimal_llama_config(tmp_path) + model_config = ModelConfig( + model=str(tmp_path), quantization="fp8", quantization_config=overrides + ) + + result = weight_utils.get_quant_config(model_config, LoadConfig()) + + assert isinstance(result, OnlineQuantizationConfig) + assert result.args == resolve_quantization_config("fp8_per_tensor", overrides) + assert "--quantization fp8 is deprecated for online quantization" in caplog.text + assert "--quantization fp8_per_tensor instead" in caplog.text + + +@pytest.mark.parametrize( + "shorthand,method_cls", + [ + ("fp8_per_tensor", Fp8PerTensorOnlineLinearMethod), + ("fp8_per_block", Fp8PerBlockOnlineLinearMethod), + ("fp8_per_channel", Fp8PtpcOnlineLinearMethod), + ], +) +def test_qianfan_online_fp8_keeps_vision_layers_unquantized( + shorthand, method_cls +) -> None: + """Model exclusions preserve the language backbone and shared defaults.""" + from vllm.model_executor.models.qianfan_ocr import ( + QianfanOCRForConditionalGeneration, + ) + + args = resolve_quantization_config(shorthand, None) + assert args is not None + config = OnlineQuantizationConfig(args) + other_config = OnlineQuantizationConfig(args) + model = QianfanOCRForConditionalGeneration.__new__( + QianfanOCRForConditionalGeneration + ) + model._patch_quant_config( + SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)), + config, + ) + + layer = Mock(spec=LinearBase) + for prefix in ( + "vision_model.encoder.layers.0.attn.qkv", + "vision_model.encoder.layers.1.mlp.fc2", + "language_model.lm_head", + "mlp1.1", + "mlp1.3", + ): + assert config.resolve_quant_method_cls(layer, prefix) is None + + resolved = config.resolve_quant_method_cls( + layer, "language_model.model.layers.0.self_attn.qkv_proj" + ) + assert resolved is not None and resolved[-1] is method_cls + assert other_config.ignored_layers == [] + assert args.ignore == [] + + def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( monkeypatch, ) -> None: @@ -579,33 +659,81 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp reason="FP8 is not supported on this GPU type.", ) @pytest.mark.parametrize( - ("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"), + ( + "model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls,extra_runner_kwargs" + ), [ # simple case - quantization='fp8_per_tensor' - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_tensor", None, Fp8PerTensorOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="fp8_per_tensor", + ), + # FP8 KV cache with the fp8_per_tensor shorthand. + pytest.param( + GRANITE_MODEL_NAME, + "fp8_per_tensor", + None, + Fp8PerTensorOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + {"kv_cache_dtype": "fp8"}, + id="fp8_per_tensor-fp8-kv-cache", + ), + pytest.param( + GRANITE_MODEL_NAME, + "fp8_per_tensor", + None, + Fp8PerTensorOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + {"linear_backend": "marlin", "moe_backend": "marlin"}, + marks=pytest.mark.skipif( + not current_platform.is_cuda(), + reason="Marlin online FP8 coverage is CUDA-only.", + ), + id="fp8_per_tensor-marlin", + ), + pytest.param( + GRANITE_MODEL_NAME, + "fp8_per_tensor", + None, + Fp8PerTensorOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + { + "kv_cache_dtype": "fp8", + "linear_backend": "marlin", + "moe_backend": "marlin", + }, + marks=pytest.mark.skipif( + not current_platform.is_cuda(), + reason="Marlin online FP8 coverage is CUDA-only.", + ), + id="fp8_per_tensor-fp8-kv-cache-marlin", ), # simple case - quantization='fp8_per_block' - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_block", None, Fp8PerBlockOnlineLinearMethod, Fp8PerBlockOnlineMoEMethod, + {}, + id="fp8_per_block", ), - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_channel", None, Fp8PtpcOnlineLinearMethod, Fp8PtpcOnlineMoEMethod, + {}, + id="fp8_per_channel", ), # quantization='online' with per-layer-kind overrides - ( + pytest.param( GRANITE_MODEL_NAME, "online", { @@ -614,9 +742,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp }, Fp8PerBlockOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="per_layer_kind_overrides", ), # quantization='online' with per-layer target patterns - ( + pytest.param( GRANITE_MODEL_NAME, "online", { @@ -627,9 +757,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp }, Fp8PerBlockOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="targets", ), # ignore with direct layer name - ( + pytest.param( GRANITE_MODEL_NAME, "fp8_per_tensor", # qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex @@ -637,13 +769,17 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp {"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]}, Fp8PerTensorOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, + {}, + id="ignore", ), - ( + pytest.param( GRANITE_MODEL_NAME, "mxfp4", None, Mxfp4OnlineLinearMethod, Mxfp4OnlineMoEMethod, + {}, + id="mxfp4", ), pytest.param( PARTIALLY_PREQUANTIZED_MODEL_NAME, @@ -651,19 +787,10 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp {"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}}, Mxfp8OnlineLinearMethod, CompressedTensorsMoEMethod, + {}, id="partially_prequantized_checkpoint", ), ], - ids=[ - "fp8_per_tensor", - "fp8_per_block", - "fp8_per_channel", - "per_layer_kind_overrides", - "targets", - "ignore", - "mxfp4", - "partially_prequantized_checkpoint", - ], ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] @@ -674,6 +801,7 @@ def test_online_quantization( online_quant_args: dict | None, expected_linear_cls, expected_moe_cls, + extra_runner_kwargs: dict, use_rocm_aiter: bool, monkeypatch, dist_init, @@ -700,14 +828,31 @@ def test_online_quantization( ): pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.") + runner_kwargs = dict(extra_runner_kwargs) + kv_cache_dtype = runner_kwargs.get("kv_cache_dtype", "auto") + force_marlin = extra_runner_kwargs.get("linear_backend") == "marlin" + if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90): + # FA3 requires BF16 output when the query input is FP8. + runner_kwargs["dtype"] = "bfloat16" + + vllm_config_kwargs = { + "cache_config": CacheConfig(cache_dtype=kv_cache_dtype), + "kernel_config": KernelConfig( + linear_backend=runner_kwargs.get("linear_backend", "auto"), + moe_backend=runner_kwargs.get("moe_backend", "auto"), + ), + } + if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: model, vllm_config = load_model_without_vllm_runner( model_name, + dtype=runner_kwargs.get("dtype", "bfloat16"), model_config_kwargs={ "quantization_config": resolve_quantization_config( None, online_quant_args ) }, + vllm_config_kwargs=vllm_config_kwargs, model_loader_cls=DummyModelLoader, ) assert isinstance( @@ -725,7 +870,7 @@ def test_online_quantization( else: model, vllm_config = load_model_without_vllm_runner( model_name, - dtype="bfloat16", + dtype=runner_kwargs.get("dtype", "bfloat16"), quantization=quant_scheme, model_config_kwargs={ "quantization_config": resolve_quantization_config( @@ -743,6 +888,7 @@ def test_online_quantization( "num_experts_per_tok": 2, }, }, + vllm_config_kwargs=vllm_config_kwargs, model_loader_cls=DummyModelLoader, ) @@ -770,7 +916,16 @@ def test_online_quantization( elif quant_scheme == "mxfp4": assert o_proj.weight.dtype == torch.uint8 elif current_platform.is_cuda() or current_platform.is_xpu(): - assert o_proj.weight.dtype == torch.float8_e4m3fn + if current_platform.supports_fp8() and not force_marlin: + assert o_proj.weight.dtype == torch.float8_e4m3fn + assert not isinstance( + o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel + ) + else: + assert o_proj.weight.dtype == torch.int32 + assert isinstance( + o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel + ) elif current_platform.is_rocm(): assert o_proj.weight.dtype == current_platform.fp8_dtype() else: @@ -794,6 +949,11 @@ def test_online_quantization( layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod ) + if kv_cache_dtype == "fp8": + attn = model.model.layers[0].self_attn.attn + assert attn._k_scale == 1.0 + assert attn._v_scale == 1.0 + if ( model_name == GRANITE_MODEL_NAME and isinstance(online_quant_args, dict) diff --git a/tests/v1/spec_decode/test_speculators_correctness.py b/tests/v1/spec_decode/test_speculators_correctness.py index 4cc66d5cd3..7d7ea53f71 100644 --- a/tests/v1/spec_decode/test_speculators_correctness.py +++ b/tests/v1/spec_decode/test_speculators_correctness.py @@ -37,7 +37,7 @@ DFLASH_CONFIG = SpeculatorTestConfig( acceptance_len_rtol=0.15, expected_per_pos_acceptance_rates=(0.795, 0.611, 0.429, 0.282), per_pos_rtol=0.15, - quantization="fp8", + quantization="fp8_per_tensor", ) PEAGLE_CONFIG = SpeculatorTestConfig( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 47df735e89..860c41e501 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -93,7 +93,7 @@ class Fp8Config(QuantizationConfig): def __init__( self, - is_checkpoint_fp8_serialized: bool = False, + is_checkpoint_fp8_serialized: bool = True, activation_scheme: str = "dynamic", ignored_layers: list[str] | None = None, weight_block_size: list[int] | None = None, @@ -102,6 +102,13 @@ class Fp8Config(QuantizationConfig): super().__init__() self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + if not is_checkpoint_fp8_serialized: + raise ValueError( + "The `fp8` quantization method no longer supports online " + "quantization. Please use `--quantization fp8_per_tensor` " + "instead. See " + "https://docs.vllm.ai/en/stable/features/quantization/online/" + ) if activation_scheme not in ACTIVATION_SCHEMES: raise ValueError(f"Unsupported activation scheme {activation_scheme}") @@ -112,11 +119,6 @@ class Fp8Config(QuantizationConfig): ) self.store_dtype = store_dtype if weight_block_size is not None: - if not is_checkpoint_fp8_serialized: - raise ValueError( - "The block-wise quantization only supports fp8-serialized " - "checkpoint for now." - ) if len(weight_block_size) != 2: raise ValueError( "The quantization block size of weight must have 2 " @@ -182,18 +184,9 @@ class Fp8Config(QuantizationConfig): match_mode=self.ignored_layers_match_mode, ): return UnquantizedLinearMethod() - if not self.is_checkpoint_fp8_serialized: - from vllm.model_executor.layers.quantization.online.fp8 import ( - Fp8PerTensorOnlineLinearMethod, - ) - - online_method = Fp8PerTensorOnlineLinearMethod() - online_method.marlin_input_dtype = get_marlin_input_dtype(prefix) - return online_method - else: - offline_method = Fp8LinearMethod(self) - offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix) - return offline_method + offline_method = Fp8LinearMethod(self) + offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix) + return offline_method elif isinstance(layer, RoutedExperts): if is_layer_skipped( prefix=prefix, @@ -208,14 +201,7 @@ class Fp8Config(QuantizationConfig): ) return Mxfp4MoEMethod(layer.moe_config) - if self.is_checkpoint_fp8_serialized: - return Fp8MoEMethod(self, layer) - else: - from vllm.model_executor.layers.quantization.online.fp8 import ( - Fp8PerTensorOnlineMoEMethod, - ) - - return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config) + return Fp8MoEMethod(self, layer) elif isinstance(layer, Attention): return Fp8KVCacheMethod(self) return None @@ -548,7 +534,6 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer.orig_dtype = params_dtype layer.weight_block_size = None - assert self.quant_config.is_checkpoint_fp8_serialized params_dtype = torch.float8_e4m3fn if self.block_quant: diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 2fa79a4e1c..9414abb991 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -195,7 +195,11 @@ def get_quant_config( if model_config.quantization is None: raise ValueError("Model quantization method is not specified in the config.") quant_cls = get_quantization_config(model_config.quantization) - from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs + from vllm.config.quantization import ( + _ONLINE_SHORTHANDS, + QuantizationConfigArgs, + resolve_quantization_config, + ) from vllm.model_executor.layers.quantization.online.base import ( OnlineQuantizationConfig, ) @@ -335,6 +339,14 @@ def get_quant_config( # If the quantization config is not found, use the default config. if not possible_config_filenames: + if model_config.quantization == "fp8": + logger.warning_once( + "--quantization fp8 is deprecated for online quantization; " + "use --quantization fp8_per_tensor instead." + ) + fp8_args = resolve_quantization_config("fp8_per_tensor", online_args) + assert fp8_args is not None + return OnlineQuantizationConfig(args=fp8_args) if model_config.quantization in _ONLINE_SHORTHANDS: args = online_args or _ONLINE_SHORTHANDS[model_config.quantization] assert isinstance(args, QuantizationConfigArgs) diff --git a/vllm/model_executor/models/qianfan_ocr.py b/vllm/model_executor/models/qianfan_ocr.py index 8367737c95..335a3f0f98 100644 --- a/vllm/model_executor/models/qianfan_ocr.py +++ b/vllm/model_executor/models/qianfan_ocr.py @@ -9,6 +9,12 @@ from transformers import PreTrainedConfig from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8Static128BlockSym, + kFp8StaticChannelSym, + kFp8StaticTensorSym, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.transformers_utils.processors.internvl import ( InternVLImageProcessor, @@ -71,7 +77,12 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel): ) -> None: super()._patch_quant_config(config, quant_config) # ignore vit layers to preserve model performance - if isinstance(quant_config, Fp8Config): + if isinstance(quant_config, Fp8Config) or ( + isinstance(quant_config, OnlineQuantizationConfig) + and quant_config.args.linear is not None + and quant_config.args.linear.weight + in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym) + ): _FP8_IGNORED_LAYERS = [ *( layer @@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel): "mlp1.1", "mlp1.3", ] + quant_config.ignored_layers = list(quant_config.ignored_layers) for layer in _FP8_IGNORED_LAYERS: if layer not in quant_config.ignored_layers: quant_config.ignored_layers.append(layer)