[Cleanup] Remove online quantization support in fp8.py in favor of online shorthands (#53585)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.buildkite/intel_jobs/test-intel.yaml

@@ -38,7 +38,7 @@ steps:
         python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE &&
         python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
         python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
-        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
+        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor &&
         python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
         python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
         python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
@@ -74,8 +74,8 @@ steps:
         python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
         python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
         python3 examples/basic/offline_inference/generate.py --linear-backend torch --model Qwen/Qwen3-4B-Instruct-2507-FP8 --enforce-eager --max-model-len 4096 &&
-        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 &&
-        python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
+        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 &&
+        python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
         '
   - label: "XPU V1 test"
     key: xpu-v1-test

.buildkite/scripts/hardware_ci/run-intel-ci-test.sh

@@ -17,7 +17,7 @@ case "${test_suite}" in
     python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
     python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
     python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
+    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor
     python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
     python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
     python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
@@ -30,8 +30,8 @@ case "${test_suite}" in
     python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096
     python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096
     python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096
-    python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
-    python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
+    python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
+    python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
     ;;
   v1)
     cd tests

docs/features/quantization/llm_compressor/fp8.md

@@ -143,14 +143,14 @@ If you encounter any issues or have feature requests, please open an issue on th
 
 ## Online Dynamic Quantization
 
-Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8"` in the command line or setting `quantization="fp8"` in the LLM constructor.
+Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8_per_tensor"` in the command line or setting `quantization="fp8_per_tensor"` in the LLM constructor.
 
 In this mode, all Linear modules (except for the final `lm_head`) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode.
 
 ```python
 from vllm import LLM
 
-llm = LLM("facebook/opt-125m", quantization="fp8")
+llm = LLM("facebook/opt-125m", quantization="fp8_per_tensor")
 # INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB
 result = llm.generate("Hello, my name is")
 print(result[0].outputs[0].text)

tests/compile/passes/test_mla_attn_quant_fusion.py

@@ -271,7 +271,6 @@ class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel):
 
     quant_key = kFp8Dynamic128Sym
     quant_config = Fp8Config(
-        is_checkpoint_fp8_serialized=True,
         weight_block_size=[128, 128],
     )
 

tests/kernels/moe/test_moe_layer.py

@@ -796,11 +796,11 @@ def make_quant_config(
         return None, QuantizedWeights(w13_weight=w1, w2_weight=w2)
 
     if quantization == "fp8":
-        return Fp8Config(True), _quantize_fp8_halves(w1, w2)
+        return Fp8Config(), _quantize_fp8_halves(w1, w2)
 
     if quantization == "fp8_blocked":
         block_shape = [128, 128]
-        return Fp8Config(True, weight_block_size=block_shape), _quantize_fp8_halves(
+        return Fp8Config(weight_block_size=block_shape), _quantize_fp8_halves(
             w1, w2, block_shape
         )
 

tests/kernels/moe/utils.py

@@ -641,7 +641,7 @@ def make_shared_experts_with_weights(
         if quant_dtype == torch.float8_e4m3fn:
             from vllm.model_executor.layers.quantization.fp8 import Fp8Config
 
-            quant_config = Fp8Config(True)
+            quant_config = Fp8Config()
         else:
             quant_config = None
 

tests/model_executor/layers/test_fused_shared_expert.py

@@ -1227,7 +1227,6 @@ def test_non_quark_shared_expert_fse_is_incompatible() -> None:
 
 def _fp8_config(**kwargs: Any) -> Fp8Config:
     return Fp8Config(
-        is_checkpoint_fp8_serialized=True,
         activation_scheme="dynamic",
         weight_block_size=[128, 128],
         **kwargs,
@@ -1248,7 +1247,7 @@ def test_block_fp8_shared_expert_fse_is_compatible() -> None:
 def test_per_tensor_fp8_shared_expert_fse_is_incompatible() -> None:
     """Per-tensor scales are 0-D, so the shared-expert chunker cannot slice them."""
     compatible, reason = is_shared_expert_quant_fse_compatible(
-        Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"),
+        Fp8Config(activation_scheme="dynamic"),
         "model.layers.0.mlp.experts",
         "model.layers.0.mlp.shared_experts",
     )

tests/model_executor/model_loader/test_reload.py

@@ -1158,13 +1158,13 @@ def test_kv_scale_reload(vllm_runner):
             "Qwen/Qwen3-0.6B",
             "inference-optimization/Qwen3-0.6B-debug-multiply",
             "inference-optimization/Qwen3-0.6B-debug-add",
-            "fp8",
+            "fp8_per_tensor",
         ),
         pytest.param(
             "inference-optimization/DeepSeek-V3-debug-empty",
             "inference-optimization/DeepSeek-V3-debug-multiply",
             "inference-optimization/DeepSeek-V3-debug-add",
-            "fp8",
+            "fp8_per_tensor",
             marks=[pytest.mark.slow_test],
         ),
         pytest.param(
@@ -1192,7 +1192,7 @@ def test_online_quantize_reload(
     if current_platform.device_count() < tp_size:
         pytest.skip(reason="Not enough GPU devices")
 
-    if quantization == "fp8" and _fp8_reload_unsupported():
+    if quantization == "fp8_per_tensor" and _fp8_reload_unsupported():
         pytest.skip(reason="Requires FP8 support")
 
     with vllm_runner(

tests/models/qwen4_exp/test_ple.py

@@ -14,11 +14,13 @@ from torch.nn import functional as F
 import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
 import vllm.model_executor.parameter as parameter_module
 import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module
+from vllm.config.quantization import QuantizationConfigArgs
 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
 from vllm.model_executor.layers.quantization.modelopt import (
     ModelOptMixedPrecisionConfig,
     ModelOptNvFp4Config,
 )
+from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
 from vllm.models.qwen4_exp.common.ple import (
     PLEShardOverlap,
     compute_ple_shard_overlap,
@@ -396,7 +398,6 @@ def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:
 def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:
     prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
     quant_config = Fp8Config(
-        is_checkpoint_fp8_serialized=True,
         ignored_layers=[],
         weight_block_size=[128, 128],
     )
@@ -422,12 +423,11 @@ def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:
     with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):
         Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)
 
-    dynamic_fp8_config = Fp8Config(
-        is_checkpoint_fp8_serialized=False,
-        ignored_layers=[],
+    online_fp8_config = OnlineQuantizationConfig(
+        QuantizationConfigArgs(linear="fp8_per_tensor")
     )
-    with pytest.raises(NotImplementedError, match="serialized FP8"):
-        Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)
+    with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):
+        Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)
 
 
 def test_ple_embedding_respects_modelopt_exclusion() -> None:

tests/quantization/test_fp8.py

@@ -5,11 +5,9 @@
 Run `pytest tests/quantization/test_fp8.py --forked`.
 """
 
-import logging
 from types import SimpleNamespace
 
 import pytest
-import regex as re
 import torch
 
 from tests.quantization.utils import (
@@ -22,9 +20,6 @@ from vllm.config.cache import CacheConfig
 from vllm.config.kernel import KernelConfig
 from vllm.config.model import ModelConfig
 from vllm.forward_context import set_forward_context
-from vllm.model_executor.kernels.linear.scaled_mm import (
-    MarlinFP8ScaledMMLinearKernel,
-)
 from vllm.model_executor.layers.attention import Attention
 from vllm.model_executor.layers.attention.attention import (
     set_default_quant_scales,
@@ -32,15 +27,11 @@ from vllm.model_executor.layers.attention.attention import (
 from vllm.model_executor.layers.fused_moe import FusedMoEFactory
 from vllm.model_executor.layers.quantization.fp8 import (
     Fp8Config,
-    Fp8KVCacheMethod,
     Fp8LinearMethod,
     Fp8MoEMethod,
 )
 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
 from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod
-from vllm.model_executor.layers.quantization.online.fp8 import (
-    Fp8PerTensorOnlineLinearMethod,
-)
 from vllm.model_executor.layers.quantization.utils import flashinfer_utils
 from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
     prepare_fp8_moe_layer_for_fi,
@@ -531,187 +522,6 @@ def test_model_load_and_run(
         model(input_ids, positions, None)
 
 
-@pytest.mark.skipif(
-    not is_quant_method_supported("fp8"),
-    reason="FP8 is not supported on this GPU type.",
-)
-@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
-@pytest.mark.parametrize(
-    "force_marlin", [True, False] if current_platform.is_cuda() else [False]
-)
-@pytest.mark.parametrize(
-    "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
-)
-def test_online_quantization(
-    vllm_runner,
-    kv_cache_dtype: str,
-    force_marlin: bool,
-    use_rocm_aiter: bool,
-    monkeypatch,
-) -> None:
-    if use_rocm_aiter:
-        monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
-
-    # `LLM.apply_model` requires pickling a function.
-    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
-
-    kwargs = {}
-    if force_marlin:
-        kwargs["linear_backend"] = "marlin"
-        kwargs["moe_backend"] = "marlin"
-
-    model_dtype = "auto"
-    if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
-        # FA3 requires BF16 output when the query input is FP8.
-        model_dtype = "bfloat16"
-
-    with vllm_runner(
-        "facebook/opt-125m",
-        quantization="fp8",
-        dtype=model_dtype,
-        enforce_eager=True,
-        kv_cache_dtype=kv_cache_dtype,
-        **kwargs,
-    ) as llm:
-
-        def check_model(model):
-            fc1 = model.model.decoder.layers[0].fc1
-            assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod)
-            if kv_cache_dtype == "fp8":
-                attn = model.model.decoder.layers[0].self_attn.attn
-                assert isinstance(attn.quant_method, Fp8KVCacheMethod)
-                assert attn._k_scale == 1.0
-                assert attn._v_scale == 1.0
-
-            if current_platform.is_cuda() or current_platform.is_xpu():
-                if current_platform.supports_fp8() and not force_marlin:
-                    # For GPUs with hardware support, we keep weights in fp8
-                    assert fc1.weight.dtype == torch.float8_e4m3fn
-                    assert not isinstance(
-                        fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
-                    )
-                else:
-                    # For GPUs without hardware support, we pack the fp8 weights
-                    # for weight-only quantization using Marlin kernels
-                    assert fc1.weight.dtype == torch.int32
-                    assert isinstance(
-                        fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
-                    )
-            elif current_platform.is_rocm():
-                if current_platform.supports_fp8() and not force_marlin:
-                    # For GPUs with hardware support, we keep weights in fp8
-                    assert fc1.weight.dtype == current_platform.fp8_dtype()
-                else:  # unsupported ROCm platform
-                    pytest.skip(
-                        "Skip `test_load_fp16_model`. "
-                        "It only runs on ROCm platform with FP8 compute."
-                        " e.g. MI300X and above."
-                    )
-            else:  # unsupported platform
-                pytest.skip(
-                    "Skip `test_load_fp16_model`. "
-                    "It only runs on CUDA and ROCm platform."
-                )
-
-        llm.apply_model(check_model)
-
-        outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
-        print(outputs[0][1])
-
-
-@pytest.mark.skipif(
-    not is_quant_method_supported("fp8"),
-    reason="FP8 is not supported on this GPU type.",
-)
-def test_online_quant_peak_mem(
-    vllm_runner,
-    caplog_mp_spawn,
-    monkeypatch,
-) -> None:
-    # Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
-    # 1. it covers both Linear and MoE paths
-    # 2. it is already used by other tests in CI, so adding it here
-    #    does not increase disk space for CI runners
-    # I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
-    # which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
-    # 1.3 GiB fp8), but could not as adding one more model makes CI
-    # run out of disk space.
-    model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
-
-    # Force spawn to ensure caplog_mp_spawn works consistently
-    # (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
-    monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
-
-    with (
-        caplog_mp_spawn(logging.DEBUG) as log_holder,
-        vllm_runner(
-            model_name,
-            quantization="fp8",
-            enforce_eager=True,
-        ) as llm,
-    ):
-        outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
-        print(outputs[0][1])
-
-    log_text = log_holder.text
-
-    # Parse memory usage from captured logs
-    model_memory_gib = None
-    peak_memory_gib = None
-    for line in log_text.splitlines():
-        if model_memory_gib is None:
-            match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
-            if match:
-                model_memory_gib = float(match.group(1))
-        if peak_memory_gib is None:
-            match = re.search(
-                r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
-            )
-            if match:
-                peak_memory_gib = float(match.group(1))
-
-    assert model_memory_gib is not None, "Could not find model loading memory log"
-    assert peak_memory_gib is not None, "Could not find peak memory log"
-    print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
-    print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
-
-    # model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant
-    # uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB)
-    expected_model_memory_gib = 6.7
-
-    # for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
-    # GiB, which is 1.36x above model_memory_gib. A slightly higher number is
-    # expected as when we load and quantize weights in a streaming fashion we
-    # need to have individual weights in bf16 + fp8 alive at the same time.
-    expected_peak_memory_gib = expected_model_memory_gib * 1.4
-
-    assert model_memory_gib < expected_model_memory_gib, (
-        f"{model_memory_gib=} higher than {expected_model_memory_gib}"
-    )
-    assert peak_memory_gib < expected_peak_memory_gib, (
-        f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
-    )
-
-
-@pytest.mark.skipif(
-    not is_quant_method_supported("fp8"),
-    reason="FP8 is not supported on this GPU type.",
-)
-def test_online_quant_load_format_dummy(
-    vllm_runner,
-    monkeypatch,
-    caplog,
-) -> None:
-    with vllm_runner(
-        "ibm-granite/granite-3.0-1b-a400m-base",
-        quantization="fp8",
-        enforce_eager=True,
-        load_format="dummy",
-    ) as llm:
-        outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
-        print(outputs[0][1])
-
-
 @pytest.mark.skipif(
     not is_quant_method_supported("fp8"),
     reason="FP8 is not supported on this GPU type.",
@@ -799,7 +609,6 @@ def test_scaled_fp8_quant(dtype) -> None:
 )
 @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
 # FP8 weight reloading does not support online quantization
-@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True])  # skip False
 @pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
 # any postprocessing that is applied to the weights such as padding and repacking
 # (excluding device sharding) must also be applied to the reloaded weights
@@ -809,7 +618,6 @@ def test_scaled_fp8_quant(dtype) -> None:
 def test_fp8_reloading(
     default_vllm_config,
     method_cls,
-    is_checkpoint_fp8_serialized,
     weight_block_size,
     use_marlin,
     dist_init,
@@ -820,9 +628,6 @@ def test_fp8_reloading(
     # we set fp8_backend to None, which sidestepped the issue.
     monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
 
-    if is_checkpoint_fp8_serialized is False:
-        pytest.skip("FP8 weight reloading does not support online quantization")
-
     if method_cls is Fp8MoEMethod and weight_block_size is None:
         pytest.skip(
             "FP8 Tensor weight reloading does not support fusing w13_weight_scale. "
@@ -835,7 +640,6 @@ def test_fp8_reloading(
     layer_size = 128 if weight_block_size is not None else 1
     with torch.device(f"{DEVICE_TYPE}:0"):
         config = Fp8Config(
-            is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
             weight_block_size=weight_block_size,
         )
 

tests/quantization/test_online.py

@@ -20,6 +20,8 @@ from tests.quantization.utils import (
 from vllm import _custom_ops as ops
 from vllm._aiter_ops import rocm_aiter_ops
 from vllm._custom_ops import scaled_fp4_quant
+from vllm.config.cache import CacheConfig
+from vllm.config.kernel import KernelConfig
 from vllm.config.load import LoadConfig
 from vllm.config.model import ModelConfig
 from vllm.config.quantization import (
@@ -34,6 +36,9 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (
 from vllm.model_executor.kernels.linear.mxfp8.marlin import (
     MarlinMxfp8LinearKernel,
 )
+from vllm.model_executor.kernels.linear.scaled_mm import (
+    MarlinFP8ScaledMMLinearKernel,
+)
 from vllm.model_executor.layers.attention import Attention
 from vllm.model_executor.layers.fused_moe import FusedMoEFactory
 from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
@@ -125,6 +130,81 @@ PARTIALLY_PREQUANTIZED_MODEL_NAME = (
 )
 
 
+@pytest.mark.parametrize(
+    "overrides",
+    [
+        None,
+        {"ignore": ["model.layers.0.self_attn.o_proj"]},
+        {
+            "linear": "fp8_per_block",
+            "ignore": ["model.layers.0.self_attn.o_proj"],
+        },
+    ],
+    ids=["defaults", "ignore", "linear-and-ignore"],
+)
+def test_legacy_fp8_online_quantization_uses_per_tensor_shorthand(
+    tmp_path, caplog, disable_log_dedup, overrides
+) -> None:
+    """The legacy alias preserves overrides and fills in per-tensor defaults."""
+    _write_minimal_llama_config(tmp_path)
+    model_config = ModelConfig(
+        model=str(tmp_path), quantization="fp8", quantization_config=overrides
+    )
+
+    result = weight_utils.get_quant_config(model_config, LoadConfig())
+
+    assert isinstance(result, OnlineQuantizationConfig)
+    assert result.args == resolve_quantization_config("fp8_per_tensor", overrides)
+    assert "--quantization fp8 is deprecated for online quantization" in caplog.text
+    assert "--quantization fp8_per_tensor instead" in caplog.text
+
+
+@pytest.mark.parametrize(
+    "shorthand,method_cls",
+    [
+        ("fp8_per_tensor", Fp8PerTensorOnlineLinearMethod),
+        ("fp8_per_block", Fp8PerBlockOnlineLinearMethod),
+        ("fp8_per_channel", Fp8PtpcOnlineLinearMethod),
+    ],
+)
+def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
+    shorthand, method_cls
+) -> None:
+    """Model exclusions preserve the language backbone and shared defaults."""
+    from vllm.model_executor.models.qianfan_ocr import (
+        QianfanOCRForConditionalGeneration,
+    )
+
+    args = resolve_quantization_config(shorthand, None)
+    assert args is not None
+    config = OnlineQuantizationConfig(args)
+    other_config = OnlineQuantizationConfig(args)
+    model = QianfanOCRForConditionalGeneration.__new__(
+        QianfanOCRForConditionalGeneration
+    )
+    model._patch_quant_config(
+        SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),
+        config,
+    )
+
+    layer = Mock(spec=LinearBase)
+    for prefix in (
+        "vision_model.encoder.layers.0.attn.qkv",
+        "vision_model.encoder.layers.1.mlp.fc2",
+        "language_model.lm_head",
+        "mlp1.1",
+        "mlp1.3",
+    ):
+        assert config.resolve_quant_method_cls(layer, prefix) is None
+
+    resolved = config.resolve_quant_method_cls(
+        layer, "language_model.model.layers.0.self_attn.qkv_proj"
+    )
+    assert resolved is not None and resolved[-1] is method_cls
+    assert other_config.ignored_layers == []
+    assert args.ignore == []
+
+
 def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
     monkeypatch,
 ) -> None:
@@ -579,33 +659,81 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
     reason="FP8 is not supported on this GPU type.",
 )
 @pytest.mark.parametrize(
-    ("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"),
+    (
+        "model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls,extra_runner_kwargs"
+    ),
     [
         # simple case - quantization='fp8_per_tensor'
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "fp8_per_tensor",
             None,
             Fp8PerTensorOnlineLinearMethod,
             Fp8PerTensorOnlineMoEMethod,
+            {},
+            id="fp8_per_tensor",
+        ),
+        # FP8 KV cache with the fp8_per_tensor shorthand.
+        pytest.param(
+            GRANITE_MODEL_NAME,
+            "fp8_per_tensor",
+            None,
+            Fp8PerTensorOnlineLinearMethod,
+            Fp8PerTensorOnlineMoEMethod,
+            {"kv_cache_dtype": "fp8"},
+            id="fp8_per_tensor-fp8-kv-cache",
+        ),
+        pytest.param(
+            GRANITE_MODEL_NAME,
+            "fp8_per_tensor",
+            None,
+            Fp8PerTensorOnlineLinearMethod,
+            Fp8PerTensorOnlineMoEMethod,
+            {"linear_backend": "marlin", "moe_backend": "marlin"},
+            marks=pytest.mark.skipif(
+                not current_platform.is_cuda(),
+                reason="Marlin online FP8 coverage is CUDA-only.",
+            ),
+            id="fp8_per_tensor-marlin",
+        ),
+        pytest.param(
+            GRANITE_MODEL_NAME,
+            "fp8_per_tensor",
+            None,
+            Fp8PerTensorOnlineLinearMethod,
+            Fp8PerTensorOnlineMoEMethod,
+            {
+                "kv_cache_dtype": "fp8",
+                "linear_backend": "marlin",
+                "moe_backend": "marlin",
+            },
+            marks=pytest.mark.skipif(
+                not current_platform.is_cuda(),
+                reason="Marlin online FP8 coverage is CUDA-only.",
+            ),
+            id="fp8_per_tensor-fp8-kv-cache-marlin",
         ),
         # simple case - quantization='fp8_per_block'
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "fp8_per_block",
             None,
             Fp8PerBlockOnlineLinearMethod,
             Fp8PerBlockOnlineMoEMethod,
+            {},
+            id="fp8_per_block",
         ),
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "fp8_per_channel",
             None,
             Fp8PtpcOnlineLinearMethod,
             Fp8PtpcOnlineMoEMethod,
+            {},
+            id="fp8_per_channel",
         ),
         # quantization='online' with per-layer-kind overrides
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "online",
             {
@@ -614,9 +742,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
             },
             Fp8PerBlockOnlineLinearMethod,
             Fp8PerTensorOnlineMoEMethod,
+            {},
+            id="per_layer_kind_overrides",
         ),
         # quantization='online' with per-layer target patterns
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "online",
             {
@@ -627,9 +757,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
             },
             Fp8PerBlockOnlineLinearMethod,
             Fp8PerTensorOnlineMoEMethod,
+            {},
+            id="targets",
         ),
         # ignore with direct layer name
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "fp8_per_tensor",
             # qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
@@ -637,13 +769,17 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
             {"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]},
             Fp8PerTensorOnlineLinearMethod,
             Fp8PerTensorOnlineMoEMethod,
+            {},
+            id="ignore",
         ),
-        (
+        pytest.param(
             GRANITE_MODEL_NAME,
             "mxfp4",
             None,
             Mxfp4OnlineLinearMethod,
             Mxfp4OnlineMoEMethod,
+            {},
+            id="mxfp4",
         ),
         pytest.param(
             PARTIALLY_PREQUANTIZED_MODEL_NAME,
@@ -651,19 +787,10 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
             {"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
             Mxfp8OnlineLinearMethod,
             CompressedTensorsMoEMethod,
+            {},
             id="partially_prequantized_checkpoint",
         ),
     ],
-    ids=[
-        "fp8_per_tensor",
-        "fp8_per_block",
-        "fp8_per_channel",
-        "per_layer_kind_overrides",
-        "targets",
-        "ignore",
-        "mxfp4",
-        "partially_prequantized_checkpoint",
-    ],
 )
 @pytest.mark.parametrize(
     "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
@@ -674,6 +801,7 @@ def test_online_quantization(
     online_quant_args: dict | None,
     expected_linear_cls,
     expected_moe_cls,
+    extra_runner_kwargs: dict,
     use_rocm_aiter: bool,
     monkeypatch,
     dist_init,
@@ -700,14 +828,31 @@ def test_online_quantization(
     ):
         pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.")
 
+    runner_kwargs = dict(extra_runner_kwargs)
+    kv_cache_dtype = runner_kwargs.get("kv_cache_dtype", "auto")
+    force_marlin = extra_runner_kwargs.get("linear_backend") == "marlin"
+    if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
+        # FA3 requires BF16 output when the query input is FP8.
+        runner_kwargs["dtype"] = "bfloat16"
+
+    vllm_config_kwargs = {
+        "cache_config": CacheConfig(cache_dtype=kv_cache_dtype),
+        "kernel_config": KernelConfig(
+            linear_backend=runner_kwargs.get("linear_backend", "auto"),
+            moe_backend=runner_kwargs.get("moe_backend", "auto"),
+        ),
+    }
+
     if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
         model, vllm_config = load_model_without_vllm_runner(
             model_name,
+            dtype=runner_kwargs.get("dtype", "bfloat16"),
             model_config_kwargs={
                 "quantization_config": resolve_quantization_config(
                     None, online_quant_args
                 )
             },
+            vllm_config_kwargs=vllm_config_kwargs,
             model_loader_cls=DummyModelLoader,
         )
         assert isinstance(
@@ -725,7 +870,7 @@ def test_online_quantization(
     else:
         model, vllm_config = load_model_without_vllm_runner(
             model_name,
-            dtype="bfloat16",
+            dtype=runner_kwargs.get("dtype", "bfloat16"),
             quantization=quant_scheme,
             model_config_kwargs={
                 "quantization_config": resolve_quantization_config(
@@ -743,6 +888,7 @@ def test_online_quantization(
                     "num_experts_per_tok": 2,
                 },
             },
+            vllm_config_kwargs=vllm_config_kwargs,
             model_loader_cls=DummyModelLoader,
         )
 
@@ -770,7 +916,16 @@ def test_online_quantization(
     elif quant_scheme == "mxfp4":
         assert o_proj.weight.dtype == torch.uint8
     elif current_platform.is_cuda() or current_platform.is_xpu():
-        assert o_proj.weight.dtype == torch.float8_e4m3fn
+        if current_platform.supports_fp8() and not force_marlin:
+            assert o_proj.weight.dtype == torch.float8_e4m3fn
+            assert not isinstance(
+                o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
+            )
+        else:
+            assert o_proj.weight.dtype == torch.int32
+            assert isinstance(
+                o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
+            )
     elif current_platform.is_rocm():
         assert o_proj.weight.dtype == current_platform.fp8_dtype()
     else:
@@ -794,6 +949,11 @@ def test_online_quantization(
                 layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
             )
 
+    if kv_cache_dtype == "fp8":
+        attn = model.model.layers[0].self_attn.attn
+        assert attn._k_scale == 1.0
+        assert attn._v_scale == 1.0
+
     if (
         model_name == GRANITE_MODEL_NAME
         and isinstance(online_quant_args, dict)

tests/v1/spec_decode/test_speculators_correctness.py

@@ -37,7 +37,7 @@ DFLASH_CONFIG = SpeculatorTestConfig(
     acceptance_len_rtol=0.15,
     expected_per_pos_acceptance_rates=(0.795, 0.611, 0.429, 0.282),
     per_pos_rtol=0.15,
-    quantization="fp8",
+    quantization="fp8_per_tensor",
 )
 
 PEAGLE_CONFIG = SpeculatorTestConfig(

vllm/model_executor/layers/quantization/fp8.py

@@ -93,7 +93,7 @@ class Fp8Config(QuantizationConfig):
 
     def __init__(
         self,
-        is_checkpoint_fp8_serialized: bool = False,
+        is_checkpoint_fp8_serialized: bool = True,
         activation_scheme: str = "dynamic",
         ignored_layers: list[str] | None = None,
         weight_block_size: list[int] | None = None,
@@ -102,6 +102,13 @@ class Fp8Config(QuantizationConfig):
         super().__init__()
 
         self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized
+        if not is_checkpoint_fp8_serialized:
+            raise ValueError(
+                "The `fp8` quantization method no longer supports online "
+                "quantization. Please use `--quantization fp8_per_tensor` "
+                "instead. See "
+                "https://docs.vllm.ai/en/stable/features/quantization/online/"
+            )
 
         if activation_scheme not in ACTIVATION_SCHEMES:
             raise ValueError(f"Unsupported activation scheme {activation_scheme}")
@@ -112,11 +119,6 @@ class Fp8Config(QuantizationConfig):
         )
         self.store_dtype = store_dtype
         if weight_block_size is not None:
-            if not is_checkpoint_fp8_serialized:
-                raise ValueError(
-                    "The block-wise quantization only supports fp8-serialized "
-                    "checkpoint for now."
-                )
             if len(weight_block_size) != 2:
                 raise ValueError(
                     "The quantization block size of weight must have 2 "
@@ -182,18 +184,9 @@ class Fp8Config(QuantizationConfig):
                 match_mode=self.ignored_layers_match_mode,
             ):
                 return UnquantizedLinearMethod()
-            if not self.is_checkpoint_fp8_serialized:
-                from vllm.model_executor.layers.quantization.online.fp8 import (
-                    Fp8PerTensorOnlineLinearMethod,
-                )
-
-                online_method = Fp8PerTensorOnlineLinearMethod()
-                online_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
-                return online_method
-            else:
-                offline_method = Fp8LinearMethod(self)
-                offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
-                return offline_method
+            offline_method = Fp8LinearMethod(self)
+            offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
+            return offline_method
         elif isinstance(layer, RoutedExperts):
             if is_layer_skipped(
                 prefix=prefix,
@@ -208,14 +201,7 @@ class Fp8Config(QuantizationConfig):
                 )
 
                 return Mxfp4MoEMethod(layer.moe_config)
-            if self.is_checkpoint_fp8_serialized:
-                return Fp8MoEMethod(self, layer)
-            else:
-                from vllm.model_executor.layers.quantization.online.fp8 import (
-                    Fp8PerTensorOnlineMoEMethod,
-                )
-
-                return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
+            return Fp8MoEMethod(self, layer)
         elif isinstance(layer, Attention):
             return Fp8KVCacheMethod(self)
         return None
@@ -548,7 +534,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
         layer.orig_dtype = params_dtype
         layer.weight_block_size = None
 
-        assert self.quant_config.is_checkpoint_fp8_serialized
         params_dtype = torch.float8_e4m3fn
 
         if self.block_quant:

vllm/model_executor/model_loader/weight_utils.py

@@ -195,7 +195,11 @@ def get_quant_config(
     if model_config.quantization is None:
         raise ValueError("Model quantization method is not specified in the config.")
     quant_cls = get_quantization_config(model_config.quantization)
-    from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs
+    from vllm.config.quantization import (
+        _ONLINE_SHORTHANDS,
+        QuantizationConfigArgs,
+        resolve_quantization_config,
+    )
     from vllm.model_executor.layers.quantization.online.base import (
         OnlineQuantizationConfig,
     )
@@ -335,6 +339,14 @@ def get_quant_config(
 
     # If the quantization config is not found, use the default config.
     if not possible_config_filenames:
+        if model_config.quantization == "fp8":
+            logger.warning_once(
+                "--quantization fp8 is deprecated for online quantization; "
+                "use --quantization fp8_per_tensor instead."
+            )
+            fp8_args = resolve_quantization_config("fp8_per_tensor", online_args)
+            assert fp8_args is not None
+            return OnlineQuantizationConfig(args=fp8_args)
         if model_config.quantization in _ONLINE_SHORTHANDS:
             args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
             assert isinstance(args, QuantizationConfigArgs)

vllm/model_executor/models/qianfan_ocr.py

@@ -9,6 +9,12 @@ from transformers import PreTrainedConfig
 
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.quantization.fp8 import Fp8Config
+from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    kFp8Static128BlockSym,
+    kFp8StaticChannelSym,
+    kFp8StaticTensorSym,
+)
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.transformers_utils.processors.internvl import (
     InternVLImageProcessor,
@@ -71,7 +77,12 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
     ) -> None:
         super()._patch_quant_config(config, quant_config)
         # ignore vit layers to preserve model performance
-        if isinstance(quant_config, Fp8Config):
+        if isinstance(quant_config, Fp8Config) or (
+            isinstance(quant_config, OnlineQuantizationConfig)
+            and quant_config.args.linear is not None
+            and quant_config.args.linear.weight
+            in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym)
+        ):
             _FP8_IGNORED_LAYERS = [
                 *(
                     layer
@@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
                 "mlp1.1",
                 "mlp1.3",
             ]
+            quant_config.ignored_layers = list(quant_config.ignored_layers)
             for layer in _FP8_IGNORED_LAYERS:
                 if layer not in quant_config.ignored_layers:
                     quant_config.ignored_layers.append(layer)