[Quantization] add online fp8 ptpc (#44132)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-06-08 22:42:11 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ac3409d162
commit 753e9d55e6
6 changed files with 385 additions and 8 deletions
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for online FP8 per-channel quantization.
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
quantization) and compares log-probabilities against the same model served in
BF16 without quantization. This exercises the full pipeline: config parsing,
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
loading, online quantization / shuffling, and inference.
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
diverse prompts from ``tests/prompts/example.txt``.
"""
import pytest
from tests.quantization.utils import is_quant_method_supported
from ..utils import check_logprobs_close
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
# Small dense model (no MoE) to validate the linear-only path.
DENSE_MODEL = "Qwen/Qwen3-0.6B"
MAX_MODEL_LEN = 1024
MAX_TOKENS = 4
NUM_LOG_PROBS = 8
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported on this GPU type.",
)
@pytest.mark.quant_model
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_fp8_per_channel_logprobs(
vllm_runner,
example_prompts,
model: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Compare BF16 baseline logprobs against online per-channel-quantized
model.
Runs the same model twice -- once in BF16 (baseline) and once with online
FP8 per-channel quantization -- then checks that the top log-probabilities
are close. Only 4 tokens are generated to keep the test fast while still
catching numerical divergence beyond expected per-channel error.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
with vllm_runner(
model,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
) as vllm_model:
baseline_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
with vllm_runner(
model,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
quantization="fp8_per_channel",
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="bf16",
name_1="fp8_per_channel",
)
+103
View File
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FP8 per-channel online quantization.
Per-output-channel weight scale + dynamic per-token activation scale.
bf16/fp16 checkpoints are quantized at load time with one fp32 scale per
output channel for weights and one fp32 scale per token for activations
(computed dynamically inside the kernel). Run via
`pytest tests/quantization/test_fp8_per_channel.py --forked`.
"""
import pytest
import torch
from tests.quantization.utils import is_quant_method_supported
from vllm import _custom_ops as ops
from vllm.config.quantization import (
_ONLINE_SHORTHANDS,
QUANT_KEY_NAMES,
QuantizationConfigArgs,
)
from vllm.model_executor.layers.quantization.online.base import (
_ONLINE_LINEAR_METHODS,
_ONLINE_MOE_METHODS,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PtpcOnlineLinearMethod,
Fp8PtpcOnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticChannelSym,
)
from vllm.platforms import current_platform
def test_fp8_per_channel_shorthand_registered() -> None:
"""The `fp8_per_channel` CLI shorthand must resolve to a config that
dispatches the per-channel methods. Guards against regressions in
`_ONLINE_SHORTHANDS` / `_ONLINE_LINEAR_METHODS` / `_ONLINE_MOE_METHODS`
drifting out of sync.
"""
args = _ONLINE_SHORTHANDS["fp8_per_channel"]
assert isinstance(args, QuantizationConfigArgs)
assert args.linear is not None
assert args.moe is not None
assert args.linear.weight is kFp8StaticChannelSym
assert args.moe.weight is kFp8StaticChannelSym
assert _ONLINE_LINEAR_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineLinearMethod
assert _ONLINE_MOE_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineMoEMethod
assert QUANT_KEY_NAMES["fp8_per_channel_static"] is kFp8StaticChannelSym
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_scaled_fp8_quant_per_channel_shape() -> None:
"""Verify the kernel call per-channel quant depends on: passing a 2D
weight to `ops.scaled_fp8_quant` with `use_per_token_if_dynamic=True`
yields one scale per output row -- a [N, 1] fp32 tensor.
"""
x = (torch.randn(size=(96, 256), device="cuda") * 13).to(torch.bfloat16)
y, s = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
assert y.shape == (96, 256)
assert y.dtype == current_platform.fp8_dtype()
assert s.shape == (96, 1)
assert s.dtype == torch.float32
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_fp8_per_channel_online_quantization(
vllm_runner,
monkeypatch,
) -> None:
"""End-to-end smoke: load `facebook/opt-125m` bf16 with
`quantization='fp8_per_channel'`, check a dense Linear is wrapped by
`Fp8PtpcOnlineLinearMethod`, its weights are fp8 with per-channel
scales (shape `[N, 1]`), and a short greedy generation works.
"""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
"facebook/opt-125m",
quantization="fp8_per_channel",
enforce_eager=True,
) as llm:
def check_model(model):
fc1 = model.model.decoder.layers[0].fc1
assert isinstance(fc1.quant_method, Fp8PtpcOnlineLinearMethod)
assert fc1.weight.dtype == current_platform.fp8_dtype()
assert fc1.weight_scale.ndim == 2
assert fc1.weight_scale.shape[-1] == 1
assert fc1.input_scale is None
llm.apply_model(check_model)
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
print(outputs[0][1])