[Quantization] add online fp8 ptpc (#44132)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ac3409d162
commit
753e9d55e6
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for online FP8 per-channel quantization.
|
||||
|
||||
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
|
||||
quantization) and compares log-probabilities against the same model served in
|
||||
BF16 without quantization. This exercises the full pipeline: config parsing,
|
||||
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
|
||||
loading, online quantization / shuffling, and inference.
|
||||
|
||||
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
|
||||
diverse prompts from ``tests/prompts/example.txt``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
|
||||
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
# Small dense model (no MoE) to validate the linear-only path.
|
||||
DENSE_MODEL = "Qwen/Qwen3-0.6B"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_TOKENS = 4
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_fp8_per_channel_logprobs(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare BF16 baseline logprobs against online per-channel-quantized
|
||||
model.
|
||||
|
||||
Runs the same model twice -- once in BF16 (baseline) and once with online
|
||||
FP8 per-channel quantization -- then checks that the top log-probabilities
|
||||
are close. Only 4 tokens are generated to keep the test fast while still
|
||||
catching numerical divergence beyond expected per-channel error.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
quantization="fp8_per_channel",
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16",
|
||||
name_1="fp8_per_channel",
|
||||
)
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FP8 per-channel online quantization.
|
||||
|
||||
Per-output-channel weight scale + dynamic per-token activation scale.
|
||||
bf16/fp16 checkpoints are quantized at load time with one fp32 scale per
|
||||
output channel for weights and one fp32 scale per token for activations
|
||||
(computed dynamically inside the kernel). Run via
|
||||
`pytest tests/quantization/test_fp8_per_channel.py --forked`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config.quantization import (
|
||||
_ONLINE_SHORTHANDS,
|
||||
QUANT_KEY_NAMES,
|
||||
QuantizationConfigArgs,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
_ONLINE_LINEAR_METHODS,
|
||||
_ONLINE_MOE_METHODS,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
Fp8PtpcOnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticChannelSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_fp8_per_channel_shorthand_registered() -> None:
|
||||
"""The `fp8_per_channel` CLI shorthand must resolve to a config that
|
||||
dispatches the per-channel methods. Guards against regressions in
|
||||
`_ONLINE_SHORTHANDS` / `_ONLINE_LINEAR_METHODS` / `_ONLINE_MOE_METHODS`
|
||||
drifting out of sync.
|
||||
"""
|
||||
args = _ONLINE_SHORTHANDS["fp8_per_channel"]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
assert args.linear is not None
|
||||
assert args.moe is not None
|
||||
assert args.linear.weight is kFp8StaticChannelSym
|
||||
assert args.moe.weight is kFp8StaticChannelSym
|
||||
|
||||
assert _ONLINE_LINEAR_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineLinearMethod
|
||||
assert _ONLINE_MOE_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineMoEMethod
|
||||
|
||||
assert QUANT_KEY_NAMES["fp8_per_channel_static"] is kFp8StaticChannelSym
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_scaled_fp8_quant_per_channel_shape() -> None:
|
||||
"""Verify the kernel call per-channel quant depends on: passing a 2D
|
||||
weight to `ops.scaled_fp8_quant` with `use_per_token_if_dynamic=True`
|
||||
yields one scale per output row -- a [N, 1] fp32 tensor.
|
||||
"""
|
||||
x = (torch.randn(size=(96, 256), device="cuda") * 13).to(torch.bfloat16)
|
||||
y, s = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
|
||||
assert y.shape == (96, 256)
|
||||
assert y.dtype == current_platform.fp8_dtype()
|
||||
assert s.shape == (96, 1)
|
||||
assert s.dtype == torch.float32
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_fp8_per_channel_online_quantization(
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""End-to-end smoke: load `facebook/opt-125m` bf16 with
|
||||
`quantization='fp8_per_channel'`, check a dense Linear is wrapped by
|
||||
`Fp8PtpcOnlineLinearMethod`, its weights are fp8 with per-channel
|
||||
scales (shape `[N, 1]`), and a short greedy generation works.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
with vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
quantization="fp8_per_channel",
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
|
||||
def check_model(model):
|
||||
fc1 = model.model.decoder.layers[0].fc1
|
||||
assert isinstance(fc1.quant_method, Fp8PtpcOnlineLinearMethod)
|
||||
assert fc1.weight.dtype == current_platform.fp8_dtype()
|
||||
assert fc1.weight_scale.ndim == 2
|
||||
assert fc1.weight_scale.shape[-1] == 1
|
||||
assert fc1.input_scale is None
|
||||
|
||||
llm.apply_model(check_model)
|
||||
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
Reference in New Issue
Block a user