[Cleanup] Remove online quantization support in fp8.py in favor of online shorthands (#53585)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mgoin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
58aa2f1ef1
commit
90a9515006
@@ -5,11 +5,9 @@
|
||||
Run `pytest tests/quantization/test_fp8.py --forked`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import (
|
||||
@@ -22,9 +20,6 @@ from vllm.config.cache import CacheConfig
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
@@ -32,15 +27,11 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8KVCacheMethod,
|
||||
Fp8LinearMethod,
|
||||
Fp8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
prepare_fp8_moe_layer_for_fi,
|
||||
@@ -531,187 +522,6 @@ def test_model_load_and_run(
|
||||
model(input_ids, positions, None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize(
|
||||
"force_marlin", [True, False] if current_platform.is_cuda() else [False]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
def test_online_quantization(
|
||||
vllm_runner,
|
||||
kv_cache_dtype: str,
|
||||
force_marlin: bool,
|
||||
use_rocm_aiter: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
if use_rocm_aiter:
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
# `LLM.apply_model` requires pickling a function.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
kwargs = {}
|
||||
if force_marlin:
|
||||
kwargs["linear_backend"] = "marlin"
|
||||
kwargs["moe_backend"] = "marlin"
|
||||
|
||||
model_dtype = "auto"
|
||||
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
||||
# FA3 requires BF16 output when the query input is FP8.
|
||||
model_dtype = "bfloat16"
|
||||
|
||||
with vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
quantization="fp8",
|
||||
dtype=model_dtype,
|
||||
enforce_eager=True,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
**kwargs,
|
||||
) as llm:
|
||||
|
||||
def check_model(model):
|
||||
fc1 = model.model.decoder.layers[0].fc1
|
||||
assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod)
|
||||
if kv_cache_dtype == "fp8":
|
||||
attn = model.model.decoder.layers[0].self_attn.attn
|
||||
assert isinstance(attn.quant_method, Fp8KVCacheMethod)
|
||||
assert attn._k_scale == 1.0
|
||||
assert attn._v_scale == 1.0
|
||||
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
if current_platform.supports_fp8() and not force_marlin:
|
||||
# For GPUs with hardware support, we keep weights in fp8
|
||||
assert fc1.weight.dtype == torch.float8_e4m3fn
|
||||
assert not isinstance(
|
||||
fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||
)
|
||||
else:
|
||||
# For GPUs without hardware support, we pack the fp8 weights
|
||||
# for weight-only quantization using Marlin kernels
|
||||
assert fc1.weight.dtype == torch.int32
|
||||
assert isinstance(
|
||||
fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
if current_platform.supports_fp8() and not force_marlin:
|
||||
# For GPUs with hardware support, we keep weights in fp8
|
||||
assert fc1.weight.dtype == current_platform.fp8_dtype()
|
||||
else: # unsupported ROCm platform
|
||||
pytest.skip(
|
||||
"Skip `test_load_fp16_model`. "
|
||||
"It only runs on ROCm platform with FP8 compute."
|
||||
" e.g. MI300X and above."
|
||||
)
|
||||
else: # unsupported platform
|
||||
pytest.skip(
|
||||
"Skip `test_load_fp16_model`. "
|
||||
"It only runs on CUDA and ROCm platform."
|
||||
)
|
||||
|
||||
llm.apply_model(check_model)
|
||||
|
||||
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_online_quant_peak_mem(
|
||||
vllm_runner,
|
||||
caplog_mp_spawn,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
# Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
|
||||
# 1. it covers both Linear and MoE paths
|
||||
# 2. it is already used by other tests in CI, so adding it here
|
||||
# does not increase disk space for CI runners
|
||||
# I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
|
||||
# which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
|
||||
# 1.3 GiB fp8), but could not as adding one more model makes CI
|
||||
# run out of disk space.
|
||||
model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
|
||||
# Force spawn to ensure caplog_mp_spawn works consistently
|
||||
# (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
with (
|
||||
caplog_mp_spawn(logging.DEBUG) as log_holder,
|
||||
vllm_runner(
|
||||
model_name,
|
||||
quantization="fp8",
|
||||
enforce_eager=True,
|
||||
) as llm,
|
||||
):
|
||||
outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
|
||||
log_text = log_holder.text
|
||||
|
||||
# Parse memory usage from captured logs
|
||||
model_memory_gib = None
|
||||
peak_memory_gib = None
|
||||
for line in log_text.splitlines():
|
||||
if model_memory_gib is None:
|
||||
match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
|
||||
if match:
|
||||
model_memory_gib = float(match.group(1))
|
||||
if peak_memory_gib is None:
|
||||
match = re.search(
|
||||
r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
|
||||
)
|
||||
if match:
|
||||
peak_memory_gib = float(match.group(1))
|
||||
|
||||
assert model_memory_gib is not None, "Could not find model loading memory log"
|
||||
assert peak_memory_gib is not None, "Could not find peak memory log"
|
||||
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
|
||||
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
|
||||
|
||||
# model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant
|
||||
# uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB)
|
||||
expected_model_memory_gib = 6.7
|
||||
|
||||
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
|
||||
# GiB, which is 1.36x above model_memory_gib. A slightly higher number is
|
||||
# expected as when we load and quantize weights in a streaming fashion we
|
||||
# need to have individual weights in bf16 + fp8 alive at the same time.
|
||||
expected_peak_memory_gib = expected_model_memory_gib * 1.4
|
||||
|
||||
assert model_memory_gib < expected_model_memory_gib, (
|
||||
f"{model_memory_gib=} higher than {expected_model_memory_gib}"
|
||||
)
|
||||
assert peak_memory_gib < expected_peak_memory_gib, (
|
||||
f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_online_quant_load_format_dummy(
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
caplog,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
"ibm-granite/granite-3.0-1b-a400m-base",
|
||||
quantization="fp8",
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
) as llm:
|
||||
outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
@@ -799,7 +609,6 @@ def test_scaled_fp8_quant(dtype) -> None:
|
||||
)
|
||||
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
|
||||
# FP8 weight reloading does not support online quantization
|
||||
@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False
|
||||
@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
|
||||
# any postprocessing that is applied to the weights such as padding and repacking
|
||||
# (excluding device sharding) must also be applied to the reloaded weights
|
||||
@@ -809,7 +618,6 @@ def test_scaled_fp8_quant(dtype) -> None:
|
||||
def test_fp8_reloading(
|
||||
default_vllm_config,
|
||||
method_cls,
|
||||
is_checkpoint_fp8_serialized,
|
||||
weight_block_size,
|
||||
use_marlin,
|
||||
dist_init,
|
||||
@@ -820,9 +628,6 @@ def test_fp8_reloading(
|
||||
# we set fp8_backend to None, which sidestepped the issue.
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
|
||||
|
||||
if is_checkpoint_fp8_serialized is False:
|
||||
pytest.skip("FP8 weight reloading does not support online quantization")
|
||||
|
||||
if method_cls is Fp8MoEMethod and weight_block_size is None:
|
||||
pytest.skip(
|
||||
"FP8 Tensor weight reloading does not support fusing w13_weight_scale. "
|
||||
@@ -835,7 +640,6 @@ def test_fp8_reloading(
|
||||
layer_size = 128 if weight_block_size is not None else 1
|
||||
with torch.device(f"{DEVICE_TYPE}:0"):
|
||||
config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
|
||||
weight_block_size=weight_block_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -20,6 +20,8 @@ from tests.quantization.utils import (
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.quantization import (
|
||||
@@ -34,6 +36,9 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
MarlinMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
|
||||
@@ -125,6 +130,81 @@ PARTIALLY_PREQUANTIZED_MODEL_NAME = (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
None,
|
||||
{"ignore": ["model.layers.0.self_attn.o_proj"]},
|
||||
{
|
||||
"linear": "fp8_per_block",
|
||||
"ignore": ["model.layers.0.self_attn.o_proj"],
|
||||
},
|
||||
],
|
||||
ids=["defaults", "ignore", "linear-and-ignore"],
|
||||
)
|
||||
def test_legacy_fp8_online_quantization_uses_per_tensor_shorthand(
|
||||
tmp_path, caplog, disable_log_dedup, overrides
|
||||
) -> None:
|
||||
"""The legacy alias preserves overrides and fills in per-tensor defaults."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path), quantization="fp8", quantization_config=overrides
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
assert isinstance(result, OnlineQuantizationConfig)
|
||||
assert result.args == resolve_quantization_config("fp8_per_tensor", overrides)
|
||||
assert "--quantization fp8 is deprecated for online quantization" in caplog.text
|
||||
assert "--quantization fp8_per_tensor instead" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shorthand,method_cls",
|
||||
[
|
||||
("fp8_per_tensor", Fp8PerTensorOnlineLinearMethod),
|
||||
("fp8_per_block", Fp8PerBlockOnlineLinearMethod),
|
||||
("fp8_per_channel", Fp8PtpcOnlineLinearMethod),
|
||||
],
|
||||
)
|
||||
def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
|
||||
shorthand, method_cls
|
||||
) -> None:
|
||||
"""Model exclusions preserve the language backbone and shared defaults."""
|
||||
from vllm.model_executor.models.qianfan_ocr import (
|
||||
QianfanOCRForConditionalGeneration,
|
||||
)
|
||||
|
||||
args = resolve_quantization_config(shorthand, None)
|
||||
assert args is not None
|
||||
config = OnlineQuantizationConfig(args)
|
||||
other_config = OnlineQuantizationConfig(args)
|
||||
model = QianfanOCRForConditionalGeneration.__new__(
|
||||
QianfanOCRForConditionalGeneration
|
||||
)
|
||||
model._patch_quant_config(
|
||||
SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),
|
||||
config,
|
||||
)
|
||||
|
||||
layer = Mock(spec=LinearBase)
|
||||
for prefix in (
|
||||
"vision_model.encoder.layers.0.attn.qkv",
|
||||
"vision_model.encoder.layers.1.mlp.fc2",
|
||||
"language_model.lm_head",
|
||||
"mlp1.1",
|
||||
"mlp1.3",
|
||||
):
|
||||
assert config.resolve_quant_method_cls(layer, prefix) is None
|
||||
|
||||
resolved = config.resolve_quant_method_cls(
|
||||
layer, "language_model.model.layers.0.self_attn.qkv_proj"
|
||||
)
|
||||
assert resolved is not None and resolved[-1] is method_cls
|
||||
assert other_config.ignored_layers == []
|
||||
assert args.ignore == []
|
||||
|
||||
|
||||
def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
@@ -579,33 +659,81 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"),
|
||||
(
|
||||
"model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls,extra_runner_kwargs"
|
||||
),
|
||||
[
|
||||
# simple case - quantization='fp8_per_tensor'
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
None,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{},
|
||||
id="fp8_per_tensor",
|
||||
),
|
||||
# FP8 KV cache with the fp8_per_tensor shorthand.
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
None,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{"kv_cache_dtype": "fp8"},
|
||||
id="fp8_per_tensor-fp8-kv-cache",
|
||||
),
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
None,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{"linear_backend": "marlin", "moe_backend": "marlin"},
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Marlin online FP8 coverage is CUDA-only.",
|
||||
),
|
||||
id="fp8_per_tensor-marlin",
|
||||
),
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
None,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{
|
||||
"kv_cache_dtype": "fp8",
|
||||
"linear_backend": "marlin",
|
||||
"moe_backend": "marlin",
|
||||
},
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Marlin online FP8 coverage is CUDA-only.",
|
||||
),
|
||||
id="fp8_per_tensor-fp8-kv-cache-marlin",
|
||||
),
|
||||
# simple case - quantization='fp8_per_block'
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_block",
|
||||
None,
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerBlockOnlineMoEMethod,
|
||||
{},
|
||||
id="fp8_per_block",
|
||||
),
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_channel",
|
||||
None,
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
Fp8PtpcOnlineMoEMethod,
|
||||
{},
|
||||
id="fp8_per_channel",
|
||||
),
|
||||
# quantization='online' with per-layer-kind overrides
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"online",
|
||||
{
|
||||
@@ -614,9 +742,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
},
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{},
|
||||
id="per_layer_kind_overrides",
|
||||
),
|
||||
# quantization='online' with per-layer target patterns
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"online",
|
||||
{
|
||||
@@ -627,9 +757,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
},
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{},
|
||||
id="targets",
|
||||
),
|
||||
# ignore with direct layer name
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
# qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
|
||||
@@ -637,13 +769,17 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
{"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]},
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
{},
|
||||
id="ignore",
|
||||
),
|
||||
(
|
||||
pytest.param(
|
||||
GRANITE_MODEL_NAME,
|
||||
"mxfp4",
|
||||
None,
|
||||
Mxfp4OnlineLinearMethod,
|
||||
Mxfp4OnlineMoEMethod,
|
||||
{},
|
||||
id="mxfp4",
|
||||
),
|
||||
pytest.param(
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME,
|
||||
@@ -651,19 +787,10 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
|
||||
Mxfp8OnlineLinearMethod,
|
||||
CompressedTensorsMoEMethod,
|
||||
{},
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"fp8_per_tensor",
|
||||
"fp8_per_block",
|
||||
"fp8_per_channel",
|
||||
"per_layer_kind_overrides",
|
||||
"targets",
|
||||
"ignore",
|
||||
"mxfp4",
|
||||
"partially_prequantized_checkpoint",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
@@ -674,6 +801,7 @@ def test_online_quantization(
|
||||
online_quant_args: dict | None,
|
||||
expected_linear_cls,
|
||||
expected_moe_cls,
|
||||
extra_runner_kwargs: dict,
|
||||
use_rocm_aiter: bool,
|
||||
monkeypatch,
|
||||
dist_init,
|
||||
@@ -700,14 +828,31 @@ def test_online_quantization(
|
||||
):
|
||||
pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.")
|
||||
|
||||
runner_kwargs = dict(extra_runner_kwargs)
|
||||
kv_cache_dtype = runner_kwargs.get("kv_cache_dtype", "auto")
|
||||
force_marlin = extra_runner_kwargs.get("linear_backend") == "marlin"
|
||||
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
||||
# FA3 requires BF16 output when the query input is FP8.
|
||||
runner_kwargs["dtype"] = "bfloat16"
|
||||
|
||||
vllm_config_kwargs = {
|
||||
"cache_config": CacheConfig(cache_dtype=kv_cache_dtype),
|
||||
"kernel_config": KernelConfig(
|
||||
linear_backend=runner_kwargs.get("linear_backend", "auto"),
|
||||
moe_backend=runner_kwargs.get("moe_backend", "auto"),
|
||||
),
|
||||
}
|
||||
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
dtype=runner_kwargs.get("dtype", "bfloat16"),
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
None, online_quant_args
|
||||
)
|
||||
},
|
||||
vllm_config_kwargs=vllm_config_kwargs,
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
assert isinstance(
|
||||
@@ -725,7 +870,7 @@ def test_online_quantization(
|
||||
else:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
dtype=runner_kwargs.get("dtype", "bfloat16"),
|
||||
quantization=quant_scheme,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
@@ -743,6 +888,7 @@ def test_online_quantization(
|
||||
"num_experts_per_tok": 2,
|
||||
},
|
||||
},
|
||||
vllm_config_kwargs=vllm_config_kwargs,
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
|
||||
@@ -770,7 +916,16 @@ def test_online_quantization(
|
||||
elif quant_scheme == "mxfp4":
|
||||
assert o_proj.weight.dtype == torch.uint8
|
||||
elif current_platform.is_cuda() or current_platform.is_xpu():
|
||||
assert o_proj.weight.dtype == torch.float8_e4m3fn
|
||||
if current_platform.supports_fp8() and not force_marlin:
|
||||
assert o_proj.weight.dtype == torch.float8_e4m3fn
|
||||
assert not isinstance(
|
||||
o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||
)
|
||||
else:
|
||||
assert o_proj.weight.dtype == torch.int32
|
||||
assert isinstance(
|
||||
o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
assert o_proj.weight.dtype == current_platform.fp8_dtype()
|
||||
else:
|
||||
@@ -794,6 +949,11 @@ def test_online_quantization(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
if kv_cache_dtype == "fp8":
|
||||
attn = model.model.layers[0].self_attn.attn
|
||||
assert attn._k_scale == 1.0
|
||||
assert attn._v_scale == 1.0
|
||||
|
||||
if (
|
||||
model_name == GRANITE_MODEL_NAME
|
||||
and isinstance(online_quant_args, dict)
|
||||
|
||||
Reference in New Issue
Block a user