Deprecate old FP8 online MoE quantization class (#44514)
Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
@@ -14,6 +14,9 @@ import torch
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
@@ -46,7 +49,7 @@ MODELS = [
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
@pytest.mark.parametrize(
|
||||
"force_marlin", [False] if current_platform.is_rocm() else [False, True]
|
||||
"force_marlin", [True, False] if current_platform.is_cuda() else [False]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
@@ -73,7 +76,7 @@ def test_model_load_and_run(
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize(
|
||||
"force_marlin", [False] if current_platform.is_rocm() else [False, True]
|
||||
"force_marlin", [True, False] if current_platform.is_cuda() else [False]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
@@ -110,14 +113,20 @@ def test_online_quantization(
|
||||
assert attn._k_scale == 1.0
|
||||
assert attn._v_scale == 1.0
|
||||
|
||||
if current_platform.is_cuda():
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
if current_platform.supports_fp8() and not force_marlin:
|
||||
# For GPUs with hardware support, we keep weights in fp8
|
||||
assert fc1.weight.dtype == torch.float8_e4m3fn
|
||||
assert not isinstance(
|
||||
fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||
)
|
||||
else:
|
||||
# For GPUs without hardware support, we pack the fp8 weights
|
||||
# for weight-only quantization using Marlin kernels
|
||||
assert fc1.weight.dtype == torch.int32
|
||||
assert isinstance(
|
||||
fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
if current_platform.supports_fp8() and not force_marlin:
|
||||
# For GPUs with hardware support, we keep weights in fp8
|
||||
|
||||
Reference in New Issue
Block a user