[ROCm][Quantization] Add Quark W4A8 (INT4-FP8) MoE CI coverage (#48050)
Signed-off-by: amd-sourjya <amd-sourjya@users.noreply.github.com> Co-authored-by: amd-sourjya <amd-sourjya@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
amd-sourjya
Cursor
Andreas Karatzas
parent
da54a5bf05
commit
80c9d5d5e0
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
|||||||
QuarkW8A8Int8,
|
QuarkW8A8Int8,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||||
|
QuarkW4A8Fp8MoEMethod,
|
||||||
QuarkW8A8Int8MoEMethod,
|
QuarkW8A8Int8MoEMethod,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||||
@@ -31,9 +32,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
if current_platform.is_rocm():
|
if current_platform.is_rocm():
|
||||||
from vllm.platforms.rocm import on_gfx950
|
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||||
else:
|
else:
|
||||||
|
|
||||||
|
def on_gfx942() -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
def on_gfx950() -> bool:
|
def on_gfx950() -> bool:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -170,6 +174,38 @@ def test_quark_int8_w8a8_moe(vllm_runner, tp):
|
|||||||
assert output
|
assert output
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
not (on_gfx950() or on_gfx942()),
|
||||||
|
reason="Quark W4A8 (INT4-FP8) MoE requires the AITER kernel on gfx942/gfx950",
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("tp", [1])
|
||||||
|
def test_quark_w4a8_fp8_moe(vllm_runner, monkeypatch, tp):
|
||||||
|
"""Test W4A8 (INT4 weight + FP8 activation) MoE with a tiny Qwen3 MoE model.
|
||||||
|
|
||||||
|
W4A8 dispatches through the AITER fused MoE kernel, so AITER must be on.
|
||||||
|
"""
|
||||||
|
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||||
|
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||||
|
model_path = "amd/tiny-qwen3-moe-w4a8"
|
||||||
|
with vllm_runner(
|
||||||
|
model_path,
|
||||||
|
enforce_eager=True,
|
||||||
|
tensor_parallel_size=tp,
|
||||||
|
gpu_memory_utilization=0.1,
|
||||||
|
) as llm:
|
||||||
|
|
||||||
|
def check_model(model):
|
||||||
|
moe = model.model.layers[0].mlp.experts
|
||||||
|
assert isinstance(moe._quant_method, QuarkW4A8Fp8MoEMethod), (
|
||||||
|
f"Expected QuarkW4A8Fp8MoEMethod, got {type(moe._quant_method)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
llm.apply_model(check_model)
|
||||||
|
|
||||||
|
output = llm.generate_greedy("Hello", max_tokens=4)
|
||||||
|
assert output
|
||||||
|
|
||||||
|
|
||||||
def test_quark_fp8_parity(vllm_runner):
|
def test_quark_fp8_parity(vllm_runner):
|
||||||
quark_model_id = "amd-quark/llama-tiny-fp8-quark-quant-method"
|
quark_model_id = "amd-quark/llama-tiny-fp8-quark-quant-method"
|
||||||
fp8_model_id = "amd-quark/llama-tiny-fp8-quant-method"
|
fp8_model_id = "amd-quark/llama-tiny-fp8-quant-method"
|
||||||
|
|||||||
Reference in New Issue
Block a user