[ROCm][Quantization][5/N] Refactor quark_moe w8a8-int8 w/ oracle (#46765)

Signed-off-by: amd-sourjya <amd-sourjya@users.noreply.github.com>
Co-authored-by: amd-sourjya <amd-sourjya@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
amd-sourjya
2026-07-27 16:01:34 -05:00
committed by GitHub
co-authored by amd-sourjya Cursor Andreas Karatzas
parent b5bcb3ce88
commit 1053e248f0
8 changed files with 234 additions and 17 deletions
@@ -45,9 +45,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt8DynamicTensorSym,
kInt8DynamicTokenSym,
kInt8Static,
kInt8StaticChannelSym,
kInt8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl
@@ -103,15 +105,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
# INT8 requires at least 7.5 (Turing).
# INT8 requires at least 7.5 (Turing) on CUDA. ROCm CDNA GPUs
# (e.g. MI2xx/MI3xx/gfx950) provide native INT8 matrix-core support and
# the Triton int8_w8a8 fused MoE kernel handles them.
device_supports_int8 = (
current_platform.is_cuda()
and current_platform.has_device_capability((7, 5))
)
) or current_platform.is_rocm()
supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)]
if device_supports_int8:
supported.append((kInt8StaticChannelSym, kInt8DynamicTokenSym))
# Activations are consumed as float and quantized to int8
# dynamically inside the kernel, so only dynamic-activation int8
# schemes are supported (static-activation int8 is not).
supported += [
# per-channel weight + dynamic per-token activation
(kInt8StaticChannelSym, kInt8DynamicTokenSym),
# per-tensor weight + dynamic per-tensor activation
(kInt8StaticTensorSym, kInt8DynamicTensorSym),
]
if current_platform.supports_fp8():
supported += [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
@@ -166,7 +166,9 @@ def select_int8_moe_backend(
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"No Int8 MoE backend supports the deployment configuration."
"No Int8 MoE backend supports the deployment configuration "
f"(weight_key={weight_key}, activation_key={activation_key}). "
"Set `VLLM_LOGGING_LEVEL=DEBUG` to see per-backend unsupported reasons."
)