[ROCm][Quantization][5/N] Refactor quark_moe w8a8-int8 w/ oracle (#46765)
Signed-off-by: amd-sourjya <amd-sourjya@users.noreply.github.com> Co-authored-by: amd-sourjya <amd-sourjya@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
amd-sourjya
Cursor
Andreas Karatzas
parent
b5bcb3ce88
commit
1053e248f0
@@ -45,9 +45,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8Static,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl
|
||||
@@ -103,15 +105,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
# INT8 requires at least 7.5 (Turing).
|
||||
# INT8 requires at least 7.5 (Turing) on CUDA. ROCm CDNA GPUs
|
||||
# (e.g. MI2xx/MI3xx/gfx950) provide native INT8 matrix-core support and
|
||||
# the Triton int8_w8a8 fused MoE kernel handles them.
|
||||
device_supports_int8 = (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.has_device_capability((7, 5))
|
||||
)
|
||||
) or current_platform.is_rocm()
|
||||
|
||||
supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)]
|
||||
if device_supports_int8:
|
||||
supported.append((kInt8StaticChannelSym, kInt8DynamicTokenSym))
|
||||
# Activations are consumed as float and quantized to int8
|
||||
# dynamically inside the kernel, so only dynamic-activation int8
|
||||
# schemes are supported (static-activation int8 is not).
|
||||
supported += [
|
||||
# per-channel weight + dynamic per-token activation
|
||||
(kInt8StaticChannelSym, kInt8DynamicTokenSym),
|
||||
# per-tensor weight + dynamic per-tensor activation
|
||||
(kInt8StaticTensorSym, kInt8DynamicTensorSym),
|
||||
]
|
||||
if current_platform.supports_fp8():
|
||||
supported += [
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
|
||||
@@ -166,7 +166,9 @@ def select_int8_moe_backend(
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
|
||||
raise NotImplementedError(
|
||||
"No Int8 MoE backend supports the deployment configuration."
|
||||
"No Int8 MoE backend supports the deployment configuration "
|
||||
f"(weight_key={weight_key}, activation_key={activation_key}). "
|
||||
"Set `VLLM_LOGGING_LEVEL=DEBUG` to see per-backend unsupported reasons."
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user