[ROCm][Kimi-K3] Add opt-in gfx942 MXFP4-to-int4 conversion (#51274)

Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com>
Signed-off-by: Hartikainen <mahartik@amd.com>
This commit is contained in:
Markus Hartikainen
2026-10-03 14:28:02 +08:00
committed by GitHub
parent ca0df1f122
commit b0e21b3083
5 changed files with 346 additions and 2 deletions
+108
View File
@@ -0,0 +1,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from collections.abc import Callable
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock, patch
import pytest
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.mxfp4 import (
_moe_weight_override_is_int4,
_use_k3_situ_int4_gfx942,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
pytestmark = pytest.mark.cpu_test
class _FakeRocmModule(ModuleType):
on_gfx942: Callable[[], bool]
class _FakeAiterOpsModule(ModuleType):
rocm_aiter_ops: SimpleNamespace
def _vllm_config(moe_weight: str | None):
quantization_config = (
None
if moe_weight is None
else QuantizationConfigArgs(moe={"weight": moe_weight})
)
return SimpleNamespace(
model_config=SimpleNamespace(quantization_config=quantization_config)
)
@pytest.mark.parametrize(
("moe_weight", "expected"),
[
(None, False),
("mxfp4", False),
("int4_per_group_32", True),
],
)
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
monkeypatch.setattr(
"vllm.config.get_current_vllm_config",
lambda: _vllm_config(moe_weight),
)
assert _moe_weight_override_is_int4() is expected
@pytest.mark.parametrize("int4_requested", [False, True])
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
moe = SimpleNamespace(
activation=MoEActivation.SITU,
activation_situ_linear_beta=25.0,
)
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
fake_rocm.on_gfx942 = lambda: True
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
with (
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
patch.dict(
sys.modules,
{
"vllm.platforms.rocm": fake_rocm,
"vllm._aiter_ops": fake_aiter_ops,
},
),
patch(
"vllm.model_executor.layers.quantization.mxfp4."
"_moe_weight_override_is_int4",
return_value=int4_requested,
),
):
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
def test_int4_per_group_32_is_not_an_online_moe_method():
# Load-time gfx942 requant. Must not match the online MoE table.
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
)
assert (
config.resolve_quant_method_cls(
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
)
is None
)
def test_unknown_online_moe_weight_still_raises():
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
)
with pytest.raises(ValueError, match="online quantization"):
config.resolve_quant_method_cls(
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
)
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt4Static32,
kInt8StaticChannelSym,
kMxfp8Dynamic,
)
@@ -46,6 +47,12 @@ def test_quant_spec_string_representation_uses_quantization_name():
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
def test_quant_spec_resolves_groupwise_int4_weight():
spec = QuantSpec(weight="int4_per_group_32")
assert spec.weight == kInt4Static32
assert spec.activation is None
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
quant_spec(weight="not_a_real_format")
+3
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static32,
kInt8StaticChannelSym,
kMxfp4Dynamic,
kMxfp4Static,
@@ -40,6 +41,8 @@ QUANT_KEY_NAMES: dict[str, QuantKey] = {
"mxfp8": kMxfp8Dynamic,
"mxfp4": kMxfp4Dynamic,
"int8_per_channel_static": kInt8StaticChannelSym,
# Load-time MXFP4-to-int4 for Kimi-K3 on gfx942.
"int4_per_group_32": kInt4Static32,
}
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import torch
@@ -22,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
backend_to_kernel_cls,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
@@ -477,6 +479,63 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
)
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
"""Route Kimi-K3 weight-only MXFP4 SiTU MoE to AITER A16W4 on gfx950."""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx950
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx950()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
)
def _use_k3_situ_int4_gfx942(moe: FusedMoEConfig) -> bool:
"""Opt-in gfx942 MXFP4-to-int4 requant for Kimi-K3 SiTU experts."""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx942
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx942()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and _moe_weight_override_is_int4()
)
def _moe_weight_override_is_int4() -> bool:
"""True when --quantization-config.moe.weight int4_per_group_32 was set."""
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
if vllm_config is None:
return False
quant_args = getattr(vllm_config.model_config, "quantization_config", None)
moe_spec = getattr(quant_args, "moe", None)
weight = getattr(moe_spec, "weight", None)
if weight is None:
return False
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static32,
)
return weight == kInt4Static32
class Mxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
@@ -484,9 +543,33 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
if self.is_k3_situ_aiter or self.is_k3_situ_int4_gfx942:
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
if self.is_k3_situ_int4_gfx942:
logger.warning_once(
"Requantizing Kimi-K3 MXFP4 experts to groupwise int4 "
"for gfx942. This conversion is lossy and was explicitly "
"enabled by --quantization-config.moe.weight "
"int4_per_group_32."
)
else:
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_fused_moe_situv2_enabled():
# a16w4 does not read this bound. Force 0 so leftover a8w4
# config cannot keep bf16 activations.
# TODO: Remove once AITER takes this as a kernel argument.
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
)
self.max_capture_size = moe.max_capture_size
@@ -784,6 +867,141 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def _setup_kernel_k3_situ_gfx942(self, layer: RoutedExperts) -> None:
import inspect
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
)
# Fail closed if this AITER build would silently run SiLU.
if "act" not in inspect.signature(flydsl_a16w4_gemm1).parameters:
raise RuntimeError(
"This AITER build ignores the SiTUv2 activation on the "
"replacement packed-int4 MoE path and would silently compute "
"SiLU. Rebuild with the replacement A16W4/int4 pipeline."
)
from aiter import dtypes as aiter_dtypes
from aiter.ops.quant import per_1x32_i4_quant
from aiter.ops.shuffle import (
pack_int8_to_packed_int4,
shuffle_scale_for_int4,
shuffle_weight,
)
from aiter.utility import fp4_utils
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# mxfp4_to_f32 grows the packed weight 8x and peaks above 20 GiB per
# rank if all experts convert at once. Convert 8 experts at a time.
_CONVERT_CHUNK = 8
def convert(
weight: torch.nn.Parameter,
scale: torch.nn.Parameter,
) -> tuple[torch.Tensor, torch.Tensor]:
# Drop the source before returning. Packed int4 is the same size as
# packed MXFP4, so holding both doubles the footprint. That fits
# under expert parallel (1/ep_size experts per rank) and does not
# fit under pure tensor parallel.
w_all = weight.data.view(fp4_dtype)
s_all = scale.data.view(e8m0_dtype)
num_experts = w_all.shape[0]
# Write chunks into a preallocated output. torch.cat of a list
# would hold the full result twice at the join.
out_packed: torch.Tensor | None = None
out_scale: torch.Tensor | None = None
scale_stride = 0
for lo in range(0, num_experts, _CONVERT_CHUNK):
hi = min(lo + _CONVERT_CHUNK, num_experts)
weight_f32 = fp4_utils.mxfp4_to_f32(w_all[lo:hi])
scale_f32 = fp4_utils.e8m0_to_f32(s_all[lo:hi])
chunk_experts, output_size, input_size = weight_f32.shape
weight_bf16 = (
(
weight_f32.view(
chunk_experts, output_size, input_size // 32, 32
)
* scale_f32.view(
chunk_experts, output_size, input_size // 32, 1
)
)
.view(chunk_experts, output_size, input_size)
.to(torch.bfloat16)
)
del weight_f32, scale_f32
weight_int4, weight_scale = per_1x32_i4_quant(weight_bf16)
del weight_bf16
weight_int4 = weight_int4.view(aiter_dtypes.i4x2).view(
chunk_experts, output_size, input_size
)
weight_packed = pack_int8_to_packed_int4(
shuffle_weight(weight_int4.view(aiter_dtypes.i8), (16, 16))
)
del weight_int4
chunk_packed = weight_packed.view(
chunk_experts, output_size, input_size // 2
).view(aiter_dtypes.i4x2)
chunk_scale = (
shuffle_scale_for_int4(weight_scale, group_size=32)
.view(-1)
.contiguous()
)
if out_packed is None:
out_packed = torch.empty(
(num_experts,) + tuple(chunk_packed.shape[1:]),
dtype=chunk_packed.dtype,
device=chunk_packed.device,
)
scale_stride = chunk_scale.numel() // chunk_experts
out_scale = torch.empty(
num_experts * scale_stride,
dtype=chunk_scale.dtype,
device=chunk_scale.device,
)
assert out_packed is not None
assert out_scale is not None
out_packed[lo:hi].copy_(chunk_packed)
out_scale[lo * scale_stride : hi * scale_stride].copy_(chunk_scale)
del weight_packed, weight_scale, chunk_packed, chunk_scale
torch.accelerator.empty_cache()
del w_all, s_all
assert out_packed is not None
assert out_scale is not None
weight.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
scale.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
torch.accelerator.empty_cache()
return out_packed, out_scale
w13, w13_scale = convert(layer.w13_weight, layer.w13_weight_scale)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w13_weight_scale", w13_scale)
del w13, w13_scale
torch.accelerator.empty_cache()
w2, w2_scale = convert(layer.w2_weight, layer.w2_weight_scale)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w2_weight_scale", w2_scale)
del w2, w2_scale
torch.accelerator.empty_cache()
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
return
@@ -797,6 +1015,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self._build_moe_kernel(layer)
return
if self.is_k3_situ_int4_gfx942:
self._setup_kernel_k3_situ_gfx942(layer)
return
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
@@ -65,6 +65,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static32,
kInt8StaticChannelSym,
kMxfp4Static,
kMxfp8Dynamic,
@@ -231,6 +232,9 @@ class OnlineQuantizationConfig(QuantizationConfig):
"""
if spec is None or spec.weight is None:
return None
# Load-time gfx942 requant, not online conversion. Mxfp4MoEMethod owns it.
if spec.weight == kInt4Static32:
return None
cls = table.get(spec.weight)
if cls is None:
raise ValueError(