[ROCm][Kimi-K3] Add opt-in gfx942 MXFP4-to-int4 conversion (#51274)
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Signed-off-by: Hartikainen <mahartik@amd.com>
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import (
|
||||
_moe_weight_override_is_int4,
|
||||
_use_k3_situ_int4_gfx942,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeRocmModule(ModuleType):
|
||||
on_gfx942: Callable[[], bool]
|
||||
|
||||
|
||||
class _FakeAiterOpsModule(ModuleType):
|
||||
rocm_aiter_ops: SimpleNamespace
|
||||
|
||||
|
||||
def _vllm_config(moe_weight: str | None):
|
||||
quantization_config = (
|
||||
None
|
||||
if moe_weight is None
|
||||
else QuantizationConfigArgs(moe={"weight": moe_weight})
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(quantization_config=quantization_config)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("moe_weight", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
("mxfp4", False),
|
||||
("int4_per_group_32", True),
|
||||
],
|
||||
)
|
||||
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.get_current_vllm_config",
|
||||
lambda: _vllm_config(moe_weight),
|
||||
)
|
||||
assert _moe_weight_override_is_int4() is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("int4_requested", [False, True])
|
||||
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
|
||||
moe = SimpleNamespace(
|
||||
activation=MoEActivation.SITU,
|
||||
activation_situ_linear_beta=25.0,
|
||||
)
|
||||
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
|
||||
fake_rocm.on_gfx942 = lambda: True
|
||||
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
|
||||
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"vllm.platforms.rocm": fake_rocm,
|
||||
"vllm._aiter_ops": fake_aiter_ops,
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.mxfp4."
|
||||
"_moe_weight_override_is_int4",
|
||||
return_value=int4_requested,
|
||||
),
|
||||
):
|
||||
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
|
||||
|
||||
|
||||
def test_int4_per_group_32_is_not_an_online_moe_method():
|
||||
# Load-time gfx942 requant. Must not match the online MoE table.
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
|
||||
)
|
||||
assert (
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_online_moe_weight_still_raises():
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
|
||||
)
|
||||
with pytest.raises(ValueError, match="online quantization"):
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
|
||||
)
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
@@ -46,6 +47,12 @@ def test_quant_spec_string_representation_uses_quantization_name():
|
||||
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
|
||||
|
||||
|
||||
def test_quant_spec_resolves_groupwise_int4_weight():
|
||||
spec = QuantSpec(weight="int4_per_group_32")
|
||||
assert spec.weight == kInt4Static32
|
||||
assert spec.activation is None
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
quant_spec(weight="not_a_real_format")
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
@@ -40,6 +41,8 @@ QUANT_KEY_NAMES: dict[str, QuantKey] = {
|
||||
"mxfp8": kMxfp8Dynamic,
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"int8_per_channel_static": kInt8StaticChannelSym,
|
||||
# Load-time MXFP4-to-int4 for Kimi-K3 on gfx942.
|
||||
"int4_per_group_32": kInt4Static32,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
@@ -22,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
@@ -477,6 +479,63 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
|
||||
"""Route Kimi-K3 weight-only MXFP4 SiTU MoE to AITER A16W4 on gfx950."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx950()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_int4_gfx942(moe: FusedMoEConfig) -> bool:
|
||||
"""Opt-in gfx942 MXFP4-to-int4 requant for Kimi-K3 SiTU experts."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx942()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and _moe_weight_override_is_int4()
|
||||
)
|
||||
|
||||
|
||||
def _moe_weight_override_is_int4() -> bool:
|
||||
"""True when --quantization-config.moe.weight int4_per_group_32 was set."""
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config is None:
|
||||
return False
|
||||
quant_args = getattr(vllm_config.model_config, "quantization_config", None)
|
||||
moe_spec = getattr(quant_args, "moe", None)
|
||||
weight = getattr(moe_spec, "weight", None)
|
||||
if weight is None:
|
||||
return False
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static32,
|
||||
)
|
||||
|
||||
return weight == kInt4Static32
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
@@ -484,9 +543,33 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
if self.is_k3_situ_aiter or self.is_k3_situ_int4_gfx942:
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
if self.is_k3_situ_int4_gfx942:
|
||||
logger.warning_once(
|
||||
"Requantizing Kimi-K3 MXFP4 experts to groupwise int4 "
|
||||
"for gfx942. This conversion is lossy and was explicitly "
|
||||
"enabled by --quantization-config.moe.weight "
|
||||
"int4_per_group_32."
|
||||
)
|
||||
else:
|
||||
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_enabled():
|
||||
# a16w4 does not read this bound. Force 0 so leftover a8w4
|
||||
# config cannot keep bf16 activations.
|
||||
# TODO: Remove once AITER takes this as a kernel argument.
|
||||
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
|
||||
else:
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
|
||||
moe
|
||||
)
|
||||
|
||||
self.max_capture_size = moe.max_capture_size
|
||||
|
||||
@@ -784,6 +867,141 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def _setup_kernel_k3_situ_gfx942(self, layer: RoutedExperts) -> None:
|
||||
import inspect
|
||||
|
||||
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
|
||||
flydsl_a16w4_gemm1,
|
||||
)
|
||||
|
||||
# Fail closed if this AITER build would silently run SiLU.
|
||||
if "act" not in inspect.signature(flydsl_a16w4_gemm1).parameters:
|
||||
raise RuntimeError(
|
||||
"This AITER build ignores the SiTUv2 activation on the "
|
||||
"replacement packed-int4 MoE path and would silently compute "
|
||||
"SiLU. Rebuild with the replacement A16W4/int4 pipeline."
|
||||
)
|
||||
|
||||
from aiter import dtypes as aiter_dtypes
|
||||
from aiter.ops.quant import per_1x32_i4_quant
|
||||
from aiter.ops.shuffle import (
|
||||
pack_int8_to_packed_int4,
|
||||
shuffle_scale_for_int4,
|
||||
shuffle_weight,
|
||||
)
|
||||
from aiter.utility import fp4_utils
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
|
||||
# mxfp4_to_f32 grows the packed weight 8x and peaks above 20 GiB per
|
||||
# rank if all experts convert at once. Convert 8 experts at a time.
|
||||
_CONVERT_CHUNK = 8
|
||||
|
||||
def convert(
|
||||
weight: torch.nn.Parameter,
|
||||
scale: torch.nn.Parameter,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Drop the source before returning. Packed int4 is the same size as
|
||||
# packed MXFP4, so holding both doubles the footprint. That fits
|
||||
# under expert parallel (1/ep_size experts per rank) and does not
|
||||
# fit under pure tensor parallel.
|
||||
w_all = weight.data.view(fp4_dtype)
|
||||
s_all = scale.data.view(e8m0_dtype)
|
||||
num_experts = w_all.shape[0]
|
||||
|
||||
# Write chunks into a preallocated output. torch.cat of a list
|
||||
# would hold the full result twice at the join.
|
||||
out_packed: torch.Tensor | None = None
|
||||
out_scale: torch.Tensor | None = None
|
||||
scale_stride = 0
|
||||
for lo in range(0, num_experts, _CONVERT_CHUNK):
|
||||
hi = min(lo + _CONVERT_CHUNK, num_experts)
|
||||
weight_f32 = fp4_utils.mxfp4_to_f32(w_all[lo:hi])
|
||||
scale_f32 = fp4_utils.e8m0_to_f32(s_all[lo:hi])
|
||||
chunk_experts, output_size, input_size = weight_f32.shape
|
||||
weight_bf16 = (
|
||||
(
|
||||
weight_f32.view(
|
||||
chunk_experts, output_size, input_size // 32, 32
|
||||
)
|
||||
* scale_f32.view(
|
||||
chunk_experts, output_size, input_size // 32, 1
|
||||
)
|
||||
)
|
||||
.view(chunk_experts, output_size, input_size)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
del weight_f32, scale_f32
|
||||
|
||||
weight_int4, weight_scale = per_1x32_i4_quant(weight_bf16)
|
||||
del weight_bf16
|
||||
weight_int4 = weight_int4.view(aiter_dtypes.i4x2).view(
|
||||
chunk_experts, output_size, input_size
|
||||
)
|
||||
weight_packed = pack_int8_to_packed_int4(
|
||||
shuffle_weight(weight_int4.view(aiter_dtypes.i8), (16, 16))
|
||||
)
|
||||
del weight_int4
|
||||
chunk_packed = weight_packed.view(
|
||||
chunk_experts, output_size, input_size // 2
|
||||
).view(aiter_dtypes.i4x2)
|
||||
chunk_scale = (
|
||||
shuffle_scale_for_int4(weight_scale, group_size=32)
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
if out_packed is None:
|
||||
out_packed = torch.empty(
|
||||
(num_experts,) + tuple(chunk_packed.shape[1:]),
|
||||
dtype=chunk_packed.dtype,
|
||||
device=chunk_packed.device,
|
||||
)
|
||||
scale_stride = chunk_scale.numel() // chunk_experts
|
||||
out_scale = torch.empty(
|
||||
num_experts * scale_stride,
|
||||
dtype=chunk_scale.dtype,
|
||||
device=chunk_scale.device,
|
||||
)
|
||||
assert out_packed is not None
|
||||
assert out_scale is not None
|
||||
out_packed[lo:hi].copy_(chunk_packed)
|
||||
out_scale[lo * scale_stride : hi * scale_stride].copy_(chunk_scale)
|
||||
del weight_packed, weight_scale, chunk_packed, chunk_scale
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
del w_all, s_all
|
||||
assert out_packed is not None
|
||||
assert out_scale is not None
|
||||
weight.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
|
||||
scale.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
|
||||
torch.accelerator.empty_cache()
|
||||
return out_packed, out_scale
|
||||
|
||||
w13, w13_scale = convert(layer.w13_weight, layer.w13_weight_scale)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
del w13, w13_scale
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
w2, w2_scale = convert(layer.w2_weight, layer.w2_weight_scale)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
del w2, w2_scale
|
||||
torch.accelerator.empty_cache()
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
return
|
||||
@@ -797,6 +1015,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self._build_moe_kernel(layer)
|
||||
return
|
||||
|
||||
if self.is_k3_situ_int4_gfx942:
|
||||
self._setup_kernel_k3_situ_gfx942(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
|
||||
@@ -65,6 +65,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
@@ -231,6 +232,9 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
# Load-time gfx942 requant, not online conversion. Mxfp4MoEMethod owns it.
|
||||
if spec.weight == kInt4Static32:
|
||||
return None
|
||||
cls = table.get(spec.weight)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user