diff --git a/tests/models/kimi_k3/test_gfx942_int4.py b/tests/models/kimi_k3/test_gfx942_int4.py new file mode 100644 index 0000000000..20bef41eb8 --- /dev/null +++ b/tests/models/kimi_k3/test_gfx942_int4.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import sys +from collections.abc import Callable +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock, patch + +import pytest + +from vllm.config.quantization import QuantizationConfigArgs +from vllm.model_executor.layers.fused_moe import RoutedExperts +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.quantization.mxfp4 import ( + _moe_weight_override_is_int4, + _use_k3_situ_int4_gfx942, +) +from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, +) + +pytestmark = pytest.mark.cpu_test + + +class _FakeRocmModule(ModuleType): + on_gfx942: Callable[[], bool] + + +class _FakeAiterOpsModule(ModuleType): + rocm_aiter_ops: SimpleNamespace + + +def _vllm_config(moe_weight: str | None): + quantization_config = ( + None + if moe_weight is None + else QuantizationConfigArgs(moe={"weight": moe_weight}) + ) + return SimpleNamespace( + model_config=SimpleNamespace(quantization_config=quantization_config) + ) + + +@pytest.mark.parametrize( + ("moe_weight", "expected"), + [ + (None, False), + ("mxfp4", False), + ("int4_per_group_32", True), + ], +) +def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected): + monkeypatch.setattr( + "vllm.config.get_current_vllm_config", + lambda: _vllm_config(moe_weight), + ) + assert _moe_weight_override_is_int4() is expected + + +@pytest.mark.parametrize("int4_requested", [False, True]) +def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested): + moe = SimpleNamespace( + activation=MoEActivation.SITU, + activation_situ_linear_beta=25.0, + ) + fake_rocm = _FakeRocmModule("vllm.platforms.rocm") + fake_rocm.on_gfx942 = lambda: True + fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops") + fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True) + with ( + patch("vllm.platforms.current_platform.is_rocm", return_value=True), + patch.dict( + sys.modules, + { + "vllm.platforms.rocm": fake_rocm, + "vllm._aiter_ops": fake_aiter_ops, + }, + ), + patch( + "vllm.model_executor.layers.quantization.mxfp4." + "_moe_weight_override_is_int4", + return_value=int4_requested, + ), + ): + assert _use_k3_situ_int4_gfx942(moe) is int4_requested + + +def test_int4_per_group_32_is_not_an_online_moe_method(): + # Load-time gfx942 requant. Must not match the online MoE table. + config = OnlineQuantizationConfig( + QuantizationConfigArgs(moe={"weight": "int4_per_group_32"}) + ) + assert ( + config.resolve_quant_method_cls( + Mock(spec=RoutedExperts), "model.layers.0.mlp.experts" + ) + is None + ) + + +def test_unknown_online_moe_weight_still_raises(): + config = OnlineQuantizationConfig( + QuantizationConfigArgs(moe={"weight": "fp8_per_token"}) + ) + with pytest.raises(ValueError, match="online quantization"): + config.resolve_quant_method_cls( + Mock(spec=RoutedExperts), "model.layers.0.mlp.experts" + ) diff --git a/tests/quantization/test_quantization_config_args.py b/tests/quantization/test_quantization_config_args.py index 3723c6be30..1d6ced0d48 100644 --- a/tests/quantization/test_quantization_config_args.py +++ b/tests/quantization/test_quantization_config_args.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8DynamicTokenSym, kFp8Static128BlockSym, kFp8StaticTensorSym, + kInt4Static32, kInt8StaticChannelSym, kMxfp8Dynamic, ) @@ -46,6 +47,12 @@ def test_quant_spec_string_representation_uses_quantization_name(): assert str(quant_spec(weight="mxfp4")) == "mxfp4" +def test_quant_spec_resolves_groupwise_int4_weight(): + spec = QuantSpec(weight="int4_per_group_32") + assert spec.weight == kInt4Static32 + assert spec.activation is None + + def test_quant_spec_rejects_unknown_name(): with pytest.raises(ValueError, match="unknown quantization name"): quant_spec(weight="not_a_real_format") diff --git a/vllm/config/quantization.py b/vllm/config/quantization.py index 7d32a17308..d950084726 100644 --- a/vllm/config/quantization.py +++ b/vllm/config/quantization.py @@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static32, kInt8StaticChannelSym, kMxfp4Dynamic, kMxfp4Static, @@ -40,6 +41,8 @@ QUANT_KEY_NAMES: dict[str, QuantKey] = { "mxfp8": kMxfp8Dynamic, "mxfp4": kMxfp4Dynamic, "int8_per_channel_static": kInt8StaticChannelSym, + # Load-time MXFP4-to-int4 for Kimi-K3 on gfx942. + "int4_per_group_32": kInt4Static32, } diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 861d3cb8d6..69ddd0045b 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os import torch @@ -22,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, + backend_to_kernel_cls, convert_gpt_oss_weight_to_mxfp4_moe_kernel_format, convert_weight_to_mxfp4_moe_kernel_format, make_mxfp4_moe_kernel, @@ -477,6 +479,63 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): ) +def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool: + """Route Kimi-K3 weight-only MXFP4 SiTU MoE to AITER A16W4 on gfx950.""" + from vllm.platforms import current_platform + + if not current_platform.is_rocm(): + return False + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.platforms.rocm import on_gfx950 + + return ( + rocm_aiter_ops.is_fused_moe_enabled() + and on_gfx950() + and moe.activation == MoEActivation.SITU + and moe.activation_situ_linear_beta is not None + and rocm_aiter_ops.get_aiter_activation_type("situ") is not None + ) + + +def _use_k3_situ_int4_gfx942(moe: FusedMoEConfig) -> bool: + """Opt-in gfx942 MXFP4-to-int4 requant for Kimi-K3 SiTU experts.""" + from vllm.platforms import current_platform + + if not current_platform.is_rocm(): + return False + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.platforms.rocm import on_gfx942 + + return ( + rocm_aiter_ops.is_fused_moe_enabled() + and on_gfx942() + and moe.activation == MoEActivation.SITU + and moe.activation_situ_linear_beta is not None + and _moe_weight_override_is_int4() + ) + + +def _moe_weight_override_is_int4() -> bool: + """True when --quantization-config.moe.weight int4_per_group_32 was set.""" + from vllm.config import get_current_vllm_config + + vllm_config = get_current_vllm_config() + if vllm_config is None: + return False + quant_args = getattr(vllm_config.model_config, "quantization_config", None) + moe_spec = getattr(quant_args, "moe", None) + weight = getattr(moe_spec, "weight", None) + if weight is None: + return False + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static32, + ) + + return weight == kInt4Static32 + + class Mxfp4MoEMethod(FusedMoEMethodBase): """MXFP4 MoE quantization method.""" @@ -484,9 +543,33 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): def __init__(self, moe: FusedMoEConfig): super().__init__(moe) - self.weight_dtype = "mxfp4" - self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe) + self.is_k3_situ_aiter = _use_k3_situ_aiter(moe) + self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe) + self.experts_cls: type[mk.FusedMoEExperts] | None + if self.is_k3_situ_aiter or self.is_k3_situ_int4_gfx942: + self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16 + self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0] + if self.is_k3_situ_int4_gfx942: + logger.warning_once( + "Requantizing Kimi-K3 MXFP4 experts to groupwise int4 " + "for gfx942. This conversion is lossy and was explicitly " + "enabled by --quantization-config.moe.weight " + "int4_per_group_32." + ) + else: + logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.") + from vllm._aiter_ops import rocm_aiter_ops + + if rocm_aiter_ops.is_fused_moe_situv2_enabled(): + # a16w4 does not read this bound. Force 0 so leftover a8w4 + # config cannot keep bf16 activations. + # TODO: Remove once AITER takes this as a kernel argument. + os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0" + else: + self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend( + moe + ) self.max_capture_size = moe.max_capture_size @@ -784,6 +867,141 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) + def _setup_kernel_k3_situ_gfx942(self, layer: RoutedExperts) -> None: + import inspect + + from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import ( + flydsl_a16w4_gemm1, + ) + + # Fail closed if this AITER build would silently run SiLU. + if "act" not in inspect.signature(flydsl_a16w4_gemm1).parameters: + raise RuntimeError( + "This AITER build ignores the SiTUv2 activation on the " + "replacement packed-int4 MoE path and would silently compute " + "SiLU. Rebuild with the replacement A16W4/int4 pipeline." + ) + + from aiter import dtypes as aiter_dtypes + from aiter.ops.quant import per_1x32_i4_quant + from aiter.ops.shuffle import ( + pack_int8_to_packed_int4, + shuffle_scale_for_int4, + shuffle_weight, + ) + from aiter.utility import fp4_utils + + fp4_dtype = torch.float4_e2m1fn_x2 + e8m0_dtype = torch.float8_e8m0fnu + + # mxfp4_to_f32 grows the packed weight 8x and peaks above 20 GiB per + # rank if all experts convert at once. Convert 8 experts at a time. + _CONVERT_CHUNK = 8 + + def convert( + weight: torch.nn.Parameter, + scale: torch.nn.Parameter, + ) -> tuple[torch.Tensor, torch.Tensor]: + # Drop the source before returning. Packed int4 is the same size as + # packed MXFP4, so holding both doubles the footprint. That fits + # under expert parallel (1/ep_size experts per rank) and does not + # fit under pure tensor parallel. + w_all = weight.data.view(fp4_dtype) + s_all = scale.data.view(e8m0_dtype) + num_experts = w_all.shape[0] + + # Write chunks into a preallocated output. torch.cat of a list + # would hold the full result twice at the join. + out_packed: torch.Tensor | None = None + out_scale: torch.Tensor | None = None + scale_stride = 0 + for lo in range(0, num_experts, _CONVERT_CHUNK): + hi = min(lo + _CONVERT_CHUNK, num_experts) + weight_f32 = fp4_utils.mxfp4_to_f32(w_all[lo:hi]) + scale_f32 = fp4_utils.e8m0_to_f32(s_all[lo:hi]) + chunk_experts, output_size, input_size = weight_f32.shape + weight_bf16 = ( + ( + weight_f32.view( + chunk_experts, output_size, input_size // 32, 32 + ) + * scale_f32.view( + chunk_experts, output_size, input_size // 32, 1 + ) + ) + .view(chunk_experts, output_size, input_size) + .to(torch.bfloat16) + ) + del weight_f32, scale_f32 + + weight_int4, weight_scale = per_1x32_i4_quant(weight_bf16) + del weight_bf16 + weight_int4 = weight_int4.view(aiter_dtypes.i4x2).view( + chunk_experts, output_size, input_size + ) + weight_packed = pack_int8_to_packed_int4( + shuffle_weight(weight_int4.view(aiter_dtypes.i8), (16, 16)) + ) + del weight_int4 + chunk_packed = weight_packed.view( + chunk_experts, output_size, input_size // 2 + ).view(aiter_dtypes.i4x2) + chunk_scale = ( + shuffle_scale_for_int4(weight_scale, group_size=32) + .view(-1) + .contiguous() + ) + if out_packed is None: + out_packed = torch.empty( + (num_experts,) + tuple(chunk_packed.shape[1:]), + dtype=chunk_packed.dtype, + device=chunk_packed.device, + ) + scale_stride = chunk_scale.numel() // chunk_experts + out_scale = torch.empty( + num_experts * scale_stride, + dtype=chunk_scale.dtype, + device=chunk_scale.device, + ) + assert out_packed is not None + assert out_scale is not None + out_packed[lo:hi].copy_(chunk_packed) + out_scale[lo * scale_stride : hi * scale_stride].copy_(chunk_scale) + del weight_packed, weight_scale, chunk_packed, chunk_scale + torch.accelerator.empty_cache() + + del w_all, s_all + assert out_packed is not None + assert out_scale is not None + weight.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device) + scale.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device) + torch.accelerator.empty_cache() + return out_packed, out_scale + + w13, w13_scale = convert(layer.w13_weight, layer.w13_weight_scale) + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w13_weight_scale", w13_scale) + del w13, w13_scale + torch.accelerator.empty_cache() + + w2, w2_scale = convert(layer.w2_weight, layer.w2_weight_scale) + replace_parameter(layer, "w2_weight", w2) + replace_parameter(layer, "w2_weight_scale", w2_scale) + del w2, w2_scale + torch.accelerator.empty_cache() + layer.w13_weight.is_shuffled = True + layer.w2_weight.is_shuffled = True + + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + if self.moe_quant_config is not None and self.experts_cls is not None: + self.moe_kernel = make_mxfp4_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + mxfp4_backend=self.mxfp4_backend, + experts_cls=self.experts_cls, + routing_tables=layer._expert_routing_tables(), + ) + def process_weights_after_loading(self, layer: RoutedExperts) -> None: if self.mxfp4_backend == Mxfp4MoeBackend.NONE: return @@ -797,6 +1015,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): self._build_moe_kernel(layer) return + if self.is_k3_situ_int4_gfx942: + self._setup_kernel_k3_situ_gfx942(layer) + return + w13 = layer.w13_weight w2 = layer.w2_weight w13_scale = layer.w13_weight_scale diff --git a/vllm/model_executor/layers/quantization/online/base.py b/vllm/model_executor/layers/quantization/online/base.py index 41bca8d29d..8ed48b2b64 100644 --- a/vllm/model_executor/layers/quantization/online/base.py +++ b/vllm/model_executor/layers/quantization/online/base.py @@ -65,6 +65,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static32, kInt8StaticChannelSym, kMxfp4Static, kMxfp8Dynamic, @@ -231,6 +232,9 @@ class OnlineQuantizationConfig(QuantizationConfig): """ if spec is None or spec.weight is None: return None + # Load-time gfx942 requant, not online conversion. Mxfp4MoEMethod owns it. + if spec.weight == kInt4Static32: + return None cls = table.get(spec.weight) if cls is None: raise ValueError(