From f1073c050c1c0149430738fd1c7cc078d4883900 Mon Sep 17 00:00:00 2001 From: Fadi Arafeh <115173828+fadara01@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:39:20 +0100 Subject: [PATCH] [CPU][BugFix] Multiple fixes to w4a8_int8 CPU MoE path (#46739) Signed-off-by: Fadi Arafeh --- .../scripts/hardware_ci/run-cpu-test-arm.sh | 3 +- csrc/cpu/torch_bindings.cpp | 3 +- csrc/moe/dynamic_4bit_int_moe_cpu.cpp | 81 +++++---- csrc/ops.h | 6 +- tests/kernels/moe/test_cpu_int4_moe.py | 154 +++++------------- vllm/_custom_ops.py | 18 ++ .../layers/fused_moe/experts/cpu_int4_moe.py | 70 ++++++-- .../layers/fused_moe/oracle/w4a8_int8.py | 4 +- .../compressed_tensors_moe_w4a8_int8.py | 31 +--- .../layers/quantization/utils/quant_utils.py | 6 + 10 files changed, 189 insertions(+), 187 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh index 252eeeef8c..2ad3a16be8 100755 --- a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh @@ -38,7 +38,8 @@ function cpu_tests() { pytest -x -v -s tests/kernels/attention/test_cpu_attn.py pytest -x -v -s tests/kernels/core/test_cpu_activation.py pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py - pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py" + pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py + pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py" # skip tests requiring model downloads if HF_TOKEN is not set # due to rate-limits diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index e17c9ab3a7..cfa296e73b 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -278,7 +278,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def( "dynamic_4bit_int_moe(" "Tensor x, Tensor topk_ids, Tensor topk_weights," - "Tensor w13_packed, Tensor w2_packed, int H, int I, int I2," + "Tensor w13_packed, Tensor w2_packed," + "int hidden_size, int intermediate_size," "int group_size, bool apply_router_weight_on_input, int activation_kind" ") -> Tensor"); diff --git a/csrc/moe/dynamic_4bit_int_moe_cpu.cpp b/csrc/moe/dynamic_4bit_int_moe_cpu.cpp index 58dc402016..1b071d334f 100644 --- a/csrc/moe/dynamic_4bit_int_moe_cpu.cpp +++ b/csrc/moe/dynamic_4bit_int_moe_cpu.cpp @@ -29,25 +29,37 @@ enum ActivationKind : int64_t { torch::Tensor dynamic_4bit_int_moe_cpu( torch::Tensor x, torch::Tensor topk_ids, torch::Tensor topk_weights, - torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t H, int64_t I, - int64_t I2, int64_t group_size, bool apply_router_weight_on_input, - int64_t activation_kind) { + torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t hidden_size, + int64_t intermediate_size, int64_t group_size, + bool apply_router_weight_on_input, int64_t activation_kind) { TORCH_CHECK(x.dim() == 2, "x must be 2D"); TORCH_CHECK(topk_ids.dim() == 2 && topk_weights.dim() == 2, "topk tensors must be [T, K]"); TORCH_CHECK( w13_packed.size(0) == w2_packed.size(0), "w13_packed and w2_packed must have same number of experts in dim 0"); - TORCH_CHECK(I2 == 2 * I, "I2 must equal 2*I"); const int64_t T = x.size(0); const int64_t K = topk_ids.size(1); const int64_t E = w13_packed.size(0); const int64_t N = T * K; + const int64_t w13_out_features = 2 * intermediate_size; auto x_c = x.contiguous(); + // _dyn_quant_matmul_4bit kernel natively supports these pre-quant activation + // dtypes: + // - fp32: with channelwise and groupwise + // - bf16: with channelwise -> upcast to fp32 for groupwise + // - fp16: not supported -> upcast to fp32 for groupwise & channelwise + const auto output_dtype = x_c.scalar_type(); + const bool should_cast_input = + ((group_size != -1) && output_dtype == at::kBFloat16) || + output_dtype == at::kHalf; + if (should_cast_input) { + x_c = x_c.to(at::kFloat); + } auto ids_c = topk_ids.contiguous(); - auto gates_c = topk_weights.to(at::kFloat).contiguous(); + auto gates_c = topk_weights.to(x_c.scalar_type()).contiguous(); // bucketing tokens -> experts c10::SmallVector counts( @@ -63,35 +75,42 @@ torch::Tensor dynamic_4bit_int_moe_cpu( c10::SmallVector offsets(E + 1, 0); // ( E +1 ) for (int64_t e = 0; e < E; ++e) offsets[e + 1] = offsets[e] + counts[e]; + // expert_tokens = [tokens indices for expert 0, ...] + // expert_gates = [router weights for tokens assigned to expert 0, ...] auto expert_tokens = at::empty({offsets[E]}, ids_c.options()); auto expert_gates = at::empty({offsets[E]}, gates_c.options()); { c10::SmallVector cursor(E, 0); - const auto* ids_ptr = ids_c.data_ptr(); - const auto* gts_ptr = gates_c.data_ptr(); - auto* tok_ptr = expert_tokens.data_ptr(); - auto* gate_ptr = expert_gates.data_ptr(); + AT_DISPATCH_FLOATING_TYPES_AND2( + at::ScalarType::BFloat16, at::ScalarType::Half, gates_c.scalar_type(), + "bucket_expert_tokens_and_gates", [&] { + const auto* ids_ptr = ids_c.data_ptr(); + const auto* gts_ptr = gates_c.data_ptr(); + auto* tok_ptr = expert_tokens.data_ptr(); + auto* gate_ptr = expert_gates.data_ptr(); - for (int64_t t = 0; t < T; ++t) { - const int64_t base = t * K; - for (int64_t k = 0; k < K; ++k) { - const int64_t idx = base + k; - const int64_t e = ids_ptr[idx]; - const int64_t p = offsets[e] + (cursor[e]++); - tok_ptr[p] = t; - gate_ptr[p] = gts_ptr[idx]; - } - } + for (int64_t t = 0; t < T; ++t) { + const int64_t base = t * K; + for (int64_t k = 0; k < K; ++k) { + const int64_t idx = base + k; + const int64_t e = ids_ptr[idx]; + const int64_t p = offsets[e] + (cursor[e]++); + tok_ptr[p] = t; + gate_ptr[p] = gts_ptr[idx]; + } + } + }); } - const int64_t g_eff_13 = (group_size != -1) ? group_size : H; - const int64_t g_eff_2 = (group_size != -1) ? group_size : I; + const int64_t g_eff_13 = (group_size != -1) ? group_size : hidden_size; + const int64_t g_eff_2 = (group_size != -1) ? group_size : intermediate_size; + // X_all [num_tokens * K, hidden_size] auto X_all = x_c.index_select(/*dim=*/0, expert_tokens); if (apply_router_weight_on_input) { X_all = X_all.mul(expert_gates.unsqueeze(1)); } - auto Y_all = at::empty({offsets[E], H}, x_c.options()); + auto Y_all = at::empty({offsets[E], hidden_size}, x_c.options()); at::parallel_for(0, offsets[E], 0, [&](int64_t idx_begin, int64_t idx_end) { c10::InferenceMode guard; @@ -109,11 +128,13 @@ torch::Tensor dynamic_4bit_int_moe_cpu( auto w2_e = w2_packed.select(/*dim=*/0, e); // W13 - auto y13 = - mm(x_e, w13_e, g_eff_13, /*in_features=*/H, /*out_features=*/I2); + auto y13 = mm(x_e, w13_e, g_eff_13, /*in_features=*/hidden_size, + /*out_features=*/w13_out_features); - auto g_part = y13.narrow(/*dim=*/1, /*start=*/0, /*length=*/I); - auto u_part = y13.narrow(/*dim=*/1, /*start=*/I, /*length=*/I); + auto g_part = + y13.narrow(/*dim=*/1, /*start=*/0, /*length=*/intermediate_size); + auto u_part = y13.narrow(/*dim=*/1, /*start=*/intermediate_size, + /*length=*/intermediate_size); torch::Tensor act; if (activation_kind == ActivationKind::SwiGLUOAI) { // SwiGLUOAI @@ -128,7 +149,8 @@ torch::Tensor dynamic_4bit_int_moe_cpu( } // W2 - auto y = mm(act, w2_e, g_eff_2, /*in_features=*/I, /*out_features=*/H); + auto y = mm(act, w2_e, g_eff_2, /*in_features=*/intermediate_size, + /*out_features=*/hidden_size); // Store per-expert result Y_all.narrow(/*dim=*/0, /*start=*/start, /*length=*/te).copy_(y); @@ -138,8 +160,11 @@ torch::Tensor dynamic_4bit_int_moe_cpu( if (!apply_router_weight_on_input) { Y_all = Y_all.mul(expert_gates.unsqueeze(1)); } + if (Y_all.scalar_type() != output_dtype) { + Y_all = Y_all.to(output_dtype); + } - auto out = at::zeros({T, H}, x.options()); + auto out = at::zeros({T, hidden_size}, x.options()); out = at::index_add(out, /*dim=*/0, /*index=*/expert_tokens, /*source=*/Y_all); diff --git a/csrc/ops.h b/csrc/ops.h index cd18b1e5e0..274cd52bea 100644 --- a/csrc/ops.h +++ b/csrc/ops.h @@ -53,9 +53,9 @@ void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input, torch::Tensor dynamic_4bit_int_moe_cpu( torch::Tensor x, torch::Tensor topk_ids, torch::Tensor topk_weights, - torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t H, int64_t I, - int64_t I2, int64_t group_size, bool apply_router_weight_on_input, - int64_t activation_kind); + torch::Tensor w13_packed, torch::Tensor w2_packed, int64_t hidden_size, + int64_t intermediate_size, int64_t group_size, + bool apply_router_weight_on_input, int64_t activation_kind); using fptr_t = int64_t; #ifdef USE_ROCM diff --git a/tests/kernels/moe/test_cpu_int4_moe.py b/tests/kernels/moe/test_cpu_int4_moe.py index 05694eb08b..04931e386f 100644 --- a/tests/kernels/moe/test_cpu_int4_moe.py +++ b/tests/kernels/moe/test_cpu_int4_moe.py @@ -8,19 +8,21 @@ import pytest import torch import torch.nn.functional as F -from vllm.platforms import current_platform +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import ( + CPUExpertsInt4, +) +from vllm.model_executor.layers.fused_moe.oracle.w4a8_int8 import ( + convert_to_w4a8_int8_moe_format, +) +from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import set_random_seed -if not current_platform.is_cpu(): - pytest.skip("skipping CPU-only tests", allow_module_level=True) - -# Check if the dynamic_4bit_int_moe op is available -if not hasattr(torch.ops._C, "dynamic_4bit_int_moe"): - pytest.skip("dynamic_4bit_int_moe op not available", allow_module_level=True) - -# Check if KleidiAI ops are available -if not hasattr(torch.ops.aten, "_dyn_quant_pack_4bit_weight"): - pytest.skip("KleidiAI 4-bit ops not available", allow_module_level=True) +if ( + not current_platform.is_cpu() + or current_platform.get_cpu_architecture() != CpuArchEnum.ARM +): + pytest.skip("skipping Arm CPU-only tests", allow_module_level=True) # Tolerance for INT4 W4A8 @@ -34,49 +36,6 @@ def _silu_and_mul(x: torch.Tensor) -> torch.Tensor: return F.silu(x[..., :d]) * x[..., d:] -def _pack_int4_weight_to_kleidi( - int4_as_int8: torch.Tensor, - scales: torch.Tensor, - bias: torch.Tensor | None, - group_size: int, - in_features: int, - out_features: int, -) -> torch.Tensor: - """Pack INT4 weights (stored as int8 in [-8,7]) to KleidiAI format. - - Args: - int4_as_int8: [out, in] int8 tensor with values in [-8, 7] - scales: [out, in//group_size] or [out, 1] for channel-wise - bias: [out] optional bias - group_size: Quantization group size (-1 for channel-wise) - in_features: Input dimension - out_features: Output dimension - - Returns: - Packed weight tensor in KleidiAI format - """ - # Shift to unsigned nibble [0, 15] - tmp = int4_as_int8.add(8) - # Pack pairs along input dimension - uint8_nibbles = ((tmp[:, 1::2] << 4) | tmp[:, ::2]).to(torch.uint8) - - # Determine scale dtype based on group_size - scale_dtype = torch.float32 if group_size == -1 else torch.bfloat16 - scales_typed = scales.to(scale_dtype) - bias_typed = None if bias is None else bias.to(torch.float32) - - # Pack using KleidiAI op - actual_group_size = in_features if group_size == -1 else group_size - return torch.ops.aten._dyn_quant_pack_4bit_weight( - uint8_nibbles, - scales_typed, - bias_typed, - actual_group_size, - in_features, - out_features, - ) - - def _make_int4_moe_weights( E: int, N: int, @@ -124,59 +83,29 @@ def _make_int4_moe_weights( w13_bias = torch.randn(E, 2 * N, dtype=torch.float32) * 0.01 w2_bias = torch.randn(E, K, dtype=torch.float32) * 0.01 - # Pack weights for each expert - w13_packed_list = [] - w2_packed_list = [] + w13_packed, w2_packed, *_ = convert_to_w4a8_int8_moe_format( + w13_weight=w13_int4, + w2_weight=w2_int4, + w13_weight_scale=w13_scales, + w2_weight_scale=w2_scales, + group_size=group_size, + w13_bias=w13_bias if has_bias else None, + w2_bias=w2_bias if has_bias else None, + ) - for e in range(E): - w13_packed_list.append( - _pack_int4_weight_to_kleidi( - w13_int4[e], - w13_scales[e], - w13_bias[e] if (has_bias and w13_bias is not None) else None, - group_size, - K, - 2 * N, - ) - ) - w2_packed_list.append( - _pack_int4_weight_to_kleidi( - w2_int4[e], - w2_scales[e], - w2_bias[e] if (has_bias and w2_bias is not None) else None, - group_size, - N, - K, - ) - ) + if group_size == -1: + w13_scale = w13_scales.float() + w2_scale = w2_scales.float() + else: + w13_scale = w13_scales.float().repeat_interleave(group_size, dim=-1) + w2_scale = w2_scales.float().repeat_interleave(group_size, dim=-1) - w13_packed = torch.stack(w13_packed_list, dim=0) - w2_packed = torch.stack(w2_packed_list, dim=0) - - # Create reference dequantized weights - w13_ref = torch.zeros(E, 2 * N, K, dtype=torch.float32) - w2_ref = torch.zeros(E, K, N, dtype=torch.float32) - - for e in range(E): - # Dequantize w13 - for i in range(2 * N): - for j in range(K): - group_idx = 0 if group_size == -1 else (j // group_size) - w13_ref[e, i, j] = ( - w13_int4[e, i, j].float() * w13_scales[e, i, group_idx].float() - ) - if has_bias and w13_bias is not None: - w13_ref[e, i, j] += w13_bias[e, i].float() - - # Dequantize w2 - for i in range(K): - for j in range(N): - group_idx = 0 if group_size == -1 else (j // group_size) - w2_ref[e, i, j] = ( - w2_int4[e, i, j].float() * w2_scales[e, i, group_idx].float() - ) - if has_bias and w2_bias is not None: - w2_ref[e, i, j] += w2_bias[e, i].float() + w13_ref = w13_int4.float() * w13_scale + w2_ref = w2_int4.float() * w2_scale + if has_bias and w13_bias is not None: + w13_ref = w13_ref + w13_bias.float().unsqueeze(-1) + if has_bias and w2_bias is not None: + w2_ref = w2_ref + w2_bias.float().unsqueeze(-1) return w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias @@ -233,17 +162,20 @@ MoE_CONFIGS = [ (768, 2048, 16, 4, 64), ] SEEDS = [0, 42] +ACTIVATION_DTYPES = [torch.float32, torch.bfloat16, torch.float16] @pytest.mark.parametrize("M", NUM_TOKENS) @pytest.mark.parametrize("N,K,E,topk,group_size", MoE_CONFIGS) @pytest.mark.parametrize("seed", SEEDS) -def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed): +@pytest.mark.parametrize("activation_dtype", ACTIVATION_DTYPES) +def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed, activation_dtype): """Test dynamic_4bit_int_moe kernel against dequantized torch reference.""" set_random_seed(seed) + activation = MoEActivation.SILU # Generate input activations - a = torch.randn(M, K, dtype=torch.bfloat16) / (K**0.5) + a = torch.randn(M, K, dtype=activation_dtype) / (K**0.5) # Generate INT4 weights w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias = _make_int4_moe_weights( @@ -266,8 +198,6 @@ def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed): ) # Test dynamic_4bit_int_moe kernel - # Activation kind: 1 = SwiGLU_Ug (SiLU(u)*g) for OAI-style - activation_kind = 1 apply_router_weight_on_input = False out = torch.ops._C.dynamic_4bit_int_moe( @@ -278,14 +208,14 @@ def test_cpu_int4_moe_kernel(M, N, K, E, topk, group_size, seed): w2_packed, K, # H (hidden_size / w2_out_features) N, # I (intermediate_size / w2_in_features) - 2 * N, # I2 (2*intermediate_size / w13_out_features) group_size, apply_router_weight_on_input, - activation_kind, + CPUExpertsInt4._activation_kind(activation), ) + assert out.dtype == activation_dtype torch.testing.assert_close( - ref_out.bfloat16(), + ref_out, out, atol=INT4_W4A8_ATOL, rtol=INT4_W4A8_RTOL, diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 2a32a38a69..129e198423 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2974,6 +2974,24 @@ if hasattr(torch.ops._C, "fused_experts_cpu"): return torch.empty_like(hidden_states) +if hasattr(torch.ops._C, "dynamic_4bit_int_moe"): + + @register_fake("_C::dynamic_4bit_int_moe") + def dynamic_4bit_int_moe_fake( + x: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + w13_packed: torch.Tensor, + w2_packed: torch.Tensor, + hidden_size: int, + intermediate_size: int, + group_size: int, + apply_router_weight_on_input: bool, + activation_kind: int, + ) -> torch.Tensor: + return x.new_empty((x.size(0), hidden_size)) + + def fused_experts_cpu( hidden_states: torch.Tensor, w1: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py index c21072676e..f4933e1bd7 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py @@ -14,12 +14,13 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, + kInt4W4A8StaticChannelSym, kInt4W4A8StaticGroup32Sym, kInt4W4A8StaticGroup64Sym, kInt4W4A8StaticGroup128Sym, kInt4W4A8StaticGroupSym, ) -from vllm.platforms import current_platform +from vllm.platforms import CpuArchEnum, current_platform class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): @@ -48,6 +49,48 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.Standard + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + if ( + not current_platform.is_cpu() + or current_platform.get_cpu_architecture() != CpuArchEnum.ARM + ): + return False, "kernel only supports Arm CPU" + + if moe_config.in_dtype not in ( + torch.float32, + torch.bfloat16, + torch.float16, + ): + return ( + False, + f"kernel does not support {moe_config.in_dtype} input/output dtype", + ) + + try: + _ = torch.ops.aten._dyn_quant_matmul_4bit + _ = torch.ops.aten._dyn_quant_pack_4bit_weight + except AttributeError: + return ( + False, + f"PyTorch {torch.__version__} does not support " + "_dyn_quant_* 4bit ops. Install a newer version", + ) + + return mk.FusedMoEExperts.is_supported_config( + cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) + @staticmethod def _supports_current_device() -> bool: return current_platform.is_cpu() @@ -86,8 +129,9 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): Can be channel-wise or group-wise quantization - Activations: dynamic per-token 8-bit integer quantization """ - # group size must be multiple of 32 + # channelwise or groupwise with group size being a multiple of 32 SUPPORTED_W_A = [ + (kInt4W4A8StaticChannelSym, None), (kInt4W4A8StaticGroup128Sym, None), (kInt4W4A8StaticGroup64Sym, None), (kInt4W4A8StaticGroup32Sym, None), @@ -120,7 +164,8 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): """Expert parallelism not yet supported.""" return False - def _activation_kind(self, activation: MoEActivation) -> int: + @staticmethod + def _activation_kind(activation: MoEActivation) -> int: """Convert MoEActivation to kernel activation kind integer. Returns: @@ -200,29 +245,22 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): e_score_correction_bias=e_score_correction_bias, ) - # Extract dimensions from weight tensors - # w1 is w13_packed: [num_experts, packed_data...] - # w2 is w2_packed: [num_experts, packed_data...] - # These dimensions should be available from the layer - # For now, we'll extract from moe_config - K = self.moe_config.hidden_dim - N = self.moe_config.intermediate_size_per_partition + hidden_size = self.moe_config.hidden_dim + intermediate_size = self.moe_config.intermediate_size_per_partition assert self.quant_config.block_shape is not None - if self.quant_config.is_per_act_token: + # C++ kernel expects an int: -1 for channelwise, and group size for groupwise + if self.quant_config.block_shape == [-1, 1]: group_size = -1 else: group_size = self.quant_config.block_shape[1] - - # Call the dynamic 4-bit int MoE kernel return torch.ops._C.dynamic_4bit_int_moe( hidden_states, topk_ids.to(torch.long), topk_weights, w1, # w13_weight_packed w2, # w2_weight_packed - K, # hidden_size (w2_out_features) - N, # intermediate_size (w2_in_features) - N * 2, # 2*intermediate_size (w13_out_features) + hidden_size, + intermediate_size, group_size, apply_router_weight_on_input, self._activation_kind(activation), diff --git a/vllm/model_executor/layers/fused_moe/oracle/w4a8_int8.py b/vllm/model_executor/layers/fused_moe/oracle/w4a8_int8.py index e4f4d49756..c66e43ccba 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/w4a8_int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/w4a8_int8.py @@ -270,7 +270,7 @@ def convert_to_w4a8_int8_moe_format( """ # Derive dimensions from tensor shapes E = w13_weight.shape[0] # num_experts - I2 = w13_weight.shape[1] # w13_out_features (2*IN) + w13_out_features = w13_weight.shape[1] # 2 * intermediate_size H = w13_weight.shape[2] # w13_in_features (hidden_size) IN = w2_weight.shape[2] # w2_in_features (intermediate_size) w2_out_features = w2_weight.shape[1] # Should equal H @@ -286,7 +286,7 @@ def convert_to_w4a8_int8_moe_format( w13_weight_scale[e], # [2I, H/g or 1] w13_bias[e] if w13_bias is not None else None, # [2I] H, - I2, + w13_out_features, group_size, ) ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py index a64104d3ff..2d2190e614 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py @@ -5,7 +5,6 @@ import torch from compressed_tensors.quantization import ( QuantizationArgs, - QuantizationStrategy, ) from vllm.logger import init_logger @@ -32,7 +31,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ScaleDesc, ) from vllm.model_executor.utils import replace_parameter, set_weight_attrs -from vllm.platforms import CpuArchEnum, current_platform logger = init_logger(__name__) @@ -64,32 +62,17 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod): weight_quant.group_size if (weight_quant.group_size is not None) else -1 ) - # Validate scheme: weights=W4 (channel or group), - # activations=dynamic TOKEN (A8) - - # Must be dynamic per-token activations - if ( - input_quant.strategy != QuantizationStrategy.TOKEN - or not input_quant.dynamic + # make sure group size is valid + if self.group_size != -1 and ( + moe.hidden_dim % self.group_size != 0 + or moe.intermediate_size_per_partition % self.group_size != 0 ): raise ValueError( - "W4A8-int MoE needs dynamic per-token activation quantization." + f"Group size ({self.group_size}) must evenly divide both " + f"hidden size ({moe.hidden_dim}) and intermediate size per " + f"partition ({moe.intermediate_size_per_partition})." ) - if weight_quant.num_bits != 4: - raise ValueError("This method only supports 4-bit weights (num_bits=4).") - - # Arm: check _dyn ops availability - if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: - try: - _ = torch.ops.aten._dyn_quant_matmul_4bit - _ = torch.ops.aten._dyn_quant_pack_4bit_weight - except AttributeError as err: - raise RuntimeError( - f"""PyTorch {torch.__version__} lacks _dyn_quant_* 4bit ops; - install a newer build.""" - ) from err - # Construct QuantKey for weights from QuantizationArgs # W4A8 INT4: 4-bit weights (stored as int8), static quantization if self.group_size == -1: diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index 0c5cbae2a4..705da43c37 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -216,6 +216,12 @@ kInt4W4A8StaticGroupSym = QuantKey( torch.int8, kInt4W4A8StaticGroupScale, symmetric=True ) +kInt4W4A8StaticChannelSym = QuantKey( + torch.int8, + ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL), + symmetric=True, +) + def create_fp8_quant_key( static: bool,