diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 7d600dea69..d0ece26cfa 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -48,6 +48,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, kInt4Static, kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt8DynamicTensorSym, kInt8DynamicTokenSym, kInt8Static, @@ -186,8 +188,8 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): swiglu_limit_func(output, input, activation_config.clamp_limit) return - # SWIGLUOAI_UNINTERLEAVE routes to the silu_and_mul_with_clamp kernel and - # requires a clamp limit. Other activations ignore these parameters. + # SWIGLUOAI_UNINTERLEAVE routes to torch.ops._C.silu_and_mul_with_clamp + # via apply_moe_activation() and requires clamp_limit to be set. if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE: assert activation_config.clamp_limit is not None, ( "SWIGLUOAI_UNINTERLEAVE requires gemm1_clamp_limit" @@ -580,6 +582,8 @@ class TritonWNA16Experts(TritonExperts): kInt4Static, kInt8Static, kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, # other group sizes? ] return weight_key in SUPPORTED_W @@ -591,6 +595,7 @@ class TritonWNA16Experts(TritonExperts): MoEActivation.GELU, MoEActivation.GELU_TANH, MoEActivation.SWIGLUOAI, + MoEActivation.SWIGLUOAI_UNINTERLEAVE, MoEActivation.SWIGLUSTEP, MoEActivation.SILU_NO_MUL, MoEActivation.GELU_NO_MUL, diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 5ab0277afc..5e4f30b9d1 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -1649,6 +1649,42 @@ def convert_to_wna16_moe_kernel_format( w2_uint8 = w2.transpose(1, 2).contiguous().view(torch.uint8) w13_scale = w13_scale.transpose(1, 2).contiguous() w2_scale = w2_scale.transpose(1, 2).contiguous() + # Zero points from compressed-tensors checkpoints are K-first int32 + # with 8 int4 ZPs packed per element: shape (E, K//gs, N//8). + # fused_moe_kernel_gptq_awq expects N-first uint8 with 2 int4 ZPs + # per byte: shape (E, N//2, K//gs), indexed as + # (offs_bn // 2) * stride_bzn + offs_k_group * stride_bzk. + # Conversion steps: + # (E, K//gs, N//8) int32 + # → transpose(1,2) → (E, N//8, K//gs) int32 + # → view(uint8) → (E, N//8, K//gs*4) [each int32 → 4 bytes] + # → reshape(…,4) → (E, N//8, K//gs, 4) [isolate byte index] + # → permute(0,1,3,2) → (E, N//8, 4, K//gs) [byte index before K] + # → reshape → (E, N//2, K//gs) [kernel expected layout] + # After this, element [e, offs_bn//2, k_group] is the uint8 byte + # holding the two int4 ZPs for output channels offs_bn and offs_bn+1. + if w13_qzeros is not None: + E13, Kg13, Np13 = w13_qzeros.shape + w13_qzeros = ( + w13_qzeros.transpose(1, 2) + .contiguous() + .view(torch.uint8) + .reshape(E13, Np13, Kg13, 4) + .permute(0, 1, 3, 2) + .reshape(E13, Np13 * 4, Kg13) + .contiguous() + ) + if w2_qzeros is not None: + E2, Kg2, Np2 = w2_qzeros.shape + w2_qzeros = ( + w2_qzeros.transpose(1, 2) + .contiguous() + .view(torch.uint8) + .reshape(E2, Np2, Kg2, 4) + .permute(0, 1, 3, 2) + .reshape(E2, Np2 * 4, Kg2) + .contiguous() + ) else: # MoeWNA16 uses N-first uint8 weights and scales. w13_uint8 = w13.view(torch.uint8) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py index eaec0fec45..bd7ad4234a 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py @@ -123,8 +123,6 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): # grouped actorder isn't supported by this kernel assert weight_quant.actorder != "group" - assert self.symmetric, "Only symmetric quantization is supported for MoE" - # Non-Marlin WNA16 always uses bf16/fp16 inputs self.input_dtype = torch.bfloat16 @@ -317,6 +315,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): num_groups_w2 = w2_scales_size // self.group_size num_groups_w13 = hidden_size // self.group_size + if not self.symmetric: + # For asymmetric int4 quantization, packed_factor (=8) int4 ZPs are + # stored per int32. Each TP shard must contain a whole number of + # int32 elements so the packed ZP can be sliced without straddling + # an int32 boundary. + w13_n = 2 * intermediate_size_per_partition # gate+up output channels + if w13_n % self.packed_factor != 0: + raise ValueError( + f"CompressedTensors WNA16 MoE: gate+up output channels per " + f"TP rank (2 * intermediate_size_per_partition = {w13_n}) " + f"must be divisible by packed_factor ({self.packed_factor}). " + f"Use a TP size where 2 * intermediate_size is divisible by " + f"{self.packed_factor}." + ) + if hidden_size % self.packed_factor != 0: + raise ValueError( + f"CompressedTensors WNA16 MoE: hidden_size ({hidden_size}) " + f"must be divisible by packed_factor ({self.packed_factor}) " + f"for correct ZP unpacking." + ) + layer.num_groups_w13 = num_groups_w13 layer.num_groups_w2 = num_groups_w2 @@ -513,8 +532,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): replace_parameter(layer, "w13_weight_scale", w13_scales) replace_parameter(layer, "w2_weight_scale", w2_scales) - # CPU fused_experts_cpu requires zero points even for symmetric quant - if not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU: + # CPU fused_experts_cpu requires zero points even for symmetric quant. + # EMULATION bakes ZP into the dequantized bf16 weights — ZP is None. + if ( + not self.symmetric or self.wna16_backend == WNA16MoEBackend.CPU + ) and self.wna16_backend != WNA16MoEBackend.EMULATION: assert w13_qzeros is not None and w2_qzeros is not None replace_parameter(layer, "w13_weight_zero_point", w13_qzeros) replace_parameter(layer, "w2_weight_zero_point", w2_qzeros) diff --git a/vllm/model_executor/layers/quantization/moe_wna16.py b/vllm/model_executor/layers/quantization/moe_wna16.py index 02207abf3d..d2e17d4b9b 100644 --- a/vllm/model_executor/layers/quantization/moe_wna16.py +++ b/vllm/model_executor/layers/quantization/moe_wna16.py @@ -222,7 +222,6 @@ class MoeWNA16Method(FusedMoEMethodBase): else: scale = kInt4StaticGroupScale elif num_bits == 8: - assert group_size == -1 quant_type = INT8_DTYPE scale = kInt8StaticGroupScale else: