[Bugfix] Fix MiniMax-M3 compressed-tensors FP8 MoE SwiGLU params (#46845)
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
@@ -22,6 +22,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
Fp8BlockScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import UnquantizedFusedMoEMethod
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
CompressedTensorsLinearMethod,
|
||||
@@ -35,6 +36,9 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
CompressedTensorsWNA8O8Int,
|
||||
CompressedTensorsWNA16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_w8a8_fp8 import ( # noqa: E501
|
||||
CompressedTensorsW8A8Fp8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
find_matched_target,
|
||||
)
|
||||
@@ -345,6 +349,34 @@ def test_compressed_tensors_fp8(vllm_runner):
|
||||
assert output
|
||||
|
||||
|
||||
def test_compressed_tensors_w8a8_fp8_moe_forwards_swiglu_params():
|
||||
quant_method = object.__new__(CompressedTensorsW8A8Fp8MoEMethod)
|
||||
quant_method.input_quant = QuantizationArgs(
|
||||
num_bits=8,
|
||||
type=QuantizationType.FLOAT,
|
||||
strategy=QuantizationStrategy.TOKEN,
|
||||
dynamic=True,
|
||||
symmetric=True,
|
||||
)
|
||||
quant_method.weight_block_size = None
|
||||
quant_method.fp8_backend = Fp8MoeBackend.TRITON
|
||||
|
||||
layer = Mock()
|
||||
layer.w13_weight_scale = torch.ones(2, 4, 1)
|
||||
layer.w2_weight_scale = torch.ones(2, 8, 1)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
layer.swiglu_alpha = 1.702
|
||||
layer.swiglu_beta = None
|
||||
layer.swiglu_limit = 7.0
|
||||
|
||||
quant_config = quant_method.get_fused_moe_quant_config(layer)
|
||||
|
||||
assert quant_config.gemm1_alpha == 1.702
|
||||
assert quant_config.gemm1_beta is None
|
||||
assert quant_config.gemm1_clamp_limit == 7.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
|
||||
+2
@@ -352,6 +352,8 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
per_act_token_quant=is_per_token,
|
||||
per_out_ch_quant=is_per_token,
|
||||
block_shape=self.weight_block_size,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user