[Quantization] Support compressed-tensors MXFP4 MoE with FP8 activations via Mxfp4MoEMethod (#59815)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude
Kevin H. Luu
parent
fe98d90a9d
commit
984c242ede
@@ -1480,6 +1480,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
|
||||
activation: MoEActivation | None = None,
|
||||
use_separated_a4w4: bool = False,
|
||||
activation_key: QuantKey | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -1541,8 +1542,20 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
# The env override wins; otherwise use the checkpoint's activation
|
||||
# scheme instead of Humming's default BF16 activations.
|
||||
input_schema = None
|
||||
if activation_key is not None and not envs.VLLM_HUMMING_INPUT_QUANT_CONFIG:
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
|
||||
input_schema = quant_key_to_input_schema(activation_key)
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config={"quant_method": "mxfp4"}
|
||||
layer,
|
||||
quant_config={"quant_method": "mxfp4"},
|
||||
input_schema=input_schema,
|
||||
allow_input_schema_fallback=False,
|
||||
)
|
||||
return (
|
||||
layer.w13_weight,
|
||||
|
||||
Reference in New Issue
Block a user