[Quantization] Support compressed-tensors MXFP4 MoE with FP8 activations via Mxfp4MoEMethod (#59815)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude
Kevin H. Luu
parent
fe98d90a9d
commit
984c242ede
@@ -38,7 +38,10 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
@@ -541,9 +544,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
def __init__(self, moe: FusedMoEConfig, activation_key: QuantKey | None = None):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
# Checkpoint-declared activation quantization. Only backends that pick
|
||||
# their activation format from the layer (Humming) consume it.
|
||||
self.activation_key = activation_key
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
@@ -822,6 +828,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
activation=self.moe.activation,
|
||||
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
|
||||
activation_key=self.activation_key,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user