[Quantization] Support compressed-tensors MXFP4 MoE with FP8 activations via Mxfp4MoEMethod (#59815)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude
Kevin H. Luu
parent
fe98d90a9d
commit
984c242ede
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsW4A4Mxfp4,
|
||||
CompressedTensorsW4A8Fp8,
|
||||
CompressedTensorsW4A8Int,
|
||||
CompressedTensorsW4A8Mxfp4,
|
||||
CompressedTensorsW8A8Fp8,
|
||||
CompressedTensorsW8A8Int8,
|
||||
CompressedTensorsW8A8Mxfp8,
|
||||
@@ -63,6 +64,11 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
create_fp8_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -449,6 +455,39 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
and is_symmetric
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_mxfp4_w4a8_fp8(
|
||||
cls, weight_quant: QuantizationArgs, input_quant: QuantizationArgs
|
||||
) -> bool:
|
||||
if not cls._is_mxfp4(weight_quant) or input_quant is None:
|
||||
return False
|
||||
|
||||
is_float_type = input_quant.type == QuantizationType.FLOAT
|
||||
is_8_bits = input_quant.num_bits == 8
|
||||
is_static_weight = not weight_quant.dynamic
|
||||
is_dynamic_input = bool(input_quant.dynamic)
|
||||
is_token_or_group = input_quant.strategy in (
|
||||
QuantizationStrategy.TOKEN,
|
||||
QuantizationStrategy.GROUP,
|
||||
)
|
||||
|
||||
return (
|
||||
is_float_type
|
||||
and is_8_bits
|
||||
and is_static_weight
|
||||
and is_dynamic_input
|
||||
and is_token_or_group
|
||||
and input_quant.symmetric
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fp8_dynamic_input_key(input_quant: QuantizationArgs) -> QuantKey:
|
||||
if input_quant.strategy == QuantizationStrategy.GROUP:
|
||||
group_shape = GroupShape(1, input_quant.group_size)
|
||||
else:
|
||||
group_shape = GroupShape.PER_TOKEN
|
||||
return create_fp8_quant_key(static=False, group_shape=group_shape)
|
||||
|
||||
@staticmethod
|
||||
def _is_mxfp8(quant_args: QuantizationArgs) -> bool:
|
||||
if quant_args is None:
|
||||
@@ -764,6 +803,11 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
)
|
||||
return CompressedTensorsW4A4Fp4()
|
||||
|
||||
if self._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
|
||||
return CompressedTensorsW4A8Mxfp4(
|
||||
activation_quant_key=self._fp8_dynamic_input_key(input_quant)
|
||||
)
|
||||
|
||||
if self._is_mxfp4(weight_quant):
|
||||
return CompressedTensorsW4A4Mxfp4()
|
||||
|
||||
|
||||
+16
-2
@@ -34,8 +34,12 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
# make sure quantization config for Linear can target it
|
||||
quant_config._add_fused_moe_to_target_scheme_map()
|
||||
unfused_names = [
|
||||
layer_name + proj_name
|
||||
for proj_name in [".0.gate_proj", ".0.up_proj", ".0.down_proj"]
|
||||
f"{layer_name}.0.{proj_name}"
|
||||
for proj_name in (
|
||||
layer.ckpt_gate_proj_name,
|
||||
layer.ckpt_up_proj_name,
|
||||
layer.ckpt_down_proj_name,
|
||||
)
|
||||
]
|
||||
# TODO: refactor this to use expert_mapping and check all layer numbers
|
||||
all_scheme_dicts = [
|
||||
@@ -59,6 +63,16 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
input_quant = scheme_dict.get("input_activations")
|
||||
format = scheme_dict.get("format")
|
||||
|
||||
if quant_config._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
|
||||
from .compressed_tensors_moe_w4a8_mxfp4 import (
|
||||
CompressedTensorsW4A8Mxfp4MoEMethod,
|
||||
)
|
||||
|
||||
return CompressedTensorsW4A8Mxfp4MoEMethod(
|
||||
layer.moe_config,
|
||||
activation_key=quant_config._fp8_dynamic_input_key(input_quant),
|
||||
)
|
||||
|
||||
if quant_config._is_mxfp4(weight_quant):
|
||||
from .compressed_tensors_moe_w4a4_mxfp4 import (
|
||||
CompressedTensorsW4A4Mxfp4MoEMethod,
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
|
||||
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
|
||||
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
|
||||
|
||||
Only the checkpoint naming and the declared activation scheme differ from
|
||||
the native MXFP4 path, so backend selection, weight conversion and
|
||||
execution are inherited.
|
||||
"""
|
||||
|
||||
# The packed -> unpacked rename means a processed state dict no longer
|
||||
# matches the parameters registered by create_weights.
|
||||
supports_pre_processed_weights = False
|
||||
|
||||
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
|
||||
super().create_weights(layer, *args, **kwargs)
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, name)
|
||||
delattr(layer, name)
|
||||
layer.register_parameter(f"{name}_packed", param)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, f"{name}_packed")
|
||||
delattr(layer, f"{name}_packed")
|
||||
layer.register_parameter(name, param)
|
||||
super().process_weights_after_loading(layer)
|
||||
@@ -6,6 +6,7 @@ from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
|
||||
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
|
||||
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
|
||||
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
|
||||
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
|
||||
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
|
||||
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
|
||||
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
|
||||
@@ -23,6 +24,7 @@ __all__ = [
|
||||
"CompressedTensorsW8A8Int8",
|
||||
"CompressedTensorsW8A8Fp8",
|
||||
"CompressedTensorsW4A4Mxfp4",
|
||||
"CompressedTensorsW4A8Mxfp4",
|
||||
"CompressedTensorsW4A4Fp4",
|
||||
"CompressedTensorsW4A8Int",
|
||||
"CompressedTensorsW4A8Fp8",
|
||||
|
||||
+3
-2
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
@@ -35,10 +36,10 @@ class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
|
||||
Otherwise: W4A16 weight-only via Marlin.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, activation_quant_key: QuantKey = kMxfp4Dynamic):
|
||||
self.group_size = 32
|
||||
self.kernel = init_mxfp4_linear_kernel(
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
activation_quant_key=activation_quant_key,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxfp4 import ( # noqa: E501
|
||||
CompressedTensorsW4A4Mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["CompressedTensorsW4A8Mxfp4"]
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Mxfp4(CompressedTensorsW4A4Mxfp4):
|
||||
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
|
||||
|
||||
The weight layout matches W4A4; only the activation quantization differs.
|
||||
The checkpoint's FP8 activations are used when a kernel supports them,
|
||||
otherwise this falls back to the W4A4 activation format.
|
||||
"""
|
||||
|
||||
def __init__(self, activation_quant_key: QuantKey):
|
||||
try:
|
||||
super().__init__(activation_quant_key=activation_quant_key)
|
||||
except ValueError as e:
|
||||
logger.warning_once(
|
||||
"No MXFP4 linear kernel supports %s activations on this "
|
||||
"platform; falling back to MXFP4 activations. %s",
|
||||
activation_quant_key,
|
||||
e,
|
||||
)
|
||||
super().__init__()
|
||||
@@ -38,7 +38,10 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
@@ -541,9 +544,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
def __init__(self, moe: FusedMoEConfig, activation_key: QuantKey | None = None):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
# Checkpoint-declared activation quantization. Only backends that pick
|
||||
# their activation format from the layer (Humming) consume it.
|
||||
self.activation_key = activation_key
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
@@ -822,6 +828,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
activation=self.moe.activation,
|
||||
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
|
||||
activation_key=self.activation_key,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ def prepare_humming_linear_layer_config(
|
||||
input_quant_config: dict | None = None,
|
||||
*,
|
||||
input_schema: "BaseInputSchema | None" = None,
|
||||
allow_input_schema_fallback: bool = True,
|
||||
) -> "LayerConfig":
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
@@ -119,6 +120,7 @@ def prepare_humming_linear_layer_config(
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
param_dtype=layer.params_dtype,
|
||||
allow_fallback=allow_input_schema_fallback,
|
||||
)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
|
||||
Reference in New Issue
Block a user