Fix block FP8 MTP in ModelOpt mixed checkpoints (#55513)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
sychen52
2026-09-08 22:47:21 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Isotr0py
parent 28a2ccee78
commit 60ad959b6f
5 changed files with 119 additions and 1 deletions
@@ -60,6 +60,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
FP8_SCALE_SENTINEL,
@@ -107,6 +108,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
# ``FP8_PB_WO`` is ModelOpt's canonical 2D block-FP8 name. Early composed
# Qwen3.8-Flash-Next checkpoints used ``FP8_BLOCK_SCALES`` for the same tensor
# layout, so retain it as a checkpoint-compatibility alias.
_BLOCK_FP8_MOE_ALGOS = ("FP8_PB_WO", "FP8_BLOCK_SCALES")
# Single source of truth for the ModelOpt linear algos.
#
# Each entry is (owning config's name, mixed-precision sub-config attribute).
@@ -1491,11 +1497,28 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
self.w4a16_nvfp4_config = w4a16_nvfp4_config
self.mxfp8_config = mxfp8_config
block_sizes = {
int(layer_info.get("group_size", 128))
for layer_info in quantized_layers.values()
if layer_info.get("quant_algo", "").upper() in _BLOCK_FP8_MOE_ALGOS
}
if len(block_sizes) > 1:
raise ValueError(
"MIXED_PRECISION currently requires all block-FP8 MoE layers "
f"to use one group_size, got {sorted(block_sizes)}."
)
block_size = next(iter(block_sizes), 128)
self.fp8_block_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=[block_size, block_size],
)
def has_blocked_weights(self) -> bool:
# Same gate as ModelOptFp8Config.has_blocked_weights, resolved per
# layer: "+quant_fp8" must be on as soon as any layer is block-scaled.
return any(
info.get("quant_algo", "").upper() == "FP8_PB_WO"
info.get("quant_algo", "").upper() in _BLOCK_FP8_MOE_ALGOS
for info in self.quantized_layers.values()
)
@@ -1730,6 +1753,8 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
return build_linear_method(getattr(self, subcfg_attr), quant_algo, prefix)
if isinstance(layer, RoutedExperts):
if quant_algo in _BLOCK_FP8_MOE_ALGOS:
return Fp8MoEMethod(self.fp8_block_config, layer)
if quant_algo == "FP8":
return ModelOptFp8MoEMethod(
quant_config=self.fp8_config,