Fix block FP8 MTP in ModelOpt mixed checkpoints (#55513)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Isotr0py
parent
28a2ccee78
commit
60ad959b6f
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from importlib import import_module
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -19,6 +20,8 @@ from vllm.models.qwen4_exp.config import (
|
||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
from ...utils import spawn_new_process_for_each_test
|
||||
|
||||
|
||||
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
values = {
|
||||
@@ -89,6 +92,28 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
assert returned_multi_hidden is multi_hidden
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@pytest.mark.parametrize("backend", ["amd", "nvidia"])
|
||||
def test_qwen4_exp_mtp_remaps_mixed_precision_layer_indices(backend: str) -> None:
|
||||
mtp_module = import_module(f"vllm.models.qwen4_exp.{backend}.mtp")
|
||||
|
||||
quantized_layers = {
|
||||
"model.language_model.layers.0.mlp.experts": {"quant_algo": "NVFP4"},
|
||||
"mtp.layers.0.mlp.experts": {
|
||||
"quant_algo": "FP8_BLOCK_SCALES",
|
||||
"group_size": 128,
|
||||
},
|
||||
}
|
||||
|
||||
assert mtp_module._remap_quantized_layers(quantized_layers, 48) == {
|
||||
"model.language_model.layers.0.mlp.experts": {"quant_algo": "NVFP4"},
|
||||
"mtp.layers.48.mlp.experts": {
|
||||
"quant_algo": "FP8_BLOCK_SCALES",
|
||||
"group_size": 128,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapped_config", [False, True])
|
||||
def test_qwen4_exp_mtp_override_sets_draft_config(
|
||||
wrapped_config: bool,
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
MarlinNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
LINEAR_ALGOS,
|
||||
@@ -224,6 +225,37 @@ def test_modelopt_mixed_precision_dispatches_every_linear_algo(algo):
|
||||
assert isinstance(method, ModelOptLinearMethod), (algo, type(method).__name__)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("algo", ["FP8_PB_WO", "FP8_BLOCK_SCALES"])
|
||||
def test_modelopt_mixed_precision_dispatches_block_fp8_moe(algo):
|
||||
config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
"quant_algo": "MIXED_PRECISION",
|
||||
"quantized_layers": {
|
||||
"mtp.layers.48.mlp.experts": {
|
||||
"quant_algo": algo,
|
||||
"group_size": 128,
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
layer = MagicMock(spec=RoutedExperts)
|
||||
expected = object()
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.layers.quantization.modelopt.Fp8MoEMethod",
|
||||
return_value=expected,
|
||||
) as method_cls:
|
||||
method = config.get_quant_method(layer, "mtp.layers.48.mlp.experts")
|
||||
|
||||
assert method is expected
|
||||
fp8_config, called_layer = method_cls.call_args.args
|
||||
assert called_layer is layer
|
||||
assert fp8_config.weight_block_size == [128, 128]
|
||||
assert fp8_config.activation_scheme == "dynamic"
|
||||
|
||||
|
||||
def test_modelopt_nvfp4_leaves_excluded_parallel_lm_head_unquantized():
|
||||
config = ModelOptNvFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
|
||||
Reference in New Issue
Block a user