[Model] Use standard ModelOpt config for Inkling NVFP4 (#48990)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -131,6 +131,23 @@ def test_head_size_falls_back_when_head_dim_is_zero():
|
||||
assert convertor.get_head_size() == 128
|
||||
|
||||
|
||||
def test_legacy_modelopt_config_without_producer_is_normalized():
|
||||
quantization_config = {
|
||||
"quantization": {
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
"kv_cache_quant_algo": None,
|
||||
"exclude_modules": [],
|
||||
"modelopt_quant_config": {"quant_cfg": {}},
|
||||
}
|
||||
}
|
||||
hf_config = PretrainedConfig(quantization_config=quantization_config)
|
||||
|
||||
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
|
||||
|
||||
assert convertor.get_quantization_config()["quant_method"] == "modelopt_fp4"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", BASE_MODELS_TO_TEST)
|
||||
def test_base_model_arch_config(model: str):
|
||||
"""Test model architecture config for base models."""
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
|
||||
from vllm.models.inkling.nvidia import moe
|
||||
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
|
||||
from vllm.platforms import current_platform
|
||||
@@ -87,6 +88,30 @@ def test_custom_embedding_is_not_a_lora_target() -> None:
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
def test_inkling_mapper_maps_modelopt_exclusions() -> None:
|
||||
quant_config = ModelOptNvFp4Config.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
"kv_cache_quant_algo": None,
|
||||
"exclude_modules": [
|
||||
"model.llm.layers.2.mlp.experts",
|
||||
"model.llm.layers.2.mlp.shared_experts",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
quant_config.apply_vllm_mapper(
|
||||
_TmlForCausalLMBase.hf_to_vllm_mapper.get_unstacked_mapper()
|
||||
)
|
||||
|
||||
assert quant_config.is_layer_excluded("model.layers.2.mlp.experts")
|
||||
assert quant_config.is_layer_excluded("model.layers.2.mlp.shared_experts")
|
||||
assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
|
||||
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
experts = SimpleNamespace(
|
||||
|
||||
Reference in New Issue
Block a user