From d65acd83d87b1ad8d13eaa9452510488de642543 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 23 Jul 2026 23:06:01 -0400 Subject: [PATCH] [Model] Support llm-compressor Inkling NVFP4 weights (#49258) Signed-off-by: mgoin --- .../models/inkling/test_moe_weight_layout.py | 55 +++++++++++++++++++ vllm/models/inkling/nvidia/model.py | 11 +++- vllm/models/inkling/nvidia/moe.py | 27 ++++++--- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/tests/models/inkling/test_moe_weight_layout.py b/tests/models/inkling/test_moe_weight_layout.py index 53a37e3e60..a7307e167a 100644 --- a/tests/models/inkling/test_moe_weight_layout.py +++ b/tests/models/inkling/test_moe_weight_layout.py @@ -112,6 +112,30 @@ def test_inkling_mapper_maps_modelopt_exclusions() -> None: assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts") +@pytest.mark.parametrize("projection", ["w13", "w2"]) +@pytest.mark.parametrize("nested", [False, True]) +@pytest.mark.parametrize( + "suffix", + [ + "input_global_scale", + "weight_global_scale", + "weight_packed", + "weight_scale", + ], +) +def test_inkling_mapper_maps_compressed_tensors_expert_params( + projection: str, suffix: str, nested: bool +) -> None: + projection_param = ( + f"{projection}_weight.{suffix}" if nested else f"{projection}_{suffix}" + ) + source = f"model.llm.layers.2.mlp.experts.{projection_param}" + + mapped = _TmlForCausalLMBase.hf_to_vllm_mapper.apply_list([source]) + + assert mapped == [f"model.layers.2.mlp.experts.{projection}_{suffix}"] + + @pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)]) def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None: experts = SimpleNamespace( @@ -132,6 +156,37 @@ def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None: assert loaded == [f"experts.routed_experts.{projection}_input_scale"] +@pytest.mark.parametrize("projection", ["w13", "w2"]) +@pytest.mark.parametrize("scale_kind", ["input", "weight"]) +def test_moe_loads_compressed_tensors_global_scale( + projection: str, scale_kind: str +) -> None: + param = torch.nn.Parameter(torch.empty(3, 2 if projection == "w13" else 1)) + if projection == "w2": + param = torch.nn.Parameter(param.squeeze(1)) + experts = SimpleNamespace( + **{f"{projection}_{scale_kind}_global_scale": param}, + moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=0)), + ) + layer = SimpleNamespace( + experts=SimpleNamespace(routed_experts=experts), + _local_expert_slots=lambda: {0: 0, 1: 1, 2: 2}, + ) + checkpoint_scale = torch.tensor([[1.0], [2.0], [3.0]]) + + loaded = moe.InklingMoE.load_expert_weight( + layer, + f"experts.{projection}_{scale_kind}_global_scale", + checkpoint_scale, + ) + + expected = ( + checkpoint_scale.expand_as(param) if param.ndim == 2 else checkpoint_scale[:, 0] + ) + torch.testing.assert_close(param, expected) + assert loaded == [f"experts.routed_experts.{projection}_{scale_kind}_global_scale"] + + def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None: monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2) monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1) diff --git a/vllm/models/inkling/nvidia/model.py b/vllm/models/inkling/nvidia/model.py index a8506d9232..04ba0ee085 100644 --- a/vllm/models/inkling/nvidia/model.py +++ b/vllm/models/inkling/nvidia/model.py @@ -378,11 +378,20 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA): "language_model.lm_head.": "lm_head.", }, orig_to_new_suffix={ - # NVFP4 scale + # ModelOpt NVFP4 scales ".w13_weight.scale": ".w13_weight_scale", ".w13_weight.scale2": ".w13_weight_scale_2", ".w2_weight.scale": ".w2_weight_scale", ".w2_weight.scale2": ".w2_weight_scale_2", + # Compressed tensors NVFP4 parameters + ".w13_weight.input_global_scale": ".w13_input_global_scale", + ".w13_weight.weight_global_scale": ".w13_weight_global_scale", + ".w13_weight.weight_packed": ".w13_weight_packed", + ".w13_weight.weight_scale": ".w13_weight_scale", + ".w2_weight.input_global_scale": ".w2_input_global_scale", + ".w2_weight.weight_global_scale": ".w2_weight_global_scale", + ".w2_weight.weight_packed": ".w2_weight_packed", + ".w2_weight.weight_scale": ".w2_weight_scale", }, ) packed_modules_mapping = { diff --git a/vllm/models/inkling/nvidia/moe.py b/vllm/models/inkling/nvidia/moe.py index 996369f819..32f7489a4c 100644 --- a/vllm/models/inkling/nvidia/moe.py +++ b/vllm/models/inkling/nvidia/moe.py @@ -10,11 +10,10 @@ through vLLM's FusedMoE (which handles TP/EP); the sink experts run in activates every sink) and always bf16 (the checkpoint excludes every ``shared_experts`` from quantization). -NVFP4 routed experts reuse vLLM's ModelOpt NVFP4 fused-MoE method; excluded -(bf16) layers fall back to the unquantized method. The checkpoint's fused -stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` / -``.input_amax`` aux tensors) are translated to the standard per-expert loads -in :meth:`InklingMoE.load_expert_weight`. +NVFP4 routed experts reuse vLLM's fused-MoE methods; excluded (bf16) layers +fall back to the unquantized method. Checkpoint fused stacked tensors are +translated to the standard per-expert loads in +:meth:`InklingMoE.load_expert_weight`. """ from __future__ import annotations @@ -575,11 +574,21 @@ class InklingMoE(nn.Module): lids = [slots[g] for g in gids] tp_rank = experts.moe_config.moe_parallel_config.tp_rank - if key.endswith("_scale_2"): + if key.endswith(("_scale_2", "_global_scale")): # Per-expert scalars, vectorized over the local experts. The - # fused w13 param carries one slot per gate/up half. - vals = weight[gids].float().to(param.device) - param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals + # fused w13 params carry one slot per gate/up half. A single + # checkpoint value is shared by both halves. + vals = weight[gids].float().reshape(len(gids), -1) + target_width = math.prod(param.shape[1:]) + if vals.shape[1] == 1: + vals = vals.expand(-1, target_width) + elif vals.shape[1] != target_width: + raise ValueError( + f"cannot load {tuple(weight.shape)} into {tuple(param.shape)}" + ) + param.data[lids] = vals.reshape(len(gids), *param.shape[1:]).to( + param.device + ) elif key.startswith("w13"): # Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the # fused param layout is [w1(gate); w3(up)]. The TP-local rows form