[Model] Support llm-compressor Inkling NVFP4 weights (#49258)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-23 20:06:01 -07:00
committed by GitHub
parent 80c9d5d5e0
commit d65acd83d8
3 changed files with 83 additions and 10 deletions
@@ -112,6 +112,30 @@ def test_inkling_mapper_maps_modelopt_exclusions() -> None:
assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts")
@pytest.mark.parametrize("projection", ["w13", "w2"])
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize(
"suffix",
[
"input_global_scale",
"weight_global_scale",
"weight_packed",
"weight_scale",
],
)
def test_inkling_mapper_maps_compressed_tensors_expert_params(
projection: str, suffix: str, nested: bool
) -> None:
projection_param = (
f"{projection}_weight.{suffix}" if nested else f"{projection}_{suffix}"
)
source = f"model.llm.layers.2.mlp.experts.{projection_param}"
mapped = _TmlForCausalLMBase.hf_to_vllm_mapper.apply_list([source])
assert mapped == [f"model.layers.2.mlp.experts.{projection}_{suffix}"]
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
experts = SimpleNamespace(
@@ -132,6 +156,37 @@ def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
assert loaded == [f"experts.routed_experts.{projection}_input_scale"]
@pytest.mark.parametrize("projection", ["w13", "w2"])
@pytest.mark.parametrize("scale_kind", ["input", "weight"])
def test_moe_loads_compressed_tensors_global_scale(
projection: str, scale_kind: str
) -> None:
param = torch.nn.Parameter(torch.empty(3, 2 if projection == "w13" else 1))
if projection == "w2":
param = torch.nn.Parameter(param.squeeze(1))
experts = SimpleNamespace(
**{f"{projection}_{scale_kind}_global_scale": param},
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=0)),
)
layer = SimpleNamespace(
experts=SimpleNamespace(routed_experts=experts),
_local_expert_slots=lambda: {0: 0, 1: 1, 2: 2},
)
checkpoint_scale = torch.tensor([[1.0], [2.0], [3.0]])
loaded = moe.InklingMoE.load_expert_weight(
layer,
f"experts.{projection}_{scale_kind}_global_scale",
checkpoint_scale,
)
expected = (
checkpoint_scale.expand_as(param) if param.ndim == 2 else checkpoint_scale[:, 0]
)
torch.testing.assert_close(param, expected)
assert loaded == [f"experts.routed_experts.{projection}_{scale_kind}_global_scale"]
def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None:
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
+10 -1
View File
@@ -378,11 +378,20 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA):
"language_model.lm_head.": "lm_head.",
},
orig_to_new_suffix={
# NVFP4 scale
# ModelOpt NVFP4 scales
".w13_weight.scale": ".w13_weight_scale",
".w13_weight.scale2": ".w13_weight_scale_2",
".w2_weight.scale": ".w2_weight_scale",
".w2_weight.scale2": ".w2_weight_scale_2",
# Compressed tensors NVFP4 parameters
".w13_weight.input_global_scale": ".w13_input_global_scale",
".w13_weight.weight_global_scale": ".w13_weight_global_scale",
".w13_weight.weight_packed": ".w13_weight_packed",
".w13_weight.weight_scale": ".w13_weight_scale",
".w2_weight.input_global_scale": ".w2_input_global_scale",
".w2_weight.weight_global_scale": ".w2_weight_global_scale",
".w2_weight.weight_packed": ".w2_weight_packed",
".w2_weight.weight_scale": ".w2_weight_scale",
},
)
packed_modules_mapping = {
+18 -9
View File
@@ -10,11 +10,10 @@ through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
activates every sink) and always bf16 (the checkpoint excludes every
``shared_experts`` from quantization).
NVFP4 routed experts reuse vLLM's ModelOpt NVFP4 fused-MoE method; excluded
(bf16) layers fall back to the unquantized method. The checkpoint's fused
stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` /
``.input_amax`` aux tensors) are translated to the standard per-expert loads
in :meth:`InklingMoE.load_expert_weight`.
NVFP4 routed experts reuse vLLM's fused-MoE methods; excluded (bf16) layers
fall back to the unquantized method. Checkpoint fused stacked tensors are
translated to the standard per-expert loads in
:meth:`InklingMoE.load_expert_weight`.
"""
from __future__ import annotations
@@ -575,11 +574,21 @@ class InklingMoE(nn.Module):
lids = [slots[g] for g in gids]
tp_rank = experts.moe_config.moe_parallel_config.tp_rank
if key.endswith("_scale_2"):
if key.endswith(("_scale_2", "_global_scale")):
# Per-expert scalars, vectorized over the local experts. The
# fused w13 param carries one slot per gate/up half.
vals = weight[gids].float().to(param.device)
param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals
# fused w13 params carry one slot per gate/up half. A single
# checkpoint value is shared by both halves.
vals = weight[gids].float().reshape(len(gids), -1)
target_width = math.prod(param.shape[1:])
if vals.shape[1] == 1:
vals = vals.expand(-1, target_width)
elif vals.shape[1] != target_width:
raise ValueError(
f"cannot load {tuple(weight.shape)} into {tuple(param.shape)}"
)
param.data[lids] = vals.reshape(len(gids), *param.shape[1:]).to(
param.device
)
elif key.startswith("w13"):
# Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the
# fused param layout is [w1(gate); w3(up)]. The TP-local rows form