[Model] Support llm-compressor Inkling NVFP4 weights (#49258)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -112,6 +112,30 @@ def test_inkling_mapper_maps_modelopt_exclusions() -> None:
|
||||
assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection", ["w13", "w2"])
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"suffix",
|
||||
[
|
||||
"input_global_scale",
|
||||
"weight_global_scale",
|
||||
"weight_packed",
|
||||
"weight_scale",
|
||||
],
|
||||
)
|
||||
def test_inkling_mapper_maps_compressed_tensors_expert_params(
|
||||
projection: str, suffix: str, nested: bool
|
||||
) -> None:
|
||||
projection_param = (
|
||||
f"{projection}_weight.{suffix}" if nested else f"{projection}_{suffix}"
|
||||
)
|
||||
source = f"model.llm.layers.2.mlp.experts.{projection_param}"
|
||||
|
||||
mapped = _TmlForCausalLMBase.hf_to_vllm_mapper.apply_list([source])
|
||||
|
||||
assert mapped == [f"model.layers.2.mlp.experts.{projection}_{suffix}"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
|
||||
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
experts = SimpleNamespace(
|
||||
@@ -132,6 +156,37 @@ def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
assert loaded == [f"experts.routed_experts.{projection}_input_scale"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection", ["w13", "w2"])
|
||||
@pytest.mark.parametrize("scale_kind", ["input", "weight"])
|
||||
def test_moe_loads_compressed_tensors_global_scale(
|
||||
projection: str, scale_kind: str
|
||||
) -> None:
|
||||
param = torch.nn.Parameter(torch.empty(3, 2 if projection == "w13" else 1))
|
||||
if projection == "w2":
|
||||
param = torch.nn.Parameter(param.squeeze(1))
|
||||
experts = SimpleNamespace(
|
||||
**{f"{projection}_{scale_kind}_global_scale": param},
|
||||
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=0)),
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
experts=SimpleNamespace(routed_experts=experts),
|
||||
_local_expert_slots=lambda: {0: 0, 1: 1, 2: 2},
|
||||
)
|
||||
checkpoint_scale = torch.tensor([[1.0], [2.0], [3.0]])
|
||||
|
||||
loaded = moe.InklingMoE.load_expert_weight(
|
||||
layer,
|
||||
f"experts.{projection}_{scale_kind}_global_scale",
|
||||
checkpoint_scale,
|
||||
)
|
||||
|
||||
expected = (
|
||||
checkpoint_scale.expand_as(param) if param.ndim == 2 else checkpoint_scale[:, 0]
|
||||
)
|
||||
torch.testing.assert_close(param, expected)
|
||||
assert loaded == [f"experts.routed_experts.{projection}_{scale_kind}_global_scale"]
|
||||
|
||||
|
||||
def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None:
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
|
||||
@@ -378,11 +378,20 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA):
|
||||
"language_model.lm_head.": "lm_head.",
|
||||
},
|
||||
orig_to_new_suffix={
|
||||
# NVFP4 scale
|
||||
# ModelOpt NVFP4 scales
|
||||
".w13_weight.scale": ".w13_weight_scale",
|
||||
".w13_weight.scale2": ".w13_weight_scale_2",
|
||||
".w2_weight.scale": ".w2_weight_scale",
|
||||
".w2_weight.scale2": ".w2_weight_scale_2",
|
||||
# Compressed tensors NVFP4 parameters
|
||||
".w13_weight.input_global_scale": ".w13_input_global_scale",
|
||||
".w13_weight.weight_global_scale": ".w13_weight_global_scale",
|
||||
".w13_weight.weight_packed": ".w13_weight_packed",
|
||||
".w13_weight.weight_scale": ".w13_weight_scale",
|
||||
".w2_weight.input_global_scale": ".w2_input_global_scale",
|
||||
".w2_weight.weight_global_scale": ".w2_weight_global_scale",
|
||||
".w2_weight.weight_packed": ".w2_weight_packed",
|
||||
".w2_weight.weight_scale": ".w2_weight_scale",
|
||||
},
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -10,11 +10,10 @@ through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
|
||||
activates every sink) and always bf16 (the checkpoint excludes every
|
||||
``shared_experts`` from quantization).
|
||||
|
||||
NVFP4 routed experts reuse vLLM's ModelOpt NVFP4 fused-MoE method; excluded
|
||||
(bf16) layers fall back to the unquantized method. The checkpoint's fused
|
||||
stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` /
|
||||
``.input_amax`` aux tensors) are translated to the standard per-expert loads
|
||||
in :meth:`InklingMoE.load_expert_weight`.
|
||||
NVFP4 routed experts reuse vLLM's fused-MoE methods; excluded (bf16) layers
|
||||
fall back to the unquantized method. Checkpoint fused stacked tensors are
|
||||
translated to the standard per-expert loads in
|
||||
:meth:`InklingMoE.load_expert_weight`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -575,11 +574,21 @@ class InklingMoE(nn.Module):
|
||||
lids = [slots[g] for g in gids]
|
||||
tp_rank = experts.moe_config.moe_parallel_config.tp_rank
|
||||
|
||||
if key.endswith("_scale_2"):
|
||||
if key.endswith(("_scale_2", "_global_scale")):
|
||||
# Per-expert scalars, vectorized over the local experts. The
|
||||
# fused w13 param carries one slot per gate/up half.
|
||||
vals = weight[gids].float().to(param.device)
|
||||
param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals
|
||||
# fused w13 params carry one slot per gate/up half. A single
|
||||
# checkpoint value is shared by both halves.
|
||||
vals = weight[gids].float().reshape(len(gids), -1)
|
||||
target_width = math.prod(param.shape[1:])
|
||||
if vals.shape[1] == 1:
|
||||
vals = vals.expand(-1, target_width)
|
||||
elif vals.shape[1] != target_width:
|
||||
raise ValueError(
|
||||
f"cannot load {tuple(weight.shape)} into {tuple(param.shape)}"
|
||||
)
|
||||
param.data[lids] = vals.reshape(len(gids), *param.shape[1:]).to(
|
||||
param.device
|
||||
)
|
||||
elif key.startswith("w13"):
|
||||
# Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the
|
||||
# fused param layout is [w1(gate); w3(up)]. The TP-local rows form
|
||||
|
||||
Reference in New Issue
Block a user