[Model] Support llm-compressor Inkling NVFP4 weights (#49258)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -112,6 +112,30 @@ def test_inkling_mapper_maps_modelopt_exclusions() -> None:
|
||||
assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection", ["w13", "w2"])
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"suffix",
|
||||
[
|
||||
"input_global_scale",
|
||||
"weight_global_scale",
|
||||
"weight_packed",
|
||||
"weight_scale",
|
||||
],
|
||||
)
|
||||
def test_inkling_mapper_maps_compressed_tensors_expert_params(
|
||||
projection: str, suffix: str, nested: bool
|
||||
) -> None:
|
||||
projection_param = (
|
||||
f"{projection}_weight.{suffix}" if nested else f"{projection}_{suffix}"
|
||||
)
|
||||
source = f"model.llm.layers.2.mlp.experts.{projection_param}"
|
||||
|
||||
mapped = _TmlForCausalLMBase.hf_to_vllm_mapper.apply_list([source])
|
||||
|
||||
assert mapped == [f"model.layers.2.mlp.experts.{projection}_{suffix}"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
|
||||
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
experts = SimpleNamespace(
|
||||
@@ -132,6 +156,37 @@ def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
assert loaded == [f"experts.routed_experts.{projection}_input_scale"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection", ["w13", "w2"])
|
||||
@pytest.mark.parametrize("scale_kind", ["input", "weight"])
|
||||
def test_moe_loads_compressed_tensors_global_scale(
|
||||
projection: str, scale_kind: str
|
||||
) -> None:
|
||||
param = torch.nn.Parameter(torch.empty(3, 2 if projection == "w13" else 1))
|
||||
if projection == "w2":
|
||||
param = torch.nn.Parameter(param.squeeze(1))
|
||||
experts = SimpleNamespace(
|
||||
**{f"{projection}_{scale_kind}_global_scale": param},
|
||||
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=0)),
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
experts=SimpleNamespace(routed_experts=experts),
|
||||
_local_expert_slots=lambda: {0: 0, 1: 1, 2: 2},
|
||||
)
|
||||
checkpoint_scale = torch.tensor([[1.0], [2.0], [3.0]])
|
||||
|
||||
loaded = moe.InklingMoE.load_expert_weight(
|
||||
layer,
|
||||
f"experts.{projection}_{scale_kind}_global_scale",
|
||||
checkpoint_scale,
|
||||
)
|
||||
|
||||
expected = (
|
||||
checkpoint_scale.expand_as(param) if param.ndim == 2 else checkpoint_scale[:, 0]
|
||||
)
|
||||
torch.testing.assert_close(param, expected)
|
||||
assert loaded == [f"experts.routed_experts.{projection}_{scale_kind}_global_scale"]
|
||||
|
||||
|
||||
def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None:
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
|
||||
Reference in New Issue
Block a user