[MoE] Encapsulate TRT-LLM BF16 weight layout handling (#57405)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -12,7 +12,6 @@ correctly handles this mismatch.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
@@ -465,30 +464,6 @@ class TestUnquantizedTrtLlmPrePadding:
|
||||
assert hidden == 64
|
||||
assert intermediate == expected
|
||||
|
||||
@pytest.mark.parametrize("is_act_and_mul", [True, False])
|
||||
def test_create_weights_zeroes_preallocated_padding(self, is_act_and_mul: bool):
|
||||
method = self._make_method(1344)
|
||||
if not is_act_and_mul:
|
||||
method.moe.activation = MoEActivation.RELU2_NO_MUL
|
||||
method.moe.intermediate_size_per_partition = 1408
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=2,
|
||||
hidden_size=64,
|
||||
intermediate_size_per_partition=1408,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
up_mult = 2 if is_act_and_mul else 1
|
||||
assert layer.w13_weight.shape == (2, up_mult * 1408, 64)
|
||||
assert layer.w2_weight.shape == (2, 64, 1408)
|
||||
assert torch.count_nonzero(layer.w13_weight[:, 1344:1408]) == 0
|
||||
if is_act_and_mul:
|
||||
assert torch.count_nonzero(layer.w13_weight[:, 2752:]) == 0
|
||||
assert torch.count_nonzero(layer.w2_weight[:, :, 1344:]) == 0
|
||||
|
||||
|
||||
class TestLoadWeightsExpertBias:
|
||||
"""Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize
|
||||
|
||||
Reference in New Issue
Block a user