[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -12,7 +12,7 @@ correctly handles this mismatch.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
|
||||
|
||||
class TestGetHiddenDim:
|
||||
@@ -20,45 +20,45 @@ class TestGetHiddenDim:
|
||||
|
||||
def test_2d_non_transposed_w2(self):
|
||||
# w2: shard_dim=1 (intermediate), hidden=0
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=2) == 0
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2) == 0
|
||||
|
||||
def test_2d_non_transposed_w13(self):
|
||||
# w1/w3: shard_dim=0 (intermediate), hidden=1
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=0, ndim=2) == 1
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2) == 1
|
||||
|
||||
def test_2d_transposed_w2(self):
|
||||
# transposed w2: shard_dim=0, hidden=1
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=0, ndim=2) == 1
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2) == 1
|
||||
|
||||
def test_2d_transposed_w13(self):
|
||||
# transposed w1/w3: shard_dim=1, hidden=0
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=2) == 0
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2) == 0
|
||||
|
||||
def test_3d_non_transposed_w2(self):
|
||||
# 3D w2: shard_dim=2, hidden=1
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=2, ndim=3) == 1
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=2, ndim=3) == 1
|
||||
|
||||
def test_3d_non_transposed_w13(self):
|
||||
# 3D w1/w3: shard_dim=1, hidden=2
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=3) == 2
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=3) == 2
|
||||
|
||||
def test_3d_transposed_w2(self):
|
||||
# transposed 3D w2: shard_dim=1, hidden=2
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=3) == 2
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=3) == 2
|
||||
|
||||
def test_3d_transposed_w13(self):
|
||||
# transposed 3D w1/w3: shard_dim=2, hidden=1
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=2, ndim=3) == 1
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=2, ndim=3) == 1
|
||||
|
||||
def test_1d_returns_zero(self):
|
||||
# 1D per-channel scales: always returns 0
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=0, ndim=1) == 0
|
||||
assert FusedMoE._get_hidden_dim(shard_dim=1, ndim=1) == 0
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=0, ndim=1) == 0
|
||||
assert RoutedExperts._get_hidden_dim(shard_dim=1, ndim=1) == 0
|
||||
|
||||
def test_invalid_shard_dim_raises(self):
|
||||
# shard_dim outside the data dimensions should raise
|
||||
with pytest.raises(ValueError, match="not a valid data dimension"):
|
||||
FusedMoE._get_hidden_dim(shard_dim=0, ndim=3)
|
||||
RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3)
|
||||
|
||||
|
||||
class TestNarrowExpertDataForPadding:
|
||||
@@ -67,7 +67,7 @@ class TestNarrowExpertDataForPadding:
|
||||
def test_no_narrowing_when_shapes_match(self):
|
||||
expert_data = torch.zeros(1024, 1024)
|
||||
loaded_weight = torch.randn(1024, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
assert result.shape == loaded_weight.shape
|
||||
@@ -77,7 +77,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# w2: (hidden_size, intermediate_size) - hidden_size padded at dim 0
|
||||
expert_data = torch.zeros(3072, 1024)
|
||||
loaded_weight = torch.randn(2688, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
assert result.shape == (2688, 1024)
|
||||
@@ -86,7 +86,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# w1/w3: (intermediate_size, hidden_size) - hidden_size padded at dim 1
|
||||
expert_data = torch.zeros(2048, 3072)
|
||||
loaded_weight = torch.randn(2048, 2688)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=1
|
||||
)
|
||||
assert result.shape == (2048, 2688)
|
||||
@@ -95,8 +95,8 @@ class TestNarrowExpertDataForPadding:
|
||||
# transposed w2: (intermediate_size, hidden_size) - hidden at dim 1
|
||||
expert_data = torch.zeros(1024, 3072)
|
||||
loaded_weight = torch.randn(1024, 2688)
|
||||
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=0, ndim=2)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2)
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=hidden_dim
|
||||
)
|
||||
assert result.shape == (1024, 2688)
|
||||
@@ -105,7 +105,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# 3D tensor for full_load path: w2 (num_experts, hidden_size, intermediate)
|
||||
expert_data = torch.zeros(8, 3072, 1024)
|
||||
loaded_weight = torch.randn(8, 2688, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=1
|
||||
)
|
||||
assert result.shape == (8, 2688, 1024)
|
||||
@@ -114,7 +114,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# 1D scale tensor: per-channel w2 scale (hidden_size,)
|
||||
expert_data = torch.zeros(3072)
|
||||
loaded_weight = torch.randn(2688)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
assert result.shape == (2688,)
|
||||
@@ -123,7 +123,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# 0-dim tensor should be a no-op
|
||||
expert_data = torch.zeros(3072)
|
||||
loaded_weight = torch.tensor(1.0)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
# ndim == 0, so no narrowing
|
||||
@@ -133,7 +133,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# Guard: don't narrow if loaded_weight is larger than expert_data
|
||||
expert_data = torch.zeros(2688, 1024)
|
||||
loaded_weight = torch.randn(3072, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
assert result.shape == (2688, 1024)
|
||||
@@ -143,7 +143,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# Negative hidden_dim should be a safe no-op (0 <= check)
|
||||
expert_data = torch.zeros(3072, 1024)
|
||||
loaded_weight = torch.randn(2688, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=-1
|
||||
)
|
||||
# -1 fails the 0 <= check, so no narrowing
|
||||
@@ -155,7 +155,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# even when other dimensions also differ
|
||||
expert_data = torch.zeros(3072, 2048)
|
||||
loaded_weight = torch.randn(2688, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
# Only dim 0 (hidden) should be narrowed; dim 1 stays at 2048
|
||||
@@ -165,7 +165,7 @@ class TestNarrowExpertDataForPadding:
|
||||
# Verify narrowing returns a view (writes go to original tensor)
|
||||
expert_data = torch.zeros(3072, 1024)
|
||||
loaded_weight = torch.randn(2688, 1024)
|
||||
result = FusedMoE._narrow_expert_data_for_padding(
|
||||
result = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data, loaded_weight, hidden_dim=0
|
||||
)
|
||||
result.copy_(loaded_weight)
|
||||
@@ -188,8 +188,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
loaded_weight = torch.randn(original_hidden, intermediate)
|
||||
|
||||
# w2 non-transposed: shard_dim=1, hidden_dim=0
|
||||
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=1, ndim=2)
|
||||
expert_data = FusedMoE._narrow_expert_data_for_padding(
|
||||
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2)
|
||||
expert_data = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data_full, loaded_weight, hidden_dim=hidden_dim
|
||||
)
|
||||
expert_data.copy_(loaded_weight)
|
||||
@@ -211,8 +211,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
loaded_weight = torch.randn(intermediate, original_hidden)
|
||||
|
||||
# w1 non-transposed: shard_dim=0, hidden_dim=1
|
||||
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=0, ndim=2)
|
||||
expert_data = FusedMoE._narrow_expert_data_for_padding(
|
||||
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2)
|
||||
expert_data = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data_full, loaded_weight, hidden_dim=hidden_dim
|
||||
)
|
||||
expert_data.copy_(loaded_weight)
|
||||
@@ -233,8 +233,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
expert_data_full = torch.zeros(intermediate, padded_hidden)
|
||||
loaded_weight = torch.randn(intermediate, original_hidden)
|
||||
|
||||
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=0, ndim=2)
|
||||
expert_data = FusedMoE._narrow_expert_data_for_padding(
|
||||
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=0, ndim=2)
|
||||
expert_data = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data_full, loaded_weight, hidden_dim=hidden_dim
|
||||
)
|
||||
expert_data.copy_(loaded_weight)
|
||||
@@ -249,8 +249,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
expert_data_full = torch.zeros(hidden, intermediate)
|
||||
loaded_weight = torch.randn(hidden, intermediate)
|
||||
|
||||
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=1, ndim=2)
|
||||
expert_data = FusedMoE._narrow_expert_data_for_padding(
|
||||
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=1, ndim=2)
|
||||
expert_data = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data_full, loaded_weight, hidden_dim=hidden_dim
|
||||
)
|
||||
expert_data.copy_(loaded_weight)
|
||||
@@ -270,8 +270,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
loaded_weight = torch.randn(original_hidden, original_intermediate)
|
||||
|
||||
shard_dim = 1
|
||||
hidden_dim = FusedMoE._get_hidden_dim(shard_dim=shard_dim, ndim=2)
|
||||
expert_data = FusedMoE._narrow_expert_data_for_padding(
|
||||
hidden_dim = RoutedExperts._get_hidden_dim(shard_dim=shard_dim, ndim=2)
|
||||
expert_data = RoutedExperts._narrow_expert_data_for_padding(
|
||||
expert_data_full,
|
||||
loaded_weight,
|
||||
hidden_dim=hidden_dim,
|
||||
@@ -307,8 +307,8 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
|
||||
loaded_weight = torch.randint(0, 255, (original_packed, 1), dtype=torch.uint8)
|
||||
|
||||
# Minimal FusedMoE mock so weight_loader reaches the BnB path.
|
||||
moe = MagicMock(spec=FusedMoE)
|
||||
# Minimal RoutedExperts mock so weight_loader reaches the BnB path.
|
||||
moe = MagicMock(spec=RoutedExperts)
|
||||
moe.quant_config = None
|
||||
moe.quant_method = MagicMock()
|
||||
moe.quant_method.__class__.__name__ = "BitsAndBytesMethod"
|
||||
@@ -317,7 +317,7 @@ class TestWeightLoadingWithPaddedHiddenSize:
|
||||
|
||||
# Call the real weight_loader (unbound) with our mock as self.
|
||||
with pytest.raises(ValueError, match="BitsAndBytes"):
|
||||
FusedMoE.weight_loader(
|
||||
RoutedExperts.weight_loader(
|
||||
moe,
|
||||
param,
|
||||
loaded_weight,
|
||||
|
||||
Reference in New Issue
Block a user