[MoE] Generalize masked activation for padded layouts (#51217)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a232e29e9d
commit
82b7d49a6e
@@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import (
|
||||
SwigluStepAndMul,
|
||||
swiglustep_and_mul_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16, torch.float]
|
||||
@@ -34,6 +40,100 @@ CUDA_DEVICES = [
|
||||
]
|
||||
|
||||
|
||||
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
|
||||
input = torch.empty(1, 1, 2)
|
||||
output = torch.empty(1, 1, 1)
|
||||
valid_token_counts = torch.ones(1, dtype=torch.int32)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="relu2"):
|
||||
apply_moe_activation(
|
||||
MoEActivation.RELU2,
|
||||
output,
|
||||
input,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
|
||||
def test_moe_silu_clamp_uses_native_xpu_fallback(
|
||||
default_vllm_config, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
clamp_limit = 3.0
|
||||
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
apply_moe_activation(
|
||||
MoEActivation.SILU,
|
||||
output,
|
||||
input,
|
||||
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
|
||||
)
|
||||
|
||||
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
|
||||
input
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _assert_masked_moe_activation(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
mask_layout: str,
|
||||
d: int,
|
||||
max_num_tokens: int,
|
||||
) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts = 4
|
||||
input_dim = 2 * d if activation.is_gated else d
|
||||
if mask_layout == "flat":
|
||||
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
|
||||
valid_token_counts = torch.tensor(
|
||||
[max_num_tokens // 2], dtype=torch.int32, device=device
|
||||
)
|
||||
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
|
||||
else:
|
||||
input = torch.randn(
|
||||
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
|
||||
)
|
||||
valid_token_counts = torch.tensor(
|
||||
[0, 1, max_num_tokens // 2, max_num_tokens],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
activation_config=activation_config,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
batched_input = input.view(-1, max_num_tokens, input_dim)
|
||||
batched_output = output.view(-1, max_num_tokens, d)
|
||||
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
|
||||
if num_tokens:
|
||||
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
expected,
|
||||
batched_input[expert, :num_tokens].clone(),
|
||||
activation_config=activation_config,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
batched_output[expert, :num_tokens],
|
||||
expected,
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
@@ -252,6 +352,114 @@ def test_masked_situ_and_mul(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config"),
|
||||
[
|
||||
(MoEActivation.SILU, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0),
|
||||
),
|
||||
(MoEActivation.GELU, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
),
|
||||
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
|
||||
(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
),
|
||||
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
|
||||
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
|
||||
],
|
||||
ids=[
|
||||
"silu",
|
||||
"silu_clamp",
|
||||
"gelu",
|
||||
"gelu_tanh",
|
||||
"situ",
|
||||
"swigluoai",
|
||||
"swigluoai_uninterleave",
|
||||
"swiglustep",
|
||||
"silu_no_mul",
|
||||
"gelu_no_mul",
|
||||
"gelu_tanh_no_mul",
|
||||
"relu2_no_mul",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_dispatch(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.bfloat16,
|
||||
mask_layout="batched_experts",
|
||||
d=513,
|
||||
max_num_tokens=7,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config", "mask_layout"),
|
||||
[
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(),
|
||||
"flat",
|
||||
id="flat",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
"batched_experts",
|
||||
id="batched-experts",
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_grid_stride(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
mask_layout: str,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.half,
|
||||
mask_layout=mask_layout,
|
||||
d=513,
|
||||
max_num_tokens=67,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
|
||||
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
|
||||
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
|
||||
opcheck(
|
||||
torch.ops._C.masked_moe_activation,
|
||||
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
from tests.kernels.moe.utils import (
|
||||
batched_moe,
|
||||
make_dummy_moe_config,
|
||||
make_quantized_test_activations,
|
||||
make_test_weights,
|
||||
naive_batched_moe,
|
||||
@@ -16,7 +17,13 @@ from tests.kernels.quant_utils import native_batched_masked_quant_matmul
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
invoke_moe_batched_triton_kernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -496,11 +503,146 @@ def test_batched_mm_td_zero_expert_tokens():
|
||||
|
||||
# BatchedTritonExperts device enablement (XPU)
|
||||
def test_batched_triton_experts_supports_current_device():
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
MoEActivation.SITU,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
],
|
||||
)
|
||||
def test_batched_triton_supports_masked_activations(
|
||||
monkeypatch, activation: MoEActivation
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
|
||||
assert BatchedTritonExperts._supports_activation(activation)
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
assert not BatchedTritonExperts._supports_activation(activation)
|
||||
|
||||
|
||||
def _torch_experts_with_shared_activation(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = a.shape
|
||||
topk = topk_ids.shape[1]
|
||||
routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim)
|
||||
routed_experts = topk_ids.reshape(-1)
|
||||
routed_output = torch.zeros_like(routed_input)
|
||||
|
||||
for expert in range(w1.shape[0]):
|
||||
expert_mask = routed_experts == expert
|
||||
if not expert_mask.any():
|
||||
continue
|
||||
projected = routed_input[expert_mask] @ w1[expert].T
|
||||
activated = torch.empty(
|
||||
projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device
|
||||
)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
activated,
|
||||
projected,
|
||||
activation_config=activation_config,
|
||||
)
|
||||
routed_output[expert_mask] = activated @ w2[expert].T
|
||||
|
||||
return (
|
||||
(
|
||||
routed_output.view(num_tokens, topk, hidden_dim).float()
|
||||
* topk_weights[..., None]
|
||||
)
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
|
||||
_MASKED_BATCHED_ACTIVATION_CASES = [
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
id="situ",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
id="swigluoai-uninterleave",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
ApplyMoEActivationConfig(),
|
||||
id="swiglustep",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_batched_experts_masked_activation_end_to_end(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
):
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("Masked activations are not enabled on XPU")
|
||||
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
device = current_platform.device_type
|
||||
init_workspace_manager(torch.device(f"{device}:0"))
|
||||
m, n, k, e, topk = 16, 128, 128, 8, 2
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
in_dtype=a.dtype,
|
||||
activation=activation,
|
||||
)
|
||||
moe_config.swiglu_limit = activation_config.clamp_limit
|
||||
moe_config.swiglu_alpha = activation_config.alpha
|
||||
moe_config.swiglu_beta = activation_config.beta
|
||||
moe_config.activation_situ_beta = activation_config.activation_situ_beta
|
||||
moe_config.activation_situ_linear_beta = (
|
||||
activation_config.activation_situ_linear_beta
|
||||
)
|
||||
|
||||
routing_weights = torch.softmax(score.float(), dim=-1)
|
||||
topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
baseline_output = _torch_experts_with_shared_activation(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation,
|
||||
activation_config,
|
||||
)
|
||||
triton_output = batched_moe(
|
||||
a, w1, w2, topk_weights, topk_ids, moe_config=moe_config
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
|
||||
|
||||
@@ -1377,6 +1377,102 @@ def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivati
|
||||
) == apply_moe_activation_supported(activation)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_override", "use_ep", "expected"),
|
||||
[
|
||||
(None, False, "indexed"),
|
||||
(None, True, "grouped_contiguous"),
|
||||
("auto", True, "grouped_contiguous"),
|
||||
("indexed", True, "indexed"),
|
||||
("grouped", False, "grouped_contiguous"),
|
||||
],
|
||||
)
|
||||
def test_humming_selects_gemm_from_parallelism_and_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
env_override: str | None,
|
||||
use_ep: bool,
|
||||
expected: str,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
get_humming_moe_gemm_type,
|
||||
)
|
||||
|
||||
if env_override is None:
|
||||
monkeypatch.delenv("VLLM_HUMMING_MOE_GEMM_TYPE", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_HUMMING_MOE_GEMM_TYPE", env_override)
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_parallel_config.use_ep = use_ep
|
||||
|
||||
assert get_humming_moe_gemm_type(moe_config) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dp_token_counts", "topk_shape", "expected"),
|
||||
[
|
||||
pytest.param(None, (7, 3), 21, id="runtime-layout"),
|
||||
pytest.param([3, 5], (48, 1), 48, id="expanded-dp-layout"),
|
||||
],
|
||||
)
|
||||
def test_humming_global_valid_shape_m(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dp_token_counts: list[int] | None,
|
||||
topk_shape: tuple[int, int],
|
||||
expected: int,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
|
||||
dp_metadata = (
|
||||
None
|
||||
if dp_token_counts is None
|
||||
else SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor(dp_token_counts, dtype=torch.int32)
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
humming,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(dp_metadata=dp_metadata),
|
||||
)
|
||||
experts = SimpleNamespace(moe_config=make_dummy_moe_config(experts_per_token=6))
|
||||
topk_ids = torch.empty(topk_shape, dtype=torch.int64)
|
||||
|
||||
result = humming.HummingExpertsBase.get_global_valid_shape_m(experts, topk_ids)
|
||||
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_humming_permute_scratch_is_keyed_by_runtime_topk(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
|
||||
scratch_topk6 = Mock()
|
||||
scratch_topk1 = Mock()
|
||||
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
|
||||
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
|
||||
monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type)
|
||||
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config)
|
||||
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
|
||||
assert scratch_type.call_count == 2
|
||||
first_call, second_call = scratch_type.call_args_list
|
||||
assert first_call.kwargs["max_num_tokens"] == 1024
|
||||
assert first_call.kwargs["topk"] == 6
|
||||
assert second_call.kwargs["max_num_tokens"] == 6144
|
||||
assert second_call.kwargs["topk"] == 1
|
||||
|
||||
|
||||
def test_humming_delegates_to_instance_activation():
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
@@ -1408,10 +1504,173 @@ def test_humming_delegates_to_instance_activation():
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
input=input,
|
||||
output=output,
|
||||
valid_rows=None,
|
||||
)
|
||||
|
||||
|
||||
def test_humming_grouped_apply_forwards_valid_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
HummingGroupedExperts,
|
||||
)
|
||||
|
||||
activation_func = Mock()
|
||||
buffers = {
|
||||
"gate_up_output": torch.empty(6, 4),
|
||||
"activation_output": torch.empty(6, 2),
|
||||
"down_output": torch.empty(6, 4),
|
||||
}
|
||||
experts = SimpleNamespace(
|
||||
activation=activation_func,
|
||||
num_experts=2,
|
||||
estimate_local_valid_shape_m=lambda _: 6,
|
||||
prepare_buffers=lambda *_: buffers,
|
||||
_get_permute_scratch=lambda _: None,
|
||||
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
|
||||
inputs,
|
||||
input_scale,
|
||||
),
|
||||
humming_forward=Mock(),
|
||||
compute_config_str="",
|
||||
w13_tuning_config_str="",
|
||||
w2_tuning_config_str="",
|
||||
)
|
||||
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
|
||||
|
||||
hidden_states = torch.empty(3, 4)
|
||||
output = torch.empty_like(hidden_states)
|
||||
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
|
||||
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
|
||||
expected_counts = torch.tensor([4], dtype=torch.int32)
|
||||
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
|
||||
monkeypatch.setattr(
|
||||
humming,
|
||||
"moe_permute",
|
||||
lambda **_: (
|
||||
torch.empty(6, 4),
|
||||
None,
|
||||
expert_offsets,
|
||||
torch.arange(6),
|
||||
None,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
|
||||
|
||||
HummingGroupedExperts.apply(
|
||||
experts,
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=torch.empty(0),
|
||||
w2=torch.empty(0),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SITU,
|
||||
global_num_experts=2,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
activation_func.assert_called_once()
|
||||
call_kwargs = activation_func.call_args.kwargs
|
||||
assert call_kwargs["activation"] == MoEActivation.SITU
|
||||
assert call_kwargs["input"].shape == (6, 4)
|
||||
assert call_kwargs["output"].shape == (6, 2)
|
||||
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
|
||||
|
||||
|
||||
def test_batched_marlin_activation_uses_expert_token_counts(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.marlin_moe as marlin
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
BatchedMarlinExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
ExpertTokensMetadata,
|
||||
)
|
||||
|
||||
activation_func = Mock()
|
||||
experts = SimpleNamespace(
|
||||
activation=activation_func,
|
||||
w1_bias=None,
|
||||
w2_bias=None,
|
||||
w1_scale=None,
|
||||
w2_scale=None,
|
||||
quant_type_id=0,
|
||||
a1_gscale=None,
|
||||
a2_gscale=None,
|
||||
w13_g_idx=None,
|
||||
w2_g_idx=None,
|
||||
w13_g_idx_sort_indices=None,
|
||||
w2_g_idx_sort_indices=None,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
input_dtype=None,
|
||||
is_k_full=True,
|
||||
activation_config=ApplyMoEActivationConfig(
|
||||
clamp_limit=3.0, alpha=1.3, beta=0.5
|
||||
),
|
||||
)
|
||||
hidden_states = torch.empty(2, 3, 4)
|
||||
expert_num_tokens = torch.tensor([1, 2], dtype=torch.int32)
|
||||
topk_ids = torch.zeros((2, 3), dtype=torch.int64)
|
||||
expert_map = torch.arange(2, dtype=torch.int32)
|
||||
|
||||
def fake_batched_fused_marlin_moe(**kwargs):
|
||||
kwargs["activation_func"](
|
||||
kwargs["activation"],
|
||||
torch.empty(6, 4),
|
||||
torch.empty(6, 8),
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
return kwargs["output"]
|
||||
|
||||
monkeypatch.setattr(
|
||||
marlin, "batched_fused_marlin_moe", fake_batched_fused_marlin_moe
|
||||
)
|
||||
output = torch.empty_like(hidden_states)
|
||||
|
||||
BatchedMarlinExperts.apply(
|
||||
experts,
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=torch.empty(0),
|
||||
w2=torch.empty(0),
|
||||
topk_weights=torch.ones_like(topk_ids, dtype=torch.float32),
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
global_num_experts=2,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=ExpertTokensMetadata(expert_num_tokens, None),
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
activation_func.assert_called_once()
|
||||
call = activation_func.call_args
|
||||
assert call.args[0] == MoEActivation.SWIGLUOAI_UNINTERLEAVE
|
||||
assert call.args[1].shape == (2, 3, 4)
|
||||
assert call.args[2].shape == (2, 3, 8)
|
||||
torch.testing.assert_close(call.kwargs["valid_token_counts"], expert_num_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -286,3 +286,39 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
|
||||
permuted_idx_1.untyped_storage().data_ptr()
|
||||
== scratch.permuted_idx.untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view(
|
||||
5, 16
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
|
||||
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=5,
|
||||
topk=1,
|
||||
num_experts=4,
|
||||
num_local_experts=2,
|
||||
device=hidden_states.device,
|
||||
hidden_size=16,
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
permuted, _, expert_offsets, _, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=4,
|
||||
n_local_expert=2,
|
||||
expert_map=expert_map,
|
||||
scratch=scratch,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
expert_offsets,
|
||||
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
|
||||
|
||||
@@ -5,7 +5,7 @@ quant kernel (``situ_and_mul_quant``).
|
||||
|
||||
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
|
||||
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
|
||||
the masked (``valid_rows``) contiguous layout, and compare per valid row.
|
||||
the masked contiguous layout, and compare per valid row.
|
||||
|
||||
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
|
||||
which torch cannot reproduce bit-exactly. It is therefore validated against the
|
||||
@@ -40,7 +40,7 @@ SITU_LINEAR_BETA = 25.0
|
||||
SITU_D = 3072
|
||||
GROUP_SIZE = 128
|
||||
|
||||
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
|
||||
# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows
|
||||
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
|
||||
CASES = [
|
||||
(1, None),
|
||||
@@ -111,7 +111,7 @@ def test_situ_and_mul_quant_pipelined(
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
valid_rows = (
|
||||
num_valid_tokens = (
|
||||
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
|
||||
)
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_situ_and_mul_quant_pipelined(
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
valid_rows=valid_rows,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
)
|
||||
|
||||
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
|
||||
@@ -157,7 +157,7 @@ def test_situ_and_mul_quant_pipelined(
|
||||
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
|
||||
@torch.inference_mode()
|
||||
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
|
||||
"""The kernel expands num_valid_tokens by topk to bound rows."""
|
||||
set_random_seed(7)
|
||||
d = SITU_D
|
||||
num_tokens = tokens * topk + 37 # padding tail past the valid rows
|
||||
@@ -168,7 +168,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
out_sentinel = out.clone()
|
||||
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||
num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
situ_and_mul_quant(
|
||||
out,
|
||||
@@ -177,7 +177,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
|
||||
beta=SITU_BETA,
|
||||
linear_beta=SITU_LINEAR_BETA,
|
||||
group_size=GROUP_SIZE,
|
||||
valid_rows=valid_rows,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
|
||||
|
||||
@@ -128,6 +128,7 @@ def batched_moe(
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
moe_config: FusedMoEConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
max_num_tokens = round_up(a.shape[0], 64)
|
||||
|
||||
@@ -141,7 +142,8 @@ def batched_moe(
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
if moe_config is None:
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
fused_experts = FusedMoEKernel(
|
||||
BatchedPrepareAndFinalize(
|
||||
|
||||
Reference in New Issue
Block a user