[MoE] Generalize masked activation for padded layouts (#51217)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-09-01 11:34:13 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent a232e29e9d
commit 82b7d49a6e
15 changed files with 1217 additions and 196 deletions
+208
View File
@@ -22,7 +22,13 @@ from vllm.model_executor.layers.activation import (
SwigluStepAndMul,
swiglustep_and_mul_triton,
)
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -34,6 +40,100 @@ CUDA_DEVICES = [
]
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
input = torch.empty(1, 1, 2)
output = torch.empty(1, 1, 1)
valid_token_counts = torch.ones(1, dtype=torch.int32)
with pytest.raises(NotImplementedError, match="relu2"):
apply_moe_activation(
MoEActivation.RELU2,
output,
input,
valid_token_counts=valid_token_counts,
)
def test_moe_silu_clamp_uses_native_xpu_fallback(
default_vllm_config, monkeypatch
) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
clamp_limit = 3.0
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
output = torch.empty(2, 2)
apply_moe_activation(
MoEActivation.SILU,
output,
input,
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
)
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
input
)
torch.testing.assert_close(output, expected)
def _assert_masked_moe_activation(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
*,
dtype: torch.dtype,
mask_layout: str,
d: int,
max_num_tokens: int,
) -> None:
device = CUDA_DEVICES[0]
num_experts = 4
input_dim = 2 * d if activation.is_gated else d
if mask_layout == "flat":
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
valid_token_counts = torch.tensor(
[max_num_tokens // 2], dtype=torch.int32, device=device
)
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
else:
input = torch.randn(
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
)
valid_token_counts = torch.tensor(
[0, 1, max_num_tokens // 2, max_num_tokens],
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
)
apply_moe_activation(
activation,
output,
input,
activation_config=activation_config,
valid_token_counts=valid_token_counts,
)
batched_input = input.view(-1, max_num_tokens, input_dim)
batched_output = output.view(-1, max_num_tokens, d)
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
if num_tokens:
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
apply_moe_activation(
activation,
expected,
batched_input[expert, :num_tokens].clone(),
activation_config=activation_config,
)
torch.testing.assert_close(
batched_output[expert, :num_tokens],
expected,
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
@pytest.mark.parametrize(
"activation",
[
@@ -252,6 +352,114 @@ def test_masked_situ_and_mul(
)
@pytest.mark.parametrize(
("activation", "activation_config"),
[
(MoEActivation.SILU, ApplyMoEActivationConfig()),
(
MoEActivation.SILU,
ApplyMoEActivationConfig(clamp_limit=3.0),
),
(MoEActivation.GELU, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
),
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
),
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
],
ids=[
"silu",
"silu_clamp",
"gelu",
"gelu_tanh",
"situ",
"swigluoai",
"swigluoai_uninterleave",
"swiglustep",
"silu_no_mul",
"gelu_no_mul",
"gelu_tanh_no_mul",
"relu2_no_mul",
],
)
@torch.inference_mode()
def test_masked_moe_activation_dispatch(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.bfloat16,
mask_layout="batched_experts",
d=513,
max_num_tokens=7,
)
@pytest.mark.parametrize(
("activation", "activation_config", "mask_layout"),
[
pytest.param(
MoEActivation.SILU,
ApplyMoEActivationConfig(),
"flat",
id="flat",
),
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
"batched_experts",
id="batched-experts",
),
],
)
@torch.inference_mode()
def test_masked_moe_activation_grid_stride(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
mask_layout: str,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.half,
mask_layout=mask_layout,
d=513,
max_num_tokens=67,
)
@torch.inference_mode()
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
device = CUDA_DEVICES[0]
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
opcheck(
torch.ops._C.masked_moe_activation,
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
)
@pytest.mark.parametrize(
"activation",
[
+145 -3
View File
@@ -8,6 +8,7 @@ import torch
from tests.kernels.moe.utils import (
batched_moe,
make_dummy_moe_config,
make_quantized_test_activations,
make_test_weights,
naive_batched_moe,
@@ -16,7 +17,13 @@ from tests.kernels.quant_utils import native_batched_masked_quant_matmul
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
invoke_moe_batched_triton_kernel,
)
from vllm.platforms import current_platform
@@ -496,11 +503,146 @@ def test_batched_mm_td_zero_expert_tokens():
# BatchedTritonExperts device enablement (XPU)
def test_batched_triton_experts_supports_current_device():
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
assert BatchedTritonExperts._supports_current_device()
@pytest.mark.parametrize(
"activation",
[
MoEActivation.SITU,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
],
)
def test_batched_triton_supports_masked_activations(
monkeypatch, activation: MoEActivation
):
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
assert BatchedTritonExperts._supports_activation(activation)
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
assert not BatchedTritonExperts._supports_activation(activation)
def _torch_experts_with_shared_activation(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> torch.Tensor:
num_tokens, hidden_dim = a.shape
topk = topk_ids.shape[1]
routed_input = a[:, None, :].expand(-1, topk, -1).reshape(-1, hidden_dim)
routed_experts = topk_ids.reshape(-1)
routed_output = torch.zeros_like(routed_input)
for expert in range(w1.shape[0]):
expert_mask = routed_experts == expert
if not expert_mask.any():
continue
projected = routed_input[expert_mask] @ w1[expert].T
activated = torch.empty(
projected.shape[0], w2.shape[2], dtype=a.dtype, device=a.device
)
apply_moe_activation(
activation,
activated,
projected,
activation_config=activation_config,
)
routed_output[expert_mask] = activated @ w2[expert].T
return (
(
routed_output.view(num_tokens, topk, hidden_dim).float()
* topk_weights[..., None]
)
.sum(dim=1)
.to(a.dtype)
)
assert BatchedTritonExperts._supports_current_device()
_MASKED_BATCHED_ACTIVATION_CASES = [
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
id="situ",
),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
id="swigluoai-uninterleave",
),
pytest.param(
MoEActivation.SWIGLUSTEP,
ApplyMoEActivationConfig(),
id="swiglustep",
),
]
@pytest.mark.parametrize(
("activation", "activation_config"), _MASKED_BATCHED_ACTIVATION_CASES
)
@torch.inference_mode()
def test_batched_experts_masked_activation_end_to_end(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
):
if current_platform.is_xpu():
pytest.skip("Masked activations are not enabled on XPU")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
device = current_platform.device_type
init_workspace_manager(torch.device(f"{device}:0"))
m, n, k, e, topk = 16, 128, 128, 8, 2
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
moe_config = make_dummy_moe_config(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=a.dtype,
activation=activation,
)
moe_config.swiglu_limit = activation_config.clamp_limit
moe_config.swiglu_alpha = activation_config.alpha
moe_config.swiglu_beta = activation_config.beta
moe_config.activation_situ_beta = activation_config.activation_situ_beta
moe_config.activation_situ_linear_beta = (
activation_config.activation_situ_linear_beta
)
routing_weights = torch.softmax(score.float(), dim=-1)
topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
topk_ids = topk_ids.to(torch.int32)
with set_current_vllm_config(vllm_config):
baseline_output = _torch_experts_with_shared_activation(
a,
w1,
w2,
topk_weights,
topk_ids,
activation,
activation_config,
)
triton_output = batched_moe(
a, w1, w2, topk_weights, topk_ids, moe_config=moe_config
)
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
+260 -1
View File
@@ -1377,6 +1377,102 @@ def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivati
) == apply_moe_activation_supported(activation)
@pytest.mark.parametrize(
("env_override", "use_ep", "expected"),
[
(None, False, "indexed"),
(None, True, "grouped_contiguous"),
("auto", True, "grouped_contiguous"),
("indexed", True, "indexed"),
("grouped", False, "grouped_contiguous"),
],
)
def test_humming_selects_gemm_from_parallelism_and_override(
monkeypatch: pytest.MonkeyPatch,
env_override: str | None,
use_ep: bool,
expected: str,
):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
get_humming_moe_gemm_type,
)
if env_override is None:
monkeypatch.delenv("VLLM_HUMMING_MOE_GEMM_TYPE", raising=False)
else:
monkeypatch.setenv("VLLM_HUMMING_MOE_GEMM_TYPE", env_override)
moe_config = make_dummy_moe_config()
moe_config.moe_parallel_config.use_ep = use_ep
assert get_humming_moe_gemm_type(moe_config) == expected
@pytest.mark.parametrize(
("dp_token_counts", "topk_shape", "expected"),
[
pytest.param(None, (7, 3), 21, id="runtime-layout"),
pytest.param([3, 5], (48, 1), 48, id="expanded-dp-layout"),
],
)
def test_humming_global_valid_shape_m(
monkeypatch: pytest.MonkeyPatch,
dp_token_counts: list[int] | None,
topk_shape: tuple[int, int],
expected: int,
):
from types import SimpleNamespace
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
dp_metadata = (
None
if dp_token_counts is None
else SimpleNamespace(
num_tokens_across_dp_cpu=torch.tensor(dp_token_counts, dtype=torch.int32)
)
)
monkeypatch.setattr(
humming,
"get_forward_context",
lambda: SimpleNamespace(dp_metadata=dp_metadata),
)
experts = SimpleNamespace(moe_config=make_dummy_moe_config(experts_per_token=6))
topk_ids = torch.empty(topk_shape, dtype=torch.int64)
result = humming.HummingExpertsBase.get_global_valid_shape_m(experts, topk_ids)
assert result == expected
def test_humming_permute_scratch_is_keyed_by_runtime_topk(
monkeypatch: pytest.MonkeyPatch,
):
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
scratch_topk6 = Mock()
scratch_topk1 = Mock()
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type)
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
moe_config.moe_parallel_config.dp_size = 2
experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config)
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert scratch_type.call_count == 2
first_call, second_call = scratch_type.call_args_list
assert first_call.kwargs["max_num_tokens"] == 1024
assert first_call.kwargs["topk"] == 6
assert second_call.kwargs["max_num_tokens"] == 6144
assert second_call.kwargs["topk"] == 1
def test_humming_delegates_to_instance_activation():
from types import SimpleNamespace
from unittest.mock import Mock
@@ -1408,10 +1504,173 @@ def test_humming_delegates_to_instance_activation():
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
input=input,
output=output,
valid_rows=None,
)
def test_humming_grouped_apply_forwards_valid_prefix(
monkeypatch: pytest.MonkeyPatch,
):
from types import MethodType, SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
HummingGroupedExperts,
)
activation_func = Mock()
buffers = {
"gate_up_output": torch.empty(6, 4),
"activation_output": torch.empty(6, 2),
"down_output": torch.empty(6, 4),
}
experts = SimpleNamespace(
activation=activation_func,
num_experts=2,
estimate_local_valid_shape_m=lambda _: 6,
prepare_buffers=lambda *_: buffers,
_get_permute_scratch=lambda _: None,
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
inputs,
input_scale,
),
humming_forward=Mock(),
compute_config_str="",
w13_tuning_config_str="",
w2_tuning_config_str="",
)
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
hidden_states = torch.empty(3, 4)
output = torch.empty_like(hidden_states)
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
expected_counts = torch.tensor([4], dtype=torch.int32)
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
monkeypatch.setattr(
humming,
"moe_permute",
lambda **_: (
torch.empty(6, 4),
None,
expert_offsets,
torch.arange(6),
None,
),
)
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
HummingGroupedExperts.apply(
experts,
output=output,
hidden_states=hidden_states,
w1=torch.empty(0),
w2=torch.empty(0),
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SITU,
global_num_experts=2,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
activation_func.assert_called_once()
call_kwargs = activation_func.call_args.kwargs
assert call_kwargs["activation"] == MoEActivation.SITU
assert call_kwargs["input"].shape == (6, 4)
assert call_kwargs["output"].shape == (6, 2)
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
def test_batched_marlin_activation_uses_expert_token_counts(
monkeypatch: pytest.MonkeyPatch,
):
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.marlin_moe as marlin
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
BatchedMarlinExperts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
ExpertTokensMetadata,
)
activation_func = Mock()
experts = SimpleNamespace(
activation=activation_func,
w1_bias=None,
w2_bias=None,
w1_scale=None,
w2_scale=None,
quant_type_id=0,
a1_gscale=None,
a2_gscale=None,
w13_g_idx=None,
w2_g_idx=None,
w13_g_idx_sort_indices=None,
w2_g_idx_sort_indices=None,
w1_zp=None,
w2_zp=None,
input_dtype=None,
is_k_full=True,
activation_config=ApplyMoEActivationConfig(
clamp_limit=3.0, alpha=1.3, beta=0.5
),
)
hidden_states = torch.empty(2, 3, 4)
expert_num_tokens = torch.tensor([1, 2], dtype=torch.int32)
topk_ids = torch.zeros((2, 3), dtype=torch.int64)
expert_map = torch.arange(2, dtype=torch.int32)
def fake_batched_fused_marlin_moe(**kwargs):
kwargs["activation_func"](
kwargs["activation"],
torch.empty(6, 4),
torch.empty(6, 8),
topk_ids=topk_ids,
expert_map=expert_map,
)
return kwargs["output"]
monkeypatch.setattr(
marlin, "batched_fused_marlin_moe", fake_batched_fused_marlin_moe
)
output = torch.empty_like(hidden_states)
BatchedMarlinExperts.apply(
experts,
output=output,
hidden_states=hidden_states,
w1=torch.empty(0),
w2=torch.empty(0),
topk_weights=torch.ones_like(topk_ids, dtype=torch.float32),
topk_ids=topk_ids,
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
global_num_experts=2,
expert_map=expert_map,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=ExpertTokensMetadata(expert_num_tokens, None),
apply_router_weight_on_input=False,
)
activation_func.assert_called_once()
call = activation_func.call_args
assert call.args[0] == MoEActivation.SWIGLUOAI_UNINTERLEAVE
assert call.args[1].shape == (2, 3, 4)
assert call.args[2].shape == (2, 3, 8)
torch.testing.assert_close(call.kwargs["valid_token_counts"], expert_num_tokens)
@pytest.mark.parametrize(
"activation",
[
@@ -286,3 +286,39 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
permuted_idx_1.untyped_storage().data_ptr()
== scratch.permuted_idx.untyped_storage().data_ptr()
)
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
hidden_states = torch.arange(5 * 16, dtype=torch.bfloat16, device="cuda").view(
5, 16
)
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
scratch = MoEPermuteScratch(
max_num_tokens=5,
topk=1,
num_experts=4,
num_local_experts=2,
device=hidden_states.device,
hidden_size=16,
hidden_dtype=hidden_states.dtype,
)
permuted, _, expert_offsets, _, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=4,
n_local_expert=2,
expert_map=expert_map,
scratch=scratch,
)
torch.testing.assert_close(
expert_offsets,
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
)
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
+7 -7
View File
@@ -5,7 +5,7 @@ quant kernel (``situ_and_mul_quant``).
Mirrors ``test_silu_mul_fp8_quant_deep_gemm.py`` in structure: build a reference
activation + block-FP8 quant, drive random per-row scale magnitudes, exercise
the masked (``valid_rows``) contiguous layout, and compare per valid row.
the masked contiguous layout, and compare per valid row.
The kernel computes SITU in fp32 and converts to fp8 with hardware rounding,
which torch cannot reproduce bit-exactly. It is therefore validated against the
@@ -40,7 +40,7 @@ SITU_LINEAR_BETA = 25.0
SITU_D = 3072
GROUP_SIZE = 128
# (num_tokens, valid_rows). valid_rows=None => all rows valid. 2048/4096 rows
# (num_tokens, num_valid_tokens). None means all rows are valid. 2048/4096 rows
# span multiple grid-stride waves of the persistent grid (GRID_DIM = 132*8).
CASES = [
(1, None),
@@ -111,7 +111,7 @@ def test_situ_and_mul_quant_pipelined(
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
valid_rows = (
num_valid_tokens = (
None if valid is None else torch.tensor(valid, dtype=torch.int32, device=DEVICE)
)
@@ -122,7 +122,7 @@ def test_situ_and_mul_quant_pipelined(
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
valid_rows=valid_rows,
num_valid_tokens=num_valid_tokens,
)
# Padding contract: skipped rows keep their bytes; their scale becomes 1.0.
@@ -157,7 +157,7 @@ def test_situ_and_mul_quant_pipelined(
@pytest.mark.parametrize("tokens,topk", [(300, 8), (1, 4), (256, 6)])
@torch.inference_mode()
def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
"""valid_rows is a token count; the kernel bounds rows at tokens*topk."""
"""The kernel expands num_valid_tokens by topk to bound rows."""
set_random_seed(7)
d = SITU_D
num_tokens = tokens * topk + 37 # padding tail past the valid rows
@@ -168,7 +168,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
(num_tokens, d // GROUP_SIZE), -1.0, dtype=torch.float32, device=DEVICE
)
out_sentinel = out.clone()
valid_rows = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
num_valid_tokens = torch.tensor(tokens, dtype=torch.int32, device=DEVICE)
situ_and_mul_quant(
out,
@@ -177,7 +177,7 @@ def test_situ_and_mul_quant_topk_row_bound(tokens: int, topk: int) -> None:
beta=SITU_BETA,
linear_beta=SITU_LINEAR_BETA,
group_size=GROUP_SIZE,
valid_rows=valid_rows,
num_valid_tokens=num_valid_tokens,
topk=topk,
)
+3 -1
View File
@@ -128,6 +128,7 @@ def batched_moe(
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
moe_config: FusedMoEConfig | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
@@ -141,7 +142,8 @@ def batched_moe(
a2_scale=a2_scale,
)
moe_config = make_dummy_moe_config()
if moe_config is None:
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(