[Bugfix][MoE] Size the permute scratch from its inputs (#60447)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-08 15:32:07 -04:00
committed by GitHub
co-authored by Claude
parent fff10d2072
commit e3a29aa9b3
7 changed files with 128 additions and 191 deletions
+8 -22
View File
@@ -1437,6 +1437,7 @@ def test_humming_global_valid_shape_m(
def test_humming_permute_scratch_is_shared_by_config(
monkeypatch: pytest.MonkeyPatch,
):
"""Layers share one scratch whose key is independent of batch shape."""
from types import SimpleNamespace
from unittest.mock import Mock
@@ -1447,34 +1448,19 @@ def test_humming_permute_scratch_is_shared_by_config(
manager = workspace.WorkspaceManager(torch.device("cpu"))
monkeypatch.setattr(workspace, "_manager", manager)
scratch_topk6 = Mock()
scratch_topk1 = Mock()
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
scratch = Mock()
scratch_type = Mock(side_effect=[scratch])
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
monkeypatch.setattr(permute, "MoEPermuteScratch", scratch_type)
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
moe_config.moe_parallel_config.dp_size = 2
experts = SimpleNamespace(moe_config=moe_config)
other_layer = SimpleNamespace(moe_config=moe_config)
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert (
humming.HummingExpertsBase._get_permute_scratch(other_layer, 6) is scratch_topk6
)
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert scratch_type.call_count == 2
first_call, second_call = scratch_type.call_args_list
assert first_call.kwargs["max_num_tokens"] == 1024
assert first_call.kwargs["topk"] == 6
assert second_call.kwargs["max_num_tokens"] == 6144
assert second_call.kwargs["topk"] == 1
assert humming.HummingExpertsBase._get_permute_scratch(experts) is scratch
assert humming.HummingExpertsBase._get_permute_scratch(other_layer) is scratch
manager.lock()
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
with pytest.raises(AssertionError, match="was not allocated during warmup"):
humming.HummingExpertsBase._get_permute_scratch(experts, 2)
assert humming.HummingExpertsBase._get_permute_scratch(experts) is scratch
assert scratch_type.call_count == 1
def test_humming_delegates_to_instance_activation():
@@ -1531,7 +1517,7 @@ def test_humming_grouped_apply_forwards_valid_prefix(
num_experts=2,
estimate_local_valid_shape_m=lambda _: 6,
prepare_buffers=lambda *_: buffers,
_get_permute_scratch=lambda _, *, indices_only=False: object(),
_get_permute_scratch=lambda: object(),
process_input=Mock(
side_effect=lambda _, **kwargs: (
kwargs["inputs"],
+40 -28
View File
@@ -169,13 +169,9 @@ def test_moe_permute_unpermute(
scratch = None
if use_scratch:
scratch = get_moe_permute_scratch(
max_num_tokens=n_token,
topk=topk,
num_experts=n_expert,
num_local_experts=n_local_expert,
device=hidden_states.device,
hidden_size=n_hidden,
hidden_dtype=dtype,
)
(
@@ -247,13 +243,9 @@ def test_moe_permute_reuses_scratch_buffers(
_, topk_ids, _ = fused_topk(hidden_states, gating_output, topk, False)
scratch_config = dict(
max_num_tokens=n_token,
topk=topk,
num_experts=n_expert,
num_local_experts=n_expert,
device=hidden_states.device,
hidden_size=n_hidden,
hidden_dtype=hidden_states.dtype,
)
scratch = get_moe_permute_scratch(**scratch_config)
@@ -300,15 +292,10 @@ def test_moe_permute_reuses_scratch_buffers(
permuted_idx_2,
) = second
torch.testing.assert_close(permuted_hidden_states_1, permuted_hidden_states_2)
torch.testing.assert_close(expert_first_token_offset_1, expert_first_token_offset_2)
torch.testing.assert_close(inv_permuted_idx_1, inv_permuted_idx_2)
torch.testing.assert_close(permuted_idx_1, permuted_idx_2)
assert (
permuted_hidden_states_1.untyped_storage().data_ptr()
== permuted_hidden_states_2.untyped_storage().data_ptr()
)
assert (
expert_first_token_offset_1.untyped_storage().data_ptr()
== expert_first_token_offset_2.untyped_storage().data_ptr()
@@ -330,14 +317,11 @@ def test_moe_permute_scratch_reused_across_graph_sizes(workspace_init) -> None:
device = torch.device("cuda:0")
scratch = get_moe_permute_scratch(
max_num_tokens=128,
topk=6,
num_experts=16,
num_local_experts=16,
device=device,
hidden_size=2048,
hidden_dtype=torch.bfloat16,
)
scratch.reserve(128 * 6)
current_workspace_manager().lock()
runs = []
for n_token in (128, 1, 33):
@@ -372,13 +356,9 @@ def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> Non
ubatch = 0
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: ubatch)
scratch_config = dict(
max_num_tokens=64,
topk=6,
num_experts=16,
num_local_experts=16,
device=device,
hidden_size=2048,
hidden_dtype=torch.bfloat16,
)
runs = []
for i, n_token in enumerate((1, 7, 33, 64)):
@@ -396,7 +376,7 @@ def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> Non
runs.append((stream, graph, hidden, topk_ids, output, scratch))
torch.cuda.current_stream().wait_stream(stream)
assert len({run[-1].permuted_hidden_states.data_ptr() for run in runs}) == 4
assert len({run[-1].permuted_idx.data_ptr() for run in runs}) == 4
manager.lock()
for _ in range(3):
for i, (stream, graph, hidden, topk_ids, _, scratch) in enumerate(runs):
@@ -423,8 +403,6 @@ def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
monkeypatch.setattr(workspace, "_manager", None)
config = dict(
max_num_tokens=4,
topk=2,
num_experts=4,
num_local_experts=4,
device=torch.device("cuda"),
@@ -432,6 +410,8 @@ def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
first = get_moe_permute_scratch(**config)
second = get_moe_permute_scratch(**config)
first.reserve(8)
second.reserve(8)
assert (
first.token_expert_indices.data_ptr() != second.token_expert_indices.data_ptr()
)
@@ -450,13 +430,9 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
scratch = MoEPermuteScratch(
max_num_tokens=5,
topk=1,
num_experts=4,
num_local_experts=2,
device=hidden_states.device,
hidden_size=16,
hidden_dtype=hidden_states.dtype,
)
permuted, _, expert_offsets, inverse, _ = moe_permute(
@@ -498,3 +474,39 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)
def test_moe_permute_scratch_grows_until_workspace_lock(workspace_init) -> None:
"""Capacity follows the inputs seen in warmup, whatever their topk or size."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
device = torch.device("cuda:0")
scratch = get_moe_permute_scratch(
num_experts=16, num_local_experts=16, device=device
)
def run(n_token: int, topk: int, dtype: torch.dtype) -> None:
hidden = torch.randn((n_token, 64), device=device).to(dtype)
topk_ids = torch.randint(16, (n_token, topk), device=device)
actual = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
expected = moe_permute(hidden, None, topk_ids, 16)
for a, e in zip(actual, expected):
if a is not None:
torch.testing.assert_close(a.float(), e.float())
run(8, 6, torch.bfloat16)
run(64, 6, torch.bfloat16)
run(384, 1, torch.float8_e4m3fn)
assert scratch.max_expanded_rows == 384
current_workspace_manager().lock()
run(64, 6, torch.bfloat16)
with pytest.raises(AssertionError, match="Scratch growth is not allowed"):
run(65, 6, torch.bfloat16)
# Experts rebuilt after the lock (e.g. a weight reload) keep the capacity.
rebuilt = get_moe_permute_scratch(
num_experts=16, num_local_experts=16, device=device
)
assert rebuilt.max_expanded_rows == 384