[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
Misha Goin
2026-09-20 17:28:33 +08:00
committed by GitHub
co-authored by ErinYin
parent 0e110f696d
commit e5fce7b56b
21 changed files with 598 additions and 211 deletions
+17 -3
View File
@@ -1384,24 +1384,33 @@ def test_humming_global_valid_shape_m(
assert result == expected
def test_humming_permute_scratch_is_keyed_by_runtime_topk(
def test_humming_permute_scratch_is_shared_by_config(
monkeypatch: pytest.MonkeyPatch,
):
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
import vllm.model_executor.layers.fused_moe.moe_permute_unpermute as permute
import vllm.v1.worker.workspace as workspace
manager = workspace.WorkspaceManager(torch.device("cpu"))
monkeypatch.setattr(workspace, "_manager", manager)
scratch_topk6 = Mock()
scratch_topk1 = Mock()
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type)
monkeypatch.setattr(permute, "MoEPermuteScratch", scratch_type)
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
moe_config.moe_parallel_config.dp_size = 2
experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config)
experts = SimpleNamespace(moe_config=moe_config)
other_layer = SimpleNamespace(moe_config=moe_config)
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
assert (
humming.HummingExpertsBase._get_permute_scratch(other_layer, 6) is scratch_topk6
)
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
@@ -1412,6 +1421,11 @@ def test_humming_permute_scratch_is_keyed_by_runtime_topk(
assert second_call.kwargs["max_num_tokens"] == 6144
assert second_call.kwargs["topk"] == 1
manager.lock()
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
with pytest.raises(AssertionError, match="was not allocated during warmup"):
humming.HummingExpertsBase._get_permute_scratch(experts, 2)
def test_humming_delegates_to_instance_activation():
from types import SimpleNamespace
+163 -12
View File
@@ -15,12 +15,14 @@ from vllm.model_executor.layers.fused_moe.expert_map_manager import (
)
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
MoEPermuteScratch,
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_unpermute,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import current_workspace_manager
NUM_EXPERTS = [16, 64, 256]
TOP_KS = [2, 6, 8]
@@ -120,6 +122,7 @@ def torch_unpermute(
@pytest.mark.parametrize("topk", TOP_KS)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("ep_size", EP_SIZE)
@pytest.mark.parametrize("use_scratch", [False, True])
def test_moe_permute_unpermute(
n_token: int,
n_hidden: int,
@@ -127,6 +130,8 @@ def test_moe_permute_unpermute(
n_expert: int,
ep_size: int,
dtype: torch.dtype,
use_scratch: bool,
workspace_init,
):
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
@@ -160,6 +165,18 @@ def test_moe_permute_unpermute(
expert_map=expert_map,
)
scratch = None
if use_scratch:
scratch = get_moe_permute_scratch(
max_num_tokens=n_token,
topk=topk,
num_experts=n_expert,
num_local_experts=n_local_expert,
device=hidden_states.device,
hidden_size=n_hidden,
hidden_dtype=dtype,
)
(
permuted_hidden_states,
_,
@@ -173,6 +190,7 @@ def test_moe_permute_unpermute(
n_expert=n_expert,
n_local_expert=n_local_expert,
expert_map=expert_map,
scratch=scratch,
)
# check expert_first_token_offset
@@ -212,20 +230,22 @@ def test_moe_permute_unpermute(
torch.testing.assert_close(result4, gold4, atol=2e-2, rtol=0)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("n_token", [1, 33, 128])
@pytest.mark.parametrize("topk", [1, 6, 8])
def test_moe_permute_reuses_scratch_buffers(
dtype: torch.dtype, n_token: int, topk: int, workspace_init
):
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
n_token = 64
n_hidden = 2048
n_expert = 16
topk = 4
hidden_states = torch.randn((n_token, n_hidden), device="cuda").to(dtype)
gating_output = torch.randn((n_token, n_expert), device="cuda").to(dtype)
_, topk_ids, _ = fused_topk(hidden_states, gating_output, topk, False)
scratch = MoEPermuteScratch(
scratch_config = dict(
max_num_tokens=n_token,
topk=topk,
num_experts=n_expert,
@@ -234,6 +254,7 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
hidden_size=n_hidden,
hidden_dtype=hidden_states.dtype,
)
scratch = get_moe_permute_scratch(**scratch_config)
first = moe_permute(
hidden_states=hidden_states,
@@ -242,13 +263,26 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
n_expert=n_expert,
scratch=scratch,
)
second = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=n_expert,
scratch=scratch,
)
current_workspace_manager().lock()
assert get_moe_permute_scratch(**scratch_config) is scratch
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
second = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
n_expert=n_expert,
scratch=get_moe_permute_scratch(**scratch_config),
)
for _ in range(2):
hidden_states.add_(1)
topk_ids.copy_(topk_ids.roll(1, 0))
graph.replay()
expected = moe_permute(hidden_states, None, topk_ids, n_expert)
for actual, reference in zip(second, expected):
if actual is not None:
torch.testing.assert_close(actual, reference)
(
permuted_hidden_states_1,
@@ -288,6 +322,123 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
)
def test_moe_permute_scratch_reused_across_graph_sizes(workspace_init) -> None:
"""Switching graph sizes must not expose metadata left by a larger batch."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
device = torch.device("cuda:0")
scratch = get_moe_permute_scratch(
max_num_tokens=128,
topk=6,
num_experts=16,
num_local_experts=16,
device=device,
hidden_size=2048,
hidden_dtype=torch.bfloat16,
)
current_workspace_manager().lock()
runs = []
for n_token in (128, 1, 33):
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
topk_ids = torch.randint(16, (n_token, 6), device=device)
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
runs.append((graph, hidden, topk_ids, output))
for graph, hidden, topk_ids, output in runs + runs[::-1]:
hidden.add_(1)
topk_ids.copy_((topk_ids + 1) % 16)
graph.replay()
expected = moe_permute(hidden, None, topk_ids, 16)
for actual, reference in zip(output, expected):
if actual is not None:
torch.testing.assert_close(actual, reference)
def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> None:
"""Concurrent graph replays in different ubatches/lanes cannot share scratch."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
from vllm.v1.worker import workspace
device = torch.device("cuda:0")
manager = workspace.WorkspaceManager(device, num_ubatches=2, num_lanes=2)
monkeypatch.setattr(workspace, "_manager", manager)
ubatch = 0
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: ubatch)
scratch_config = dict(
max_num_tokens=64,
topk=6,
num_experts=16,
num_local_experts=16,
device=device,
hidden_size=2048,
hidden_dtype=torch.bfloat16,
)
runs = []
for i, n_token in enumerate((1, 7, 33, 64)):
ubatch = i // 2
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
topk_ids = torch.randint(16, (n_token, 6), device=device)
scratch = get_moe_permute_scratch(**scratch_config)
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
runs.append((stream, graph, hidden, topk_ids, output, scratch))
torch.cuda.current_stream().wait_stream(stream)
assert len({run[-1].permuted_hidden_states.data_ptr() for run in runs}) == 4
manager.lock()
for _ in range(3):
for i, (stream, graph, hidden, topk_ids, _, scratch) in enumerate(runs):
ubatch = i // 2
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
assert get_moe_permute_scratch(**scratch_config) is scratch
hidden.add_(1)
topk_ids.copy_((topk_ids + 1) % 16)
graph.replay()
for stream, _, hidden, topk_ids, output, _ in runs:
torch.cuda.current_stream().wait_stream(stream)
expected = moe_permute(hidden, None, topk_ids, 16)
for actual, reference in zip(output, expected):
if actual is not None:
torch.testing.assert_close(actual, reference)
def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
"""Standalone calls get independent scratch with initialized row indices."""
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")
from vllm.v1.worker import workspace
monkeypatch.setattr(workspace, "_manager", None)
config = dict(
max_num_tokens=4,
topk=2,
num_experts=4,
num_local_experts=4,
device=torch.device("cuda"),
)
first = get_moe_permute_scratch(**config)
second = get_moe_permute_scratch(**config)
assert (
first.token_expert_indices.data_ptr() != second.token_expert_indices.data_ptr()
)
torch.testing.assert_close(
first.token_expert_indices, torch.arange(8, dtype=torch.int32, device="cuda")
)
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
if not moe_permute_unpermute_supported():
pytest.skip("moe_permute_unpermute is not supported on this platform.")