[Feature][Humming] Humming feature integration (#56685)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-09-20 08:11:01 -07:00
committed by GitHub
co-authored by Codex mgoin
parent 27757dde02
commit 7d99c2c4fd
62 changed files with 2040 additions and 1086 deletions
+75 -26
View File
@@ -1245,7 +1245,7 @@ def _make_humming_indexed_experts(activation: MoEActivation):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingIndexedExperts,
)
from vllm.model_executor.layers.quantization.utils import humming_utils
from vllm.model_executor.layers.quantization.utils import humming as humming_utils
from vllm.utils import humming
top_k, num_experts = 6, 12
@@ -1306,6 +1306,56 @@ def _make_humming_indexed_experts(activation: MoEActivation):
return experts, layer
@pytest.mark.parametrize("valid_shape_m", [8192, 8193])
@pytest.mark.parametrize("w2_block_size", [64, 96, 128])
def test_humming_indexed_routing_matches_each_projection_tile(
valid_shape_m: int, w2_block_size: int
):
"""Independently tuned projections preserve routed rows and expert identity."""
from types import SimpleNamespace
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingIndexedExperts,
)
topk_ids = torch.tensor([[0, 1], [1, 2], [2, 0]], device="cuda")
expert_map = torch.tensor([1, -1, 0], dtype=torch.int32, device="cuda")
experts = SimpleNamespace(
global_num_experts=3,
estimate_local_valid_shape_m=lambda _: valid_shape_m,
w13_tuning_config=[
[0, 8192, {"block_shape": [64, 128, 64]}],
[8192, 16384, {"block_shape": [128, 128, 64]}],
],
w2_tuning_config=[
[0, 8192, {"block_shape": [64, 128, 64]}],
[8192, 16384, {"block_shape": [w2_block_size, 128, 64]}],
],
compute_config_str="compute",
w13_tuning_config_str="w13",
w2_tuning_config_str="w2",
)
first, second, _ = HummingIndexedExperts.prepare_humming_moe_kwargs(
experts, topk_ids, expert_map, None
)
block_sizes = (64, 64) if valid_shape_m == 8192 else (128, w2_block_size)
expected_rows = torch.where(expert_map[topk_ids.flatten()] >= 0)[0]
for kwargs, block_size in zip((first, second), block_sizes):
padded = kwargs["num_tokens_padded"].item()
assert padded % block_size == 0
rows = kwargs["sorted_ids"][:padded].long()
valid = rows < topk_ids.numel()
torch.testing.assert_close(rows[valid].sort().values, expected_rows)
routed_experts = kwargs["expert_ids"][: padded // block_size]
torch.testing.assert_close(
routed_experts.repeat_interleave(block_size)[valid],
expert_map[topk_ids.flatten()[rows[valid]]],
)
assert (first["top_k"], second["top_k"]) == (2, 1)
for key in ("sorted_ids", "expert_ids", "num_tokens_padded"):
assert (first[key] is second[key]) == (block_sizes[0] == block_sizes[1])
@pytest.mark.parametrize("activation", list(MoEActivation))
def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivation):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
@@ -1464,54 +1514,46 @@ def test_humming_delegates_to_instance_activation():
def test_humming_grouped_apply_forwards_valid_prefix(
monkeypatch: pytest.MonkeyPatch,
):
from types import MethodType, SimpleNamespace
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
HummingGroupedExperts,
)
activation_func = Mock()
buffers = {
"gate_up_output": torch.empty(6, 4),
"activation_output": torch.empty(6, 2),
"quanted_down_input": torch.empty(6, 2),
"down_output": torch.empty(6, 4),
}
experts = SimpleNamespace(
activation=activation_func,
num_experts=2,
estimate_local_valid_shape_m=lambda _: 6,
prepare_buffers=lambda *_: buffers,
_get_permute_scratch=lambda _: None,
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
inputs,
input_scale,
_get_permute_scratch=lambda _, *, indices_only=False: object(),
process_input=Mock(
side_effect=lambda _, **kwargs: (
kwargs["inputs"],
kwargs.get("input_scale"),
None,
)
),
humming_forward=Mock(),
compute_config_str="",
w13_tuning_config_str="",
w2_tuning_config_str="",
)
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
hidden_states = torch.empty(3, 4)
output = torch.empty_like(hidden_states)
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
expected_counts = torch.tensor([4], dtype=torch.int32)
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
monkeypatch.setattr(
humming,
"moe_permute",
lambda **_: (
torch.empty(6, 4),
None,
expert_offsets,
torch.arange(6),
None,
),
"moe_prepare_scatter",
lambda *_: (expert_offsets, torch.arange(6).view(3, 2)),
)
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
@@ -1534,12 +1576,11 @@ def test_humming_grouped_apply_forwards_valid_prefix(
apply_router_weight_on_input=False,
)
activation_func.assert_called_once()
call_kwargs = activation_func.call_args.kwargs
call_kwargs = experts.process_input.call_args.kwargs
assert call_kwargs["activation"] == MoEActivation.SITU
assert call_kwargs["input"].shape == (6, 4)
assert call_kwargs["output"].shape == (6, 2)
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
assert call_kwargs["inputs"].shape == (6, 4)
assert call_kwargs["quanted_input"].shape == (6, 2)
torch.testing.assert_close(call_kwargs["num_valid_tokens"], expert_offsets[-1:])
def test_batched_marlin_activation_uses_expert_token_counts(
@@ -1653,7 +1694,8 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
activation=moe_config.activation,
)
assert buffer_metas["gate_up_output"]["shape"][-1] == gate_up_size
assert buffer_metas["activation_output"]["shape"][-1] == intermediate_size
assert "activation_output" not in buffer_metas
assert buffer_metas["quanted_down_input"]["shape"][-1] == intermediate_size
assert experts.moe_problem_size(
a1=torch.empty(1, hidden_size),
w1=torch.empty(num_experts, 1),
@@ -1661,6 +1703,13 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
topk_ids=torch.empty(1, top_k, dtype=torch.long),
) == (num_experts, 1, intermediate_size, hidden_size, top_k)
# Quantized batched input storage must cover every dispatcher's token slots.
experts.is_batched = lambda: True
experts.max_num_tokens = 17
experts.num_dispatchers = 2
batched_metas, _ = experts.get_buffer_metas(17, top_k, activation)
assert batched_metas["quanted_gate_up_input"]["shape"][0] == num_experts * 17 * 2
def test_humming_indexed_writes_supplied_output_buffer():
from vllm.forward_context import set_forward_context
@@ -22,6 +22,46 @@ BLOCK_SIZES = [32, 128]
set_random_seed(0)
@pytest.mark.parametrize("num_experts,num_tokens", [(32, 8), (256, 8), (256, 1024)])
@pytest.mark.parametrize("use_expert_map", [False, True])
def test_moe_align_scatter_masks_invalid_routes(
num_experts, num_tokens, use_expert_map
):
topk_ids = torch.arange(num_tokens * 4, device="cuda").reshape(-1, 4)
topk_ids %= num_experts
topk_ids[0] = torch.tensor([-1, num_experts, 0, 1], device="cuda")
expert_map = None
valid = (topk_ids >= 0) & (topk_ids < num_experts)
if use_expert_map:
expert_map = torch.full((num_experts,), -1, dtype=torch.int32, device="cuda")
expert_map[::2] = torch.arange(
num_experts // 2, device="cuda", dtype=torch.int32
)
valid &= topk_ids % 2 == 0
sorted_ids, expert_ids, num_padded, scatter_idx = moe_align_block_size(
topk_ids,
32,
num_experts,
expert_map,
ignore_invalid_experts=True,
return_scatter_idx=True,
)
expected = torch.arange(topk_ids.numel(), device="cuda", dtype=torch.int32)
expected[~valid.flatten()] = -1
torch.testing.assert_close(scatter_idx, expected[:, None])
active_sorted = sorted_ids[: num_padded.item()]
active_sorted = active_sorted[active_sorted < topk_ids.numel()]
torch.testing.assert_close(active_sorted.sort().values, expected[expected >= 0])
baseline = moe_align_block_size(
topk_ids, 32, num_experts, expert_map, ignore_invalid_experts=True
)
assert len(baseline) == 3
torch.testing.assert_close(num_padded, baseline[2])
torch.testing.assert_close(expert_ids, baseline[1])
def _group_tokens_by_expert(
sorted_ids: torch.Tensor,
expert_ids: torch.Tensor,
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_prepare_scatter,
moe_unpermute,
)
from vllm.platforms import current_platform
@@ -458,7 +459,7 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
hidden_dtype=hidden_states.dtype,
)
permuted, _, expert_offsets, _, _ = moe_permute(
permuted, _, expert_offsets, inverse, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
@@ -473,3 +474,27 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
)
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
assert torch.all(inverse[[1, 3, 4]] >= expert_offsets[-1])
expected = torch.zeros_like(hidden_states)
expected[[0, 2]] = hidden_states[[0, 2]]
output = torch.empty_like(hidden_states)
moe_unpermute(
output, permuted, torch.ones(5, 1, device="cuda"), inverse, expert_offsets
)
torch.testing.assert_close(output, expected)
expected_inverse = inverse.clone()
expected_offsets = expert_offsets.clone()
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
topk_ids.zero_()
graph.replay()
expected_inverse = torch.arange(5, dtype=torch.int32, device="cuda")
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)