[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -1245,7 +1245,7 @@ def _make_humming_indexed_experts(activation: MoEActivation):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import humming_utils
|
||||
from vllm.model_executor.layers.quantization.utils import humming as humming_utils
|
||||
from vllm.utils import humming
|
||||
|
||||
top_k, num_experts = 6, 12
|
||||
@@ -1306,6 +1306,56 @@ def _make_humming_indexed_experts(activation: MoEActivation):
|
||||
return experts, layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("valid_shape_m", [8192, 8193])
|
||||
@pytest.mark.parametrize("w2_block_size", [64, 96, 128])
|
||||
def test_humming_indexed_routing_matches_each_projection_tile(
|
||||
valid_shape_m: int, w2_block_size: int
|
||||
):
|
||||
"""Independently tuned projections preserve routed rows and expert identity."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
topk_ids = torch.tensor([[0, 1], [1, 2], [2, 0]], device="cuda")
|
||||
expert_map = torch.tensor([1, -1, 0], dtype=torch.int32, device="cuda")
|
||||
experts = SimpleNamespace(
|
||||
global_num_experts=3,
|
||||
estimate_local_valid_shape_m=lambda _: valid_shape_m,
|
||||
w13_tuning_config=[
|
||||
[0, 8192, {"block_shape": [64, 128, 64]}],
|
||||
[8192, 16384, {"block_shape": [128, 128, 64]}],
|
||||
],
|
||||
w2_tuning_config=[
|
||||
[0, 8192, {"block_shape": [64, 128, 64]}],
|
||||
[8192, 16384, {"block_shape": [w2_block_size, 128, 64]}],
|
||||
],
|
||||
compute_config_str="compute",
|
||||
w13_tuning_config_str="w13",
|
||||
w2_tuning_config_str="w2",
|
||||
)
|
||||
first, second, _ = HummingIndexedExperts.prepare_humming_moe_kwargs(
|
||||
experts, topk_ids, expert_map, None
|
||||
)
|
||||
block_sizes = (64, 64) if valid_shape_m == 8192 else (128, w2_block_size)
|
||||
expected_rows = torch.where(expert_map[topk_ids.flatten()] >= 0)[0]
|
||||
for kwargs, block_size in zip((first, second), block_sizes):
|
||||
padded = kwargs["num_tokens_padded"].item()
|
||||
assert padded % block_size == 0
|
||||
rows = kwargs["sorted_ids"][:padded].long()
|
||||
valid = rows < topk_ids.numel()
|
||||
torch.testing.assert_close(rows[valid].sort().values, expected_rows)
|
||||
routed_experts = kwargs["expert_ids"][: padded // block_size]
|
||||
torch.testing.assert_close(
|
||||
routed_experts.repeat_interleave(block_size)[valid],
|
||||
expert_map[topk_ids.flatten()[rows[valid]]],
|
||||
)
|
||||
assert (first["top_k"], second["top_k"]) == (2, 1)
|
||||
for key in ("sorted_ids", "expert_ids", "num_tokens_padded"):
|
||||
assert (first[key] is second[key]) == (block_sizes[0] == block_sizes[1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivation):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
@@ -1464,54 +1514,46 @@ def test_humming_delegates_to_instance_activation():
|
||||
def test_humming_grouped_apply_forwards_valid_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import MethodType, SimpleNamespace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
HummingGroupedExperts,
|
||||
)
|
||||
|
||||
activation_func = Mock()
|
||||
buffers = {
|
||||
"gate_up_output": torch.empty(6, 4),
|
||||
"activation_output": torch.empty(6, 2),
|
||||
"quanted_down_input": torch.empty(6, 2),
|
||||
"down_output": torch.empty(6, 4),
|
||||
}
|
||||
experts = SimpleNamespace(
|
||||
activation=activation_func,
|
||||
num_experts=2,
|
||||
estimate_local_valid_shape_m=lambda _: 6,
|
||||
prepare_buffers=lambda *_: buffers,
|
||||
_get_permute_scratch=lambda _: None,
|
||||
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
|
||||
inputs,
|
||||
input_scale,
|
||||
_get_permute_scratch=lambda _, *, indices_only=False: object(),
|
||||
process_input=Mock(
|
||||
side_effect=lambda _, **kwargs: (
|
||||
kwargs["inputs"],
|
||||
kwargs.get("input_scale"),
|
||||
None,
|
||||
)
|
||||
),
|
||||
humming_forward=Mock(),
|
||||
compute_config_str="",
|
||||
w13_tuning_config_str="",
|
||||
w2_tuning_config_str="",
|
||||
)
|
||||
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
|
||||
|
||||
hidden_states = torch.empty(3, 4)
|
||||
output = torch.empty_like(hidden_states)
|
||||
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
|
||||
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
|
||||
expected_counts = torch.tensor([4], dtype=torch.int32)
|
||||
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
|
||||
monkeypatch.setattr(
|
||||
humming,
|
||||
"moe_permute",
|
||||
lambda **_: (
|
||||
torch.empty(6, 4),
|
||||
None,
|
||||
expert_offsets,
|
||||
torch.arange(6),
|
||||
None,
|
||||
),
|
||||
"moe_prepare_scatter",
|
||||
lambda *_: (expert_offsets, torch.arange(6).view(3, 2)),
|
||||
)
|
||||
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
|
||||
|
||||
@@ -1534,12 +1576,11 @@ def test_humming_grouped_apply_forwards_valid_prefix(
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
activation_func.assert_called_once()
|
||||
call_kwargs = activation_func.call_args.kwargs
|
||||
call_kwargs = experts.process_input.call_args.kwargs
|
||||
assert call_kwargs["activation"] == MoEActivation.SITU
|
||||
assert call_kwargs["input"].shape == (6, 4)
|
||||
assert call_kwargs["output"].shape == (6, 2)
|
||||
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
|
||||
assert call_kwargs["inputs"].shape == (6, 4)
|
||||
assert call_kwargs["quanted_input"].shape == (6, 2)
|
||||
torch.testing.assert_close(call_kwargs["num_valid_tokens"], expert_offsets[-1:])
|
||||
|
||||
|
||||
def test_batched_marlin_activation_uses_expert_token_counts(
|
||||
@@ -1653,7 +1694,8 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
|
||||
activation=moe_config.activation,
|
||||
)
|
||||
assert buffer_metas["gate_up_output"]["shape"][-1] == gate_up_size
|
||||
assert buffer_metas["activation_output"]["shape"][-1] == intermediate_size
|
||||
assert "activation_output" not in buffer_metas
|
||||
assert buffer_metas["quanted_down_input"]["shape"][-1] == intermediate_size
|
||||
assert experts.moe_problem_size(
|
||||
a1=torch.empty(1, hidden_size),
|
||||
w1=torch.empty(num_experts, 1),
|
||||
@@ -1661,6 +1703,13 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
|
||||
topk_ids=torch.empty(1, top_k, dtype=torch.long),
|
||||
) == (num_experts, 1, intermediate_size, hidden_size, top_k)
|
||||
|
||||
# Quantized batched input storage must cover every dispatcher's token slots.
|
||||
experts.is_batched = lambda: True
|
||||
experts.max_num_tokens = 17
|
||||
experts.num_dispatchers = 2
|
||||
batched_metas, _ = experts.get_buffer_metas(17, top_k, activation)
|
||||
assert batched_metas["quanted_gate_up_input"]["shape"][0] == num_experts * 17 * 2
|
||||
|
||||
|
||||
def test_humming_indexed_writes_supplied_output_buffer():
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
@@ -22,6 +22,46 @@ BLOCK_SIZES = [32, 128]
|
||||
set_random_seed(0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts,num_tokens", [(32, 8), (256, 8), (256, 1024)])
|
||||
@pytest.mark.parametrize("use_expert_map", [False, True])
|
||||
def test_moe_align_scatter_masks_invalid_routes(
|
||||
num_experts, num_tokens, use_expert_map
|
||||
):
|
||||
topk_ids = torch.arange(num_tokens * 4, device="cuda").reshape(-1, 4)
|
||||
topk_ids %= num_experts
|
||||
topk_ids[0] = torch.tensor([-1, num_experts, 0, 1], device="cuda")
|
||||
expert_map = None
|
||||
valid = (topk_ids >= 0) & (topk_ids < num_experts)
|
||||
if use_expert_map:
|
||||
expert_map = torch.full((num_experts,), -1, dtype=torch.int32, device="cuda")
|
||||
expert_map[::2] = torch.arange(
|
||||
num_experts // 2, device="cuda", dtype=torch.int32
|
||||
)
|
||||
valid &= topk_ids % 2 == 0
|
||||
|
||||
sorted_ids, expert_ids, num_padded, scatter_idx = moe_align_block_size(
|
||||
topk_ids,
|
||||
32,
|
||||
num_experts,
|
||||
expert_map,
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=True,
|
||||
)
|
||||
expected = torch.arange(topk_ids.numel(), device="cuda", dtype=torch.int32)
|
||||
expected[~valid.flatten()] = -1
|
||||
torch.testing.assert_close(scatter_idx, expected[:, None])
|
||||
active_sorted = sorted_ids[: num_padded.item()]
|
||||
active_sorted = active_sorted[active_sorted < topk_ids.numel()]
|
||||
torch.testing.assert_close(active_sorted.sort().values, expected[expected >= 0])
|
||||
|
||||
baseline = moe_align_block_size(
|
||||
topk_ids, 32, num_experts, expert_map, ignore_invalid_experts=True
|
||||
)
|
||||
assert len(baseline) == 3
|
||||
torch.testing.assert_close(num_padded, baseline[2])
|
||||
torch.testing.assert_close(expert_ids, baseline[1])
|
||||
|
||||
|
||||
def _group_tokens_by_expert(
|
||||
sorted_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_prepare_scatter,
|
||||
moe_unpermute,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -458,7 +459,7 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
permuted, _, expert_offsets, _, _ = moe_permute(
|
||||
permuted, _, expert_offsets, inverse, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
@@ -473,3 +474,27 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
|
||||
assert torch.all(inverse[[1, 3, 4]] >= expert_offsets[-1])
|
||||
expected = torch.zeros_like(hidden_states)
|
||||
expected[[0, 2]] = hidden_states[[0, 2]]
|
||||
output = torch.empty_like(hidden_states)
|
||||
moe_unpermute(
|
||||
output, permuted, torch.ones(5, 1, device="cuda"), inverse, expert_offsets
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
expected_inverse = inverse.clone()
|
||||
expected_offsets = expert_offsets.clone()
|
||||
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
topk_ids.zero_()
|
||||
graph.replay()
|
||||
|
||||
expected_inverse = torch.arange(5, dtype=torch.int32, device="cuda")
|
||||
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
|
||||
Reference in New Issue
Block a user