[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_prepare_scatter,
|
||||
moe_unpermute,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -458,7 +459,7 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
permuted, _, expert_offsets, _, _ = moe_permute(
|
||||
permuted, _, expert_offsets, inverse, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
@@ -473,3 +474,27 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
|
||||
assert torch.all(inverse[[1, 3, 4]] >= expert_offsets[-1])
|
||||
expected = torch.zeros_like(hidden_states)
|
||||
expected[[0, 2]] = hidden_states[[0, 2]]
|
||||
output = torch.empty_like(hidden_states)
|
||||
moe_unpermute(
|
||||
output, permuted, torch.ones(5, 1, device="cuda"), inverse, expert_offsets
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
expected_inverse = inverse.clone()
|
||||
expected_offsets = expert_offsets.clone()
|
||||
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
|
||||
topk_ids.zero_()
|
||||
graph.replay()
|
||||
|
||||
expected_inverse = torch.arange(5, dtype=torch.int32, device="cuda")
|
||||
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
|
||||
Reference in New Issue
Block a user