[Kimi-K3] DCP support (#50484)
Signed-off-by: girasoley <girasoley@inferact.ai> Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5
云挚
foraxe
Woosuk Kwon
parent
3f142bd85e
commit
63ac04a61e
@@ -222,6 +222,49 @@ class TestLSEWeightedCombine:
|
||||
assert result.shape == (B, H, D)
|
||||
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
|
||||
|
||||
def test_empty_shard_ignores_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
|
||||
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
|
||||
|
||||
result = _lse_weighted_combine(outputs, lses)
|
||||
|
||||
torch.testing.assert_close(result, outputs[1])
|
||||
|
||||
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
|
||||
import vllm.v1.attention.ops.common as common
|
||||
|
||||
class FakeGroup:
|
||||
world_size = 2
|
||||
rank_in_group = 0
|
||||
|
||||
def all_gather(self, tensor, dim):
|
||||
assert dim == 0
|
||||
return torch.cat((tensor, tensor), dim=dim)
|
||||
|
||||
monkeypatch.setattr(
|
||||
common,
|
||||
"correct_attn_out",
|
||||
lambda output, lses, *args, **kwargs: (output, lses[0]),
|
||||
)
|
||||
output = torch.ones(7, 1, 1)
|
||||
lse = torch.ones(7, 1)
|
||||
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
|
||||
_, masked_lse = common._cp_lse_common(
|
||||
output,
|
||||
lse,
|
||||
FakeGroup(),
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
|
||||
assert torch.isneginf(masked_lse[:1]).all()
|
||||
torch.testing.assert_close(masked_lse[1:5], torch.ones_like(masked_lse[1:5]))
|
||||
assert torch.isneginf(masked_lse[5:]).all()
|
||||
|
||||
def test_mathematically_correct(self):
|
||||
"""Verify mathematical correctness of LSE combination."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
@@ -344,7 +387,7 @@ class TestPackedA2AKernels:
|
||||
world_size, B, h_per_rank, D = 4, 7, 2, 32
|
||||
H = world_size * h_per_rank
|
||||
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=dtype)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
|
||||
send_buffer = torch.empty(
|
||||
(world_size, B, h_per_rank, D + lse_pack_dim),
|
||||
@@ -375,6 +418,67 @@ class TestPackedA2AKernels:
|
||||
else:
|
||||
_assert_packed_a2a_close(actual, expected_out, dtype)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 1, reason="CUDA is required."
|
||||
)
|
||||
def test_empty_seq_lens_ignore_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
_dcp_a2a_unpack_combine,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
world_size, num_tokens, h_per_rank, head_dim = 2, 5, 1, 32
|
||||
num_heads = world_size * h_per_rank
|
||||
output = torch.randn(
|
||||
num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
output[:1] = float("nan")
|
||||
lse = torch.randn(num_tokens, num_heads, device=device, dtype=output.dtype)
|
||||
seq_lens = torch.tensor([0, 2], device=device, dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], device=device, dtype=torch.int32)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(output.dtype)
|
||||
send_buffer = torch.empty(
|
||||
(
|
||||
world_size,
|
||||
num_tokens,
|
||||
h_per_rank,
|
||||
head_dim + lse_pack_dim,
|
||||
),
|
||||
device=device,
|
||||
dtype=output.dtype,
|
||||
)
|
||||
|
||||
_dcp_a2a_pack_send(
|
||||
output,
|
||||
lse,
|
||||
send_buffer,
|
||||
world_size,
|
||||
h_per_rank,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
actual_output, actual_lse = _dcp_a2a_unpack_combine(
|
||||
send_buffer,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
return_lse=True,
|
||||
is_lse_base_on_e=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_output[:1], torch.zeros_like(actual_output[:1])
|
||||
)
|
||||
assert torch.isneginf(actual_lse[:1]).all()
|
||||
assert torch.isfinite(actual_output[1:]).all()
|
||||
|
||||
|
||||
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
@@ -417,7 +521,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
B,
|
||||
H,
|
||||
device=f"cuda:{local_rank}",
|
||||
dtype=torch.float32,
|
||||
dtype=dtype,
|
||||
generator=generator,
|
||||
)
|
||||
actual = dcp_a2a_lse_reduce(
|
||||
|
||||
Reference in New Issue
Block a user