[Kimi-K3] DCP support (#50484)

Signed-off-by: girasoley <girasoley@inferact.ai>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-10 12:54:26 -07:00
committed by GitHub
co-authored by Claude Fable 5 云挚 foraxe Woosuk Kwon
parent 3f142bd85e
commit 63ac04a61e
19 changed files with 3529 additions and 82 deletions
+106 -2
View File
@@ -222,6 +222,49 @@ class TestLSEWeightedCombine:
assert result.shape == (B, H, D)
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
def test_empty_shard_ignores_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
result = _lse_weighted_combine(outputs, lses)
torch.testing.assert_close(result, outputs[1])
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
import vllm.v1.attention.ops.common as common
class FakeGroup:
world_size = 2
rank_in_group = 0
def all_gather(self, tensor, dim):
assert dim == 0
return torch.cat((tensor, tensor), dim=dim)
monkeypatch.setattr(
common,
"correct_attn_out",
lambda output, lses, *args, **kwargs: (output, lses[0]),
)
output = torch.ones(7, 1, 1)
lse = torch.ones(7, 1)
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
_, masked_lse = common._cp_lse_common(
output,
lse,
FakeGroup(),
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
assert torch.isneginf(masked_lse[:1]).all()
torch.testing.assert_close(masked_lse[1:5], torch.ones_like(masked_lse[1:5]))
assert torch.isneginf(masked_lse[5:]).all()
def test_mathematically_correct(self):
"""Verify mathematical correctness of LSE combination."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
@@ -344,7 +387,7 @@ class TestPackedA2AKernels:
world_size, B, h_per_rank, D = 4, 7, 2, 32
H = world_size * h_per_rank
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
cp_attn_lse = torch.randn(B, H, device=device, dtype=dtype)
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
send_buffer = torch.empty(
(world_size, B, h_per_rank, D + lse_pack_dim),
@@ -375,6 +418,67 @@ class TestPackedA2AKernels:
else:
_assert_packed_a2a_close(actual, expected_out, dtype)
@pytest.mark.skipif(
torch.accelerator.device_count() < 1, reason="CUDA is required."
)
def test_empty_seq_lens_ignore_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
)
device = torch.device("cuda")
world_size, num_tokens, h_per_rank, head_dim = 2, 5, 1, 32
num_heads = world_size * h_per_rank
output = torch.randn(
num_tokens,
num_heads,
head_dim,
device=device,
dtype=torch.bfloat16,
)
output[:1] = float("nan")
lse = torch.randn(num_tokens, num_heads, device=device, dtype=output.dtype)
seq_lens = torch.tensor([0, 2], device=device, dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 5], device=device, dtype=torch.int32)
lse_pack_dim = _dcp_a2a_lse_pack_dim(output.dtype)
send_buffer = torch.empty(
(
world_size,
num_tokens,
h_per_rank,
head_dim + lse_pack_dim,
),
device=device,
dtype=output.dtype,
)
_dcp_a2a_pack_send(
output,
lse,
send_buffer,
world_size,
h_per_rank,
head_dim,
lse_pack_dim,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
actual_output, actual_lse = _dcp_a2a_unpack_combine(
send_buffer,
head_dim,
lse_pack_dim,
return_lse=True,
is_lse_base_on_e=True,
)
torch.testing.assert_close(
actual_output[:1], torch.zeros_like(actual_output[:1])
)
assert torch.isneginf(actual_lse[:1]).all()
assert torch.isfinite(actual_output[1:]).all()
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
update_environment_variables(env)
@@ -417,7 +521,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
B,
H,
device=f"cuda:{local_rank}",
dtype=torch.float32,
dtype=dtype,
generator=generator,
)
actual = dcp_a2a_lse_reduce(
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
from tests.conftest import VllmRunner
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams, TokensPrompt
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
def _make_tiny_overrides() -> dict:
linear_attn_config = {
"full_attn_layers": [4],
"head_dim": 128,
"kda_layers": [1, 2, 3],
"num_heads": 8,
"short_conv_kernel_size": 4,
"use_full_rank_gate": True,
}
return {
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"vocab_size": 256,
"hidden_size": 256,
"head_dim": 32,
"intermediate_size": 512,
"num_hidden_layers": 4,
"num_attention_heads": 8,
"num_key_value_heads": 8,
"num_experts": None,
"num_experts_per_token": None,
"num_shared_experts": 0,
"q_lora_rank": 128,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"mla_use_nope": True,
"mla_use_output_gate": True,
"linear_attn_config": linear_attn_config,
"max_position_embeddings": 512,
"model_max_length": 512,
}
def _run_tiny_model(
vllm_runner: type[VllmRunner], dcp_size: int
) -> list[tuple[list[int], list[float]]]:
with vllm_runner(
model_name=MODEL,
skip_tokenizer_init=True,
load_format="dummy",
hf_overrides=_make_tiny_overrides(),
tensor_parallel_size=2,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
distributed_executor_backend="mp",
dtype="bfloat16",
seed=0,
enforce_eager=True,
max_model_len=512,
max_num_seqs=8,
max_num_batched_tokens=1024,
gpu_memory_utilization=0.25,
enable_chunked_prefill=True,
enable_prefix_caching=False,
) as runner:
lengths = [1, 32, 33, 129]
prompts = [
TokensPrompt(
prompt_token_ids=[
3 + ((request_idx + token_idx) % 251) for token_idx in range(length)
]
)
for request_idx, length in enumerate(lengths)
]
outputs = runner.llm.generate(
prompts,
SamplingParams(
temperature=0,
max_tokens=8,
seed=0,
ignore_eos=True,
logprobs=20,
),
use_tqdm=False,
)
results = []
for request_output in outputs:
completion = request_output.outputs[0]
token_ids = list(completion.token_ids)
assert completion.logprobs is not None
selected_logprobs = [
step_logprobs[token_id].logprob
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
]
results.append((token_ids, selected_logprobs))
return results
@create_new_process_for_each_test()
@pytest.mark.distributed(num_gpus=2)
def test_kimi_linear_dcp_tiny(
vllm_runner: type[VllmRunner],
num_gpus_available: int,
) -> None:
if num_gpus_available < 2:
pytest.skip("Need at least 2 GPUs")
baseline = _run_tiny_model(vllm_runner, dcp_size=1)
dcp = _run_tiny_model(vllm_runner, dcp_size=2)
assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline]
logprob_drifts = []
for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp):
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
assert math.isfinite(baseline_logprob)
assert math.isfinite(dcp_logprob)
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
assert max(logprob_drifts) <= 1e-2