[Kimi-K3] support DCP partial prefix cache hit (#50493)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-17 17:05:51 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Yifan Qiao
parent 0e8989b416
commit 0db502c8d8
8 changed files with 523 additions and 37 deletions
@@ -2,12 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from pathlib import Path
import pytest
from tests.conftest import VllmRunner
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams, TokensPrompt
from vllm.config import CUDAGraphMode
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
@@ -126,3 +129,111 @@ def test_kimi_linear_dcp_tiny(
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
assert max(logprob_drifts) <= 1e-2
def _make_tiny_k3_config(model_dir: Path) -> str:
config = KimiK3Config(
text_config=_make_tiny_overrides(),
vision_config={
"vt_num_attention_heads": 2,
"vt_num_hidden_layers": 1,
"vt_hidden_size": 32,
"vt_intermediate_size": 64,
"qkv_hidden_size": 48,
},
architectures=["KimiK3ForConditionalGeneration"],
)
config.save_pretrained(model_dir)
return str(model_dir)
def _run_k3_partial_prefix_reuse(
vllm_runner: type[VllmRunner], model_name: str, dcp_size: int
) -> tuple[list[int], list[float], int]:
block_size = 256
common_prefix = [3 + (token_idx % 251) for token_idx in range(block_size)]
prime_prompt = TokensPrompt(prompt_token_ids=common_prefix + [17])
replay_prompt = TokensPrompt(prompt_token_ids=common_prefix + [18, 19])
sampling_params = SamplingParams(
temperature=0,
max_tokens=8,
seed=0,
ignore_eos=True,
logprobs=20,
)
with vllm_runner(
model_name=model_name,
skip_tokenizer_init=True,
load_format="dummy",
language_model_only=True,
tensor_parallel_size=2,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
distributed_executor_backend="mp",
dtype="bfloat16",
seed=0,
max_model_len=512,
max_num_seqs=4,
max_num_batched_tokens=256,
gpu_memory_utilization=0.85,
block_size=block_size,
enable_chunked_prefill=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
) as runner:
assert (
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
== CUDAGraphMode.FULL_AND_PIECEWISE
)
runner.llm.generate(
[prime_prompt],
sampling_params,
use_tqdm=False,
)
replay_output = runner.llm.generate(
[replay_prompt],
sampling_params,
use_tqdm=False,
)[0]
completion = replay_output.outputs[0]
token_ids = list(completion.token_ids)
assert len(token_ids) == sampling_params.max_tokens
assert completion.logprobs is not None
assert len(completion.logprobs) == sampling_params.max_tokens
selected_logprobs = [
step_logprobs[token_id].logprob
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
]
return token_ids, selected_logprobs, replay_output.num_cached_tokens
@create_new_process_for_each_test()
@pytest.mark.distributed(num_gpus=2)
def test_kimi_k3_dcp_partial_prefix_reuse(
vllm_runner: type[VllmRunner],
num_gpus_available: int,
tmp_path: Path,
) -> None:
if num_gpus_available < 2:
pytest.skip("Need at least 2 GPUs")
model_name = _make_tiny_k3_config(tmp_path / "tiny-kimi-k3")
baseline_tokens, baseline_logprobs, baseline_cached = _run_k3_partial_prefix_reuse(
vllm_runner, model_name, dcp_size=1
)
dcp_tokens, dcp_logprobs, dcp_cached = _run_k3_partial_prefix_reuse(
vllm_runner, model_name, dcp_size=2
)
assert baseline_cached == 256
assert dcp_cached == 256
assert dcp_tokens == baseline_tokens
assert len(dcp_logprobs) == len(baseline_logprobs) == 8
logprob_drifts = []
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
assert math.isfinite(baseline_logprob)
assert math.isfinite(dcp_logprob)
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
assert max(logprob_drifts) <= 1e-2