[Kimi-K3] support DCP partial prefix cache hit (#50493)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
0e8989b416
commit
0db502c8d8
@@ -2,12 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
|
||||
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
|
||||
|
||||
@@ -126,3 +129,111 @@ def test_kimi_linear_dcp_tiny(
|
||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||
|
||||
assert max(logprob_drifts) <= 1e-2
|
||||
|
||||
|
||||
def _make_tiny_k3_config(model_dir: Path) -> str:
|
||||
config = KimiK3Config(
|
||||
text_config=_make_tiny_overrides(),
|
||||
vision_config={
|
||||
"vt_num_attention_heads": 2,
|
||||
"vt_num_hidden_layers": 1,
|
||||
"vt_hidden_size": 32,
|
||||
"vt_intermediate_size": 64,
|
||||
"qkv_hidden_size": 48,
|
||||
},
|
||||
architectures=["KimiK3ForConditionalGeneration"],
|
||||
)
|
||||
config.save_pretrained(model_dir)
|
||||
return str(model_dir)
|
||||
|
||||
|
||||
def _run_k3_partial_prefix_reuse(
|
||||
vllm_runner: type[VllmRunner], model_name: str, dcp_size: int
|
||||
) -> tuple[list[int], list[float], int]:
|
||||
block_size = 256
|
||||
common_prefix = [3 + (token_idx % 251) for token_idx in range(block_size)]
|
||||
prime_prompt = TokensPrompt(prompt_token_ids=common_prefix + [17])
|
||||
replay_prompt = TokensPrompt(prompt_token_ids=common_prefix + [18, 19])
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=8,
|
||||
seed=0,
|
||||
ignore_eos=True,
|
||||
logprobs=20,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
language_model_only=True,
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
distributed_executor_backend="mp",
|
||||
dtype="bfloat16",
|
||||
seed=0,
|
||||
max_model_len=512,
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=256,
|
||||
gpu_memory_utilization=0.85,
|
||||
block_size=block_size,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
) as runner:
|
||||
assert (
|
||||
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
== CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
runner.llm.generate(
|
||||
[prime_prompt],
|
||||
sampling_params,
|
||||
use_tqdm=False,
|
||||
)
|
||||
replay_output = runner.llm.generate(
|
||||
[replay_prompt],
|
||||
sampling_params,
|
||||
use_tqdm=False,
|
||||
)[0]
|
||||
|
||||
completion = replay_output.outputs[0]
|
||||
token_ids = list(completion.token_ids)
|
||||
assert len(token_ids) == sampling_params.max_tokens
|
||||
assert completion.logprobs is not None
|
||||
assert len(completion.logprobs) == sampling_params.max_tokens
|
||||
selected_logprobs = [
|
||||
step_logprobs[token_id].logprob
|
||||
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
|
||||
]
|
||||
return token_ids, selected_logprobs, replay_output.num_cached_tokens
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
def test_kimi_k3_dcp_partial_prefix_reuse(
|
||||
vllm_runner: type[VllmRunner],
|
||||
num_gpus_available: int,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip("Need at least 2 GPUs")
|
||||
|
||||
model_name = _make_tiny_k3_config(tmp_path / "tiny-kimi-k3")
|
||||
baseline_tokens, baseline_logprobs, baseline_cached = _run_k3_partial_prefix_reuse(
|
||||
vllm_runner, model_name, dcp_size=1
|
||||
)
|
||||
dcp_tokens, dcp_logprobs, dcp_cached = _run_k3_partial_prefix_reuse(
|
||||
vllm_runner, model_name, dcp_size=2
|
||||
)
|
||||
|
||||
assert baseline_cached == 256
|
||||
assert dcp_cached == 256
|
||||
assert dcp_tokens == baseline_tokens
|
||||
assert len(dcp_logprobs) == len(baseline_logprobs) == 8
|
||||
logprob_drifts = []
|
||||
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
|
||||
assert math.isfinite(baseline_logprob)
|
||||
assert math.isfinite(dcp_logprob)
|
||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||
assert max(logprob_drifts) <= 1e-2
|
||||
|
||||
Reference in New Issue
Block a user