[Kimi-K3] support DCP partial prefix cache hit (#50493)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
0e8989b416
commit
0db502c8d8
@@ -2,12 +2,15 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import math
|
import math
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.conftest import VllmRunner
|
from tests.conftest import VllmRunner
|
||||||
from tests.utils import create_new_process_for_each_test
|
from tests.utils import create_new_process_for_each_test
|
||||||
from vllm import SamplingParams, TokensPrompt
|
from vllm import SamplingParams, TokensPrompt
|
||||||
|
from vllm.config import CUDAGraphMode
|
||||||
|
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||||
|
|
||||||
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
|
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
|
||||||
|
|
||||||
@@ -126,3 +129,111 @@ def test_kimi_linear_dcp_tiny(
|
|||||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||||
|
|
||||||
assert max(logprob_drifts) <= 1e-2
|
assert max(logprob_drifts) <= 1e-2
|
||||||
|
|
||||||
|
|
||||||
|
def _make_tiny_k3_config(model_dir: Path) -> str:
|
||||||
|
config = KimiK3Config(
|
||||||
|
text_config=_make_tiny_overrides(),
|
||||||
|
vision_config={
|
||||||
|
"vt_num_attention_heads": 2,
|
||||||
|
"vt_num_hidden_layers": 1,
|
||||||
|
"vt_hidden_size": 32,
|
||||||
|
"vt_intermediate_size": 64,
|
||||||
|
"qkv_hidden_size": 48,
|
||||||
|
},
|
||||||
|
architectures=["KimiK3ForConditionalGeneration"],
|
||||||
|
)
|
||||||
|
config.save_pretrained(model_dir)
|
||||||
|
return str(model_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_k3_partial_prefix_reuse(
|
||||||
|
vllm_runner: type[VllmRunner], model_name: str, dcp_size: int
|
||||||
|
) -> tuple[list[int], list[float], int]:
|
||||||
|
block_size = 256
|
||||||
|
common_prefix = [3 + (token_idx % 251) for token_idx in range(block_size)]
|
||||||
|
prime_prompt = TokensPrompt(prompt_token_ids=common_prefix + [17])
|
||||||
|
replay_prompt = TokensPrompt(prompt_token_ids=common_prefix + [18, 19])
|
||||||
|
sampling_params = SamplingParams(
|
||||||
|
temperature=0,
|
||||||
|
max_tokens=8,
|
||||||
|
seed=0,
|
||||||
|
ignore_eos=True,
|
||||||
|
logprobs=20,
|
||||||
|
)
|
||||||
|
|
||||||
|
with vllm_runner(
|
||||||
|
model_name=model_name,
|
||||||
|
skip_tokenizer_init=True,
|
||||||
|
load_format="dummy",
|
||||||
|
language_model_only=True,
|
||||||
|
tensor_parallel_size=2,
|
||||||
|
decode_context_parallel_size=dcp_size,
|
||||||
|
cp_kv_cache_interleave_size=1,
|
||||||
|
distributed_executor_backend="mp",
|
||||||
|
dtype="bfloat16",
|
||||||
|
seed=0,
|
||||||
|
max_model_len=512,
|
||||||
|
max_num_seqs=4,
|
||||||
|
max_num_batched_tokens=256,
|
||||||
|
gpu_memory_utilization=0.85,
|
||||||
|
block_size=block_size,
|
||||||
|
enable_chunked_prefill=True,
|
||||||
|
enable_prefix_caching=True,
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
) as runner:
|
||||||
|
assert (
|
||||||
|
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||||
|
== CUDAGraphMode.FULL_AND_PIECEWISE
|
||||||
|
)
|
||||||
|
runner.llm.generate(
|
||||||
|
[prime_prompt],
|
||||||
|
sampling_params,
|
||||||
|
use_tqdm=False,
|
||||||
|
)
|
||||||
|
replay_output = runner.llm.generate(
|
||||||
|
[replay_prompt],
|
||||||
|
sampling_params,
|
||||||
|
use_tqdm=False,
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
completion = replay_output.outputs[0]
|
||||||
|
token_ids = list(completion.token_ids)
|
||||||
|
assert len(token_ids) == sampling_params.max_tokens
|
||||||
|
assert completion.logprobs is not None
|
||||||
|
assert len(completion.logprobs) == sampling_params.max_tokens
|
||||||
|
selected_logprobs = [
|
||||||
|
step_logprobs[token_id].logprob
|
||||||
|
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
|
||||||
|
]
|
||||||
|
return token_ids, selected_logprobs, replay_output.num_cached_tokens
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test()
|
||||||
|
@pytest.mark.distributed(num_gpus=2)
|
||||||
|
def test_kimi_k3_dcp_partial_prefix_reuse(
|
||||||
|
vllm_runner: type[VllmRunner],
|
||||||
|
num_gpus_available: int,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
if num_gpus_available < 2:
|
||||||
|
pytest.skip("Need at least 2 GPUs")
|
||||||
|
|
||||||
|
model_name = _make_tiny_k3_config(tmp_path / "tiny-kimi-k3")
|
||||||
|
baseline_tokens, baseline_logprobs, baseline_cached = _run_k3_partial_prefix_reuse(
|
||||||
|
vllm_runner, model_name, dcp_size=1
|
||||||
|
)
|
||||||
|
dcp_tokens, dcp_logprobs, dcp_cached = _run_k3_partial_prefix_reuse(
|
||||||
|
vllm_runner, model_name, dcp_size=2
|
||||||
|
)
|
||||||
|
|
||||||
|
assert baseline_cached == 256
|
||||||
|
assert dcp_cached == 256
|
||||||
|
assert dcp_tokens == baseline_tokens
|
||||||
|
assert len(dcp_logprobs) == len(baseline_logprobs) == 8
|
||||||
|
logprob_drifts = []
|
||||||
|
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
|
||||||
|
assert math.isfinite(baseline_logprob)
|
||||||
|
assert math.isfinite(dcp_logprob)
|
||||||
|
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||||
|
assert max(logprob_drifts) <= 1e-2
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
|||||||
MLAAttention,
|
MLAAttention,
|
||||||
QueryLenSupport,
|
QueryLenSupport,
|
||||||
_DecodeConcatQuantFP8,
|
_DecodeConcatQuantFP8,
|
||||||
|
build_mla_chunked_context_metadata,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
@@ -93,6 +94,48 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
|
|||||||
assert spec.page_size_bytes == 64 * 656
|
assert spec.page_size_bytes == 64 * 656
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.cpu_test
|
||||||
|
def test_dcp_chunked_context_accepts_non_virtual_block_aligned_prefix():
|
||||||
|
context_lens_cpu = torch.tensor([65, 96], dtype=torch.int32)
|
||||||
|
prefill_query_start_loc_cpu = torch.tensor([0, 1, 2], dtype=torch.int32)
|
||||||
|
|
||||||
|
metadata = build_mla_chunked_context_metadata(
|
||||||
|
context_lens_cpu=context_lens_cpu,
|
||||||
|
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||||
|
chunked_prefill_workspace=torch.empty(0),
|
||||||
|
chunked_prefill_workspace_size=128,
|
||||||
|
block_size=64,
|
||||||
|
align_chunk_to_block=True,
|
||||||
|
device=torch.device("cpu"),
|
||||||
|
dcp_world_size=4,
|
||||||
|
dcp_local_block_size=1,
|
||||||
|
dcp_virtual_block_size=4,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert metadata is not None
|
||||||
|
assert metadata.context_lens.tolist() == [65, 96]
|
||||||
|
assert [chunk.seq_lens.tolist() for chunk in metadata.chunks] == [[65], [96]]
|
||||||
|
assert [chunk.starts.tolist() for chunk in metadata.chunks] == [[0], [0]]
|
||||||
|
assert [chunk.local_context_lens_allranks for chunk in metadata.chunks] == [
|
||||||
|
[[17, 16, 16, 16]],
|
||||||
|
[[24, 24, 24, 24]],
|
||||||
|
]
|
||||||
|
assert [chunk.padded_local_seq_lens for chunk in metadata.chunks] == [
|
||||||
|
[17],
|
||||||
|
[24],
|
||||||
|
]
|
||||||
|
assert [chunk.padded_local_cu_seq_lens.tolist() for chunk in metadata.chunks] == [
|
||||||
|
[0, 17],
|
||||||
|
[0, 24],
|
||||||
|
]
|
||||||
|
assert [chunk.cu_seq_lens.tolist() for chunk in metadata.chunks] == [
|
||||||
|
[0, 65],
|
||||||
|
[0, 96],
|
||||||
|
]
|
||||||
|
assert [chunk.num_context_tokens for chunk in metadata.chunks] == [65, 96]
|
||||||
|
assert [chunk.num_local_context_tokens for chunk in metadata.chunks] == [17, 24]
|
||||||
|
|
||||||
|
|
||||||
# Remove sm100 backends from the list if not using sm100
|
# Remove sm100 backends from the list if not using sm100
|
||||||
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
|
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
|
||||||
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
|
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||||
on partial hits, and same-step deferral."""
|
on partial hits, and same-step deferral."""
|
||||||
|
|
||||||
|
from math import lcm
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
@@ -68,12 +69,62 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
|
|||||||
manager.get_computed_blocks.assert_not_called()
|
manager.get_computed_blocks.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def test_mamba_align_split_partial_tail_schedule():
|
def make_full_mamba_manager(
|
||||||
|
*,
|
||||||
|
dcp_world_size: int,
|
||||||
|
hash_block_size: int = 2,
|
||||||
|
full_block_size: int = 4,
|
||||||
|
mamba_block_size: int = 4,
|
||||||
|
num_blocks: int = 32,
|
||||||
|
use_eagle: bool = False,
|
||||||
|
):
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=num_blocks,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["full"],
|
||||||
|
FullAttentionSpec(
|
||||||
|
block_size=full_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
KVCacheGroupSpec(
|
||||||
|
["mamba"],
|
||||||
|
MambaSpec(
|
||||||
|
block_size=mamba_block_size,
|
||||||
|
shapes=(1, 1),
|
||||||
|
dtypes=(torch.float32,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
scheduler_block_size = lcm(
|
||||||
|
full_block_size * dcp_world_size,
|
||||||
|
mamba_block_size,
|
||||||
|
)
|
||||||
|
return make_kv_cache_manager(
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_model_len=8192,
|
||||||
|
enable_caching=True,
|
||||||
|
dcp_world_size=dcp_world_size,
|
||||||
|
scheduler_block_size=scheduler_block_size,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
use_eagle=use_eagle,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("dcp_world_size", [1, 4])
|
||||||
|
def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||||
at the prompt's last hash boundary (registering the partial tail), then
|
at the prompt's last hash boundary (registering the partial tail), then
|
||||||
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||||
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||||
block_size = 512
|
block_size = 512
|
||||||
|
scheduler_block_size = block_size * dcp_world_size
|
||||||
hash_block_size = 32
|
hash_block_size = 32
|
||||||
mock = SimpleNamespace(
|
mock = SimpleNamespace(
|
||||||
cache_config=SimpleNamespace(block_size=block_size),
|
cache_config=SimpleNamespace(block_size=block_size),
|
||||||
@@ -81,6 +132,8 @@ def test_mamba_align_split_partial_tail_schedule():
|
|||||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||||
use_eagle=False,
|
use_eagle=False,
|
||||||
hash_block_size=hash_block_size,
|
hash_block_size=hash_block_size,
|
||||||
|
dcp_world_size=dcp_world_size,
|
||||||
|
scheduler_block_size=scheduler_block_size,
|
||||||
mamba_partial_cache_hit=True,
|
mamba_partial_cache_hit=True,
|
||||||
)
|
)
|
||||||
split = Scheduler._mamba_block_aligned_split
|
split = Scheduler._mamba_block_aligned_split
|
||||||
@@ -109,6 +162,7 @@ def test_mamba_align_split_partial_tail_schedule():
|
|||||||
# stops at the next block boundary (10240), later chunk ends re-align.
|
# stops at the next block boundary (10240), later chunk ends re-align.
|
||||||
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
|
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
|
||||||
req2.num_computed_tokens = 9984
|
req2.num_computed_tokens = 9984
|
||||||
|
assert req2.num_computed_tokens % scheduler_block_size != 0
|
||||||
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
|
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
|
||||||
req2.num_computed_tokens = 10240
|
req2.num_computed_tokens = 10240
|
||||||
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
||||||
@@ -1499,3 +1553,151 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
|||||||
|
|
||||||
assert num_computed == mamba_block_size
|
assert num_computed == mamba_block_size
|
||||||
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||||
|
def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
|
||||||
|
hash_block_size = 2
|
||||||
|
physical_block_size = 4
|
||||||
|
manager = make_full_mamba_manager(
|
||||||
|
dcp_world_size=dcp_world_size,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
full_block_size=physical_block_size,
|
||||||
|
mamba_block_size=physical_block_size,
|
||||||
|
)
|
||||||
|
assert manager.coordinator.enable_partial_hash_hits
|
||||||
|
|
||||||
|
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||||
|
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||||
|
manager.free(req0)
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
partial_hash = req0.block_hashes[2]
|
||||||
|
partial_full_block = manager.block_pool.get_cached_block(partial_hash, [0])
|
||||||
|
partial_mamba_block = manager.block_pool.get_cached_block(partial_hash, [1])
|
||||||
|
assert partial_full_block is not None
|
||||||
|
assert partial_mamba_block is not None
|
||||||
|
|
||||||
|
req1 = make_request("dcp-replay", [0, 0, 1, 1, 2, 2, 3, 3], 2, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 6
|
||||||
|
full_block_size = physical_block_size * dcp_world_size
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [
|
||||||
|
(6 + full_block_size - 1) // full_block_size,
|
||||||
|
2,
|
||||||
|
]
|
||||||
|
|
||||||
|
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||||
|
assert new_blocks is not None
|
||||||
|
full_new_block_id = new_blocks.get_block_ids()[0][0]
|
||||||
|
mamba_new_block_id = new_blocks.get_block_ids()[1][0]
|
||||||
|
copies, retained = manager.take_kv_cache_block_copies()
|
||||||
|
assert KVCacheBlockCopy(partial_full_block[0].block_id, full_new_block_id) in copies
|
||||||
|
assert (
|
||||||
|
KVCacheBlockCopy(partial_mamba_block[0].block_id, mamba_new_block_id) in copies
|
||||||
|
)
|
||||||
|
manager.block_pool.free_blocks(retained)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("dcp_world_size", [2, 4])
|
||||||
|
def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
|
||||||
|
dcp_world_size: int,
|
||||||
|
):
|
||||||
|
block_size = 4
|
||||||
|
manager = make_full_mamba_manager(
|
||||||
|
dcp_world_size=dcp_world_size,
|
||||||
|
hash_block_size=block_size,
|
||||||
|
full_block_size=block_size,
|
||||||
|
mamba_block_size=block_size,
|
||||||
|
)
|
||||||
|
assert manager.coordinator.enable_partial_hash_hits
|
||||||
|
assert manager.coordinator._cache_hit_alignment_tokens == block_size
|
||||||
|
assert manager.coordinator.single_type_managers[0].block_size == (
|
||||||
|
block_size * dcp_world_size
|
||||||
|
)
|
||||||
|
assert manager.coordinator.single_type_managers[1].block_size == block_size
|
||||||
|
|
||||||
|
prefix = list(range(12))
|
||||||
|
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||||
|
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
|
||||||
|
manager.free(req0)
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
req1 = make_request(
|
||||||
|
"snapshot-replay", prefix + list(range(12, 16)), block_size, sha256
|
||||||
|
)
|
||||||
|
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 12
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [
|
||||||
|
(12 + block_size * dcp_world_size - 1) // (block_size * dcp_world_size),
|
||||||
|
3,
|
||||||
|
]
|
||||||
|
partial_full_block = computed_blocks.blocks[0][-1]
|
||||||
|
new_blocks = manager.allocate_slots(req1, 4, num_computed, computed_blocks)
|
||||||
|
assert new_blocks is not None
|
||||||
|
full_new_block_id = new_blocks.get_block_ids()[0][0]
|
||||||
|
copies, retained = manager.take_kv_cache_block_copies()
|
||||||
|
assert KVCacheBlockCopy(partial_full_block.block_id, full_new_block_id) in copies
|
||||||
|
assert all(
|
||||||
|
copy.src_block_id != computed_blocks.blocks[1][-1].block_id for copy in copies
|
||||||
|
)
|
||||||
|
manager.block_pool.free_blocks(retained)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
|
||||||
|
block_size = 4
|
||||||
|
manager = make_full_mamba_manager(
|
||||||
|
dcp_world_size=2,
|
||||||
|
hash_block_size=block_size,
|
||||||
|
full_block_size=block_size,
|
||||||
|
mamba_block_size=block_size,
|
||||||
|
)
|
||||||
|
prefix = list(range(12))
|
||||||
|
req0 = make_request("joint-owner", prefix, block_size, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||||
|
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
replay_tokens = prefix + list(range(12, 16))
|
||||||
|
req1 = make_request("joint-before", replay_tokens, block_size, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 8
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [1, 2]
|
||||||
|
|
||||||
|
req0.num_computed_tokens = 8
|
||||||
|
assert manager.allocate_slots(req0, 4) is not None
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
req2 = make_request("joint-after", replay_tokens, block_size, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req2)
|
||||||
|
assert num_computed == 12
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [2, 3]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dcp_partial_hit_with_eagle_rewinds_one_hash_unit():
|
||||||
|
hash_block_size = 2
|
||||||
|
manager = make_full_mamba_manager(
|
||||||
|
dcp_world_size=2,
|
||||||
|
hash_block_size=hash_block_size,
|
||||||
|
full_block_size=4,
|
||||||
|
mamba_block_size=4,
|
||||||
|
use_eagle=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
req0 = make_request("eagle-owner", [7] * 6, hash_block_size, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||||
|
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||||
|
req0.num_computed_tokens = 4
|
||||||
|
manager.new_step_starts()
|
||||||
|
assert manager.allocate_slots(req0, 2) is not None
|
||||||
|
req0.num_computed_tokens = 6
|
||||||
|
manager.new_step_starts()
|
||||||
|
|
||||||
|
req1 = make_request("eagle-replay", [7] * 6 + [9] * 2, 2, sha256)
|
||||||
|
computed_blocks, num_computed, _ = manager.get_computed_blocks(req1)
|
||||||
|
assert num_computed == 4
|
||||||
|
assert [len(group) for group in computed_blocks.blocks] == [1, 1]
|
||||||
|
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||||
|
|||||||
@@ -55,9 +55,9 @@ def cache_full_block_and_partial_tail(
|
|||||||
token_ids: list[int],
|
token_ids: list[int],
|
||||||
*,
|
*,
|
||||||
enable_kv_cache_events: bool = False,
|
enable_kv_cache_events: bool = False,
|
||||||
|
block_size: int = 6,
|
||||||
) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]:
|
) -> tuple[BlockPool, Request, list[KVCacheBlock], BlockHash]:
|
||||||
hash_block_size = 2
|
hash_block_size = 2
|
||||||
block_size = 6
|
|
||||||
kv_cache_group_id = 0
|
kv_cache_group_id = 0
|
||||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||||
pool = BlockPool(
|
pool = BlockPool(
|
||||||
@@ -389,27 +389,34 @@ def test_reset_prefix_cache_clears_partial_entry_metadata():
|
|||||||
assert pool.cached_block_hashes_by_block == {}
|
assert pool.cached_block_hashes_by_block == {}
|
||||||
|
|
||||||
|
|
||||||
def test_evict_cached_block_removes_full_hash_and_partial_entry():
|
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||||
pool, req, blocks, partial_hash_10 = cache_full_block_and_partial_tail(
|
def test_evict_cached_block_removes_full_hash_and_partial_entry(
|
||||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
dcp_world_size: int,
|
||||||
|
):
|
||||||
|
block_size = 6 * dcp_world_size
|
||||||
|
partial_num_tokens = 2 * block_size - 2
|
||||||
|
pool, req, blocks, partial_hash = cache_full_block_and_partial_tail(
|
||||||
|
list(range(partial_num_tokens)), block_size=block_size
|
||||||
)
|
)
|
||||||
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, 6)[0]
|
full_hash = BlockHashListWithBlockSize(req.block_hashes, 2, block_size)[0]
|
||||||
|
|
||||||
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
|
assert pool.get_cached_block(full_hash, [0]) == [blocks[0]]
|
||||||
assert pool.get_cached_block(partial_hash_10, [0]) == [blocks[1]]
|
assert pool.get_cached_block(partial_hash, [0]) == [blocks[1]]
|
||||||
|
|
||||||
pool.evict_blocks({blocks[0].block_id, blocks[1].block_id})
|
pool.evict_blocks({blocks[0].block_id, blocks[1].block_id})
|
||||||
|
|
||||||
assert pool.get_cached_block(full_hash, [0]) is None
|
assert pool.get_cached_block(full_hash, [0]) is None
|
||||||
assert pool.get_cached_block(partial_hash_10, [0]) is None
|
assert pool.get_cached_block(partial_hash, [0]) is None
|
||||||
assert pool.cached_block_hashes_by_block == {}
|
assert pool.cached_block_hashes_by_block == {}
|
||||||
|
|
||||||
|
|
||||||
def test_partial_block_promotes_to_direct_full_block_hash():
|
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||||
|
def test_partial_block_promotes_to_direct_full_block_hash(dcp_world_size: int):
|
||||||
hash_block_size = 2
|
hash_block_size = 2
|
||||||
block_size = 6
|
block_size = 6 * dcp_world_size
|
||||||
kv_cache_group_id = 0
|
kv_cache_group_id = 0
|
||||||
token_ids = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4]
|
partial_num_tokens = 2 * block_size - hash_block_size
|
||||||
|
token_ids = list(range(partial_num_tokens))
|
||||||
req = make_request("0", token_ids, hash_block_size, sha256)
|
req = make_request("0", token_ids, hash_block_size, sha256)
|
||||||
pool = BlockPool(
|
pool = BlockPool(
|
||||||
num_gpu_blocks=3,
|
num_gpu_blocks=3,
|
||||||
@@ -426,27 +433,22 @@ def test_partial_block_promotes_to_direct_full_block_hash():
|
|||||||
block_size=block_size,
|
block_size=block_size,
|
||||||
kv_cache_group_id=kv_cache_group_id,
|
kv_cache_group_id=kv_cache_group_id,
|
||||||
)
|
)
|
||||||
partial_hash_10 = boundary_hash(req, hash_block_size, 10)
|
partial_hash = boundary_hash(req, hash_block_size, partial_num_tokens)
|
||||||
assert pool.cache_partial_block(
|
assert pool.cache_partial_block(
|
||||||
request=req,
|
request=req,
|
||||||
block=blocks[1],
|
block=blocks[1],
|
||||||
num_tokens=10,
|
num_tokens=partial_num_tokens,
|
||||||
kv_cache_group_id=kv_cache_group_id,
|
kv_cache_group_id=kv_cache_group_id,
|
||||||
block_size=block_size,
|
block_size=block_size,
|
||||||
)
|
)
|
||||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) == [blocks[1]]
|
assert pool.get_cached_block(partial_hash, [kv_cache_group_id]) == [blocks[1]]
|
||||||
|
|
||||||
req.append_output_token_ids([5, 5])
|
req.append_output_token_ids(list(range(partial_num_tokens, 2 * block_size)))
|
||||||
full_hashes = BlockHashListWithBlockSize(
|
full_hashes = BlockHashListWithBlockSize(
|
||||||
req.block_hashes, hash_block_size, block_size
|
req.block_hashes, hash_block_size, block_size
|
||||||
)
|
)
|
||||||
promoted_full_hash = full_hashes[1]
|
promoted_full_hash = full_hashes[1]
|
||||||
# The promoted full-block hash is the fine hash at the 12-token boundary,
|
assert promoted_full_hash == req.block_hashes[2 * block_size // hash_block_size - 1]
|
||||||
# not a concatenation of the fine hashes inside the block.
|
|
||||||
assert promoted_full_hash == req.block_hashes[12 // hash_block_size - 1]
|
|
||||||
assert promoted_full_hash != BlockHash(
|
|
||||||
req.block_hashes[3] + req.block_hashes[4] + req.block_hashes[5]
|
|
||||||
)
|
|
||||||
|
|
||||||
pool.cache_full_blocks(
|
pool.cache_full_blocks(
|
||||||
request=req,
|
request=req,
|
||||||
@@ -457,4 +459,4 @@ def test_partial_block_promotes_to_direct_full_block_hash():
|
|||||||
kv_cache_group_id=kv_cache_group_id,
|
kv_cache_group_id=kv_cache_group_id,
|
||||||
)
|
)
|
||||||
assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]]
|
assert pool.get_cached_block(promoted_full_hash, [kv_cache_group_id]) == [blocks[1]]
|
||||||
assert pool.get_cached_block(partial_hash_10, [kv_cache_group_id]) is None
|
assert pool.get_cached_block(partial_hash, [kv_cache_group_id]) is None
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
import vllm.v1.worker.gpu.model_runner as model_runner_module
|
||||||
|
from vllm.v1.kv_cache_interface import (
|
||||||
|
FullAttentionSpec,
|
||||||
|
KVCacheConfig,
|
||||||
|
KVCacheGroupSpec,
|
||||||
|
MambaSpec,
|
||||||
|
)
|
||||||
|
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||||
|
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("mamba_cache_mode", "num_speculative_blocks", "expected"),
|
||||||
|
[
|
||||||
|
pytest.param("align", 0, 65_536, id="align-prefix-cache"),
|
||||||
|
pytest.param("none", 7, 8, id="no-prefix-cache-with-speculation"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_initialize_kv_cache_does_not_dcp_shard_mamba_block_table(
|
||||||
|
monkeypatch,
|
||||||
|
mamba_cache_mode: str,
|
||||||
|
num_speculative_blocks: int,
|
||||||
|
expected: int,
|
||||||
|
):
|
||||||
|
"""Mamba/GDN block-table rows index global positions, unlike DCP KV."""
|
||||||
|
|
||||||
|
max_model_len = 1_048_576
|
||||||
|
attention_block_size = 1_536
|
||||||
|
mamba_block_size = 16
|
||||||
|
dcp_size = 8
|
||||||
|
full_attention_spec = FullAttentionSpec(
|
||||||
|
block_size=attention_block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
)
|
||||||
|
mamba_spec = MambaSpec(
|
||||||
|
shapes=((1,),),
|
||||||
|
dtypes=(torch.bfloat16,),
|
||||||
|
block_size=mamba_block_size,
|
||||||
|
mamba_cache_mode=mamba_cache_mode,
|
||||||
|
num_speculative_blocks=num_speculative_blocks,
|
||||||
|
)
|
||||||
|
kv_cache_config = KVCacheConfig(
|
||||||
|
num_blocks=1,
|
||||||
|
kv_cache_tensors=[],
|
||||||
|
kv_cache_groups=[
|
||||||
|
KVCacheGroupSpec(["attention"], full_attention_spec),
|
||||||
|
KVCacheGroupSpec(["kda"], mamba_spec),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
vllm_config = SimpleNamespace(
|
||||||
|
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
|
||||||
|
cache_config=SimpleNamespace(mamba_cache_mode=mamba_cache_mode),
|
||||||
|
)
|
||||||
|
runner = SimpleNamespace(
|
||||||
|
max_model_len=max_model_len,
|
||||||
|
is_encoder_decoder=False,
|
||||||
|
vllm_config=vllm_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
class _CapturedWidths(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
captured: list[int] = []
|
||||||
|
|
||||||
|
def capture_width(max_num_blocks: int, *_args, **_kwargs) -> int:
|
||||||
|
captured.append(max_num_blocks)
|
||||||
|
if len(captured) == 2:
|
||||||
|
raise _CapturedWidths
|
||||||
|
return max_num_blocks
|
||||||
|
|
||||||
|
monkeypatch.setattr(model_runner_module, "get_block_table_width", capture_width)
|
||||||
|
|
||||||
|
with pytest.raises(_CapturedWidths):
|
||||||
|
GPUModelRunner.initialize_kv_cache(runner, kv_cache_config)
|
||||||
|
|
||||||
|
# Attention KV is local to one of eight DCP ranks; KDA state is replicated
|
||||||
|
# and therefore needs one table entry for every global 16-token page.
|
||||||
|
assert captured == [86, expected]
|
||||||
|
|
||||||
|
|
||||||
|
def test_append_block_ids_rejects_write_past_row_capacity():
|
||||||
|
"""Reject an oversized staged write before it can corrupt the next row."""
|
||||||
|
|
||||||
|
class _BlockTable:
|
||||||
|
gpu = torch.empty((2, 4), dtype=torch.int32)
|
||||||
|
|
||||||
|
def stage_write(self, *_args):
|
||||||
|
pytest.fail("an oversized write must not be staged")
|
||||||
|
|
||||||
|
block_tables = BlockTables.__new__(BlockTables)
|
||||||
|
block_tables.num_kv_cache_groups = 1
|
||||||
|
block_tables.blocks_per_kv_block = [1]
|
||||||
|
block_tables.block_tables = [_BlockTable()]
|
||||||
|
block_tables.num_blocks = SimpleNamespace(
|
||||||
|
np=torch.tensor([[0, 3]], dtype=torch.int32)
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
RuntimeError,
|
||||||
|
match=r"request 1, group 0 exceeds row capacity \(5 > 4\)",
|
||||||
|
):
|
||||||
|
block_tables.append_block_ids(
|
||||||
|
req_index=1,
|
||||||
|
new_block_ids=([4, 5],),
|
||||||
|
overwrite=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert block_tables.num_blocks.np[0, 1] == 3
|
||||||
@@ -619,15 +619,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
|||||||
"full-attention and Mamba groups, got: "
|
"full-attention and Mamba groups, got: "
|
||||||
f"{type(g.kv_cache_spec).__name__}."
|
f"{type(g.kv_cache_spec).__name__}."
|
||||||
)
|
)
|
||||||
# Fine-grained hash hits require Mamba "align", no context
|
# Fine-grained hash hits require Mamba "align" and compatible cache
|
||||||
# parallelism, and compatible cache managers in every group.
|
# managers in every group. TP needs hashing finer than the Mamba block;
|
||||||
|
# DCP accepts equality because it scales the effective full-attention
|
||||||
|
# block instead.
|
||||||
has_partial_mamba_group = any(
|
has_partial_mamba_group = any(
|
||||||
isinstance(g.kv_cache_spec, MambaSpec)
|
isinstance(g.kv_cache_spec, MambaSpec)
|
||||||
and g.kv_cache_spec.mamba_cache_mode == "align"
|
and g.kv_cache_spec.mamba_cache_mode == "align"
|
||||||
and g.kv_cache_spec.block_size > hash_block_size
|
and (
|
||||||
|
(dcp_world_size == 1 and g.kv_cache_spec.block_size > hash_block_size)
|
||||||
|
or (
|
||||||
|
dcp_world_size > 1 and g.kv_cache_spec.block_size >= hash_block_size
|
||||||
|
)
|
||||||
|
)
|
||||||
for g in kv_cache_config.kv_cache_groups
|
for g in kv_cache_config.kv_cache_groups
|
||||||
)
|
)
|
||||||
self.enable_partial_hash_hits = dcp_world_size == 1 and has_partial_mamba_group
|
self.enable_partial_hash_hits = has_partial_mamba_group
|
||||||
if self.enable_partial_hash_hits:
|
if self.enable_partial_hash_hits:
|
||||||
unsupported_partial_hit_managers = {
|
unsupported_partial_hit_managers = {
|
||||||
type(manager).__name__
|
type(manager).__name__
|
||||||
|
|||||||
@@ -116,8 +116,15 @@ class BlockTables:
|
|||||||
bpk = self.blocks_per_kv_block[i]
|
bpk = self.blocks_per_kv_block[i]
|
||||||
if bpk > 1:
|
if bpk > 1:
|
||||||
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
|
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
|
||||||
|
end = start + len(block_ids)
|
||||||
|
row_capacity = self.block_tables[i].gpu.shape[1]
|
||||||
|
if end > row_capacity:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Block table write for request {req_index}, group {i} exceeds "
|
||||||
|
f"row capacity ({end} > {row_capacity})"
|
||||||
|
)
|
||||||
self.block_tables[i].stage_write(req_index, start, block_ids)
|
self.block_tables[i].stage_write(req_index, start, block_ids)
|
||||||
self.num_blocks.np[i, req_index] = start + len(block_ids)
|
self.num_blocks.np[i, req_index] = end
|
||||||
|
|
||||||
def apply_staged_writes(self) -> None:
|
def apply_staged_writes(self) -> None:
|
||||||
if self.num_kv_cache_groups == 0:
|
if self.num_kv_cache_groups == 0:
|
||||||
|
|||||||
@@ -60,7 +60,6 @@ from vllm.multimodal.encoder_budget import (
|
|||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tasks import SupportedTask
|
from vllm.tasks import SupportedTask
|
||||||
from vllm.utils.math_utils import cdiv
|
|
||||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||||
@@ -517,17 +516,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||||
spec = kv_cache_group.kv_cache_spec
|
spec = kv_cache_group.kv_cache_spec
|
||||||
block_sizes.append(spec.block_size)
|
block_sizes.append(spec.block_size)
|
||||||
# When using DCP, each request's KV cache is sharded among different ranks.
|
# Let each cache type account for CP. Attention KV is DCP-sharded,
|
||||||
# As a result, one block on the current rank covers `block_size * cp_size`
|
# while Mamba/GDN recurrent state is replicated across DCP ranks.
|
||||||
# tokens in the full, global (unsharded) sequence.
|
max_num_blocks = spec.max_num_blocks_per_req(
|
||||||
max_num_blocks = cdiv(
|
self.vllm_config, block_table_max_model_len
|
||||||
block_table_max_model_len, spec.block_size * self.dcp_size
|
|
||||||
)
|
)
|
||||||
# For Mamba/Hybrid Model, KVCaches need extra blocks for speculative tokens
|
# Preserve each cache type's alignment requirements after applying
|
||||||
|
# its topology-aware block-table width.
|
||||||
if isinstance(spec, MambaSpec):
|
if isinstance(spec, MambaSpec):
|
||||||
max_num_blocks = (
|
|
||||||
max_num_blocks if self.cache_config.enable_prefix_caching else 1
|
|
||||||
) + spec.num_speculative_blocks
|
|
||||||
max_num_blocks = get_block_table_width(
|
max_num_blocks = get_block_table_width(
|
||||||
max_num_blocks, spec.block_size, token_alignment=None
|
max_num_blocks, spec.block_size, token_alignment=None
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user