[PCP][DCP] Support DCP target model with non-DCP Dspark (#56723)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5
Codex
parent
77871126f9
commit
927c87b347
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
|
||||
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
|
||||
|
||||
|
||||
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
|
||||
dcp = 8
|
||||
full = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
@pytest.mark.parametrize("sliding_window", [None, 256])
|
||||
@pytest.mark.parametrize("disable_hybrid", [False, True])
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
def test_dcp_target_allocates_replicated_draft_independently(
|
||||
monkeypatch, sliding_window, disable_hybrid, pcp_size
|
||||
):
|
||||
"""A draft must retain all positions even when the target shards them."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.parallel_config.prefill_context_parallel_size = pcp_size
|
||||
config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.kv_cache_layout = None
|
||||
draft_args = dict(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
mla = new_mla_spec()
|
||||
mamba = new_mamba_spec()
|
||||
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
|
||||
draft = (
|
||||
FullAttentionSpec(**draft_args)
|
||||
if sliding_window is None
|
||||
else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
|
||||
)
|
||||
specs = {"target": new_mla_spec(), "draft": draft}
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
|
||||
assert layout == KVCacheLayout.BLHNC
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
assert [group.layer_names for group in groups] == [["target"], ["draft"]]
|
||||
widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
|
||||
assert widths == [16, 64]
|
||||
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=16 * 1024 * 1024
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([cache_config])
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
|
||||
get_max_concurrency_for_kv_cache_config(config, scheduler_config)
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
scheduler_config,
|
||||
max_model_len=1024,
|
||||
hash_block_size=16,
|
||||
scheduler_block_size=64,
|
||||
dcp_world_size=4,
|
||||
pcp_world_size=pcp_size,
|
||||
enable_caching=True,
|
||||
)
|
||||
assert manager.coordinator.group_block_sizes == (64, 16)
|
||||
request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
|
||||
blocks = manager.allocate_slots(request, 65)
|
||||
assert blocks is not None
|
||||
assert [len(group) for group in blocks.blocks] == [2, 5]
|
||||
manager.cache_blocks(request, 64)
|
||||
cached_request = make_request(
|
||||
"cached-draft", [1] * 65, block_size=16, hash_fn=sha256
|
||||
)
|
||||
cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
|
||||
assert num_cached == 64
|
||||
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
draft = replace(
|
||||
new_mla_spec() if use_mla else new_kv_cache_spec(),
|
||||
block_size=16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
merged = type(draft).merge([draft, draft])
|
||||
wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
|
||||
assert wrapped is not None and not wrapped.dcp_sharded
|
||||
assert merged.max_num_blocks_per_req(config, 1024) == 64
|
||||
assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
|
||||
with pytest.raises(AssertionError):
|
||||
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_draft", [False, True])
|
||||
@pytest.mark.parametrize("indexer_alignment", [None, 512])
|
||||
def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
|
||||
specs = {
|
||||
"target": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=576,
|
||||
state_content_bytes=656,
|
||||
block_stride_alignment=656,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"indexer": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=132,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
block_stride_alignment=indexer_alignment,
|
||||
),
|
||||
"draft": SlidingWindowSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=64,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=2048,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
}
|
||||
if not with_draft:
|
||||
del specs["draft"]
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * 1024 * 1024
|
||||
)
|
||||
caches = allocate_kv_cache(
|
||||
cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
|
||||
)
|
||||
cache = caches["target"].squeeze(1)
|
||||
if indexer_alignment is not None:
|
||||
assert caches["indexer"].stride(0) % indexer_alignment == 0
|
||||
rows, stride_rows = flat_kv_row_view(cache, 64)
|
||||
cache[1, 0].fill_(7)
|
||||
torch.testing.assert_close(rows[stride_rows], cache[1, 0])
|
||||
assert torch.all(rows[stride_rows] == 7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
"layer_1": before_spec_1,
|
||||
"layer_2": before_spec_2,
|
||||
}
|
||||
kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
|
||||
kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
expected_spec_1 = new_kv_cache_spec(block_size=64)
|
||||
expected_spec_2 = new_kv_cache_spec(
|
||||
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
assert kv_cache_spec["layer_1"] == expected_spec_1
|
||||
assert kv_cache_spec["layer_2"] == expected_spec_2
|
||||
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
|
||||
assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
|
||||
|
||||
# 2. has_full_attention and has_chunked_local_attention
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
# Replicated Mamba state still requires the hybrid cache manager.
|
||||
kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
|
||||
def test_unify_kv_cache_spec_page_size_mamba():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
|
||||
|
||||
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
@pytest.mark.parametrize("draft_sharded", [True, False])
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
"""Prefix lookup must use each group's DCP size, not the process-wide one.
|
||||
|
||||
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
|
||||
(DCP=1). Hits then align to the sharded full-attention block, not the
|
||||
unsharded page size.
|
||||
Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
|
||||
Hits must align to the target block in either case.
|
||||
"""
|
||||
block_size = 16
|
||||
dcp = 8
|
||||
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
KVCacheGroupSpec(
|
||||
["draft_mla"],
|
||||
MLAAttentionSpec(
|
||||
dcp_sharded=draft_sharded,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
)
|
||||
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
|
||||
assert target_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
|
||||
assert mamba_mgr.dcp_world_size == 1
|
||||
assert target_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
|
||||
assert mamba_mgr.block_size == block_size
|
||||
|
||||
hash_fn = sha256
|
||||
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * sharded_block
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
|
||||
assert [len(group) for group in computed_blocks.blocks] == [
|
||||
2,
|
||||
2 if draft_sharded else 16,
|
||||
16,
|
||||
]
|
||||
|
||||
manager.free(req0)
|
||||
manager.free(req1)
|
||||
|
||||
Reference in New Issue
Block a user