[PCP][DCP] Support DCP target model with non-DCP Dspark (#56723)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5
Codex
parent
77871126f9
commit
927c87b347
@@ -942,8 +942,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
impl.o_sf_scale = None
|
||||
impl.window_left = -1
|
||||
impl.sinks = None
|
||||
impl.cache_config = unittest.mock.Mock()
|
||||
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
|
||||
|
||||
layer = unittest.mock.Mock(
|
||||
_q_scale=torch.tensor(1.0),
|
||||
@@ -959,6 +957,7 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
q_len_per_req=1,
|
||||
)
|
||||
attn_metadata = flashinfer_backend.FlashInferMetadata(
|
||||
kv_cache_layout=KVCacheLayout.LBHNC,
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.empty(0, dtype=torch.int64),
|
||||
q_data_type_prefill=torch.bfloat16,
|
||||
|
||||
@@ -101,6 +101,7 @@ def test_flash_attention_geometry_comes_from_the_group():
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=True,
|
||||
)
|
||||
|
||||
with (
|
||||
|
||||
@@ -1599,19 +1599,142 @@ def test_project_kv_cache_groups_to_worker():
|
||||
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
|
||||
|
||||
|
||||
def test_dcp_world_size_for_kv_cache_spec_shards_full_attention_only():
|
||||
dcp = 8
|
||||
full = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
@pytest.mark.parametrize("sliding_window", [None, 256])
|
||||
@pytest.mark.parametrize("disable_hybrid", [False, True])
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
def test_dcp_target_allocates_replicated_draft_independently(
|
||||
monkeypatch, sliding_window, disable_hybrid, pcp_size
|
||||
):
|
||||
"""A draft must retain all positions even when the target shards them."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.parallel_config.prefill_context_parallel_size = pcp_size
|
||||
config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hybrid
|
||||
config.cache_config.block_size = 16
|
||||
config.cache_config.kv_cache_layout = None
|
||||
draft_args = dict(
|
||||
block_size=16,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
mla = new_mla_spec()
|
||||
mamba = new_mamba_spec()
|
||||
uniform_mla = UniformTypeKVCacheSpecs(block_size=16, kv_cache_specs={"layer": mla})
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(uniform_mla, dcp) == dcp
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(mamba, dcp) == 1
|
||||
assert kv_cache_utils.dcp_world_size_for_kv_cache_spec(full, 1) == 1
|
||||
draft = (
|
||||
FullAttentionSpec(**draft_args)
|
||||
if sliding_window is None
|
||||
else SlidingWindowSpec(**draft_args, sliding_window=sliding_window)
|
||||
)
|
||||
specs = {"target": new_mla_spec(), "draft": draft}
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC", "BLHNC"]], specs.values())
|
||||
assert layout == KVCacheLayout.BLHNC
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
assert [group.layer_names for group in groups] == [["target"], ["draft"]]
|
||||
widths = [g.kv_cache_spec.max_num_blocks_per_req(config, 1024) for g in groups]
|
||||
assert widths == [16, 64]
|
||||
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=16 * 1024 * 1024
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([cache_config])
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) > 0
|
||||
assert get_max_concurrency_for_kv_cache_config(config, cache_config) == (
|
||||
get_max_concurrency_for_kv_cache_config(config, scheduler_config)
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
scheduler_config,
|
||||
max_model_len=1024,
|
||||
hash_block_size=16,
|
||||
scheduler_block_size=64,
|
||||
dcp_world_size=4,
|
||||
pcp_world_size=pcp_size,
|
||||
enable_caching=True,
|
||||
)
|
||||
assert manager.coordinator.group_block_sizes == (64, 16)
|
||||
request = make_request("replicated-draft", [1] * 65, block_size=16, hash_fn=sha256)
|
||||
blocks = manager.allocate_slots(request, 65)
|
||||
assert blocks is not None
|
||||
assert [len(group) for group in blocks.blocks] == [2, 5]
|
||||
manager.cache_blocks(request, 64)
|
||||
cached_request = make_request(
|
||||
"cached-draft", [1] * 65, block_size=16, hash_fn=sha256
|
||||
)
|
||||
cached_blocks, num_cached, _ = manager.get_computed_blocks(cached_request)
|
||||
assert num_cached == 64
|
||||
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
draft = replace(
|
||||
new_mla_spec() if use_mla else new_kv_cache_spec(),
|
||||
block_size=16,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
merged = type(draft).merge([draft, draft])
|
||||
wrapped = UniformTypeKVCacheSpecs.from_specs({"draft": merged})
|
||||
assert wrapped is not None and not wrapped.dcp_sharded
|
||||
assert merged.max_num_blocks_per_req(config, 1024) == 64
|
||||
assert merged.max_memory_usage_bytes(config) == 64 * draft.page_size_bytes
|
||||
assert kv_cache_utils.resolve_dcp_kv_block_size(merged, 4) == 16
|
||||
with pytest.raises(AssertionError):
|
||||
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_draft", [False, True])
|
||||
@pytest.mark.parametrize("indexer_alignment", [None, 512])
|
||||
def test_sparse_mla_preserves_physical_row_addressing(with_draft, indexer_alignment):
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import flat_kv_row_view
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
common = dict(block_size=64, num_kv_heads=1, dtype=torch.uint8)
|
||||
specs = {
|
||||
"target": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=576,
|
||||
state_content_bytes=656,
|
||||
block_stride_alignment=656,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"indexer": MLAAttentionSpec(
|
||||
**common,
|
||||
head_size=132,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
block_stride_alignment=indexer_alignment,
|
||||
),
|
||||
"draft": SlidingWindowSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=64,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=2048,
|
||||
dcp_sharded=False,
|
||||
),
|
||||
}
|
||||
if not with_draft:
|
||||
del specs["draft"]
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=8 * 1024 * 1024
|
||||
)
|
||||
caches = allocate_kv_cache(
|
||||
cache_config, torch.device("cpu"), KVCacheLayout.BLHNC, [64] * len(groups)
|
||||
)
|
||||
cache = caches["target"].squeeze(1)
|
||||
if indexer_alignment is not None:
|
||||
assert caches["indexer"].stride(0) % indexer_alignment == 0
|
||||
rows, stride_rows = flat_kv_row_view(cache, 64)
|
||||
cache[1, 0].fill_(7)
|
||||
torch.testing.assert_close(rows[stride_rows], cache[1, 0])
|
||||
assert torch.all(rows[stride_rows] == 7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -3815,6 +3938,8 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
"layer_1": before_spec_1,
|
||||
"layer_2": before_spec_2,
|
||||
}
|
||||
kv_cache_spec["draft_layer_1"] = replace(before_spec_1, dcp_sharded=False)
|
||||
kv_cache_spec["draft_layer_2"] = replace(before_spec_2, dcp_sharded=False)
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
expected_spec_1 = new_kv_cache_spec(block_size=64)
|
||||
expected_spec_2 = new_kv_cache_spec(
|
||||
@@ -3823,6 +3948,7 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
assert kv_cache_spec["layer_1"] == expected_spec_1
|
||||
assert kv_cache_spec["layer_2"] == expected_spec_2
|
||||
assert kv_cache_spec["layer_2"].page_size_bytes == 64 * 1024
|
||||
assert kv_cache_spec["draft_layer_2"] == replace(expected_spec_2, dcp_sharded=False)
|
||||
|
||||
# 2. has_full_attention and has_chunked_local_attention
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
@@ -3874,6 +4000,11 @@ def test_unify_hybrid_kv_cache_specs():
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
# Replicated Mamba state still requires the hybrid cache manager.
|
||||
kv_cache_spec = {"attention": new_kv_cache_spec(), "mamba": new_mamba_spec()}
|
||||
with pytest.raises(ValueError):
|
||||
kv_cache_utils.unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
|
||||
|
||||
def test_unify_kv_cache_spec_page_size_mamba():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/43626.
|
||||
|
||||
@@ -1336,12 +1336,12 @@ def make_kv_cache_config_three_types(
|
||||
)
|
||||
|
||||
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
@pytest.mark.parametrize("draft_sharded", [True, False])
|
||||
def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
"""Prefix lookup must use each group's DCP size, not the process-wide one.
|
||||
|
||||
Target and draft MLA are both sharded (DCP=8); Mamba stays replicated
|
||||
(DCP=1). Hits then align to the sharded full-attention block, not the
|
||||
unsharded page size.
|
||||
Target MLA is sharded; draft MLA can opt out and Mamba stays replicated.
|
||||
Hits must align to the target block in either case.
|
||||
"""
|
||||
block_size = 16
|
||||
dcp = 8
|
||||
@@ -1362,6 +1362,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
KVCacheGroupSpec(
|
||||
["draft_mla"],
|
||||
MLAAttentionSpec(
|
||||
dcp_sharded=draft_sharded,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
@@ -1388,10 +1389,10 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
)
|
||||
target_mgr, draft_mgr, mamba_mgr = manager.coordinator.single_type_managers
|
||||
assert target_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == dcp
|
||||
assert draft_mgr.dcp_world_size == (dcp if draft_sharded else 1)
|
||||
assert mamba_mgr.dcp_world_size == 1
|
||||
assert target_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == sharded_block
|
||||
assert draft_mgr.block_size == (sharded_block if draft_sharded else block_size)
|
||||
assert mamba_mgr.block_size == block_size
|
||||
|
||||
hash_fn = sha256
|
||||
@@ -1407,7 +1408,11 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry():
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * sharded_block
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2, 16]
|
||||
assert [len(group) for group in computed_blocks.blocks] == [
|
||||
2,
|
||||
2 if draft_sharded else 16,
|
||||
16,
|
||||
]
|
||||
|
||||
manager.free(req0)
|
||||
manager.free(req1)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.config import CompilationConfig, VllmConfig, replace
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
|
||||
from ..utils import (
|
||||
Messages,
|
||||
@@ -196,6 +197,38 @@ def test_draft_model_engine_args_tensor_parallelism():
|
||||
assert draft_config.quant_config is None
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("dcp_size", [1, 2])
|
||||
def test_draft_model_inherits_target_dcp_config(monkeypatch, dcp_size):
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 16)
|
||||
target_config = EngineArgs(
|
||||
model="Qwen/Qwen3-1.7B",
|
||||
tensor_parallel_size=16,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
dcp_comm_backend="ag_rs",
|
||||
distributed_executor_backend="mp",
|
||||
speculative_config={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"method": "draft_model",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
).create_engine_config()
|
||||
proposer = object.__new__(DraftModelProposer)
|
||||
proposer.vllm_config = target_config
|
||||
proposer.speculative_config = target_config.speculative_config
|
||||
draft_config = proposer._create_draft_vllm_config()
|
||||
draft_parallel = draft_config.parallel_config
|
||||
assert draft_parallel.decode_context_parallel_size == dcp_size
|
||||
assert draft_parallel.cp_kv_cache_interleave_size == 16
|
||||
assert draft_parallel.dcp_comm_backend == "ag_rs"
|
||||
assert (
|
||||
draft_parallel.dcp_q_replicate == target_config.parallel_config.dcp_q_replicate
|
||||
)
|
||||
assert draft_parallel is not target_config.parallel_config
|
||||
assert target_config.parallel_config.decode_context_parallel_size == dcp_size
|
||||
|
||||
|
||||
def _apply_draft_moe_backend(vllm_config: VllmConfig) -> VllmConfig:
|
||||
"""Replicate SpecDecodeBaseProposer._create_draft_vllm_config logic
|
||||
so we can test it without instantiating a full proposer."""
|
||||
|
||||
@@ -500,7 +500,7 @@ def test_decode_bench_connector_skips_null_padding():
|
||||
|
||||
|
||||
def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
"""DCP scales full-attention blocks but not sliding-window blocks."""
|
||||
"""DCP scales target blocks but not the replicated draft's window."""
|
||||
block_size = 16
|
||||
dcp_world_size = 2
|
||||
vllm_config = create_vllm_config(
|
||||
@@ -515,6 +515,7 @@ def test_decode_bench_connector_uses_per_group_dcp_block_sizes():
|
||||
dtype=torch.float32,
|
||||
)
|
||||
sliding_window = SlidingWindowSpec(
|
||||
dcp_sharded=False,
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
|
||||
@@ -2316,8 +2316,8 @@ def _make_hybrid_attention_mamba_scheduler(
|
||||
def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
"""Replicated groups must not be scaled by the DCP world size.
|
||||
|
||||
``dcp_world_size_for_kv_cache_spec`` gives full attention the process DCP
|
||||
size and every other spec 1. Reconstructing a group's geometry as
|
||||
The spec's ``dcp_sharded`` flag controls its effective block size.
|
||||
Reconstructing a group's geometry as
|
||||
``spec.block_size * cp_world_size`` over-scales the replicated groups, so
|
||||
the eager store scan believes far fewer of their blocks are ready and
|
||||
silently offloads only a fraction of them.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -9,7 +10,7 @@ import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
prepare_dflash_inputs,
|
||||
DFlashSpeculator,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -24,6 +25,7 @@ def _run_prepare(
|
||||
cp_rank: int = 0,
|
||||
cp_size: int = 1,
|
||||
cp_interleave: int = 1,
|
||||
draft_dcp_size: int | None = None,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
@@ -40,7 +42,9 @@ def _run_prepare(
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_tokens=4,
|
||||
num_scheduled_tokens=np.array([4], dtype=np.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([target_positions[-1] + 1]),
|
||||
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
|
||||
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
|
||||
@@ -74,36 +78,65 @@ def _run_prepare(
|
||||
next_prefill_tokens = torch.zeros_like(last_sampled)
|
||||
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
|
||||
|
||||
prepare_dflash_inputs(
|
||||
input_buffers,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
input_batch,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
cp_rank,
|
||||
cp_size,
|
||||
cp_interleave,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
128,
|
||||
class InputsPrepared(Exception):
|
||||
pass
|
||||
|
||||
draft = SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
context_positions=context_positions,
|
||||
_context_slot_mappings=context_slot_mapping.unsqueeze(0),
|
||||
sample_indices=sample_indices,
|
||||
sample_pos=sample_pos,
|
||||
sample_idx_mapping=sample_idx_mapping,
|
||||
temperature=temperature,
|
||||
seeds=seeds,
|
||||
hidden_states=torch.zeros(4, 1, device=device),
|
||||
prepare_context_anchor=lambda *args: None,
|
||||
query_cudagraph_manager=None,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
pcp_manager=None,
|
||||
draft_kv_cache_group_id=0,
|
||||
draft_kv_cache_group_ids=[0],
|
||||
_layer_group_idx=None,
|
||||
block_tables=SimpleNamespace(
|
||||
slot_mappings=query_slot_mapping.unsqueeze(0),
|
||||
input_block_tables=[block_table],
|
||||
kernel_block_sizes=[4],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=cp_interleave,
|
||||
),
|
||||
dcp_size=cp_size if draft_dcp_size is None else draft_dcp_size,
|
||||
parallel_drafting_token_id=123,
|
||||
num_query_per_req=num_speculative_steps,
|
||||
num_speculative_steps=num_speculative_steps,
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_num_tokens=max_num_tokens,
|
||||
max_model_len=128,
|
||||
sample_from_anchor=True,
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"vllm.v1.worker.gpu.spec_decode.dflash.speculator.dispatch_cg_and_sync_dp",
|
||||
side_effect=InputsPrepared,
|
||||
),
|
||||
pytest.raises(InputsPrepared),
|
||||
):
|
||||
DFlashSpeculator.propose(
|
||||
draft,
|
||||
input_batch,
|
||||
{},
|
||||
{},
|
||||
draft.hidden_states,
|
||||
None,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
@@ -118,12 +151,17 @@ def _run_prepare(
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
@pytest.mark.parametrize("cp_rank,cp_size", [(0, 1), (0, 2), (1, 2)])
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size):
|
||||
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
|
||||
# block 8, but must be PAD context rather than contaminating draft KV.
|
||||
out = _run_prepare(
|
||||
target_positions=[10, 11, 12, 13],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=2,
|
||||
draft_dcp_size=1,
|
||||
)
|
||||
|
||||
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
|
||||
|
||||
@@ -17,6 +17,7 @@ from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -53,6 +54,7 @@ def _make_fake_speculator(
|
||||
arange_np=np.arange(max_num_reqs + 1, dtype=np.int32),
|
||||
draft_is_prefilling=torch.zeros(max_num_reqs, dtype=torch.bool),
|
||||
block_tables=fake_block_tables,
|
||||
dcp_size=1,
|
||||
input_buffers=fake_input_buffers,
|
||||
attn_groups=[],
|
||||
kv_cache_config=SimpleNamespace(kv_cache_groups=[]),
|
||||
@@ -152,14 +154,17 @@ def test_build_draft_attn_metadata_clamps_to_max_model_len():
|
||||
assert torch.equal(bound, torch.tensor([1024, 503], dtype=torch.int32))
|
||||
|
||||
|
||||
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
@pytest.mark.parametrize("draft_dcp_size", [1, 2])
|
||||
def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens(draft_dcp_size):
|
||||
fake = _make_fake_speculator()
|
||||
fake.dcp_size = draft_dcp_size
|
||||
fake.block_tables.cp_size = 2
|
||||
fake.block_tables.cp_rank = 1
|
||||
fake.block_tables.cp_interleave = 4
|
||||
fake.input_buffers.seq_lens[:3] = torch.tensor([5, 9, 16])
|
||||
|
||||
def fake_prepare(out, seq_lens, num_reqs, dcp_size, dcp_rank, cp_interleave):
|
||||
assert draft_dcp_size > 1
|
||||
assert seq_lens is fake.input_buffers.seq_lens
|
||||
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
|
||||
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
|
||||
@@ -176,6 +181,9 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
|
||||
)
|
||||
|
||||
local = captured["dcp_local_seq_lens"]
|
||||
if draft_dcp_size == 1:
|
||||
assert local is None
|
||||
return
|
||||
assert isinstance(local, torch.Tensor)
|
||||
assert local.data_ptr() == fake.input_buffers.dcp_local_seq_lens.data_ptr()
|
||||
assert local.tolist() == [1, 4, 8, 0]
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -7,6 +9,38 @@ from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.worker.cp_utils import should_skip_dcp_context_attention
|
||||
|
||||
|
||||
def test_cp_checks_allow_a_replicated_draft_backend_without_dcp(monkeypatch):
|
||||
from vllm.v1.worker import cp_utils
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=4,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
),
|
||||
speculative_config=object(),
|
||||
)
|
||||
target = SimpleNamespace(
|
||||
impl=SimpleNamespace(
|
||||
dcp_world_size=4,
|
||||
supports_mtp_with_cp_non_trivial_interleave_size=True,
|
||||
need_to_return_lse_for_decode=True,
|
||||
)
|
||||
)
|
||||
draft = SimpleNamespace(
|
||||
impl=SimpleNamespace(
|
||||
dcp_world_size=1,
|
||||
supports_mtp_with_cp_non_trivial_interleave_size=False,
|
||||
need_to_return_lse_for_decode=False,
|
||||
)
|
||||
)
|
||||
layers = {"target": target, "draft": draft}
|
||||
monkeypatch.setattr(cp_utils, "get_layers_from_vllm_config", lambda *args: layers)
|
||||
cp_utils.check_attention_cp_compatibility(config, target_layer_names={"target"})
|
||||
with pytest.raises(AssertionError, match="MTP with cp_kv_cache_interleave_size"):
|
||||
cp_utils.check_attention_cp_compatibility(config)
|
||||
|
||||
|
||||
def test_skip_gate_only_for_zero_context():
|
||||
assert should_skip_dcp_context_attention(torch.zeros(3, dtype=torch.int32))
|
||||
assert not should_skip_dcp_context_attention(
|
||||
|
||||
@@ -2,8 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.worker.gpu import model_runner
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.utils import _get_dspark_parallel_config
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -49,3 +58,86 @@ def test_dspark_parallel_config_disables_eplb_atomically():
|
||||
assert draft_config.eplb_config.num_redundant_experts == 0
|
||||
assert not draft_config.enable_elastic_ep
|
||||
assert draft_config.eplb_config is not target_config.eplb_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [1, 4])
|
||||
@pytest.mark.parametrize("dcp_size", [1, 4])
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_draft_context_parallelism_without_changing_target(
|
||||
monkeypatch, pcp_size, dcp_size, use_mla
|
||||
):
|
||||
target_parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
prefill_context_parallel_size=pcp_size,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=16,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
target_config = SimpleNamespace(
|
||||
parallel_config=target_parallel,
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(use_mla=use_mla)
|
||||
),
|
||||
)
|
||||
|
||||
class CapturedConfig(Exception):
|
||||
pass
|
||||
|
||||
def capture_init(self, config, device):
|
||||
raise CapturedConfig(config)
|
||||
|
||||
monkeypatch.setattr(DraftModelSpeculator, "__init__", capture_init)
|
||||
with pytest.raises(CapturedConfig) as captured:
|
||||
DFlashSpeculator(target_config, device=None)
|
||||
draft_parallel = captured.value.args[0].parallel_config
|
||||
assert draft_parallel.tensor_parallel_size == 4
|
||||
assert draft_parallel.prefill_context_parallel_size == 1
|
||||
assert draft_parallel.decode_context_parallel_size == (dcp_size if use_mla else 1)
|
||||
assert draft_parallel.cp_kv_cache_interleave_size == 16
|
||||
assert target_parallel.prefill_context_parallel_size == pcp_size
|
||||
assert target_parallel.decode_context_parallel_size == dcp_size
|
||||
assert target_parallel.cp_kv_cache_interleave_size == 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured_dcp", [1, 4])
|
||||
def test_attention_uses_draft_dcp_setting_inside_target_process_group(
|
||||
monkeypatch, configured_dcp
|
||||
):
|
||||
import vllm.config as config_module
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.v1.attention.backend import AttentionImplBase
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=configured_dcp)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
config_module, "get_current_vllm_config_or_none", lambda: config
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
parallel_state, "_DCP", SimpleNamespace(world_size=4, rank_in_group=2)
|
||||
)
|
||||
impl = AttentionImplBase()
|
||||
assert impl.dcp_world_size == configured_dcp
|
||||
assert impl.dcp_rank == (0 if configured_dcp == 1 else 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_dcp_size", [1, 4])
|
||||
def test_runner_marks_only_replicated_draft_caches(monkeypatch, draft_dcp_size):
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.bfloat16
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
model_runner, "get_kv_cache_spec", lambda _: {"target": spec, "draft": spec}
|
||||
)
|
||||
runner = object.__new__(model_runner.GPUModelRunner)
|
||||
runner.vllm_config = None
|
||||
runner.dcp_size = 4
|
||||
runner.speculator = object.__new__(DFlashSpeculator)
|
||||
runner.speculator.dcp_size = draft_dcp_size
|
||||
runner.speculator.draft_attn_layer_names = {"draft"}
|
||||
|
||||
specs = runner.get_kv_cache_spec()
|
||||
|
||||
assert specs["target"].dcp_sharded
|
||||
assert specs["draft"].dcp_sharded == (draft_dcp_size == 4)
|
||||
|
||||
@@ -88,6 +88,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
|
||||
)
|
||||
speculator.dcp_size = 1
|
||||
speculator.block_tables = SimpleNamespace(
|
||||
cp_size=1,
|
||||
input_block_tables=[torch.zeros(4, 1, dtype=torch.int32)],
|
||||
@@ -183,6 +184,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank=0,
|
||||
decode_context_parallel_size=1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -171,22 +171,23 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
"""DCP interleave is expressed in logical-block token coordinates."""
|
||||
"""Only sharded groups use DCP interleave in logical-block coordinates."""
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
block_sizes=[128],
|
||||
block_sizes=[128, 128],
|
||||
max_num_reqs=1,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_blocks_per_group=[2],
|
||||
max_num_blocks_per_group=[2, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[64],
|
||||
kernel_block_sizes=[64, 64],
|
||||
dcp_sharded=[True, False],
|
||||
cp_size=4,
|
||||
cp_rank=cp_rank,
|
||||
cp_interleave=128,
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
req_index=0,
|
||||
new_block_ids=([5, 9],),
|
||||
new_block_ids=([5, 9], list(range(10, 18))),
|
||||
overwrite=True,
|
||||
)
|
||||
block_tables.apply_staged_writes()
|
||||
@@ -199,7 +200,7 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_tokens_padded=1024,
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected = torch.full((1024,), -1, dtype=torch.int64, device=device)
|
||||
first_start = cp_rank * 128
|
||||
@@ -210,7 +211,8 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
expected[second_start : second_start + 128] = torch.arange(
|
||||
9 * 128, 10 * 128, dtype=torch.int64, device=device
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
assert torch.equal(actual[0], expected)
|
||||
assert torch.equal(actual[1], positions + 10 * 128)
|
||||
|
||||
|
||||
def test_v1_block_table_move_row_clears_vacated_row():
|
||||
|
||||
@@ -75,6 +75,24 @@ def test_validate_config_rejects_full_graph_for_prefills():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("draft_uses_mla", [False, True])
|
||||
@pytest.mark.parametrize("dcp_size", [1, 2, 4])
|
||||
def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
|
||||
config = _make_config(CUDAGraphMode.NONE)
|
||||
config.parallel_config.prefill_context_parallel_size = 4
|
||||
config.parallel_config.decode_context_parallel_size = dcp_size
|
||||
config.model_config.hf_text_config.index_topk = 1
|
||||
config.speculative_config = SimpleNamespace(
|
||||
use_dspark=lambda: True,
|
||||
draft_model_config=SimpleNamespace(use_mla=draft_uses_mla),
|
||||
)
|
||||
if draft_uses_mla and dcp_size == 2:
|
||||
with pytest.raises(NotImplementedError, match="requires DCP=1 or DCP=PCP"):
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
return
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
|
||||
|
||||
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
|
||||
Reference in New Issue
Block a user