[Bugfix][MRV2] Initialize DCP metadata after batch partitioning (#55212)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Ziyang Zhang <hafuhafu@qq.com>
Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Raphael Rialland <raphael.rialland@mistral.ai>
Co-authored-by: Ziyang Zhang <hafuhafu@qq.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Sy03
2026-09-10 06:50:09 +00:00
committed by GitHub
co-authored by Raphael Rialland Ziyang Zhang Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 912d2e79fe
commit 83990f5bcc
12 changed files with 460 additions and 77 deletions
@@ -144,8 +144,11 @@ def test_build_draft_attn_metadata_recomputes_dcp_local_seq_lens():
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (3, 2, 1, 4)
out[:num_reqs].copy_(torch.tensor([1, 4, 8], dtype=torch.int32))
out[num_reqs:].zero_()
return out
with patch.object(base_speculator, "prepare_dcp_local_seq_lens", fake_prepare):
with patch.object(
base_speculator, "maybe_prepare_dcp_local_seq_lens", fake_prepare
):
captured = _run_build(
fake,
num_reqs=3,
+110
View File
@@ -2,10 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the V2 model runner's InputBatch (vllm.v1.worker.gpu.input_batch)."""
import numpy as np
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.worker.gpu import cp_utils
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
DEVICE = current_platform.device_type
@@ -51,3 +54,110 @@ def test_make_dummy_distributes_remainder(num_reqs: int, num_tokens: int):
assert torch.equal(
batch.query_start_loc.cpu(), torch.from_numpy(batch.query_start_loc_np)
)
def test_maybe_prepare_dcp_local_seq_lens_uses_shared_buffer(monkeypatch):
"""The batch must view the caller-owned buffer, sliced to padded length.
Runtime (and capture) paths all funnel through this helper; attention
metadata indexes padded rows, so the view must reach
num_reqs_after_padding, and it must alias the persistent buffer so CUDA
graph replay sees the recomputed values.
"""
buffers = InputBuffers(max_num_reqs=4, max_num_tokens=4, device=torch.device("cpu"))
batch = InputBatch.make_dummy(2, 4, buffers)
batch.num_reqs_after_padding = 4
def fake_kernel(
output,
seq_lens,
dcp_size,
dcp_rank,
cp_interleave,
num_reqs,
max_num_reqs,
block_size,
):
assert output is buffers.dcp_local_seq_lens
assert seq_lens is batch.seq_lens
assert (num_reqs, dcp_size, dcp_rank, cp_interleave) == (2, 4, 1, 16)
assert (max_num_reqs, block_size) == (4, 128)
output[:] = torch.tensor([1, 2, 0, 0], dtype=output.dtype)
class FakeKernel:
def __getitem__(self, grid):
assert grid == (1,)
return fake_kernel
monkeypatch.setattr(cp_utils, "_dcp_local_seq_lens_kernel", FakeKernel())
batch.dcp_local_seq_lens = cp_utils.maybe_prepare_dcp_local_seq_lens(
buffers.dcp_local_seq_lens,
batch.seq_lens,
batch.num_reqs,
4,
1,
16,
num_reqs_padded=batch.num_reqs_after_padding,
)
assert batch.dcp_local_seq_lens is not None
assert batch.dcp_local_seq_lens.data_ptr() == buffers.dcp_local_seq_lens.data_ptr()
assert batch.dcp_local_seq_lens.tolist() == [1, 2, 0, 0]
def test_maybe_prepare_dcp_local_seq_lens_clears_stale_metadata(monkeypatch):
"""dcp_size == 1 resets the field to None instead of leaving a leftover.
DCP is toggled per deployment, and batches are recycled; a value from an
earlier DCP batch would otherwise be consumed as if it were current.
"""
buffers = InputBuffers(max_num_reqs=2, max_num_tokens=2, device=torch.device("cpu"))
batch = InputBatch.make_dummy(1, 1, buffers)
batch.dcp_local_seq_lens = buffers.dcp_local_seq_lens[:1]
def fail_if_called(*args, **kwargs):
raise AssertionError("kernel must not run with dcp_size == 1")
monkeypatch.setattr(cp_utils, "_dcp_local_seq_lens_kernel", fail_if_called)
batch.dcp_local_seq_lens = cp_utils.maybe_prepare_dcp_local_seq_lens(
buffers.dcp_local_seq_lens,
batch.seq_lens,
batch.num_reqs,
1,
0,
1,
num_reqs_padded=batch.num_reqs_after_padding,
)
assert batch.dcp_local_seq_lens is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="triton kernel needs CUDA")
@pytest.mark.parametrize("dcp_size", [2, 4])
@pytest.mark.parametrize("cp_interleave", [1, 16])
def test_maybe_prepare_dcp_local_seq_lens_matches_reference(
dcp_size: int, cp_interleave: int
):
"""Every rank's local lengths must equal the reference torch formula."""
device = torch.device("cuda:0")
seq_lens_np = np.array([7, 16, 33, 64, 512, 1023], dtype=np.int32)
buffers = InputBuffers(max_num_reqs=8, max_num_tokens=32, device=device)
batch = InputBatch.make_dummy(6, 12, buffers)
buffers.seq_lens[: len(seq_lens_np)] = torch.from_numpy(seq_lens_np).to(device)
for dcp_rank in range(dcp_size):
buffers.dcp_local_seq_lens.fill_(-1)
batch.dcp_local_seq_lens = cp_utils.maybe_prepare_dcp_local_seq_lens(
buffers.dcp_local_seq_lens,
batch.seq_lens,
batch.num_reqs,
dcp_size,
dcp_rank,
cp_interleave,
num_reqs_padded=batch.num_reqs_after_padding,
)
expected = get_dcp_local_seq_lens(
torch.from_numpy(seq_lens_np), dcp_size, dcp_rank, cp_interleave
)
assert batch.dcp_local_seq_lens is not None
assert torch.equal(batch.dcp_local_seq_lens.cpu(), expected.to(torch.int32))
+105
View File
@@ -1,11 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import replace
import numpy as np
import pytest
import torch
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.pcp_manager import PCPManager
from vllm.v1.worker.gpu.states import RequestState
def _copy_to_cpu(value, out=None, device=None):
@@ -107,3 +113,102 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
query_start_loc_np=np.arange(4, dtype=np.int32),
padded_num_tokens=2,
)
def _make_global_decode_batch(
num_computed_tokens: list[int], buffers: InputBuffers, device: torch.device
) -> InputBatch:
"""A replicate-decode global batch as `prepare_inputs` would build it."""
num_reqs = len(num_computed_tokens)
num_tokens = num_reqs
seq_lens_np = np.asarray(num_computed_tokens, dtype=np.int32) + 1
base = InputBatch.make_dummy(num_reqs, num_tokens, buffers)
buffers.seq_lens[:num_reqs] = torch.from_numpy(seq_lens_np).to(device)
query_start_loc_np = np.arange(num_reqs + 1, dtype=np.int32)
buffers.query_start_loc[: num_reqs + 1] = torch.from_numpy(query_start_loc_np).to(
device
)
return replace(
base,
req_ids=[f"req_{i}" for i in range(num_reqs)],
num_reqs=num_reqs,
num_reqs_after_padding=num_reqs,
idx_mapping=torch.arange(num_reqs, dtype=torch.int32, device=device),
idx_mapping_np=np.arange(num_reqs, dtype=np.int32),
num_scheduled_tokens=np.ones(num_reqs, dtype=np.int32),
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens,
num_draft_tokens=0,
num_draft_tokens_per_req=np.zeros(num_reqs, dtype=np.int32),
query_start_loc=buffers.query_start_loc[: num_reqs + 1],
query_start_loc_np=query_start_loc_np,
seq_lens=buffers.seq_lens[:num_reqs],
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_np),
dcp_local_seq_lens=None,
num_computed_tokens_np=np.asarray(num_computed_tokens, dtype=np.int32),
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
input_ids=buffers.input_ids[:num_tokens],
positions=buffers.positions[:num_tokens],
is_padding=buffers.is_padding[:num_tokens],
prompt_lens=None,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs GPU kernels")
def test_partition_defers_dcp_metadata_to_post_partition_batch():
"""DCP-local lengths must derive from the partitioned batch, not the
global one: the partition replaces seq_lens, so a pre-partition value is
stale. partition_batch therefore returns None, and the runtime populates
the field afterwards from the PCP-owned buffers.
"""
device = torch.device("cuda:0")
req_states = RequestState(
max_num_reqs=4,
max_model_len=64,
max_num_batched_tokens=8,
num_speculative_steps=1,
vocab_size=8,
device=device,
)
manager = PCPManager(
pcp_world_size=2,
pcp_rank=0,
device=device,
req_states=req_states,
max_num_reqs=4,
max_num_tokens=8,
dcp_world_size=2,
dcp_rank=0,
)
global_buffers = InputBuffers(4, 8, device)
global_batch = _make_global_decode_batch([16, 24], global_buffers, device)
# A leftover from an earlier DCP batch must not survive the partition.
global_batch.dcp_local_seq_lens = global_buffers.dcp_local_seq_lens[:2]
global_batch.dcp_local_seq_lens.fill_(-1)
local_batch = manager.partition_batch(global_batch, padded_num_tokens=2)
assert local_batch.dcp_local_seq_lens is None
assert local_batch.seq_lens.tolist() == [17, 25]
# What execute_model does next: derive DCP metadata from the final batch
# on the PCP-owned buffers.
local_batch.dcp_local_seq_lens = gpu_cp_utils.maybe_prepare_dcp_local_seq_lens(
manager.input_buffers.dcp_local_seq_lens,
local_batch.seq_lens,
local_batch.num_reqs,
dcp_size=2,
dcp_rank=0,
cp_interleave=1,
num_reqs_padded=local_batch.num_reqs_after_padding,
)
expected = get_dcp_local_seq_lens(
torch.tensor([17, 25], dtype=torch.int32), 2, 0, 1
)
assert local_batch.dcp_local_seq_lens is not None
assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected)
+137
View File
@@ -22,7 +22,9 @@ import torch
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import CUDAGraphMode, ModelConfig, ParallelConfig, VllmConfig
from vllm.forward_context import create_forward_context
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
from vllm.v1.worker.gpu import dp_utils
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
@@ -581,3 +583,138 @@ def test_ubatch_runner_names_the_microbatch_that_failed():
assert isinstance(result["error"], RuntimeError)
assert "Microbatch 0" in str(result["error"])
assert isinstance(result["error"].__cause__, ValueError)
# DCP x DBO: each microbatch recomputes DCP-local seq_lens from its own
# truncated seq_lens into its own persistent buffer (the parent's values for
# a straddling request are wrong for the leading microbatch).
DCP_SIZE = 2
CP_INTERLEAVE = 1
def _make_cuda_input_batch(
query_lens: list[int], seq_lens: list[int]
) -> tuple[InputBatch, InputBuffers]:
buffers = InputBuffers(
max_num_reqs=MAX_NUM_REQS,
max_num_tokens=MAX_NUM_TOKENS,
device=torch.device("cuda:0"),
)
return _make_input_batch(query_lens, seq_lens, buffers), buffers
def _make_dcp_ubatch_buffers(
num_ubatches: int = 2,
) -> list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
"""Per-microbatch (query_start_loc, seq_lens, dcp_local_seq_lens)."""
return [
(
torch.zeros(MAX_NUM_REQS + 1, dtype=torch.int32, device="cuda:0"),
torch.zeros(MAX_NUM_REQS, dtype=torch.int32, device="cuda:0"),
torch.zeros(MAX_NUM_REQS, dtype=torch.int32, device="cuda:0"),
)
for _ in range(num_ubatches)
]
def _slice_with_dcp(
input_batch: InputBatch, dcp_rank: int, num_ubatches: int = 2
) -> tuple[list[InputBatch], list[torch.Tensor]]:
dcp_buffers = [
torch.zeros(MAX_NUM_REQS, dtype=torch.int32, device="cuda:0")
for _ in range(num_ubatches)
]
ubatch_buffers = _make_dcp_ubatch_buffers(num_ubatches)
ubatches = [
_slice_input_batch(
input_batch,
ubatch_slice,
ubatch_buffers[i][0],
ubatch_buffers[i][1],
dcp_buffers[i],
dcp_size=DCP_SIZE,
dcp_rank=dcp_rank,
cp_interleave=CP_INTERLEAVE,
)
for i, ubatch_slice in enumerate(
create_ubatch_slices(input_batch, num_ubatches)
)
]
return ubatches, dcp_buffers
@pytest.mark.skipif(not torch.cuda.is_available(), reason="triton kernel needs CUDA")
@pytest.mark.parametrize("dcp_rank", [0, 1])
def test_microbatches_recompute_dcp_lens_from_truncated_seq_lens(dcp_rank: int):
"""A boundary-split request keeps a truncated seq_len on the leading
microbatch, whose DCP-local length may differ (in parity) from the full
batch's. Microbatches must therefore derive DCP lengths from their own
seq_lens, not reuse the parent's row.
"""
input_batch, buffers = _make_cuda_input_batch(
[1, 1, 10, 1, 1], [64, 96, 512, 32, 48]
)
# execute_model has populated the merged batch's DCP metadata already.
input_batch.dcp_local_seq_lens = gpu_cp_utils.maybe_prepare_dcp_local_seq_lens(
buffers.dcp_local_seq_lens,
input_batch.seq_lens,
input_batch.num_reqs,
DCP_SIZE,
dcp_rank,
CP_INTERLEAVE,
num_reqs_padded=input_batch.num_reqs_after_padding,
)
parent_lens = input_batch.dcp_local_seq_lens.clone()
ubatches, _ = _slice_with_dcp(input_batch, dcp_rank)
for ubatch in ubatches:
assert ubatch.dcp_local_seq_lens is not None
expected = get_dcp_local_seq_lens(
ubatch.seq_lens.cpu(), DCP_SIZE, dcp_rank, CP_INTERLEAVE
)
assert torch.equal(
ubatch.dcp_local_seq_lens[: ubatch.num_reqs].cpu(),
expected.to(torch.int32),
)
# The straddling request (512 tokens, 5 of its 10 query tokens truncated)
# lands on this rank with a different local length than the full batch
# reports (507 vs 512 -> 254 vs 256 for dcp_size=2, interleave=1); copying
# the parent's row would ship the stale value.
assert ubatches[0].num_reqs == 3
assert ubatches[0].seq_lens[2].item() == 507
assert ubatches[0].dcp_local_seq_lens[2].item() != parent_lens[2].item()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="triton kernel needs CUDA")
def test_each_microbatch_owns_its_dcp_buffer():
"""Microbatch DCP lengths live in per-microbatch persistent buffers so
CPU-visible shapes stay valid under CUDA graph capture and replay."""
input_batch, _ = _make_cuda_input_batch([4, 4, 4, 4], [8, 9, 10, 11])
ubatches, dcp_buffers = _slice_with_dcp(input_batch, dcp_rank=0)
for ubatch, dcp_buffer in zip(ubatches, dcp_buffers):
assert ubatch.dcp_local_seq_lens is not None
assert ubatch.dcp_local_seq_lens.data_ptr() == dcp_buffer.data_ptr()
assert dcp_buffers[0].data_ptr() != dcp_buffers[1].data_ptr()
def test_slicing_drops_stale_dcp_metadata_when_dcp_is_off():
"""dcp_size == 1 yields None even when the parent carries a value, so a
microbatch can never consume DCP metadata this deployment did not
enable.
"""
buffers = _make_buffers()
input_batch = _make_input_batch([4, 4, 4, 4], [8, 9, 10, 11], buffers)
input_batch.dcp_local_seq_lens = buffers.dcp_local_seq_lens[:4]
ubatch_buffers = _make_ubatch_buffers()
ubatches = [
_slice_input_batch(input_batch, ubatch_slice, *ubatch_buffers[i])
for i, ubatch_slice in enumerate(create_ubatch_slices(input_batch, 2))
]
assert all(ubatch.dcp_local_seq_lens is None for ubatch in ubatches)