[Bugfix][MRV2] Initialize DCP metadata after batch partitioning (#55212)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Ziyang Zhang <hafuhafu@qq.com>
Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Raphael Rialland <raphael.rialland@mistral.ai>
Co-authored-by: Ziyang Zhang <hafuhafu@qq.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Sy03
2026-09-10 06:50:09 +00:00
committed by GitHub
co-authored by Raphael Rialland Ziyang Zhang Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 912d2e79fe
commit 83990f5bcc
12 changed files with 460 additions and 77 deletions
+35 -9
View File
@@ -10,6 +10,7 @@ import numpy as np
import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_dcp_group
from vllm.forward_context import (
DPMetadata,
ForwardContext,
@@ -21,6 +22,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.ubatch_utils import (
@@ -76,16 +78,20 @@ def _slice_input_batch(
ubatch_slice: UBatchSlice,
query_start_loc_buf: torch.Tensor,
seq_lens_buf: torch.Tensor,
dcp_local_seq_lens_buf: torch.Tensor | None = None,
*,
dcp_size: int = 1,
dcp_rank: int = 0,
cp_interleave: int = 1,
) -> InputBatch:
"""Build the sub-`InputBatch` a single microbatch runs on.
A request straddling the boundary appears in both microbatches, its query
truncated to the tokens each one owns.
Everything except `query_start_loc` and `seq_lens` is a view of the full
batch's buffers, so slicing costs no allocation. Those two are rebased onto
the microbatch's token range and written to the caller's buffers, which
keeps them valid under CUDA graph capture and replay.
Query offsets and sequence lengths (including DCP-local lengths) are
written to the caller's persistent buffers after truncation. Other GPU
inputs are views of the full batch's buffers.
The sub-batch describes the forward pass only. Sampling runs once over the
merged batch, so `logits_indices`, `cu_num_logits` and the draft-token
@@ -139,11 +145,18 @@ def _slice_input_batch(
# max_query_len from this array see the microbatch's own lengths.
num_scheduled_tokens = np.diff(query_start_loc_np)[:num_reqs]
dcp_local_seq_lens = input_batch.dcp_local_seq_lens
if dcp_local_seq_lens is not None:
# NOTE: a request split across microbatches keeps its full local
# seq_len here. DCP is not yet validated with DBO.
dcp_local_seq_lens = dcp_local_seq_lens[req_start:req_stop]
dcp_local_seq_lens = None
if dcp_size > 1:
assert dcp_local_seq_lens_buf is not None
dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
dcp_local_seq_lens_buf,
seq_lens,
num_reqs,
dcp_size,
dcp_rank,
cp_interleave,
num_reqs_padded=num_reqs_padded,
)
return replace(
input_batch,
@@ -278,6 +291,9 @@ class UBatchRunner:
self.vllm_config = vllm_config
self.parallel_config = vllm_config.parallel_config
self.num_ubatches = self.parallel_config.num_ubatches
self.dcp_size = self.parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_size > 1 else 0
self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size
self.device = device
self.model_state = model_state
self.attn_groups = attn_groups
@@ -293,6 +309,12 @@ class UBatchRunner:
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
for _ in range(self.num_ubatches)
]
self.ubatch_dcp_local_seq_lens = [
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
if self.dcp_size > 1
else None
for _ in range(self.num_ubatches)
]
self.comm_stream = torch.cuda.Stream(device=device)
# The microbatch threads plus the thread that starts them.
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
@@ -320,6 +342,10 @@ class UBatchRunner:
ubatch_slice,
self.ubatch_query_start_loc[i],
self.ubatch_seq_lens[i],
self.ubatch_dcp_local_seq_lens[i],
dcp_size=self.dcp_size,
dcp_rank=self.dcp_rank,
cp_interleave=self.cp_interleave,
)
ubatch_slot_mappings = slot_mappings[:, ubatch_slice.token_slice]
ubatch_block_tables = tuple(