[Bugfix][MRV2] Initialize DCP metadata after batch partitioning (#55212)

Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai>
Signed-off-by: Ziyang Zhang <hafuhafu@qq.com>
Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Raphael Rialland <raphael.rialland@mistral.ai>
Co-authored-by: Ziyang Zhang <hafuhafu@qq.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Sy03
2026-09-10 06:50:09 +00:00
committed by GitHub
co-authored by Raphael Rialland Ziyang Zhang Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 912d2e79fe
commit 83990f5bcc
12 changed files with 460 additions and 77 deletions
+7 -4
View File
@@ -5,17 +5,19 @@ import torch
from vllm.triton_utils import tl, triton
def prepare_dcp_local_seq_lens(
def maybe_prepare_dcp_local_seq_lens(
dcp_local_seq_lens: torch.Tensor,
seq_lens: torch.Tensor,
num_reqs: int,
dcp_size: int,
dcp_rank: int,
cp_interleave: int,
) -> None:
"""Populate the persistent DCP local seq_lens buffer (CUDA graph safe)."""
*,
num_reqs_padded: int | None = None,
) -> torch.Tensor | None:
"""Populate caller-owned storage and return its padded view, or None without DCP."""
if dcp_size == 1:
return
return None
max_num_reqs = dcp_local_seq_lens.shape[0]
BLOCK_SIZE = 128
@@ -30,6 +32,7 @@ def prepare_dcp_local_seq_lens(
max_num_reqs,
BLOCK_SIZE,
)
return dcp_local_seq_lens[:num_reqs_padded]
@triton.jit
+10 -12
View File
@@ -38,7 +38,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup, clear_layer_kv_caches
@@ -699,17 +699,15 @@ def prepare_inputs_to_capture(
slot_mappings, kv_cache_config
)
# HACK(woosuk): Special handling for DCP.
if block_tables.cp_size > 1:
prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
input_batch.seq_lens,
num_reqs,
block_tables.cp_size,
block_tables.cp_rank,
block_tables.cp_interleave,
)
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
input_batch.seq_lens,
input_batch.num_reqs,
block_tables.cp_size,
block_tables.cp_rank,
block_tables.cp_interleave,
num_reqs_padded=input_batch.num_reqs_after_padding,
)
# NOTE(woosuk): Attention metadata is required not just by standard attention
# kernels, but also by specialized attention-like operations (e.g., Inkling's sconv,
+18 -16
View File
@@ -99,7 +99,7 @@ from vllm.v1.worker.gpu.buffer_utils import (
async_copy_to_gpu,
set_default_max_concurrency,
)
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
ModelCudaGraphManager,
@@ -1324,19 +1324,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
seq_lens = self.input_buffers.seq_lens[:num_reqs_padded]
dcp_local_seq_lens = None
if self.use_dcp:
# Prepare dcp local seq_lens.
prepare_dcp_local_seq_lens(
self.input_buffers.dcp_local_seq_lens,
self.input_buffers.seq_lens,
num_reqs,
self.dcp_size,
self.dcp_rank,
self.cp_interleave,
)
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens[:num_reqs_padded]
# Some input token ids are directly read from the last sampled tokens
# and draft tokens. Also, get the logits indices to sample tokens from.
logits_indices = combine_sampled_and_draft_tokens(
@@ -1384,7 +1371,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
query_start_loc_np=query_start_loc_np,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
dcp_local_seq_lens=None,
num_computed_tokens_np=num_computed_tokens_np,
prefill_len_np=batch_req_state.prefill_len_np,
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
@@ -1404,11 +1391,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
else None
),
)
return pcp.maybe_partition_pcp_batch(
input_batch = pcp.maybe_partition_pcp_batch(
self.pcp_manager,
input_batch,
padded_num_tokens=batch_desc.num_tokens,
)
return input_batch
def prepare_attn(
self, input_batch: InputBatch
@@ -1705,6 +1693,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
attn_metadata = None
slot_mappings_by_layer = None
if not (dummy_run and skip_attn_for_dummy_run):
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
(
self.pcp_manager.input_buffers
if self.pcp_manager is not None
else self.input_buffers
).dcp_local_seq_lens,
input_batch.seq_lens,
input_batch.num_reqs,
self.dcp_size,
self.dcp_rank,
self.cp_interleave,
num_reqs_padded=input_batch.num_reqs_after_padding,
)
ubatch_state: UBatchState | None = None
if batch_desc.num_ubatches > 1:
assert self.ubatch_runner is not None
+1 -14
View File
@@ -12,7 +12,6 @@ from vllm.logger import init_logger
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
InputBuffers,
@@ -537,18 +536,6 @@ class PCPManager:
seq_lens_cpu_upper_bound_np = np.zeros(num_local_reqs, dtype=np.int32)
seq_lens_cpu_upper_bound_np[:] = local_start_pos_np + local_num_scheduled_tokens
dcp_local_seq_lens = None
if self.dcp_world_size > 1:
prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
seq_lens,
num_local_reqs,
self.dcp_world_size,
self.dcp_rank,
self.cp_interleave,
)
dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_local_reqs]
return replace(
input_batch,
req_ids=local_req_ids,
@@ -569,7 +556,7 @@ class PCPManager:
query_start_loc_np=local_query_start_loc_np[: num_local_reqs + 1],
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_cpu_upper_bound_np),
dcp_local_seq_lens=dcp_local_seq_lens,
dcp_local_seq_lens=None,
num_computed_tokens_np=local_start_pos_np,
prefill_len_np=local_prefill_len_np,
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
@@ -11,7 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import (
build_slot_mappings_by_layer,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionState,
BatchExecutionDescriptor,
@@ -42,17 +42,15 @@ def _prepare_dflash_inputs_to_capture(
attn_metadata = None
if not skip_attn:
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
dcp_local_seq_lens = None
if block_tables.cp_size > 1:
prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
input_buffers.seq_lens,
num_reqs,
block_tables.cp_size,
block_tables.cp_rank,
block_tables.cp_interleave,
)
dcp_local_seq_lens = input_buffers.dcp_local_seq_lens
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
input_batch.seq_lens,
input_batch.num_reqs,
block_tables.cp_size,
block_tables.cp_rank,
block_tables.cp_interleave,
num_reqs_padded=input_batch.num_reqs_after_padding,
)
attn_metadata = build_attn_metadata(
attn_groups=attn_groups,
num_reqs=num_reqs,
@@ -61,7 +59,7 @@ def _prepare_dflash_inputs_to_capture(
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=num_tokens // num_reqs,
seq_lens=input_batch.seq_lens,
dcp_local_seq_lens=dcp_local_seq_lens,
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
max_seq_len=max_model_len,
block_tables=input_block_tables,
slot_mappings=slot_mappings,
+2 -3
View File
@@ -22,7 +22,7 @@ from vllm.v1.worker.gpu.attn_utils import (
init_attn_backend,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.dp_utils import DPSyncState
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
@@ -297,7 +297,7 @@ class DraftModelSpeculator(BaseSpeculator):
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
# Draft steps advance and rewind their own global sequence lengths,
# so the target model's DCP-local lengths may already be stale.
prepare_dcp_local_seq_lens(
dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
self.input_buffers.dcp_local_seq_lens,
self.input_buffers.seq_lens,
num_reqs,
@@ -305,7 +305,6 @@ class DraftModelSpeculator(BaseSpeculator):
self.block_tables.cp_rank,
self.block_tables.cp_interleave,
)
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens
attn_metadata = build_attn_metadata(
attn_groups=self.attn_groups,
num_reqs=num_reqs_padded,
+35 -9
View File
@@ -10,6 +10,7 @@ import numpy as np
import torch
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed import get_dcp_group
from vllm.forward_context import (
DPMetadata,
ForwardContext,
@@ -21,6 +22,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.ubatch_utils import (
@@ -76,16 +78,20 @@ def _slice_input_batch(
ubatch_slice: UBatchSlice,
query_start_loc_buf: torch.Tensor,
seq_lens_buf: torch.Tensor,
dcp_local_seq_lens_buf: torch.Tensor | None = None,
*,
dcp_size: int = 1,
dcp_rank: int = 0,
cp_interleave: int = 1,
) -> InputBatch:
"""Build the sub-`InputBatch` a single microbatch runs on.
A request straddling the boundary appears in both microbatches, its query
truncated to the tokens each one owns.
Everything except `query_start_loc` and `seq_lens` is a view of the full
batch's buffers, so slicing costs no allocation. Those two are rebased onto
the microbatch's token range and written to the caller's buffers, which
keeps them valid under CUDA graph capture and replay.
Query offsets and sequence lengths (including DCP-local lengths) are
written to the caller's persistent buffers after truncation. Other GPU
inputs are views of the full batch's buffers.
The sub-batch describes the forward pass only. Sampling runs once over the
merged batch, so `logits_indices`, `cu_num_logits` and the draft-token
@@ -139,11 +145,18 @@ def _slice_input_batch(
# max_query_len from this array see the microbatch's own lengths.
num_scheduled_tokens = np.diff(query_start_loc_np)[:num_reqs]
dcp_local_seq_lens = input_batch.dcp_local_seq_lens
if dcp_local_seq_lens is not None:
# NOTE: a request split across microbatches keeps its full local
# seq_len here. DCP is not yet validated with DBO.
dcp_local_seq_lens = dcp_local_seq_lens[req_start:req_stop]
dcp_local_seq_lens = None
if dcp_size > 1:
assert dcp_local_seq_lens_buf is not None
dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
dcp_local_seq_lens_buf,
seq_lens,
num_reqs,
dcp_size,
dcp_rank,
cp_interleave,
num_reqs_padded=num_reqs_padded,
)
return replace(
input_batch,
@@ -278,6 +291,9 @@ class UBatchRunner:
self.vllm_config = vllm_config
self.parallel_config = vllm_config.parallel_config
self.num_ubatches = self.parallel_config.num_ubatches
self.dcp_size = self.parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_size > 1 else 0
self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size
self.device = device
self.model_state = model_state
self.attn_groups = attn_groups
@@ -293,6 +309,12 @@ class UBatchRunner:
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
for _ in range(self.num_ubatches)
]
self.ubatch_dcp_local_seq_lens = [
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
if self.dcp_size > 1
else None
for _ in range(self.num_ubatches)
]
self.comm_stream = torch.cuda.Stream(device=device)
# The microbatch threads plus the thread that starts them.
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
@@ -320,6 +342,10 @@ class UBatchRunner:
ubatch_slice,
self.ubatch_query_start_loc[i],
self.ubatch_seq_lens[i],
self.ubatch_dcp_local_seq_lens[i],
dcp_size=self.dcp_size,
dcp_rank=self.dcp_rank,
cp_interleave=self.cp_interleave,
)
ubatch_slot_mappings = slot_mappings[:, ubatch_slice.token_slice]
ubatch_block_tables = tuple(