[Bugfix][MRV2] Initialize DCP metadata after batch partitioning (#55212)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Ziyang Zhang <hafuhafu@qq.com> Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Raphael Rialland <raphael.rialland@mistral.ai> Co-authored-by: Ziyang Zhang <hafuhafu@qq.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Raphael Rialland
Ziyang Zhang
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
912d2e79fe
commit
83990f5bcc
@@ -5,17 +5,19 @@ import torch
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def prepare_dcp_local_seq_lens(
|
||||
def maybe_prepare_dcp_local_seq_lens(
|
||||
dcp_local_seq_lens: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
num_reqs: int,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
cp_interleave: int,
|
||||
) -> None:
|
||||
"""Populate the persistent DCP local seq_lens buffer (CUDA graph safe)."""
|
||||
*,
|
||||
num_reqs_padded: int | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Populate caller-owned storage and return its padded view, or None without DCP."""
|
||||
if dcp_size == 1:
|
||||
return
|
||||
return None
|
||||
|
||||
max_num_reqs = dcp_local_seq_lens.shape[0]
|
||||
BLOCK_SIZE = 128
|
||||
@@ -30,6 +32,7 @@ def prepare_dcp_local_seq_lens(
|
||||
max_num_reqs,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
return dcp_local_seq_lens[:num_reqs_padded]
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -38,7 +38,7 @@ from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup, clear_layer_kv_caches
|
||||
@@ -699,17 +699,15 @@ def prepare_inputs_to_capture(
|
||||
slot_mappings, kv_cache_config
|
||||
)
|
||||
|
||||
# HACK(woosuk): Special handling for DCP.
|
||||
if block_tables.cp_size > 1:
|
||||
prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
)
|
||||
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
|
||||
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
|
||||
# NOTE(woosuk): Attention metadata is required not just by standard attention
|
||||
# kernels, but also by specialized attention-like operations (e.g., Inkling's sconv,
|
||||
|
||||
@@ -99,7 +99,7 @@ from vllm.v1.worker.gpu.buffer_utils import (
|
||||
async_copy_to_gpu,
|
||||
set_default_max_concurrency,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
@@ -1324,19 +1324,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
seq_lens = self.input_buffers.seq_lens[:num_reqs_padded]
|
||||
|
||||
dcp_local_seq_lens = None
|
||||
if self.use_dcp:
|
||||
# Prepare dcp local seq_lens.
|
||||
prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
self.dcp_size,
|
||||
self.dcp_rank,
|
||||
self.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens[:num_reqs_padded]
|
||||
|
||||
# Some input token ids are directly read from the last sampled tokens
|
||||
# and draft tokens. Also, get the logits indices to sample tokens from.
|
||||
logits_indices = combine_sampled_and_draft_tokens(
|
||||
@@ -1384,7 +1371,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
dcp_local_seq_lens=None,
|
||||
num_computed_tokens_np=num_computed_tokens_np,
|
||||
prefill_len_np=batch_req_state.prefill_len_np,
|
||||
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
|
||||
@@ -1404,11 +1391,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else None
|
||||
),
|
||||
)
|
||||
return pcp.maybe_partition_pcp_batch(
|
||||
input_batch = pcp.maybe_partition_pcp_batch(
|
||||
self.pcp_manager,
|
||||
input_batch,
|
||||
padded_num_tokens=batch_desc.num_tokens,
|
||||
)
|
||||
return input_batch
|
||||
|
||||
def prepare_attn(
|
||||
self, input_batch: InputBatch
|
||||
@@ -1705,6 +1693,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
(
|
||||
self.pcp_manager.input_buffers
|
||||
if self.pcp_manager is not None
|
||||
else self.input_buffers
|
||||
).dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
self.dcp_size,
|
||||
self.dcp_rank,
|
||||
self.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
ubatch_state: UBatchState | None = None
|
||||
if batch_desc.num_ubatches > 1:
|
||||
assert self.ubatch_runner is not None
|
||||
|
||||
@@ -12,7 +12,6 @@ from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import (
|
||||
InputBatch,
|
||||
InputBuffers,
|
||||
@@ -537,18 +536,6 @@ class PCPManager:
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_local_reqs, dtype=np.int32)
|
||||
seq_lens_cpu_upper_bound_np[:] = local_start_pos_np + local_num_scheduled_tokens
|
||||
|
||||
dcp_local_seq_lens = None
|
||||
if self.dcp_world_size > 1:
|
||||
prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
seq_lens,
|
||||
num_local_reqs,
|
||||
self.dcp_world_size,
|
||||
self.dcp_rank,
|
||||
self.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_local_reqs]
|
||||
|
||||
return replace(
|
||||
input_batch,
|
||||
req_ids=local_req_ids,
|
||||
@@ -569,7 +556,7 @@ class PCPManager:
|
||||
query_start_loc_np=local_query_start_loc_np[: num_local_reqs + 1],
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens_cpu_upper_bound_np),
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
dcp_local_seq_lens=None,
|
||||
num_computed_tokens_np=local_start_pos_np,
|
||||
prefill_len_np=local_prefill_len_np,
|
||||
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_slot_mappings_by_layer,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionState,
|
||||
BatchExecutionDescriptor,
|
||||
@@ -42,17 +42,15 @@ def _prepare_dflash_inputs_to_capture(
|
||||
attn_metadata = None
|
||||
if not skip_attn:
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
dcp_local_seq_lens = None
|
||||
if block_tables.cp_size > 1:
|
||||
prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = input_buffers.dcp_local_seq_lens
|
||||
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
block_tables.cp_size,
|
||||
block_tables.cp_rank,
|
||||
block_tables.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
@@ -61,7 +59,7 @@ def _prepare_dflash_inputs_to_capture(
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=num_tokens // num_reqs,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
max_seq_len=max_model_len,
|
||||
block_tables=input_block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_attn_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
@@ -297,7 +297,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
if dcp_local_seq_lens is None and self.block_tables.cp_size > 1:
|
||||
# Draft steps advance and rewind their own global sequence lengths,
|
||||
# so the target model's DCP-local lengths may already be stale.
|
||||
prepare_dcp_local_seq_lens(
|
||||
dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
@@ -305,7 +305,6 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.block_tables.cp_rank,
|
||||
self.block_tables.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=self.attn_groups,
|
||||
num_reqs=num_reqs_padded,
|
||||
|
||||
@@ -10,6 +10,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.forward_context import (
|
||||
DPMetadata,
|
||||
ForwardContext,
|
||||
@@ -21,6 +22,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
@@ -76,16 +78,20 @@ def _slice_input_batch(
|
||||
ubatch_slice: UBatchSlice,
|
||||
query_start_loc_buf: torch.Tensor,
|
||||
seq_lens_buf: torch.Tensor,
|
||||
dcp_local_seq_lens_buf: torch.Tensor | None = None,
|
||||
*,
|
||||
dcp_size: int = 1,
|
||||
dcp_rank: int = 0,
|
||||
cp_interleave: int = 1,
|
||||
) -> InputBatch:
|
||||
"""Build the sub-`InputBatch` a single microbatch runs on.
|
||||
|
||||
A request straddling the boundary appears in both microbatches, its query
|
||||
truncated to the tokens each one owns.
|
||||
|
||||
Everything except `query_start_loc` and `seq_lens` is a view of the full
|
||||
batch's buffers, so slicing costs no allocation. Those two are rebased onto
|
||||
the microbatch's token range and written to the caller's buffers, which
|
||||
keeps them valid under CUDA graph capture and replay.
|
||||
Query offsets and sequence lengths (including DCP-local lengths) are
|
||||
written to the caller's persistent buffers after truncation. Other GPU
|
||||
inputs are views of the full batch's buffers.
|
||||
|
||||
The sub-batch describes the forward pass only. Sampling runs once over the
|
||||
merged batch, so `logits_indices`, `cu_num_logits` and the draft-token
|
||||
@@ -139,11 +145,18 @@ def _slice_input_batch(
|
||||
# max_query_len from this array see the microbatch's own lengths.
|
||||
num_scheduled_tokens = np.diff(query_start_loc_np)[:num_reqs]
|
||||
|
||||
dcp_local_seq_lens = input_batch.dcp_local_seq_lens
|
||||
if dcp_local_seq_lens is not None:
|
||||
# NOTE: a request split across microbatches keeps its full local
|
||||
# seq_len here. DCP is not yet validated with DBO.
|
||||
dcp_local_seq_lens = dcp_local_seq_lens[req_start:req_stop]
|
||||
dcp_local_seq_lens = None
|
||||
if dcp_size > 1:
|
||||
assert dcp_local_seq_lens_buf is not None
|
||||
dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
dcp_local_seq_lens_buf,
|
||||
seq_lens,
|
||||
num_reqs,
|
||||
dcp_size,
|
||||
dcp_rank,
|
||||
cp_interleave,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
)
|
||||
|
||||
return replace(
|
||||
input_batch,
|
||||
@@ -278,6 +291,9 @@ class UBatchRunner:
|
||||
self.vllm_config = vllm_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.num_ubatches = self.parallel_config.num_ubatches
|
||||
self.dcp_size = self.parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_size > 1 else 0
|
||||
self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size
|
||||
self.device = device
|
||||
self.model_state = model_state
|
||||
self.attn_groups = attn_groups
|
||||
@@ -293,6 +309,12 @@ class UBatchRunner:
|
||||
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
|
||||
for _ in range(self.num_ubatches)
|
||||
]
|
||||
self.ubatch_dcp_local_seq_lens = [
|
||||
torch.zeros(max_num_reqs, dtype=torch.int32, device=device)
|
||||
if self.dcp_size > 1
|
||||
else None
|
||||
for _ in range(self.num_ubatches)
|
||||
]
|
||||
self.comm_stream = torch.cuda.Stream(device=device)
|
||||
# The microbatch threads plus the thread that starts them.
|
||||
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
|
||||
@@ -320,6 +342,10 @@ class UBatchRunner:
|
||||
ubatch_slice,
|
||||
self.ubatch_query_start_loc[i],
|
||||
self.ubatch_seq_lens[i],
|
||||
self.ubatch_dcp_local_seq_lens[i],
|
||||
dcp_size=self.dcp_size,
|
||||
dcp_rank=self.dcp_rank,
|
||||
cp_interleave=self.cp_interleave,
|
||||
)
|
||||
ubatch_slot_mappings = slot_mappings[:, ubatch_slice.token_slice]
|
||||
ubatch_block_tables = tuple(
|
||||
|
||||
Reference in New Issue
Block a user