[Bugfix][MRV2] Initialize DCP metadata after batch partitioning (#55212)
Signed-off-by: Raphael Rialland <raphael.rialland@mistral.ai> Signed-off-by: Ziyang Zhang <hafuhafu@qq.com> Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Raphael Rialland <raphael.rialland@mistral.ai> Co-authored-by: Ziyang Zhang <hafuhafu@qq.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Raphael Rialland
Ziyang Zhang
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
912d2e79fe
commit
83990f5bcc
@@ -99,7 +99,7 @@ from vllm.v1.worker.gpu.buffer_utils import (
|
||||
async_copy_to_gpu,
|
||||
set_default_max_concurrency,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
@@ -1324,19 +1324,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
seq_lens = self.input_buffers.seq_lens[:num_reqs_padded]
|
||||
|
||||
dcp_local_seq_lens = None
|
||||
if self.use_dcp:
|
||||
# Prepare dcp local seq_lens.
|
||||
prepare_dcp_local_seq_lens(
|
||||
self.input_buffers.dcp_local_seq_lens,
|
||||
self.input_buffers.seq_lens,
|
||||
num_reqs,
|
||||
self.dcp_size,
|
||||
self.dcp_rank,
|
||||
self.cp_interleave,
|
||||
)
|
||||
dcp_local_seq_lens = self.input_buffers.dcp_local_seq_lens[:num_reqs_padded]
|
||||
|
||||
# Some input token ids are directly read from the last sampled tokens
|
||||
# and draft tokens. Also, get the logits indices to sample tokens from.
|
||||
logits_indices = combine_sampled_and_draft_tokens(
|
||||
@@ -1384,7 +1371,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
dcp_local_seq_lens=None,
|
||||
num_computed_tokens_np=num_computed_tokens_np,
|
||||
prefill_len_np=batch_req_state.prefill_len_np,
|
||||
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
|
||||
@@ -1404,11 +1391,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else None
|
||||
),
|
||||
)
|
||||
return pcp.maybe_partition_pcp_batch(
|
||||
input_batch = pcp.maybe_partition_pcp_batch(
|
||||
self.pcp_manager,
|
||||
input_batch,
|
||||
padded_num_tokens=batch_desc.num_tokens,
|
||||
)
|
||||
return input_batch
|
||||
|
||||
def prepare_attn(
|
||||
self, input_batch: InputBatch
|
||||
@@ -1705,6 +1693,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
input_batch.dcp_local_seq_lens = maybe_prepare_dcp_local_seq_lens(
|
||||
(
|
||||
self.pcp_manager.input_buffers
|
||||
if self.pcp_manager is not None
|
||||
else self.input_buffers
|
||||
).dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
input_batch.num_reqs,
|
||||
self.dcp_size,
|
||||
self.dcp_rank,
|
||||
self.cp_interleave,
|
||||
num_reqs_padded=input_batch.num_reqs_after_padding,
|
||||
)
|
||||
ubatch_state: UBatchState | None = None
|
||||
if batch_desc.num_ubatches > 1:
|
||||
assert self.ubatch_runner is not None
|
||||
|
||||
Reference in New Issue
Block a user