[Feature][PCP] Support decode-only FULL CUDA graphs (#53867)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Signed-off-by: QiuChunshuo <chunshuoq@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ca67438c08
commit
676650397f
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -217,9 +218,12 @@ def set_dummy_context(
|
||||
context_len: int,
|
||||
num_kv_blocks: int,
|
||||
max_model_len: int,
|
||||
input_block_tables: Sequence[torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""Give each dummy request context_len of context, used when profiling step cost."""
|
||||
if not block_tables.input_block_tables:
|
||||
if input_block_tables is None:
|
||||
input_block_tables = block_tables.input_block_tables
|
||||
if not input_block_tables:
|
||||
# Attention-free models have no KV context to fabricate.
|
||||
return
|
||||
num_reqs = input_batch.num_reqs
|
||||
@@ -241,7 +245,7 @@ def set_dummy_context(
|
||||
|
||||
seq_len = context_len + query_len
|
||||
for block_table, block_size, bpk in zip(
|
||||
block_tables.input_block_tables,
|
||||
input_block_tables,
|
||||
block_tables.kernel_block_sizes,
|
||||
block_tables.blocks_per_kv_block,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user