[Core] MRV2 support for fast-prefill (#56145)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.utils import random_uuid
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.attn_utils import FastPrefillBatchMetadata
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
|
||||
|
||||
@@ -109,6 +110,10 @@ class InputBatch:
|
||||
# stays valid for every replay the graph serves.
|
||||
max_query_len: int | None = None
|
||||
|
||||
# Arms the KV-sharing fast prefill path for this step. Absent for dummy
|
||||
# (cudagraph capture) batches, which run the KV-sharing layers in full.
|
||||
fast_prefill: "FastPrefillBatchMetadata | None" = None
|
||||
|
||||
@classmethod
|
||||
def make_dummy(
|
||||
cls,
|
||||
|
||||
Reference in New Issue
Block a user