[MM][Perf][CG] Support ViT full CUDA graph for Qwen3-VL video inference (#38061)
Signed-off-by: shen-shanshan <467638484@qq.com> Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
c0ecaed950
commit
80118853f4
@@ -1524,6 +1524,13 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
|
||||
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig": ...
|
||||
|
||||
def get_input_modality(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> str:
|
||||
"""Return the modality of the inputs."""
|
||||
...
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
@@ -1536,7 +1543,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
(e.g. max_num_batched_tokens)
|
||||
|
||||
Used when ``encoder_cudagraph_token_budgets`` and/or
|
||||
``encoder_cudagraph_max_images_per_batch`` are not explicitly
|
||||
``encoder_cudagraph_max_vision_items_per_batch`` are not explicitly
|
||||
specified by the user.
|
||||
"""
|
||||
...
|
||||
@@ -1590,6 +1597,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> "EncoderCudaGraphCaptureInputs":
|
||||
@@ -1600,6 +1608,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
) -> "EncoderCudaGraphReplayBuffers":
|
||||
"""Compute buffer values from actual batch inputs for replay."""
|
||||
...
|
||||
|
||||
Reference in New Issue
Block a user