[MM][Perf][CG] Support dual-path ViT full CUDA graph for DeepSeek-OCR (#43586)
Signed-off-by: shen-shanshan <467638484@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Roger Wang
Isotr0py
parent
b2cfae777d
commit
ad32608e24
@@ -1623,6 +1623,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
local_output: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Post-process encoder output, directly call scatter_output_slices by default.
|
||||
@@ -1643,6 +1644,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
) -> "EncoderCudaGraphCaptureInputs":
|
||||
"""Create dummy inputs and buffers for CUDA graph capture."""
|
||||
...
|
||||
@@ -1652,6 +1654,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
) -> "EncoderCudaGraphReplayBuffers":
|
||||
"""Compute buffer values from actual batch inputs for replay."""
|
||||
...
|
||||
@@ -1659,6 +1662,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
inputs: dict[str, torch.Tensor],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
"""Run the encoder forward pass with precomputed buffers.
|
||||
|
||||
@@ -1669,6 +1673,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
"""Run the encoder forward pass without precomputed buffers.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user