[1/N] Unify multiple-path encoder cuda graph support (#49934)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c67fe497a2
commit
e2fa28594f
@@ -1696,13 +1696,12 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
local_output: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Post-process encoder output, directly call scatter_output_slices by default.
|
||||
@@ -1714,7 +1713,9 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
"""
|
||||
from vllm.model_executor.models.utils import scatter_output_slices
|
||||
|
||||
scatter_output_slices(output, indices, per_item_out_tokens, dest, clone)
|
||||
scatter_output_slices(
|
||||
outputs["default"], indices, per_item_out_tokens, dest, clone
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user