[MM][CG] Enable encoder Cudagraph for Step3VL (#42224)
Signed-off-by: JisoLya <523420504@qq.com> Signed-off-by: Soyaazz <523420504@qq.com>
This commit is contained in:
@@ -14,7 +14,10 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEncoderCudaGraph,
|
||||
)
|
||||
from vllm.model_executor.models.utils import scatter_output_slices
|
||||
from vllm.model_executor.models.vision import get_load_balance_assignment
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -254,22 +257,6 @@ class EncoderCudaGraphManager:
|
||||
for t in self.model.get_encoder_cudagraph_per_item_output_tokens(mm_kwargs)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _scatter_output_slices(
|
||||
output: torch.Tensor,
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
) -> None:
|
||||
"""Slice a concatenated output tensor and scatter into dest by index."""
|
||||
offset = 0
|
||||
for idx in indices:
|
||||
n_tok = per_item_out_tokens[idx]
|
||||
sliced = output[offset : offset + n_tok]
|
||||
dest[idx] = sliced.clone() if clone else sliced
|
||||
offset += n_tok
|
||||
|
||||
def _run_budget_graph(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
@@ -414,7 +401,7 @@ class EncoderCudaGraphManager:
|
||||
self.graph_misses += len(batch_orig_indices)
|
||||
with torch.inference_mode():
|
||||
raw = self.model.encoder_eager_forward(batch_mm_kwargs)
|
||||
self._scatter_output_slices(
|
||||
scatter_output_slices(
|
||||
raw,
|
||||
batch_orig_indices,
|
||||
per_item_out_tokens,
|
||||
@@ -440,12 +427,13 @@ class EncoderCudaGraphManager:
|
||||
batch_mm_kwargs, token_budget, replay.buffers
|
||||
)
|
||||
assert output is not None
|
||||
self._scatter_output_slices(
|
||||
self.model.postprocess_encoder_output(
|
||||
output,
|
||||
batch_orig_indices,
|
||||
per_item_out_tokens,
|
||||
outputs_by_orig_idx,
|
||||
clone=True,
|
||||
batch_mm_kwargs=batch_mm_kwargs,
|
||||
)
|
||||
|
||||
# Return in original batch order (caller maps outputs to token positions)
|
||||
@@ -573,7 +561,7 @@ class EncoderCudaGraphManager:
|
||||
count = images_per_rank[rank]
|
||||
if count > 0:
|
||||
rank_items = image_rank_assignment[current_idx : current_idx + count]
|
||||
self._scatter_output_slices(
|
||||
scatter_output_slices(
|
||||
rank_outputs[rank],
|
||||
rank_items,
|
||||
per_item_out_tokens,
|
||||
|
||||
Reference in New Issue
Block a user