[Multimodal] Simplify ViT CUDA graph interfaces (#41234)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -50,6 +50,7 @@ if TYPE_CHECKING:
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
EncoderItemSpec,
|
||||
)
|
||||
else:
|
||||
VllmConfig = object
|
||||
@@ -1549,30 +1550,15 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
def get_encoder_cudagraph_num_items(
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> int:
|
||||
"""Return the number of items (e.g. images) in the batch."""
|
||||
...
|
||||
) -> list["EncoderItemSpec"]:
|
||||
"""Return specs describing each item in the batch.
|
||||
|
||||
def get_encoder_cudagraph_per_item_output_tokens(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[int]:
|
||||
"""Return output token count for each item.
|
||||
|
||||
Used for greedy packing and DP load balancing.
|
||||
"""
|
||||
...
|
||||
|
||||
def get_encoder_cudagraph_per_item_input_sizes(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[int]:
|
||||
"""Return input size (e.g. patch count) for each item.
|
||||
|
||||
Used for input tensor slicing offsets.
|
||||
Replaces the former separate methods for num_items,
|
||||
per_item_output_tokens, and per_item_input_sizes.
|
||||
The manager derives all three from this single return value.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user