[Multimodal] Optimize encoder cudagraph padding logic (#58925)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
87d9996abe
commit
8352b2427f
@@ -23,6 +23,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.model_executor.models.utils import scatter_output_slices
|
||||
from vllm.model_executor.models.vision import get_load_balance_assignment
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import round_up_to
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG,
|
||||
@@ -347,17 +348,17 @@ class EncoderCudaGraphManager:
|
||||
def _find_smallest_fitting_budget_given_tokens(
|
||||
self, total_tokens: int, budgets: list[int] | None = None
|
||||
) -> int | None:
|
||||
"""Find smallest budget >= total_tokens.
|
||||
|
||||
Returns:
|
||||
Token budget if found, None if no fitting budget.
|
||||
|
||||
"""
|
||||
"""Find smallest budget >= total_tokens, or None if none fits."""
|
||||
budgets = budgets if budgets is not None else self.token_budgets
|
||||
for budget in budgets:
|
||||
if budget >= total_tokens:
|
||||
return budget
|
||||
return None
|
||||
return round_up_to(budgets, total_tokens)
|
||||
|
||||
def _path_fitting_budget(self, path: str, tokens: int) -> int | None:
|
||||
"""Smallest captured budget on a path; 0 when idle, None if oversized."""
|
||||
if tokens == 0:
|
||||
return 0
|
||||
return self._find_smallest_fitting_budget_given_tokens(
|
||||
tokens, self.path_token_budgets[path]
|
||||
)
|
||||
|
||||
def _get_item_specs(self, mm_kwargs: dict[str, Any]) -> list[EncoderItemSpec]:
|
||||
"""Get item specs from the model."""
|
||||
@@ -479,24 +480,38 @@ class EncoderCudaGraphManager:
|
||||
if not current_batch:
|
||||
return
|
||||
path_budgets = {
|
||||
path: (
|
||||
0
|
||||
if current_tokens[path] == 0
|
||||
else self._find_smallest_fitting_budget_given_tokens(
|
||||
current_tokens[path], self.path_token_budgets[path]
|
||||
)
|
||||
)
|
||||
path: self._path_fitting_budget(path, current_tokens[path])
|
||||
for path in paths
|
||||
}
|
||||
batches.append((list(current_batch), path_budgets))
|
||||
|
||||
def should_defer_item(item_tokens: dict[str, int]) -> bool:
|
||||
"""Whether closing the batch strictly reduces summed budgets."""
|
||||
for path in paths:
|
||||
item = item_tokens[path]
|
||||
if item == 0:
|
||||
continue
|
||||
cur = current_tokens[path]
|
||||
b_cur = self._path_fitting_budget(path, cur)
|
||||
b_alone = self._path_fitting_budget(path, item)
|
||||
b_merged = self._path_fitting_budget(path, cur + item)
|
||||
# Only called when the item fits; lookups land on a budget.
|
||||
assert (
|
||||
b_cur is not None and b_alone is not None and b_merged is not None
|
||||
)
|
||||
# Only defer items that alone outweigh the whole current
|
||||
# batch; splitting smaller ones fragments flood packing.
|
||||
if b_cur + b_alone < b_merged and b_alone >= b_cur:
|
||||
return True
|
||||
return False
|
||||
|
||||
for orig_idx in sorted_indices:
|
||||
item_tokens = {path: per_item_path_tokens[path][orig_idx] for path in paths}
|
||||
fits = len(current_batch) < self.max_batch_size and all(
|
||||
current_tokens[path] + item_tokens[path] <= max_path_budgets[path]
|
||||
for path in paths
|
||||
)
|
||||
if current_batch and not fits:
|
||||
if current_batch and (not fits or should_defer_item(item_tokens)):
|
||||
append_current_batch()
|
||||
current_batch = []
|
||||
current_tokens = dict.fromkeys(paths, 0)
|
||||
|
||||
Reference in New Issue
Block a user