[Fix] Misc Fixes in ViT CUDA Graph (#38040)
Signed-off-by: Baorun Mu <bmu@nvidia.com>
This commit is contained in:
@@ -72,25 +72,65 @@ class EncoderCudaGraphManager:
|
||||
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
# Invariant: max_batch_size <= min_token_budget.
|
||||
# This ensures per_image_output = budget // max_batch_size >= 1
|
||||
# for every captured budget, preventing reshape crashes on empty
|
||||
# tensors during CUDA graph capture. Validated/enforced below for
|
||||
# each configuration path.
|
||||
if user_budgets and user_max_vision_items > 0:
|
||||
# Fully user-specified
|
||||
# Fully user-specified: validate the invariant.
|
||||
self.token_budgets = sorted(user_budgets)
|
||||
self.max_batch_size = user_max_vision_items
|
||||
min_tok = min(self.token_budgets)
|
||||
if self.max_batch_size > min_tok:
|
||||
raise ValueError(
|
||||
f"encoder_cudagraph_max_vision_items_per_batch "
|
||||
f"({self.max_batch_size}) must be <= smallest token "
|
||||
f"budget ({min_tok}). With budgets="
|
||||
f"{self.token_budgets}, per_image_output = "
|
||||
f"{min_tok} // {self.max_batch_size} = "
|
||||
f"{min_tok // self.max_batch_size}, which would cause "
|
||||
f"a capture failure. Either increase the smallest "
|
||||
f"budget or decrease max_vision_items_per_batch."
|
||||
)
|
||||
else:
|
||||
# Auto-infer missing values from model
|
||||
# Auto-infer missing values from model.
|
||||
min_budget, max_budget = model.get_encoder_cudagraph_budget_range(
|
||||
vllm_config
|
||||
)
|
||||
self.token_budgets = (
|
||||
sorted(user_budgets)
|
||||
if user_budgets
|
||||
else self._generate_budgets(min_budget, max_budget)
|
||||
)
|
||||
self.max_batch_size = (
|
||||
user_max_vision_items
|
||||
if user_max_vision_items > 0
|
||||
else max_budget // min_budget
|
||||
)
|
||||
if min_budget <= 0 or max_budget <= 0:
|
||||
raise ValueError(
|
||||
f"Invalid encoder cudagraph budget range: "
|
||||
f"min_budget={min_budget}, max_budget={max_budget}. "
|
||||
f"Both must be positive."
|
||||
)
|
||||
if min_budget > max_budget:
|
||||
raise ValueError(
|
||||
f"Invalid encoder cudagraph budget range: "
|
||||
f"min_budget={min_budget} > max_budget={max_budget}."
|
||||
)
|
||||
|
||||
if user_max_vision_items > 0:
|
||||
# User provided max_vision_items only; adjust auto-inferred
|
||||
# budgets so min(budgets) >= max_batch_size.
|
||||
self.max_batch_size = user_max_vision_items
|
||||
effective_min = max(min_budget, user_max_vision_items)
|
||||
self.token_budgets = self._generate_budgets(effective_min, max_budget)
|
||||
elif user_budgets:
|
||||
# User provided budgets only; cap auto-inferred
|
||||
# max_batch_size to min(user_budgets).
|
||||
self.token_budgets = sorted(user_budgets)
|
||||
self.max_batch_size = min(
|
||||
max_budget // min_budget,
|
||||
min(self.token_budgets),
|
||||
)
|
||||
else:
|
||||
# Fully auto-inferred.
|
||||
self.token_budgets = self._generate_budgets(min_budget, max_budget)
|
||||
self.max_batch_size = min(
|
||||
max_budget // min_budget,
|
||||
min(self.token_budgets),
|
||||
)
|
||||
|
||||
assert multimodal_config is not None
|
||||
if multimodal_config.get_limit_per_prompt("video") == 0:
|
||||
|
||||
Reference in New Issue
Block a user