[Fix] Misc Fixes in ViT CUDA Graph (#38040)

Signed-off-by: Baorun Mu <bmu@nvidia.com>
This commit is contained in:
Baorun (Lauren) Mu
2026-05-14 23:49:06 +08:00
committed by GitHub
parent b8a25d0e12
commit a7737cb4f3
4 changed files with 242 additions and 21 deletions
+52 -12
View File
@@ -72,25 +72,65 @@ class EncoderCudaGraphManager:
multimodal_config = vllm_config.model_config.multimodal_config
# Invariant: max_batch_size <= min_token_budget.
# This ensures per_image_output = budget // max_batch_size >= 1
# for every captured budget, preventing reshape crashes on empty
# tensors during CUDA graph capture. Validated/enforced below for
# each configuration path.
if user_budgets and user_max_vision_items > 0:
# Fully user-specified
# Fully user-specified: validate the invariant.
self.token_budgets = sorted(user_budgets)
self.max_batch_size = user_max_vision_items
min_tok = min(self.token_budgets)
if self.max_batch_size > min_tok:
raise ValueError(
f"encoder_cudagraph_max_vision_items_per_batch "
f"({self.max_batch_size}) must be <= smallest token "
f"budget ({min_tok}). With budgets="
f"{self.token_budgets}, per_image_output = "
f"{min_tok} // {self.max_batch_size} = "
f"{min_tok // self.max_batch_size}, which would cause "
f"a capture failure. Either increase the smallest "
f"budget or decrease max_vision_items_per_batch."
)
else:
# Auto-infer missing values from model
# Auto-infer missing values from model.
min_budget, max_budget = model.get_encoder_cudagraph_budget_range(
vllm_config
)
self.token_budgets = (
sorted(user_budgets)
if user_budgets
else self._generate_budgets(min_budget, max_budget)
)
self.max_batch_size = (
user_max_vision_items
if user_max_vision_items > 0
else max_budget // min_budget
)
if min_budget <= 0 or max_budget <= 0:
raise ValueError(
f"Invalid encoder cudagraph budget range: "
f"min_budget={min_budget}, max_budget={max_budget}. "
f"Both must be positive."
)
if min_budget > max_budget:
raise ValueError(
f"Invalid encoder cudagraph budget range: "
f"min_budget={min_budget} > max_budget={max_budget}."
)
if user_max_vision_items > 0:
# User provided max_vision_items only; adjust auto-inferred
# budgets so min(budgets) >= max_batch_size.
self.max_batch_size = user_max_vision_items
effective_min = max(min_budget, user_max_vision_items)
self.token_budgets = self._generate_budgets(effective_min, max_budget)
elif user_budgets:
# User provided budgets only; cap auto-inferred
# max_batch_size to min(user_budgets).
self.token_budgets = sorted(user_budgets)
self.max_batch_size = min(
max_budget // min_budget,
min(self.token_budgets),
)
else:
# Fully auto-inferred.
self.token_budgets = self._generate_budgets(min_budget, max_budget)
self.max_batch_size = min(
max_budget // min_budget,
min(self.token_budgets),
)
assert multimodal_config is not None
if multimodal_config.get_limit_per_prompt("video") == 0: