[Multimodal] Optimize encoder cudagraph padding logic (#58925)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Isotr0py
2026-10-08 03:38:09 +00:00
committed by GitHub
co-authored by Shanshan Shen
parent 87d9996abe
commit 8352b2427f
5 changed files with 188 additions and 66 deletions
+8 -26
View File
@@ -479,30 +479,16 @@ class KimiK25ForConditionalGeneration(
# Image-only (t == 1). Video chunks (t > 1) fall back to eager. #
# ------------------------------------------------------------------ #
@property
def _encoder_cudagraph_pad_totals(self) -> dict[int, int]:
"""Row count of each captured buffer set, keyed by cu_seqlens ptr."""
totals = self.__dict__.get("_encoder_cg_pad_totals")
if totals is None:
totals = {}
self.__dict__["_encoder_cg_pad_totals"] = totals
return totals
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
pad_totals = self._encoder_cudagraph_pad_totals
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
# Varlen attention requires cu_seqlens[-1] to equal the number of
# rows actually passed in. The captured buffers are sized for the
# full token budget, so a smaller real batch has to be completed
# with one trailing padding sequence; declaring fewer rows than the
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
total = pad_totals.get(dst.data_ptr())
# Zero-length padding (same as Qwen): tail slots repeat the last
# real offset, so FlashAttn schedules no work for padding rows.
n = min(src.shape[0], dst.shape[0])
dst[:n].copy_(src[:n])
dst[n:] = total if total is not None else src[-1]
if n < dst.shape[0]:
dst[n:] = src[-1] if n else 0
return EncoderCudaGraphConfig(
modalities=["vision_chunk"],
@@ -633,8 +619,8 @@ class KimiK25ForConditionalGeneration(
# max_seqlen must cover the worst case: one item consuming the full
# budget, i.e. token_budget * kh * kw patches.
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append
# a padding sequence covering rows the real batch does not fill.
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can
# pad the tail with zero-length sequences.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=max_batch_size + 1,
@@ -645,10 +631,6 @@ class KimiK25ForConditionalGeneration(
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
values.update({k: v for k, v in metadata.items() if v is not None})
cu_seqlens = values.get("cu_seqlens")
if cu_seqlens is not None:
self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
@@ -663,8 +645,8 @@ class KimiK25ForConditionalGeneration(
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
# so cu_seqlens[-1] matches the captured buffer's row count.
# Unpadded: pad_cu_seqlens pads the tail with zero-length sequences
# whose entries repeat the real total.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=None,