[Multimodal] Optimize encoder cudagraph padding logic (#58925)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
87d9996abe
commit
8352b2427f
@@ -479,30 +479,16 @@ class KimiK25ForConditionalGeneration(
|
||||
# Image-only (t == 1). Video chunks (t > 1) fall back to eager. #
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def _encoder_cudagraph_pad_totals(self) -> dict[int, int]:
|
||||
"""Row count of each captured buffer set, keyed by cu_seqlens ptr."""
|
||||
totals = self.__dict__.get("_encoder_cg_pad_totals")
|
||||
if totals is None:
|
||||
totals = {}
|
||||
self.__dict__["_encoder_cg_pad_totals"] = totals
|
||||
return totals
|
||||
|
||||
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
|
||||
|
||||
pad_totals = self._encoder_cudagraph_pad_totals
|
||||
|
||||
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
|
||||
# Varlen attention requires cu_seqlens[-1] to equal the number of
|
||||
# rows actually passed in. The captured buffers are sized for the
|
||||
# full token budget, so a smaller real batch has to be completed
|
||||
# with one trailing padding sequence; declaring fewer rows than the
|
||||
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
|
||||
total = pad_totals.get(dst.data_ptr())
|
||||
# Zero-length padding (same as Qwen): tail slots repeat the last
|
||||
# real offset, so FlashAttn schedules no work for padding rows.
|
||||
n = min(src.shape[0], dst.shape[0])
|
||||
dst[:n].copy_(src[:n])
|
||||
dst[n:] = total if total is not None else src[-1]
|
||||
if n < dst.shape[0]:
|
||||
dst[n:] = src[-1] if n else 0
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["vision_chunk"],
|
||||
@@ -633,8 +619,8 @@ class KimiK25ForConditionalGeneration(
|
||||
|
||||
# max_seqlen must cover the worst case: one item consuming the full
|
||||
# budget, i.e. token_budget * kh * kw patches.
|
||||
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append
|
||||
# a padding sequence covering rows the real batch does not fill.
|
||||
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can
|
||||
# pad the tail with zero-length sequences.
|
||||
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
|
||||
grid_thw_list,
|
||||
max_batch_size=max_batch_size + 1,
|
||||
@@ -645,10 +631,6 @@ class KimiK25ForConditionalGeneration(
|
||||
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
|
||||
values.update({k: v for k, v in metadata.items() if v is not None})
|
||||
|
||||
cu_seqlens = values.get("cu_seqlens")
|
||||
if cu_seqlens is not None:
|
||||
self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches
|
||||
|
||||
return EncoderCudaGraphCaptureInputs(values=values)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
@@ -663,8 +645,8 @@ class KimiK25ForConditionalGeneration(
|
||||
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
|
||||
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
|
||||
# so cu_seqlens[-1] matches the captured buffer's row count.
|
||||
# Unpadded: pad_cu_seqlens pads the tail with zero-length sequences
|
||||
# whose entries repeat the real total.
|
||||
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
|
||||
grid_thw_list,
|
||||
max_batch_size=None,
|
||||
|
||||
Reference in New Issue
Block a user