Adjust design around encoder_cudagraph_forward (#42288)

Signed-off-by: Weida Hong <wdhongtw@google.com>
This commit is contained in:
Weida Hong
2026-05-29 03:02:52 +00:00
committed by GitHub
parent 212deff2ec
commit dfe8ba7c80
8 changed files with 105 additions and 154 deletions
+16 -39
View File
@@ -32,23 +32,20 @@ class BudgetGraphMetadata:
"""Metadata for a single budget graph.
CUDA graph replay pattern:
1. Copy new batch data into input_buffer (e.g. pixel_values)
2. Copy precomputed values into metadata_buffers
3. Replay graph
4. Read encoder outputs from output_buffer
* Copy precomputed values into input_buffers
* Replay graph
* Read encoder outputs from output_buffer
"""
token_budget: int
max_batch_size: int # Max number of images/videos per batch
max_frames_per_batch: int # Max total frames per batch (for video)
graph: torch.cuda.CUDAGraph
# The input tensor updated before replay (e.g. pixel_values)
input_buffer: torch.Tensor
# Buffers recorded into the CUDA graph (e.g. embeddings, sequence metadata).
# Before replay the manager updates these in-place. By default buffers are
# zeroed before slice-copying the actual values; model-specific padding
# behavior is provided by EncoderCudaGraphConfig.padding_logics.
metadata_buffers: dict[str, torch.Tensor]
input_buffers: dict[str, torch.Tensor]
# Output written by graph, read after replay
output_buffer: torch.Tensor
@@ -214,29 +211,23 @@ class EncoderCudaGraphManager:
self.dtype,
)
mm_kwargs = capture_inputs.mm_kwargs
buffers = capture_inputs.buffers
values = capture_inputs.values
with torch.inference_mode():
output = self.model.encoder_cudagraph_forward(mm_kwargs, buffers)
output = self.model.encoder_cudagraph_forward({**values})
output_buffer = torch.empty_like(output)
graph = torch.cuda.CUDAGraph()
with torch.inference_mode(), torch.cuda.graph(graph):
output = self.model.encoder_cudagraph_forward(mm_kwargs, buffers)
output = self.model.encoder_cudagraph_forward({**values})
output_buffer.copy_(output)
# Since the image and video modalities share the same per-patch shape,
# so we can use the image dummy inputs to capture CUDA graph for both
# image and video.
input_key = self.config.input_key_by_modality["image"]
self.budget_graphs[token_budget] = BudgetGraphMetadata(
token_budget=token_budget,
max_batch_size=self.max_batch_size,
max_frames_per_batch=self.max_frames_per_batch,
graph=graph,
input_buffer=mm_kwargs[input_key],
metadata_buffers=buffers,
input_buffers=values,
output_buffer=output_buffer,
)
@@ -273,15 +264,12 @@ class EncoderCudaGraphManager:
self,
mm_kwargs: dict[str, Any],
token_budget: int,
replay_buffers: dict[str, torch.Tensor | None],
) -> torch.Tensor | None:
"""Execute budget graph.
Args:
mm_kwargs: Multimodal inputs for the batch.
token_budget: Token budget to use.
replay_buffers: Buffer values to copy into captured buffers.
None values leave the corresponding buffer unchanged.
Returns:
Encoder outputs, or None if graph not captured.
@@ -293,22 +281,18 @@ class EncoderCudaGraphManager:
graph_meta = self.budget_graphs[token_budget]
# Copy the input tensor. Buffers are sized for the full budget;
# actual inputs may be smaller. Zero then slice-copy so padded
# positions are invisible to attention (cu_seqlens masks them out).
input_key = self.config.input_key_by_modality[
self.model.get_input_modality(mm_kwargs)
]
src = mm_kwargs[input_key]
n = src.shape[0]
graph_meta.input_buffer[:n].copy_(src)
replay = self.model.prepare_encoder_cudagraph_replay_buffers(
mm_kwargs,
self.max_batch_size,
self.max_frames_per_batch,
)
# Copy metadata buffers using keys from config.buffer_keys.
for key in self.config.buffer_keys:
src = replay_buffers.get(key)
src = replay.values.get(key)
if src is None:
continue
buf = graph_meta.metadata_buffers[key]
buf = graph_meta.input_buffers[key]
if src.ndim == 0:
buf.copy_(src)
else:
@@ -430,16 +414,9 @@ class EncoderCudaGraphManager:
token_budget,
(token_budget - batch_out_tokens) / token_budget * 100,
)
replay = self.model.prepare_encoder_cudagraph_replay_buffers(
batch_mm_kwargs,
self.max_batch_size,
self.max_frames_per_batch,
)
# graph_hits counted inside _run_budget_graph after replay.
output = self._run_budget_graph(
batch_mm_kwargs, token_budget, replay.buffers
)
output = self._run_budget_graph(batch_mm_kwargs, token_budget)
assert output is not None
self.model.postprocess_encoder_output(
output,