Adjust design around encoder_cudagraph_forward (#42288)
Signed-off-by: Weida Hong <wdhongtw@google.com>
This commit is contained in:
@@ -32,23 +32,20 @@ class BudgetGraphMetadata:
|
||||
"""Metadata for a single budget graph.
|
||||
|
||||
CUDA graph replay pattern:
|
||||
1. Copy new batch data into input_buffer (e.g. pixel_values)
|
||||
2. Copy precomputed values into metadata_buffers
|
||||
3. Replay graph
|
||||
4. Read encoder outputs from output_buffer
|
||||
* Copy precomputed values into input_buffers
|
||||
* Replay graph
|
||||
* Read encoder outputs from output_buffer
|
||||
"""
|
||||
|
||||
token_budget: int
|
||||
max_batch_size: int # Max number of images/videos per batch
|
||||
max_frames_per_batch: int # Max total frames per batch (for video)
|
||||
graph: torch.cuda.CUDAGraph
|
||||
# The input tensor updated before replay (e.g. pixel_values)
|
||||
input_buffer: torch.Tensor
|
||||
# Buffers recorded into the CUDA graph (e.g. embeddings, sequence metadata).
|
||||
# Before replay the manager updates these in-place. By default buffers are
|
||||
# zeroed before slice-copying the actual values; model-specific padding
|
||||
# behavior is provided by EncoderCudaGraphConfig.padding_logics.
|
||||
metadata_buffers: dict[str, torch.Tensor]
|
||||
input_buffers: dict[str, torch.Tensor]
|
||||
# Output written by graph, read after replay
|
||||
output_buffer: torch.Tensor
|
||||
|
||||
@@ -214,29 +211,23 @@ class EncoderCudaGraphManager:
|
||||
self.dtype,
|
||||
)
|
||||
|
||||
mm_kwargs = capture_inputs.mm_kwargs
|
||||
buffers = capture_inputs.buffers
|
||||
values = capture_inputs.values
|
||||
|
||||
with torch.inference_mode():
|
||||
output = self.model.encoder_cudagraph_forward(mm_kwargs, buffers)
|
||||
output = self.model.encoder_cudagraph_forward({**values})
|
||||
output_buffer = torch.empty_like(output)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.inference_mode(), torch.cuda.graph(graph):
|
||||
output = self.model.encoder_cudagraph_forward(mm_kwargs, buffers)
|
||||
output = self.model.encoder_cudagraph_forward({**values})
|
||||
output_buffer.copy_(output)
|
||||
|
||||
# Since the image and video modalities share the same per-patch shape,
|
||||
# so we can use the image dummy inputs to capture CUDA graph for both
|
||||
# image and video.
|
||||
input_key = self.config.input_key_by_modality["image"]
|
||||
self.budget_graphs[token_budget] = BudgetGraphMetadata(
|
||||
token_budget=token_budget,
|
||||
max_batch_size=self.max_batch_size,
|
||||
max_frames_per_batch=self.max_frames_per_batch,
|
||||
graph=graph,
|
||||
input_buffer=mm_kwargs[input_key],
|
||||
metadata_buffers=buffers,
|
||||
input_buffers=values,
|
||||
output_buffer=output_buffer,
|
||||
)
|
||||
|
||||
@@ -273,15 +264,12 @@ class EncoderCudaGraphManager:
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
token_budget: int,
|
||||
replay_buffers: dict[str, torch.Tensor | None],
|
||||
) -> torch.Tensor | None:
|
||||
"""Execute budget graph.
|
||||
|
||||
Args:
|
||||
mm_kwargs: Multimodal inputs for the batch.
|
||||
token_budget: Token budget to use.
|
||||
replay_buffers: Buffer values to copy into captured buffers.
|
||||
None values leave the corresponding buffer unchanged.
|
||||
|
||||
Returns:
|
||||
Encoder outputs, or None if graph not captured.
|
||||
@@ -293,22 +281,18 @@ class EncoderCudaGraphManager:
|
||||
|
||||
graph_meta = self.budget_graphs[token_budget]
|
||||
|
||||
# Copy the input tensor. Buffers are sized for the full budget;
|
||||
# actual inputs may be smaller. Zero then slice-copy so padded
|
||||
# positions are invisible to attention (cu_seqlens masks them out).
|
||||
input_key = self.config.input_key_by_modality[
|
||||
self.model.get_input_modality(mm_kwargs)
|
||||
]
|
||||
src = mm_kwargs[input_key]
|
||||
n = src.shape[0]
|
||||
graph_meta.input_buffer[:n].copy_(src)
|
||||
replay = self.model.prepare_encoder_cudagraph_replay_buffers(
|
||||
mm_kwargs,
|
||||
self.max_batch_size,
|
||||
self.max_frames_per_batch,
|
||||
)
|
||||
|
||||
# Copy metadata buffers using keys from config.buffer_keys.
|
||||
for key in self.config.buffer_keys:
|
||||
src = replay_buffers.get(key)
|
||||
src = replay.values.get(key)
|
||||
if src is None:
|
||||
continue
|
||||
buf = graph_meta.metadata_buffers[key]
|
||||
buf = graph_meta.input_buffers[key]
|
||||
if src.ndim == 0:
|
||||
buf.copy_(src)
|
||||
else:
|
||||
@@ -430,16 +414,9 @@ class EncoderCudaGraphManager:
|
||||
token_budget,
|
||||
(token_budget - batch_out_tokens) / token_budget * 100,
|
||||
)
|
||||
replay = self.model.prepare_encoder_cudagraph_replay_buffers(
|
||||
batch_mm_kwargs,
|
||||
self.max_batch_size,
|
||||
self.max_frames_per_batch,
|
||||
)
|
||||
|
||||
# graph_hits counted inside _run_budget_graph after replay.
|
||||
output = self._run_budget_graph(
|
||||
batch_mm_kwargs, token_budget, replay.buffers
|
||||
)
|
||||
output = self._run_budget_graph(batch_mm_kwargs, token_budget)
|
||||
assert output is not None
|
||||
self.model.postprocess_encoder_output(
|
||||
output,
|
||||
|
||||
Reference in New Issue
Block a user