[2/2][Model Runner V2] FULL CUDA graph capture for microbatched steps (DBO) (#51700)

Signed-off-by: specture724 <specture724@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Anjie Hou
2026-09-13 22:42:33 -07:00
committed by GitHub
co-authored by Codex
parent 67111973ee
commit 73d2a8cf86
6 changed files with 407 additions and 80 deletions
+89 -31
View File
@@ -3,6 +3,8 @@
"""Microbatching (DBO) helpers for the V2 GPU model runner."""
import threading
from collections.abc import Callable
from contextlib import ExitStack
from dataclasses import replace
from typing import Any, NamedTuple
@@ -317,8 +319,11 @@ class UBatchRunner:
for _ in range(self.num_ubatches)
]
self.comm_stream = torch.cuda.Stream(device=device)
# Threads and graph capture must use the same compute stream.
self.capture_stream = torch.cuda.Stream(device=device)
# The microbatch threads plus the thread that starts them.
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
self._pending_finish: Callable[[], Any] | None = None
self.sm_control = create_sm_control_context(self.parallel_config)
def prepare(
@@ -326,11 +331,16 @@ class UBatchRunner:
input_batch: InputBatch,
block_tables: tuple[torch.Tensor, ...],
slot_mappings: torch.Tensor,
cg_mode: CUDAGraphMode = CUDAGraphMode.NONE,
for_capture: bool = False,
) -> UBatchState:
"""Split the batch into the microbatches the step will run on.
Attention metadata is built per microbatch and carried in the forward
contexts the threads install, not in the caller's.
FULL uses padded sizes; `for_capture` refreshes metadata from dummy
block tables.
"""
ubatch_slices = create_ubatch_slices(input_batch, self.num_ubatches)
@@ -355,12 +365,13 @@ class UBatchRunner:
attn_metadata.append(
self.model_state.prepare_attn(
ubatch,
CUDAGraphMode.NONE,
cg_mode,
ubatch_block_tables,
ubatch_slot_mappings,
self.attn_groups,
self.kv_cache_config,
ubatch_idx=i,
for_capture=for_capture,
)
)
slot_mappings_by_layer.append(
@@ -415,14 +426,34 @@ class UBatchRunner:
model_inputs: dict[str, Any],
ubatch_state: UBatchState,
) -> Any:
finish = self.begin_capturable_run(model, model_inputs, ubatch_state)
return finish()
def begin_capturable_run(
self,
model: Any,
model_inputs: dict[str, Any],
ubatch_state: UBatchState,
for_capture: bool = False,
) -> Callable[[], Any]:
"""Start microbatch threads and return a callback to run and join them.
For capture, start threads outside the graph and call the callback inside.
`for_capture` selects the capture stream and initializes cuBLAS workspaces
before capture.
Call the callback exactly once on this thread, or call `abort_pending_run`
on failure to release parked threads.
"""
ubatch_slices = ubatch_state.slices
assert len(ubatch_slices) == len(ubatch_state.forward_contexts)
assert len(ubatch_slices) == self.num_ubatches
compute_stream = self.capture_stream if for_capture else current_stream()
ubatch_contexts = make_ubatch_contexts(
num_micro_batches=self.num_ubatches,
comm_stream=self.comm_stream,
compute_stream=current_stream(),
compute_stream=compute_stream,
forward_contexts=ubatch_state.forward_contexts,
ready_barrier=self.ready_barrier,
)
@@ -433,6 +464,13 @@ class UBatchRunner:
@torch.inference_mode()
def run_ubatch(ubatch_context, inputs: dict[str, Any]) -> None:
try:
# New threads must select the worker's device.
torch.accelerator.set_device_index(self.device.index)
if for_capture:
# Allocate cuBLAS workspaces before capture.
for stream in (compute_stream, self.comm_stream):
with torch.cuda.stream(stream):
torch.cuda.current_blas_handle()
with ubatch_context:
outputs[ubatch_context.id] = model(**inputs)
except BaseException as e: # noqa: BLE001
@@ -445,33 +483,56 @@ class UBatchRunner:
# is deliberately out of scope here.
errors[ubatch_context.id] = e
# The threads manage the forward context themselves; clear it here so
# it is restored correctly once they are done.
with override_forward_context(None), self.sm_control:
threads = []
for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices):
thread = threading.Thread(
target=run_ubatch,
args=(
ubatch_context,
slice_model_inputs(model_inputs, ubatch_slice.token_slice),
),
)
threads.append(thread)
thread.start()
# Threads own the forward context; keep it cleared and SMs reserved
# until finish joins them.
stack = ExitStack()
stack.enter_context(override_forward_context(None))
stack.enter_context(self.sm_control)
# Wait for every thread to reach its context, then start the first.
self.ready_barrier.wait()
ubatch_contexts[0].cpu_wait_event.set()
for thread in threads:
thread.join()
threads = []
for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices):
thread = threading.Thread(
target=run_ubatch,
args=(
ubatch_context,
slice_model_inputs(model_inputs, ubatch_slice.token_slice),
),
)
threads.append(thread)
thread.start()
if errors:
failed = min(errors)
raise RuntimeError(
f"Microbatch {failed} of {self.num_ubatches} failed"
) from errors[failed]
return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)])
# Park all threads before the caller starts capture.
self.ready_barrier.wait()
def finish() -> Any:
try:
ubatch_contexts[0].cpu_wait_event.set()
for thread in threads:
thread.join()
finally:
self._pending_finish = None
stack.close()
if errors:
failed = min(errors)
raise RuntimeError(
f"Microbatch {failed} of {self.num_ubatches} failed"
) from errors[failed]
return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)])
self._pending_finish = finish
return finish
def abort_pending_run(self) -> None:
"""Release and join parked threads after a failed capture."""
if self._pending_finish is None:
return
try:
self._pending_finish()
except Exception:
logger.exception("Failed to join microbatch threads after an abort")
finally:
self._pending_finish = None
def maybe_build_ubatch_runner(
@@ -491,10 +552,7 @@ def maybe_build_ubatch_runner(
if not parallel_config.use_ubatching or parallel_config.data_parallel_size <= 1:
return None
logger.info_once(
"Dual batch overlap is enabled. Microbatched steps run without "
"CUDA graphs on the V2 model runner."
)
logger.info_once("Dual batch overlap is enabled.")
return UBatchRunner(
vllm_config, device, model_state, attn_groups, kv_cache_config, max_num_reqs
)