[2/2][Model Runner V2] FULL CUDA graph capture for microbatched steps (DBO) (#51700)
Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
"""Microbatching (DBO) helpers for the V2 GPU model runner."""
|
||||
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import replace
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -317,8 +319,11 @@ class UBatchRunner:
|
||||
for _ in range(self.num_ubatches)
|
||||
]
|
||||
self.comm_stream = torch.cuda.Stream(device=device)
|
||||
# Threads and graph capture must use the same compute stream.
|
||||
self.capture_stream = torch.cuda.Stream(device=device)
|
||||
# The microbatch threads plus the thread that starts them.
|
||||
self.ready_barrier = threading.Barrier(self.num_ubatches + 1)
|
||||
self._pending_finish: Callable[[], Any] | None = None
|
||||
self.sm_control = create_sm_control_context(self.parallel_config)
|
||||
|
||||
def prepare(
|
||||
@@ -326,11 +331,16 @@ class UBatchRunner:
|
||||
input_batch: InputBatch,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
slot_mappings: torch.Tensor,
|
||||
cg_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
for_capture: bool = False,
|
||||
) -> UBatchState:
|
||||
"""Split the batch into the microbatches the step will run on.
|
||||
|
||||
Attention metadata is built per microbatch and carried in the forward
|
||||
contexts the threads install, not in the caller's.
|
||||
|
||||
FULL uses padded sizes; `for_capture` refreshes metadata from dummy
|
||||
block tables.
|
||||
"""
|
||||
ubatch_slices = create_ubatch_slices(input_batch, self.num_ubatches)
|
||||
|
||||
@@ -355,12 +365,13 @@ class UBatchRunner:
|
||||
attn_metadata.append(
|
||||
self.model_state.prepare_attn(
|
||||
ubatch,
|
||||
CUDAGraphMode.NONE,
|
||||
cg_mode,
|
||||
ubatch_block_tables,
|
||||
ubatch_slot_mappings,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
ubatch_idx=i,
|
||||
for_capture=for_capture,
|
||||
)
|
||||
)
|
||||
slot_mappings_by_layer.append(
|
||||
@@ -415,14 +426,34 @@ class UBatchRunner:
|
||||
model_inputs: dict[str, Any],
|
||||
ubatch_state: UBatchState,
|
||||
) -> Any:
|
||||
finish = self.begin_capturable_run(model, model_inputs, ubatch_state)
|
||||
return finish()
|
||||
|
||||
def begin_capturable_run(
|
||||
self,
|
||||
model: Any,
|
||||
model_inputs: dict[str, Any],
|
||||
ubatch_state: UBatchState,
|
||||
for_capture: bool = False,
|
||||
) -> Callable[[], Any]:
|
||||
"""Start microbatch threads and return a callback to run and join them.
|
||||
|
||||
For capture, start threads outside the graph and call the callback inside.
|
||||
`for_capture` selects the capture stream and initializes cuBLAS workspaces
|
||||
before capture.
|
||||
|
||||
Call the callback exactly once on this thread, or call `abort_pending_run`
|
||||
on failure to release parked threads.
|
||||
"""
|
||||
ubatch_slices = ubatch_state.slices
|
||||
assert len(ubatch_slices) == len(ubatch_state.forward_contexts)
|
||||
assert len(ubatch_slices) == self.num_ubatches
|
||||
|
||||
compute_stream = self.capture_stream if for_capture else current_stream()
|
||||
ubatch_contexts = make_ubatch_contexts(
|
||||
num_micro_batches=self.num_ubatches,
|
||||
comm_stream=self.comm_stream,
|
||||
compute_stream=current_stream(),
|
||||
compute_stream=compute_stream,
|
||||
forward_contexts=ubatch_state.forward_contexts,
|
||||
ready_barrier=self.ready_barrier,
|
||||
)
|
||||
@@ -433,6 +464,13 @@ class UBatchRunner:
|
||||
@torch.inference_mode()
|
||||
def run_ubatch(ubatch_context, inputs: dict[str, Any]) -> None:
|
||||
try:
|
||||
# New threads must select the worker's device.
|
||||
torch.accelerator.set_device_index(self.device.index)
|
||||
if for_capture:
|
||||
# Allocate cuBLAS workspaces before capture.
|
||||
for stream in (compute_stream, self.comm_stream):
|
||||
with torch.cuda.stream(stream):
|
||||
torch.cuda.current_blas_handle()
|
||||
with ubatch_context:
|
||||
outputs[ubatch_context.id] = model(**inputs)
|
||||
except BaseException as e: # noqa: BLE001
|
||||
@@ -445,33 +483,56 @@ class UBatchRunner:
|
||||
# is deliberately out of scope here.
|
||||
errors[ubatch_context.id] = e
|
||||
|
||||
# The threads manage the forward context themselves; clear it here so
|
||||
# it is restored correctly once they are done.
|
||||
with override_forward_context(None), self.sm_control:
|
||||
threads = []
|
||||
for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices):
|
||||
thread = threading.Thread(
|
||||
target=run_ubatch,
|
||||
args=(
|
||||
ubatch_context,
|
||||
slice_model_inputs(model_inputs, ubatch_slice.token_slice),
|
||||
),
|
||||
)
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
# Threads own the forward context; keep it cleared and SMs reserved
|
||||
# until finish joins them.
|
||||
stack = ExitStack()
|
||||
stack.enter_context(override_forward_context(None))
|
||||
stack.enter_context(self.sm_control)
|
||||
|
||||
# Wait for every thread to reach its context, then start the first.
|
||||
self.ready_barrier.wait()
|
||||
ubatch_contexts[0].cpu_wait_event.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
threads = []
|
||||
for ubatch_context, ubatch_slice in zip(ubatch_contexts, ubatch_slices):
|
||||
thread = threading.Thread(
|
||||
target=run_ubatch,
|
||||
args=(
|
||||
ubatch_context,
|
||||
slice_model_inputs(model_inputs, ubatch_slice.token_slice),
|
||||
),
|
||||
)
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
if errors:
|
||||
failed = min(errors)
|
||||
raise RuntimeError(
|
||||
f"Microbatch {failed} of {self.num_ubatches} failed"
|
||||
) from errors[failed]
|
||||
return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)])
|
||||
# Park all threads before the caller starts capture.
|
||||
self.ready_barrier.wait()
|
||||
|
||||
def finish() -> Any:
|
||||
try:
|
||||
ubatch_contexts[0].cpu_wait_event.set()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
finally:
|
||||
self._pending_finish = None
|
||||
stack.close()
|
||||
|
||||
if errors:
|
||||
failed = min(errors)
|
||||
raise RuntimeError(
|
||||
f"Microbatch {failed} of {self.num_ubatches} failed"
|
||||
) from errors[failed]
|
||||
return merge_ubatch_outputs([outputs[i] for i in range(self.num_ubatches)])
|
||||
|
||||
self._pending_finish = finish
|
||||
return finish
|
||||
|
||||
def abort_pending_run(self) -> None:
|
||||
"""Release and join parked threads after a failed capture."""
|
||||
if self._pending_finish is None:
|
||||
return
|
||||
try:
|
||||
self._pending_finish()
|
||||
except Exception:
|
||||
logger.exception("Failed to join microbatch threads after an abort")
|
||||
finally:
|
||||
self._pending_finish = None
|
||||
|
||||
|
||||
def maybe_build_ubatch_runner(
|
||||
@@ -491,10 +552,7 @@ def maybe_build_ubatch_runner(
|
||||
if not parallel_config.use_ubatching or parallel_config.data_parallel_size <= 1:
|
||||
return None
|
||||
|
||||
logger.info_once(
|
||||
"Dual batch overlap is enabled. Microbatched steps run without "
|
||||
"CUDA graphs on the V2 model runner."
|
||||
)
|
||||
logger.info_once("Dual batch overlap is enabled.")
|
||||
return UBatchRunner(
|
||||
vllm_config, device, model_state, attn_groups, kv_cache_config, max_num_reqs
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user