[DSV4] Enable Multi-stream for Pre-Attn GEMM (#41061)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Yongye Zhu
2026-04-28 09:08:55 -07:00
committed by GitHub
parent de3da0b97c
commit 5aa371dc8e
4 changed files with 190 additions and 60 deletions
+64
View File
@@ -56,3 +56,67 @@ def maybe_execute_in_parallel(
result0 = fn0()
result1 = fn1()
return (result0, result1)
def execute_in_parallel(
default_fn: Callable[[], Any],
aux_fns: list[Callable[[], Any] | None],
start_event: torch.cuda.Event,
done_events: list[torch.cuda.Event],
aux_streams: list[torch.cuda.Stream] | None = None,
) -> tuple[Any, list[Any]]:
"""Run default_fn on the current stream and aux_fns concurrently on
aux_streams.
Generalizes maybe_execute_in_parallel to N aux callables. Slots where
aux_fns[i] is None are skipped (no stream switch, no event record); their
corresponding entry in the returned aux_results list is None.
start_event fans out from the current stream to every launched aux stream;
done_events[i] is recorded after aux_fns[i] so the current stream joins
before returning. When aux_streams is None, all aux_fns run sequentially
on the current stream.
Args:
default_fn: Callable for the default (current) stream.
aux_fns: Per-aux callables; entries may be None to skip.
start_event: CUDA event recorded on the current stream before
default_fn so each launched aux stream can wait on it.
done_events: One CUDA event per aux slot, recorded after the
corresponding aux_fn. Length must match aux_fns.
aux_streams: Per-aux CUDA streams. Length must match aux_fns.
Multi-stream is disabled when None.
Returns:
Tuple of (default_result, aux_results) where aux_results[i] is the
result of aux_fns[i] (or None when skipped).
"""
aux_results: list[Any]
if aux_streams is None:
default_result = default_fn()
aux_results = [fn() if fn is not None else None for fn in aux_fns]
return default_result, aux_results
assert len(aux_fns) == len(aux_streams) == len(done_events), (
"aux_fns, aux_streams, and done_events must be the same length"
)
aux_results = [None] * len(aux_fns)
pending: list[torch.cuda.Event] = []
start_event.record()
for i, fn in enumerate(aux_fns):
if fn is None:
continue
with torch.cuda.stream(aux_streams[i]):
start_event.wait()
aux_results[i] = fn()
done_events[i].record()
pending.append(done_events[i])
default_result = default_fn()
for ev in pending:
ev.wait()
return default_result, aux_results