[BugFix] Set graph_pool_id before FULL CUDA graph capture in ModelRunner V2 (#48843)
Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
040cbf95cc
commit
f890e1dbe2
@@ -17,6 +17,7 @@ from vllm.compilation.breakable_cudagraph import (
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
graph_capture,
|
||||
@@ -346,6 +347,10 @@ class CudaGraphManager:
|
||||
# Sync offloader's copy stream before capture.
|
||||
# Ensure any pre-capture prefetches from offloader are complete.
|
||||
get_offloader().sync_prev_onload()
|
||||
if self.pool is not None:
|
||||
set_graph_pool_id(self.pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
with torch.cuda.graph(graph, self.pool):
|
||||
forward_fn(CUDAGraphMode.NONE)
|
||||
# Join offloader's copy stream after forward to avoid
|
||||
|
||||
Reference in New Issue
Block a user