[Hardware][GPU] Profiler config additional to increase it scope and annotation details (#37524)
Signed-off-by: devalshahamd <deval.shah@amd.com> Signed-off-by: Deval Shah <devashah@amd.com> Signed-off-by: Deval Shah <deval.shah@amd.com> Co-authored-by: Deval Shah <devashah@amd.com>
This commit is contained in:
co-authored by
Deval Shah
parent
b5433b6f50
commit
fcd2255d16
@@ -8,7 +8,7 @@ import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from contextlib import contextmanager
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from copy import copy, deepcopy
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import reduce
|
||||
@@ -6756,6 +6756,44 @@ class GPUModelRunner(
|
||||
# Capture the large shapes first so that the smaller shapes
|
||||
# can reuse the memory pool allocated for the large shapes.
|
||||
set_cudagraph_capturing_enabled(True)
|
||||
|
||||
# Setup torch profiler for graph capture traces (conditional)
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
local_rank = get_world_group().local_rank
|
||||
enable_profiler = (
|
||||
local_rank == 0
|
||||
) and self.vllm_config.profiler_config.capture_torch_profiler
|
||||
if enable_profiler:
|
||||
trace_dir = (
|
||||
self.vllm_config.profiler_config.torch_profiler_dir + "/capture_traces"
|
||||
)
|
||||
profiler = torch.profiler.profile(
|
||||
activities=[
|
||||
torch.profiler.ProfilerActivity.CPU,
|
||||
torch.profiler.ProfilerActivity.CUDA,
|
||||
],
|
||||
record_shapes=True,
|
||||
profile_memory=True,
|
||||
with_stack=True,
|
||||
on_trace_ready=torch.profiler.tensorboard_trace_handler(
|
||||
trace_dir,
|
||||
worker_name=f"graph_capture_rank_{local_rank}",
|
||||
use_gzip=True,
|
||||
),
|
||||
)
|
||||
logger.info_once(
|
||||
"Rank %d: Torch profiler enabled for CUDA graph capture, "
|
||||
"traces will be saved to: %s",
|
||||
local_rank,
|
||||
trace_dir,
|
||||
)
|
||||
else:
|
||||
profiler = nullcontext()
|
||||
logger.info_once(
|
||||
"Rank %d: Torch profiler disabled for CUDA graph capture", local_rank
|
||||
)
|
||||
|
||||
with self._freeze_gc(), graph_capture(device=self.device):
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -6768,6 +6806,7 @@ class GPUModelRunner(
|
||||
self._capture_cudagraphs(
|
||||
batch_descriptors=batch_descs,
|
||||
cudagraph_runtime_mode=runtime_mode,
|
||||
profiler=profiler,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -6811,7 +6850,10 @@ class GPUModelRunner(
|
||||
profile_seq_lens: int | None = None,
|
||||
allow_microbatching: bool = False,
|
||||
num_warmups: int | None = None,
|
||||
profiler: AbstractContextManager[Any] | None = None,
|
||||
):
|
||||
if profiler is None:
|
||||
profiler = nullcontext()
|
||||
if num_warmups is None:
|
||||
num_warmups = self.compilation_config.cudagraph_num_of_warmups
|
||||
force_attention = cudagraph_runtime_mode == CUDAGraphMode.FULL
|
||||
@@ -6827,22 +6869,29 @@ class GPUModelRunner(
|
||||
num_active_loras=desc.num_active_loras,
|
||||
profile_seq_lens=profile_seq_lens,
|
||||
)
|
||||
self._dummy_run(
|
||||
desc.num_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
uniform_decode=desc.uniform,
|
||||
allow_microbatching=allow_microbatching,
|
||||
skip_eplb=True,
|
||||
remove_lora=False,
|
||||
num_active_loras=desc.num_active_loras,
|
||||
is_graph_capturing=True,
|
||||
profile_seq_lens=profile_seq_lens,
|
||||
)
|
||||
with (
|
||||
profiler,
|
||||
torch.profiler.record_function(
|
||||
f"capture_{desc.num_tokens}_{cudagraph_runtime_mode.name}"
|
||||
),
|
||||
):
|
||||
self._dummy_run(
|
||||
desc.num_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
uniform_decode=desc.uniform,
|
||||
allow_microbatching=allow_microbatching,
|
||||
skip_eplb=True,
|
||||
remove_lora=False,
|
||||
num_active_loras=desc.num_active_loras,
|
||||
is_graph_capturing=True,
|
||||
profile_seq_lens=profile_seq_lens,
|
||||
)
|
||||
|
||||
def _capture_cudagraphs(
|
||||
self,
|
||||
batch_descriptors: list[BatchDescriptor],
|
||||
cudagraph_runtime_mode: CUDAGraphMode,
|
||||
profiler: AbstractContextManager[Any] | None = None,
|
||||
):
|
||||
assert (
|
||||
cudagraph_runtime_mode != CUDAGraphMode.NONE
|
||||
@@ -6885,6 +6934,7 @@ class GPUModelRunner(
|
||||
batch_desc,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
allow_microbatching=allow_microbatching,
|
||||
profiler=profiler,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
self.maybe_remove_all_loras(self.lora_config)
|
||||
|
||||
Reference in New Issue
Block a user