[Hardware][GPU] Profiler config additional to increase it scope and annotation details (#37524)

Signed-off-by: devalshahamd <deval.shah@amd.com>
Signed-off-by: Deval Shah <devashah@amd.com>
Signed-off-by: Deval Shah <deval.shah@amd.com>
Co-authored-by: Deval Shah <devashah@amd.com>
This commit is contained in:
devalshahamd
2026-07-17 13:38:59 -07:00
committed by GitHub
co-authored by Deval Shah
parent b5433b6f50
commit fcd2255d16
4 changed files with 240 additions and 26 deletions
+63 -1
View File
@@ -1,10 +1,15 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest import pytest
from vllm.config import ProfilerConfig from vllm.config import CUDAGraphMode, ProfilerConfig
from vllm.config.profiler import _is_uri_path from vllm.config.profiler import _is_uri_path
from vllm.profiler.wrapper import WorkerProfiler from vllm.profiler.wrapper import WorkerProfiler
from vllm.v1.core.sched.output import CachedRequestData
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
class ConcreteWorkerProfiler(WorkerProfiler): class ConcreteWorkerProfiler(WorkerProfiler):
@@ -236,3 +241,60 @@ class TestIsUriPath:
def test_is_uri_path(self, path, expected): def test_is_uri_path(self, path, expected):
"""Test that _is_uri_path correctly identifies URI vs local paths.""" """Test that _is_uri_path correctly identifies URI vs local paths."""
assert _is_uri_path(path) == expected assert _is_uri_path(path) == expected
class TestAnnotateProfile:
"""Tests for Worker.annotate_profile() annotation string formatting."""
def _annotate(self, detailed: bool) -> str:
worker = MagicMock()
worker.vllm_config.profiler_config.detailed_trace_annotation = detailed
worker.profiler = MagicMock()
ctx_req = MagicMock(req_id="ctx1", num_computed_tokens=0)
cached = CachedRequestData(
req_ids=["gen1"],
resumed_req_ids=set(),
new_token_ids=[],
all_token_ids={},
new_block_ids=[],
num_computed_tokens=[10],
num_output_tokens=[1],
)
sched = MagicMock(
scheduled_new_reqs=[ctx_req],
scheduled_cached_reqs=cached,
num_scheduled_tokens={"ctx1": 4, "gen1": 1},
)
Worker.annotate_profile(worker, sched)
return worker.profiler.annotate_context_manager.call_args[0][0]
def test_simple_format_mixed(self):
assert self._annotate(detailed=False) == (
"execute_context_1(4)_generation_1(1)"
)
def test_detailed_format_mixed(self):
# ctx1: sq=4, sk=4, sqsq=16, sqsk=16 | gen1: sq=1, sk=11, sqsq=1, sqsk=11 | bs=5
assert self._annotate(detailed=True) == (
"execute_5_context_1(sq4sk4sqsq16sqsk16)_generation_1(sq1sk11sqsq1sqsk11)"
)
def test_profiler_entered_during_capture():
"""Profiler is used as a context manager in _warmup_and_capture,
confirming it is active during the actual graph capture run."""
runner = MagicMock()
runner.compilation_config.cudagraph_num_of_warmups = 0
mock_profiler = MagicMock()
GPUModelRunner._warmup_and_capture(
runner,
desc=MagicMock(num_tokens=4, uniform=True),
cudagraph_runtime_mode=CUDAGraphMode.FULL,
profiler=mock_profiler,
)
mock_profiler.__enter__.assert_called_once()
mock_profiler.__exit__.assert_called_once()
+17
View File
@@ -66,6 +66,17 @@ class ProfilerConfig:
"""If `True`, enables memory profiling in the torch profiler. """If `True`, enables memory profiling in the torch profiler.
Disabled by default.""" Disabled by default."""
capture_torch_profiler: bool = False
"""If `True`, enables a torch profiler during CUDA graph capture on rank 0.
Traces are saved to a `capture_traces` subdirectory under `torch_profiler_dir`.
Requires `profiler` to be set to 'torch'."""
detailed_trace_annotation: bool = False
"""If `True`, uses detailed annotations with roofline metrics (sk, sqsq,
sqsk) in profiler trace events. If `False`, uses simple annotations with
only context/generation request counts and token counts.
Disabled by default."""
ignore_frontend: bool = False ignore_frontend: bool = False
"""If `True`, disables the front-end profiling of AsyncLLM when using the """If `True`, disables the front-end profiling of AsyncLLM when using the
'torch' profiler. This is needed to reduce overhead when using delay/limit options, 'torch' profiler. This is needed to reduce overhead when using delay/limit options,
@@ -144,4 +155,10 @@ class ProfilerConfig:
if profiler_dir and not _is_uri_path(profiler_dir): if profiler_dir and not _is_uri_path(profiler_dir):
self.torch_profiler_dir = os.path.abspath(os.path.expanduser(profiler_dir)) self.torch_profiler_dir = os.path.abspath(os.path.expanduser(profiler_dir))
if self.capture_torch_profiler and self.profiler != "torch":
raise ValueError(
"capture_torch_profiler is only applicable when profiler is "
"set to 'torch'"
)
return self return self
+62 -12
View File
@@ -8,7 +8,7 @@ import threading
import time import time
from collections import defaultdict from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence from collections.abc import Callable, Iterable, Iterator, Sequence
from contextlib import contextmanager from contextlib import AbstractContextManager, contextmanager, nullcontext
from copy import copy, deepcopy from copy import copy, deepcopy
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
from functools import reduce from functools import reduce
@@ -6756,6 +6756,44 @@ class GPUModelRunner(
# Capture the large shapes first so that the smaller shapes # Capture the large shapes first so that the smaller shapes
# can reuse the memory pool allocated for the large shapes. # can reuse the memory pool allocated for the large shapes.
set_cudagraph_capturing_enabled(True) set_cudagraph_capturing_enabled(True)
# Setup torch profiler for graph capture traces (conditional)
from vllm.distributed.parallel_state import get_world_group
local_rank = get_world_group().local_rank
enable_profiler = (
local_rank == 0
) and self.vllm_config.profiler_config.capture_torch_profiler
if enable_profiler:
trace_dir = (
self.vllm_config.profiler_config.torch_profiler_dir + "/capture_traces"
)
profiler = torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
record_shapes=True,
profile_memory=True,
with_stack=True,
on_trace_ready=torch.profiler.tensorboard_trace_handler(
trace_dir,
worker_name=f"graph_capture_rank_{local_rank}",
use_gzip=True,
),
)
logger.info_once(
"Rank %d: Torch profiler enabled for CUDA graph capture, "
"traces will be saved to: %s",
local_rank,
trace_dir,
)
else:
profiler = nullcontext()
logger.info_once(
"Rank %d: Torch profiler disabled for CUDA graph capture", local_rank
)
with self._freeze_gc(), graph_capture(device=self.device): with self._freeze_gc(), graph_capture(device=self.device):
torch.accelerator.synchronize() torch.accelerator.synchronize()
torch.accelerator.empty_cache() torch.accelerator.empty_cache()
@@ -6768,6 +6806,7 @@ class GPUModelRunner(
self._capture_cudagraphs( self._capture_cudagraphs(
batch_descriptors=batch_descs, batch_descriptors=batch_descs,
cudagraph_runtime_mode=runtime_mode, cudagraph_runtime_mode=runtime_mode,
profiler=profiler,
) )
torch.accelerator.synchronize() torch.accelerator.synchronize()
@@ -6811,7 +6850,10 @@ class GPUModelRunner(
profile_seq_lens: int | None = None, profile_seq_lens: int | None = None,
allow_microbatching: bool = False, allow_microbatching: bool = False,
num_warmups: int | None = None, num_warmups: int | None = None,
profiler: AbstractContextManager[Any] | None = None,
): ):
if profiler is None:
profiler = nullcontext()
if num_warmups is None: if num_warmups is None:
num_warmups = self.compilation_config.cudagraph_num_of_warmups num_warmups = self.compilation_config.cudagraph_num_of_warmups
force_attention = cudagraph_runtime_mode == CUDAGraphMode.FULL force_attention = cudagraph_runtime_mode == CUDAGraphMode.FULL
@@ -6827,22 +6869,29 @@ class GPUModelRunner(
num_active_loras=desc.num_active_loras, num_active_loras=desc.num_active_loras,
profile_seq_lens=profile_seq_lens, profile_seq_lens=profile_seq_lens,
) )
self._dummy_run( with (
desc.num_tokens, profiler,
cudagraph_runtime_mode=cudagraph_runtime_mode, torch.profiler.record_function(
uniform_decode=desc.uniform, f"capture_{desc.num_tokens}_{cudagraph_runtime_mode.name}"
allow_microbatching=allow_microbatching, ),
skip_eplb=True, ):
remove_lora=False, self._dummy_run(
num_active_loras=desc.num_active_loras, desc.num_tokens,
is_graph_capturing=True, cudagraph_runtime_mode=cudagraph_runtime_mode,
profile_seq_lens=profile_seq_lens, uniform_decode=desc.uniform,
) allow_microbatching=allow_microbatching,
skip_eplb=True,
remove_lora=False,
num_active_loras=desc.num_active_loras,
is_graph_capturing=True,
profile_seq_lens=profile_seq_lens,
)
def _capture_cudagraphs( def _capture_cudagraphs(
self, self,
batch_descriptors: list[BatchDescriptor], batch_descriptors: list[BatchDescriptor],
cudagraph_runtime_mode: CUDAGraphMode, cudagraph_runtime_mode: CUDAGraphMode,
profiler: AbstractContextManager[Any] | None = None,
): ):
assert ( assert (
cudagraph_runtime_mode != CUDAGraphMode.NONE cudagraph_runtime_mode != CUDAGraphMode.NONE
@@ -6885,6 +6934,7 @@ class GPUModelRunner(
batch_desc, batch_desc,
cudagraph_runtime_mode=cudagraph_runtime_mode, cudagraph_runtime_mode=cudagraph_runtime_mode,
allow_microbatching=allow_microbatching, allow_microbatching=allow_microbatching,
profiler=profiler,
) )
torch.accelerator.synchronize() torch.accelerator.synchronize()
self.maybe_remove_all_loras(self.lora_config) self.maybe_remove_all_loras(self.lora_config)
+98 -13
View File
@@ -975,19 +975,104 @@ class Worker(WorkerBase):
iteration_details = compute_iteration_details(scheduler_output) iteration_details = compute_iteration_details(scheduler_output)
annotation = "".join( if self.vllm_config.profiler_config.detailed_trace_annotation:
[ # Compute roofline-model metrics per request, split by phase
"execute_context_", # (context vs generation). These help estimate compute and
str(iteration_details.num_ctx_requests), # memory intensity from the trace.
"(", #
str(iteration_details.num_ctx_tokens), # Per-request quantities:
")_generation_", # query_len = number of scheduled (new) tokens for this request
str(iteration_details.num_generation_requests), # seq_len = total sequence length (computed + scheduled tokens)
"(", #
str(iteration_details.num_generation_tokens), # Aggregated across requests in each phase
")", # (ctx_=context, gen_=generation):
] # seq_len_sum = sum of seq_len (total KV length)
) # qq_compute = sum of query_len*query_len
# (proxy for QK^T compute cost)
# qk_compute = sum of query_len*seq_len
# (proxy for QK^T compute cost for decode and
# chunked prefill)
# total_scheduled_tokens = scheduled tokens across all requests
ctx_seq_len_sum = 0
ctx_qq_compute = 0
ctx_qk_compute = 0
gen_seq_len_sum = 0
gen_qq_compute = 0
gen_qk_compute = 0
total_scheduled_tokens = 0
# Build a map of req_id -> num_computed_tokens for all requests
new_req_ids = {
new_req.req_id for new_req in scheduler_output.scheduled_new_reqs
}
num_computed_tokens_ids = {
new_req.req_id: new_req.num_computed_tokens
for new_req in scheduler_output.scheduled_new_reqs
}
for req_id, num_computed_tokens in zip(
scheduler_output.scheduled_cached_reqs.req_ids,
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
):
num_computed_tokens_ids[req_id] = num_computed_tokens
# Accumulate per-phase metrics
for req_id, num_tokens in scheduler_output.num_scheduled_tokens.items():
query_len = num_tokens
total_scheduled_tokens += query_len
seq_len = num_computed_tokens_ids.get(req_id, 0) + query_len
if (
scheduler_output.scheduled_cached_reqs.is_context_phase(req_id)
or req_id in new_req_ids
):
ctx_seq_len_sum += seq_len
ctx_qq_compute += query_len * query_len
ctx_qk_compute += query_len * seq_len
else:
gen_seq_len_sum += seq_len
gen_qq_compute += query_len * query_len
gen_qk_compute += query_len * seq_len
annotation = "".join(
[
"execute_",
str(total_scheduled_tokens),
"_context_",
str(iteration_details.num_ctx_requests),
"(sq",
str(iteration_details.num_ctx_tokens),
"sk",
str(ctx_seq_len_sum),
"sqsq",
str(ctx_qq_compute),
"sqsk",
str(ctx_qk_compute),
")_generation_",
str(iteration_details.num_generation_requests),
"(sq",
str(iteration_details.num_generation_tokens),
"sk",
str(gen_seq_len_sum),
"sqsq",
str(gen_qq_compute),
"sqsk",
str(gen_qk_compute),
")",
]
)
else:
annotation = "".join(
[
"execute_context_",
str(iteration_details.num_ctx_requests),
"(",
str(iteration_details.num_ctx_tokens),
")",
"_generation_",
str(iteration_details.num_generation_requests),
"(",
str(iteration_details.num_generation_tokens),
")",
]
)
return self.profiler.annotate_context_manager(annotation) return self.profiler.annotate_context_manager(annotation)
@torch.inference_mode() @torch.inference_mode()