[Hardware][GPU] Profiler config additional to increase it scope and annotation details (#37524)

Signed-off-by: devalshahamd <deval.shah@amd.com>
Signed-off-by: Deval Shah <devashah@amd.com>
Signed-off-by: Deval Shah <deval.shah@amd.com>
Co-authored-by: Deval Shah <devashah@amd.com>
This commit is contained in:
devalshahamd
2026-07-17 13:38:59 -07:00
committed by GitHub
co-authored by Deval Shah
parent b5433b6f50
commit fcd2255d16
4 changed files with 240 additions and 26 deletions
+98 -13
View File
@@ -975,19 +975,104 @@ class Worker(WorkerBase):
iteration_details = compute_iteration_details(scheduler_output)
annotation = "".join(
[
"execute_context_",
str(iteration_details.num_ctx_requests),
"(",
str(iteration_details.num_ctx_tokens),
")_generation_",
str(iteration_details.num_generation_requests),
"(",
str(iteration_details.num_generation_tokens),
")",
]
)
if self.vllm_config.profiler_config.detailed_trace_annotation:
# Compute roofline-model metrics per request, split by phase
# (context vs generation). These help estimate compute and
# memory intensity from the trace.
#
# Per-request quantities:
# query_len = number of scheduled (new) tokens for this request
# seq_len = total sequence length (computed + scheduled tokens)
#
# Aggregated across requests in each phase
# (ctx_=context, gen_=generation):
# seq_len_sum = sum of seq_len (total KV length)
# qq_compute = sum of query_len*query_len
# (proxy for QK^T compute cost)
# qk_compute = sum of query_len*seq_len
# (proxy for QK^T compute cost for decode and
# chunked prefill)
# total_scheduled_tokens = scheduled tokens across all requests
ctx_seq_len_sum = 0
ctx_qq_compute = 0
ctx_qk_compute = 0
gen_seq_len_sum = 0
gen_qq_compute = 0
gen_qk_compute = 0
total_scheduled_tokens = 0
# Build a map of req_id -> num_computed_tokens for all requests
new_req_ids = {
new_req.req_id for new_req in scheduler_output.scheduled_new_reqs
}
num_computed_tokens_ids = {
new_req.req_id: new_req.num_computed_tokens
for new_req in scheduler_output.scheduled_new_reqs
}
for req_id, num_computed_tokens in zip(
scheduler_output.scheduled_cached_reqs.req_ids,
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
):
num_computed_tokens_ids[req_id] = num_computed_tokens
# Accumulate per-phase metrics
for req_id, num_tokens in scheduler_output.num_scheduled_tokens.items():
query_len = num_tokens
total_scheduled_tokens += query_len
seq_len = num_computed_tokens_ids.get(req_id, 0) + query_len
if (
scheduler_output.scheduled_cached_reqs.is_context_phase(req_id)
or req_id in new_req_ids
):
ctx_seq_len_sum += seq_len
ctx_qq_compute += query_len * query_len
ctx_qk_compute += query_len * seq_len
else:
gen_seq_len_sum += seq_len
gen_qq_compute += query_len * query_len
gen_qk_compute += query_len * seq_len
annotation = "".join(
[
"execute_",
str(total_scheduled_tokens),
"_context_",
str(iteration_details.num_ctx_requests),
"(sq",
str(iteration_details.num_ctx_tokens),
"sk",
str(ctx_seq_len_sum),
"sqsq",
str(ctx_qq_compute),
"sqsk",
str(ctx_qk_compute),
")_generation_",
str(iteration_details.num_generation_requests),
"(sq",
str(iteration_details.num_generation_tokens),
"sk",
str(gen_seq_len_sum),
"sqsq",
str(gen_qq_compute),
"sqsk",
str(gen_qk_compute),
")",
]
)
else:
annotation = "".join(
[
"execute_context_",
str(iteration_details.num_ctx_requests),
"(",
str(iteration_details.num_ctx_tokens),
")",
"_generation_",
str(iteration_details.num_generation_requests),
"(",
str(iteration_details.num_generation_tokens),
")",
]
)
return self.profiler.annotate_context_manager(annotation)
@torch.inference_mode()