From 8f816a3f665489d7f0d222115d4f72ebab01076b Mon Sep 17 00:00:00 2001 From: Yizhou <136800916+yiz-liu@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:19:50 +0800 Subject: [PATCH] [MRV2][Metrics] Support `CUDAGraphStat` in MRV2 (#52358) Signed-off-by: Yizhou Liu Co-authored-by: OpenAI Codex --- tests/v1/worker/test_gpu_model_runner_v2_eplb.py | 1 + vllm/v1/worker/gpu/cudagraph_utils.py | 13 ++++++++++++- vllm/v1/worker/gpu/model_runner.py | 9 +++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 559af5572d..1176857dff 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -185,6 +185,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): finished_req_ids=set(), ec_connector_output=None, routed_experts=None, + cudagraph_stats=None, ) runner.req_states = SimpleNamespace() diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 7cb1bb2526..291863779c 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -17,7 +17,7 @@ from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, ) from vllm.compilation.counter import compilation_counter -from vllm.compilation.cuda_graph import CUDAGraphWrapper +from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper from vllm.config import VllmConfig, set_current_vllm_config from vllm.config.compilation import CUDAGraphMode @@ -70,6 +70,17 @@ class BatchExecutionDescriptor: num_active_loras: int = 0 +def make_cudagraph_stats( + batch_desc: BatchExecutionDescriptor, num_tokens: int +) -> CUDAGraphStat: + return CUDAGraphStat( + num_unpadded_tokens=num_tokens, + num_padded_tokens=batch_desc.num_tokens, + num_paddings=batch_desc.num_tokens - num_tokens, + runtime_mode=str(batch_desc.cg_mode), + ) + + class CreateForwardFn(Protocol): """Factory that prepares inputs (OUTSIDE the graph) and returns a forward_fn. Called with warmup=True for the warmup pass and warmup=False diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index d7dd9441d7..15005338d4 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -30,6 +30,7 @@ import torch.nn as nn import vllm.envs as envs from vllm.compilation.counter import compilation_counter +from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode from vllm.distributed.parallel_state import ( @@ -100,6 +101,7 @@ from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, ModelCudaGraphManager, + make_cudagraph_stats, ) from vllm.v1.worker.gpu.cudagraph_utils import ( profile_cudagraph_memory as _profile_cudagraph_memory, @@ -1602,9 +1604,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): empty_output = self.kv_connector.no_forward(scheduler_output) return self._merge_ec_connector_no_forward(scheduler_output, empty_output) + cudagraph_stats = None if not dummy_run: # Common case. # Prepare all the inputs and copy to the input buffers. + if self.observability_config.cudagraph_metrics: + cudagraph_stats = make_cudagraph_stats(batch_desc, num_toks) assert batch_req_state is not None input_batch = self.prepare_inputs( scheduler_output, batch_req_state, batch_desc @@ -1826,6 +1831,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): finished_req_ids=finished_req_ids, ec_connector_output=ec_connector_output, routed_experts=routed_experts, + cudagraph_stats=cudagraph_stats, ) if not self.is_last_pp_rank: @@ -1851,6 +1857,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): finished_req_ids = self.execute_model_state.finished_req_ids ec_connector_output = self.execute_model_state.ec_connector_output routed_experts = self.execute_model_state.routed_experts + cudagraph_stats = self.execute_model_state.cudagraph_stats self.execute_model_state = None if not self.is_last_pp_rank: @@ -1909,6 +1916,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)}, sampled_token_ids=None, # type: ignore prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type] + cudagraph_stats=cudagraph_stats, ) # Start async output copy here so that it can overlap with speculator proposal. async_output = AsyncOutput( @@ -2124,6 +2132,7 @@ class ExecuteModelState(NamedTuple): finished_req_ids: set[str] ec_connector_output: ECConnectorOutput | None routed_experts: RoutedExpertsTensors | None + cudagraph_stats: CUDAGraphStat | None class BatchReqState(NamedTuple):