[MRV2][Metrics] Support CUDAGraphStat in MRV2 (#52358)

Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Yizhou
2026-09-04 07:19:50 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 29af8bd672
commit 8f816a3f66
3 changed files with 22 additions and 1 deletions
@@ -185,6 +185,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
finished_req_ids=set(),
ec_connector_output=None,
routed_experts=None,
cudagraph_stats=None,
)
runner.req_states = SimpleNamespace()
+12 -1
View File
@@ -17,7 +17,7 @@ from vllm.compilation.breakable_cudagraph import (
is_breakable_cudagraph_enabled,
)
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphWrapper
from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CUDAGraphMode
@@ -70,6 +70,17 @@ class BatchExecutionDescriptor:
num_active_loras: int = 0
def make_cudagraph_stats(
batch_desc: BatchExecutionDescriptor, num_tokens: int
) -> CUDAGraphStat:
return CUDAGraphStat(
num_unpadded_tokens=num_tokens,
num_padded_tokens=batch_desc.num_tokens,
num_paddings=batch_desc.num_tokens - num_tokens,
runtime_mode=str(batch_desc.cg_mode),
)
class CreateForwardFn(Protocol):
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
forward_fn. Called with warmup=True for the warmup pass and warmup=False
+9
View File
@@ -30,6 +30,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.parallel_state import (
@@ -100,6 +101,7 @@ from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
ModelCudaGraphManager,
make_cudagraph_stats,
)
from vllm.v1.worker.gpu.cudagraph_utils import (
profile_cudagraph_memory as _profile_cudagraph_memory,
@@ -1602,9 +1604,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
empty_output = self.kv_connector.no_forward(scheduler_output)
return self._merge_ec_connector_no_forward(scheduler_output, empty_output)
cudagraph_stats = None
if not dummy_run:
# Common case.
# Prepare all the inputs and copy to the input buffers.
if self.observability_config.cudagraph_metrics:
cudagraph_stats = make_cudagraph_stats(batch_desc, num_toks)
assert batch_req_state is not None
input_batch = self.prepare_inputs(
scheduler_output, batch_req_state, batch_desc
@@ -1826,6 +1831,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
routed_experts=routed_experts,
cudagraph_stats=cudagraph_stats,
)
if not self.is_last_pp_rank:
@@ -1851,6 +1857,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
routed_experts = self.execute_model_state.routed_experts
cudagraph_stats = self.execute_model_state.cudagraph_stats
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -1909,6 +1916,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
sampled_token_ids=None, # type: ignore
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
cudagraph_stats=cudagraph_stats,
)
# Start async output copy here so that it can overlap with speculator proposal.
async_output = AsyncOutput(
@@ -2124,6 +2132,7 @@ class ExecuteModelState(NamedTuple):
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
routed_experts: RoutedExpertsTensors | None
cudagraph_stats: CUDAGraphStat | None
class BatchReqState(NamedTuple):