[MRV2][Metrics] Support CUDAGraphStat in MRV2 (#52358)
Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -185,6 +185,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
routed_experts=None,
|
||||
cudagraph_stats=None,
|
||||
)
|
||||
runner.req_states = SimpleNamespace()
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
)
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -70,6 +70,17 @@ class BatchExecutionDescriptor:
|
||||
num_active_loras: int = 0
|
||||
|
||||
|
||||
def make_cudagraph_stats(
|
||||
batch_desc: BatchExecutionDescriptor, num_tokens: int
|
||||
) -> CUDAGraphStat:
|
||||
return CUDAGraphStat(
|
||||
num_unpadded_tokens=num_tokens,
|
||||
num_padded_tokens=batch_desc.num_tokens,
|
||||
num_paddings=batch_desc.num_tokens - num_tokens,
|
||||
runtime_mode=str(batch_desc.cg_mode),
|
||||
)
|
||||
|
||||
|
||||
class CreateForwardFn(Protocol):
|
||||
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
|
||||
forward_fn. Called with warmup=True for the warmup pass and warmup=False
|
||||
|
||||
@@ -30,6 +30,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import (
|
||||
@@ -100,6 +101,7 @@ from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
make_cudagraph_stats,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
profile_cudagraph_memory as _profile_cudagraph_memory,
|
||||
@@ -1602,9 +1604,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
empty_output = self.kv_connector.no_forward(scheduler_output)
|
||||
return self._merge_ec_connector_no_forward(scheduler_output, empty_output)
|
||||
|
||||
cudagraph_stats = None
|
||||
if not dummy_run:
|
||||
# Common case.
|
||||
# Prepare all the inputs and copy to the input buffers.
|
||||
if self.observability_config.cudagraph_metrics:
|
||||
cudagraph_stats = make_cudagraph_stats(batch_desc, num_toks)
|
||||
assert batch_req_state is not None
|
||||
input_batch = self.prepare_inputs(
|
||||
scheduler_output, batch_req_state, batch_desc
|
||||
@@ -1826,6 +1831,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
routed_experts=routed_experts,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -1851,6 +1857,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
routed_experts = self.execute_model_state.routed_experts
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -1909,6 +1916,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
|
||||
sampled_token_ids=None, # type: ignore
|
||||
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
)
|
||||
# Start async output copy here so that it can overlap with speculator proposal.
|
||||
async_output = AsyncOutput(
|
||||
@@ -2124,6 +2132,7 @@ class ExecuteModelState(NamedTuple):
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
routed_experts: RoutedExpertsTensors | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
|
||||
|
||||
class BatchReqState(NamedTuple):
|
||||
|
||||
Reference in New Issue
Block a user