[MRV2][Metrics] Support CUDAGraphStat in MRV2 (#52358)
Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -17,7 +17,7 @@ from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
)
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -70,6 +70,17 @@ class BatchExecutionDescriptor:
|
||||
num_active_loras: int = 0
|
||||
|
||||
|
||||
def make_cudagraph_stats(
|
||||
batch_desc: BatchExecutionDescriptor, num_tokens: int
|
||||
) -> CUDAGraphStat:
|
||||
return CUDAGraphStat(
|
||||
num_unpadded_tokens=num_tokens,
|
||||
num_padded_tokens=batch_desc.num_tokens,
|
||||
num_paddings=batch_desc.num_tokens - num_tokens,
|
||||
runtime_mode=str(batch_desc.cg_mode),
|
||||
)
|
||||
|
||||
|
||||
class CreateForwardFn(Protocol):
|
||||
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
|
||||
forward_fn. Called with warmup=True for the warmup pass and warmup=False
|
||||
|
||||
Reference in New Issue
Block a user