[Model Runner v2] Fix v2 compile counter num_gpu_runner_capture_triggers and num_cudagraph_captured (#41285)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import (
|
||||
@@ -237,6 +238,7 @@ class CudaGraphManager:
|
||||
# the next forward pass.
|
||||
get_offloader().join_after_forward()
|
||||
self.graphs[desc] = graph
|
||||
compilation_counter.num_cudagraph_captured += 1
|
||||
self._graphs_captured = True
|
||||
return captured_attn_states
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.parallel_state import (
|
||||
@@ -587,6 +588,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return 0
|
||||
|
||||
compilation_counter.num_gpu_runner_capture_triggers += 1
|
||||
|
||||
start_time = time.perf_counter()
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
Reference in New Issue
Block a user