[7/N] torch.compile, reduce compilation time (#10460)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
+13
-5
@@ -1,6 +1,7 @@
|
||||
"""A GPU worker class."""
|
||||
import gc
|
||||
import os
|
||||
import time
|
||||
from typing import Dict, List, Optional, Set, Tuple, Type, Union
|
||||
|
||||
import torch
|
||||
@@ -189,6 +190,7 @@ class Worker(LocalOrDistributedWorkerBase):
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
free_memory_pre_profile, total_gpu_memory = torch.cuda.mem_get_info()
|
||||
start_time = time.time()
|
||||
|
||||
# Execute a forward pass with dummy inputs to profile the memory usage
|
||||
# of the model.
|
||||
@@ -229,12 +231,18 @@ class Worker(LocalOrDistributedWorkerBase):
|
||||
num_gpu_blocks = max(num_gpu_blocks, 0)
|
||||
num_cpu_blocks = max(num_cpu_blocks, 0)
|
||||
|
||||
end_time = time.time()
|
||||
logger.info(
|
||||
"Memory profiling results: total_gpu_memory=%.2fGiB"
|
||||
" initial_memory_usage=%.2fGiB peak_torch_memory=%.2fGiB"
|
||||
" memory_usage_post_profile=%.2fGiB"
|
||||
" non_torch_memory=%.2fGiB kv_cache_size=%.2fGiB"
|
||||
" gpu_memory_utilization=%.2f", total_gpu_memory / (1024**3),
|
||||
"Memory profiling results: "
|
||||
"duration=%.2f seconds, "
|
||||
"total_gpu_memory=%.2fGiB, "
|
||||
"initial_memory_usage=%.2fGiB, "
|
||||
"peak_torch_memory=%.2fGiB, "
|
||||
"memory_usage_post_profile=%.2fGiB, "
|
||||
"non_torch_memory=%.2fGiB, "
|
||||
"kv_cache_size=%.2fGiB, "
|
||||
"gpu_memory_utilization=%.2f.", end_time - start_time,
|
||||
total_gpu_memory / (1024**3),
|
||||
(total_gpu_memory - free_memory_pre_profile) / (1024**3),
|
||||
(peak_memory - non_torch_allocations) / (1024**3),
|
||||
total_allocated_bytes / (1024**3),
|
||||
|
||||
Reference in New Issue
Block a user