[Platform] Replace torch.cuda.mem_get_info with torch.accelerator.get_memory_info (#44825)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
This commit is contained in:
@@ -63,7 +63,6 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -1274,7 +1273,7 @@ class Gemma4ForConditionalGeneration(
|
||||
# pass has already allocated activations we should account for.
|
||||
last_hidden_states_map: dict[int, torch.Tensor] = {}
|
||||
for patches, items in buckets.items():
|
||||
free, total = current_platform.mem_get_info()
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
max_batch_size = min(
|
||||
len(items),
|
||||
self._encoder_chunk(
|
||||
@@ -1382,7 +1381,7 @@ class Gemma4ForConditionalGeneration(
|
||||
fc_list = list(frame_counts)
|
||||
|
||||
total_frames = pixel_values.shape[0]
|
||||
free, total = current_platform.mem_get_info()
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
max_batch_size = min(
|
||||
total_frames,
|
||||
self._encoder_chunk(
|
||||
|
||||
@@ -92,11 +92,6 @@ class CpuPlatform(Platform):
|
||||
|
||||
return meminfo.total_memory
|
||||
|
||||
@classmethod
|
||||
def mem_get_info(cls) -> tuple[int, int]:
|
||||
meminfo = get_memory_node_info()
|
||||
return meminfo.available_memory, meminfo.total_memory
|
||||
|
||||
@classmethod
|
||||
def set_device(cls, device: torch.device) -> None:
|
||||
"""
|
||||
|
||||
@@ -143,7 +143,7 @@ class MemorySnapshot:
|
||||
"allocated_bytes.all.peak", 0
|
||||
)
|
||||
|
||||
self.free_memory, self.total_memory = current_platform.mem_get_info(device)
|
||||
self.free_memory, self.total_memory = torch.accelerator.get_memory_info(device)
|
||||
if current_platform.is_integrated_gpu(device.index):
|
||||
# On UMA (Unified Memory Architecture) platforms where CPU and
|
||||
# GPU share physical memory (e.g. GH200, DGX Spark, Jetson Orin),
|
||||
|
||||
@@ -38,6 +38,14 @@ class _StreamPlaceholder:
|
||||
pass
|
||||
|
||||
|
||||
from vllm.utils.cpu_resource_utils import get_memory_node_info
|
||||
|
||||
|
||||
def get_memory_info(*args: Any, **kwargs: Any) -> tuple[int, int]:
|
||||
meminfo = get_memory_node_info()
|
||||
return meminfo.available_memory, meminfo.total_memory
|
||||
|
||||
|
||||
torch.Event = _EventPlaceholder
|
||||
torch.cuda.Event = _EventPlaceholder
|
||||
torch.cuda.Stream = _StreamPlaceholder
|
||||
@@ -46,6 +54,7 @@ torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
|
||||
torch.accelerator.synchronize = noop
|
||||
torch.accelerator.empty_cache = noop
|
||||
torch.Tensor.pin_memory = fake_pin_memory
|
||||
torch.accelerator.get_memory_info = get_memory_info
|
||||
|
||||
# Patch vLLM torch utils
|
||||
import vllm.utils.torch_utils as torch_utils
|
||||
|
||||
@@ -701,7 +701,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
start_time = time.perf_counter()
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
attn_states = self.cudagraph_manager.capture(
|
||||
@@ -720,7 +720,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.speculator.capture(attn_states)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
elapsed_time = end_time - start_time
|
||||
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
|
||||
# This usually takes 5~20 seconds.
|
||||
|
||||
@@ -20,7 +20,7 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
|
||||
# Use the faster GPU path when there is plenty of headroom;
|
||||
# otherwise compare on CPU.
|
||||
w = draft.weight
|
||||
if w.is_cuda and torch.cuda.mem_get_info(w.device)[0] < w.numel() * 2:
|
||||
if w.is_cuda and torch.accelerator.get_memory_info(w.device)[0] < w.numel() * 2:
|
||||
return torch.equal(w.cpu(), target.weight.cpu())
|
||||
return torch.equal(w, target.weight)
|
||||
|
||||
|
||||
@@ -6527,7 +6527,7 @@ class GPUModelRunner(
|
||||
mem_samples: list[int] = []
|
||||
|
||||
for i, desc in enumerate(profile_descs):
|
||||
mem_before = torch.cuda.mem_get_info()[0]
|
||||
mem_before = torch.accelerator.get_memory_info()[0]
|
||||
self._warmup_and_capture(
|
||||
desc,
|
||||
cudagraph_runtime_mode=mode,
|
||||
@@ -6541,7 +6541,7 @@ class GPUModelRunner(
|
||||
),
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
free_after = torch.cuda.mem_get_info()[0]
|
||||
free_after = torch.accelerator.get_memory_info()[0]
|
||||
mem_samples.append(mem_before - free_after)
|
||||
|
||||
first_capture = mem_samples[0]
|
||||
@@ -6563,10 +6563,10 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if encoder_cudagraph_manager is not None:
|
||||
mem_before = torch.cuda.mem_get_info()[0]
|
||||
mem_before = torch.accelerator.get_memory_info()[0]
|
||||
encoder_cudagraph_manager.capture(graph_pool=encoder_profiling_pool)
|
||||
torch.accelerator.synchronize()
|
||||
free_after = torch.cuda.mem_get_info()[0]
|
||||
free_after = torch.accelerator.get_memory_info()[0]
|
||||
encoder_memory_estimate = max(mem_before - free_after, 0)
|
||||
|
||||
logger.debug(
|
||||
@@ -6632,7 +6632,7 @@ class GPUModelRunner(
|
||||
with self._freeze_gc(), graph_capture(device=self.device):
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.empty_cache()
|
||||
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
for (
|
||||
runtime_mode,
|
||||
@@ -6650,7 +6650,7 @@ class GPUModelRunner(
|
||||
self.encoder_cudagraph_manager.capture(graph_pool=encoder_graph_pool)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
# Disable cudagraph capturing globally, so any unexpected cudagraph
|
||||
# capturing will be detected and raise an error after here.
|
||||
|
||||
@@ -172,7 +172,7 @@ class Worker(WorkerBase):
|
||||
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
torch.accelerator.synchronize()
|
||||
free_bytes_before_sleep = current_platform.mem_get_info()[0]
|
||||
free_bytes_before_sleep = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2:
|
||||
@@ -187,7 +187,7 @@ class Worker(WorkerBase):
|
||||
torch.accelerator.synchronize()
|
||||
deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0)
|
||||
while True:
|
||||
free_bytes_after_sleep, total = current_platform.mem_get_info()
|
||||
free_bytes_after_sleep, total = torch.accelerator.get_memory_info()
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
if freed_bytes >= 0 or time.monotonic() >= deadline:
|
||||
break
|
||||
@@ -459,8 +459,8 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
# Profile CUDA graph memory if graphs will be captured.
|
||||
# Skip on ROCm/HIP/XPU as graph pool handles and mem_get_info behave
|
||||
# differently and can produce incorrect/negative estimates.
|
||||
# Skip on ROCm/HIP/XPU as graph pool handles and get_memory_info
|
||||
# behave differently and can produce incorrect/negative estimates.
|
||||
cudagraph_memory_estimate = 0
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
|
||||
@@ -45,7 +45,6 @@ def _torch_cuda_wrapper():
|
||||
torch.cuda.default_stream = torch.xpu.current_stream
|
||||
torch.cuda.current_stream = torch.xpu.current_stream
|
||||
torch.cuda.stream = torch.xpu.stream
|
||||
torch.cuda.mem_get_info = torch.xpu.mem_get_info
|
||||
torch.cuda.Event = torch.Event
|
||||
torch.cuda.set_stream = torch.xpu.set_stream
|
||||
if supports_xpu_graph():
|
||||
|
||||
Reference in New Issue
Block a user