[Platform] Replace torch.cuda.mem_get_info with torch.accelerator.get_memory_info (#44825)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
This commit is contained in:
Kunshang Ji
2026-06-30 14:39:52 +08:00
committed by GitHub
parent 0feca7ffa8
commit fb42e5219e
17 changed files with 56 additions and 52 deletions
+2 -3
View File
@@ -63,7 +63,6 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -1274,7 +1273,7 @@ class Gemma4ForConditionalGeneration(
# pass has already allocated activations we should account for.
last_hidden_states_map: dict[int, torch.Tensor] = {}
for patches, items in buckets.items():
free, total = current_platform.mem_get_info()
free, total = torch.accelerator.get_memory_info()
max_batch_size = min(
len(items),
self._encoder_chunk(
@@ -1382,7 +1381,7 @@ class Gemma4ForConditionalGeneration(
fc_list = list(frame_counts)
total_frames = pixel_values.shape[0]
free, total = current_platform.mem_get_info()
free, total = torch.accelerator.get_memory_info()
max_batch_size = min(
total_frames,
self._encoder_chunk(
-5
View File
@@ -92,11 +92,6 @@ class CpuPlatform(Platform):
return meminfo.total_memory
@classmethod
def mem_get_info(cls) -> tuple[int, int]:
meminfo = get_memory_node_info()
return meminfo.available_memory, meminfo.total_memory
@classmethod
def set_device(cls, device: torch.device) -> None:
"""
+1 -1
View File
@@ -143,7 +143,7 @@ class MemorySnapshot:
"allocated_bytes.all.peak", 0
)
self.free_memory, self.total_memory = current_platform.mem_get_info(device)
self.free_memory, self.total_memory = torch.accelerator.get_memory_info(device)
if current_platform.is_integrated_gpu(device.index):
# On UMA (Unified Memory Architecture) platforms where CPU and
# GPU share physical memory (e.g. GH200, DGX Spark, Jetson Orin),
+9
View File
@@ -38,6 +38,14 @@ class _StreamPlaceholder:
pass
from vllm.utils.cpu_resource_utils import get_memory_node_info
def get_memory_info(*args: Any, **kwargs: Any) -> tuple[int, int]:
meminfo = get_memory_node_info()
return meminfo.available_memory, meminfo.total_memory
torch.Event = _EventPlaceholder
torch.cuda.Event = _EventPlaceholder
torch.cuda.Stream = _StreamPlaceholder
@@ -46,6 +54,7 @@ torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
torch.accelerator.synchronize = noop
torch.accelerator.empty_cache = noop
torch.Tensor.pin_memory = fake_pin_memory
torch.accelerator.get_memory_info = get_memory_info
# Patch vLLM torch utils
import vllm.utils.torch_utils as torch_utils
+2 -2
View File
@@ -701,7 +701,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
start_time = time.perf_counter()
gc.collect()
torch.accelerator.empty_cache()
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
attn_states = self.cudagraph_manager.capture(
@@ -720,7 +720,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.capture(attn_states)
end_time = time.perf_counter()
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
elapsed_time = end_time - start_time
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
# This usually takes 5~20 seconds.
@@ -20,7 +20,7 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
# Use the faster GPU path when there is plenty of headroom;
# otherwise compare on CPU.
w = draft.weight
if w.is_cuda and torch.cuda.mem_get_info(w.device)[0] < w.numel() * 2:
if w.is_cuda and torch.accelerator.get_memory_info(w.device)[0] < w.numel() * 2:
return torch.equal(w.cpu(), target.weight.cpu())
return torch.equal(w, target.weight)
+6 -6
View File
@@ -6527,7 +6527,7 @@ class GPUModelRunner(
mem_samples: list[int] = []
for i, desc in enumerate(profile_descs):
mem_before = torch.cuda.mem_get_info()[0]
mem_before = torch.accelerator.get_memory_info()[0]
self._warmup_and_capture(
desc,
cudagraph_runtime_mode=mode,
@@ -6541,7 +6541,7 @@ class GPUModelRunner(
),
)
torch.accelerator.synchronize()
free_after = torch.cuda.mem_get_info()[0]
free_after = torch.accelerator.get_memory_info()[0]
mem_samples.append(mem_before - free_after)
first_capture = mem_samples[0]
@@ -6563,10 +6563,10 @@ class GPUModelRunner(
)
if encoder_cudagraph_manager is not None:
mem_before = torch.cuda.mem_get_info()[0]
mem_before = torch.accelerator.get_memory_info()[0]
encoder_cudagraph_manager.capture(graph_pool=encoder_profiling_pool)
torch.accelerator.synchronize()
free_after = torch.cuda.mem_get_info()[0]
free_after = torch.accelerator.get_memory_info()[0]
encoder_memory_estimate = max(mem_before - free_after, 0)
logger.debug(
@@ -6632,7 +6632,7 @@ class GPUModelRunner(
with self._freeze_gc(), graph_capture(device=self.device):
torch.accelerator.synchronize()
torch.accelerator.empty_cache()
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
for (
runtime_mode,
@@ -6650,7 +6650,7 @@ class GPUModelRunner(
self.encoder_cudagraph_manager.capture(graph_pool=encoder_graph_pool)
torch.accelerator.synchronize()
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
# Disable cudagraph capturing globally, so any unexpected cudagraph
# capturing will be detected and raise an error after here.
+4 -4
View File
@@ -172,7 +172,7 @@ class Worker(WorkerBase):
def sleep(self, level: int = 1) -> None:
torch.accelerator.synchronize()
free_bytes_before_sleep = current_platform.mem_get_info()[0]
free_bytes_before_sleep = torch.accelerator.get_memory_info()[0]
# Save the buffers before level 2 sleep
if level == 2:
@@ -187,7 +187,7 @@ class Worker(WorkerBase):
torch.accelerator.synchronize()
deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0)
while True:
free_bytes_after_sleep, total = current_platform.mem_get_info()
free_bytes_after_sleep, total = torch.accelerator.get_memory_info()
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
if freed_bytes >= 0 or time.monotonic() >= deadline:
break
@@ -459,8 +459,8 @@ class Worker(WorkerBase):
)
# Profile CUDA graph memory if graphs will be captured.
# Skip on ROCm/HIP/XPU as graph pool handles and mem_get_info behave
# differently and can produce incorrect/negative estimates.
# Skip on ROCm/HIP/XPU as graph pool handles and get_memory_info
# behave differently and can produce incorrect/negative estimates.
cudagraph_memory_estimate = 0
if (
current_platform.is_cuda()
-1
View File
@@ -45,7 +45,6 @@ def _torch_cuda_wrapper():
torch.cuda.default_stream = torch.xpu.current_stream
torch.cuda.current_stream = torch.xpu.current_stream
torch.cuda.stream = torch.xpu.stream
torch.cuda.mem_get_info = torch.xpu.mem_get_info
torch.cuda.Event = torch.Event
torch.cuda.set_stream = torch.xpu.set_stream
if supports_xpu_graph():