[Platform] Replace torch.cuda.mem_get_info with torch.accelerator.get_memory_info (#44825)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
This commit is contained in:
@@ -23,7 +23,7 @@ def test_python_error():
|
||||
error happening from the C++ side.
|
||||
"""
|
||||
allocator = get_mem_allocator_instance()
|
||||
total_bytes = current_platform.mem_get_info()[1]
|
||||
total_bytes = torch.accelerator.get_memory_info()[1]
|
||||
alloc_bytes = int(total_bytes * 0.7)
|
||||
tensors = []
|
||||
with allocator.use_memory_pool():
|
||||
@@ -64,9 +64,9 @@ def test_basic_cumem():
|
||||
output = x + y + z
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
free_bytes = current_platform.mem_get_info()[0]
|
||||
free_bytes = torch.accelerator.get_memory_info()[0]
|
||||
allocator.sleep()
|
||||
free_bytes_after_sleep = current_platform.mem_get_info()[0]
|
||||
free_bytes_after_sleep = torch.accelerator.get_memory_info()[0]
|
||||
assert free_bytes_after_sleep > free_bytes
|
||||
allocator.wake_up()
|
||||
|
||||
@@ -99,9 +99,9 @@ def test_cumem_with_cudagraph():
|
||||
with torch.cuda.graph(model_graph):
|
||||
y = model(x)
|
||||
|
||||
free_bytes = current_platform.mem_get_info()[0]
|
||||
free_bytes = torch.accelerator.get_memory_info()[0]
|
||||
allocator.sleep()
|
||||
free_bytes_after_sleep = current_platform.mem_get_info()[0]
|
||||
free_bytes_after_sleep = torch.accelerator.get_memory_info()[0]
|
||||
assert free_bytes_after_sleep > free_bytes
|
||||
allocator.wake_up()
|
||||
|
||||
@@ -132,7 +132,7 @@ def test_cumem_with_cudagraph():
|
||||
],
|
||||
)
|
||||
def test_end_to_end(model: str):
|
||||
free, total = current_platform.mem_get_info()
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
llm = LLM(model, enable_sleep_mode=True)
|
||||
prompt = "How are you?"
|
||||
@@ -144,7 +144,7 @@ def test_end_to_end(model: str):
|
||||
# test sleep level 1 here.
|
||||
llm.sleep(level=1)
|
||||
|
||||
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
|
||||
free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
|
||||
# now the memory usage is mostly cudagraph memory pool,
|
||||
# and it should be less than the model weights (1B model, 2GiB weights)
|
||||
@@ -164,7 +164,7 @@ def test_end_to_end(model: str):
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up(tags=["weights"])
|
||||
|
||||
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
|
||||
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
|
||||
# should just reallocate memory for weights (1B model, ~2GiB weights)
|
||||
@@ -181,7 +181,7 @@ def test_end_to_end(model: str):
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep():
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = current_platform.mem_get_info()
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
llm = LLM(model, enable_sleep_mode=True)
|
||||
prompt = "How are you?"
|
||||
@@ -191,13 +191,13 @@ def test_deep_sleep():
|
||||
# Put the engine to deep sleep
|
||||
llm.sleep(level=2)
|
||||
|
||||
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
|
||||
free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
|
||||
assert used_bytes < 3 * GiB_bytes
|
||||
|
||||
llm.wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights")
|
||||
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
|
||||
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
assert used_bytes < 4 * GiB_bytes
|
||||
|
||||
@@ -213,7 +213,7 @@ def test_deep_sleep():
|
||||
def test_deep_sleep_async():
|
||||
async def test():
|
||||
model = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
free, total = current_platform.mem_get_info()
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free # in case other process is running
|
||||
engine_args = AsyncEngineArgs(
|
||||
model=model,
|
||||
@@ -232,7 +232,7 @@ def test_deep_sleep_async():
|
||||
|
||||
await llm.wake_up(tags=["weights"])
|
||||
await llm.collective_rpc("reload_weights")
|
||||
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
|
||||
free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
|
||||
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
|
||||
assert used_bytes < 4 * GiB_bytes
|
||||
|
||||
|
||||
@@ -406,7 +406,7 @@ def test_fused_moe_int64_overflow(workspace_init):
|
||||
Reproduces the scenario from PR #34279.
|
||||
"""
|
||||
# ~12 GB GPU memory needed for intermediate caches
|
||||
free_mem = torch.cuda.mem_get_info()[0]
|
||||
free_mem = torch.accelerator.get_memory_info()[0]
|
||||
if free_mem < 12 * 1024**3:
|
||||
pytest.skip("Insufficient GPU memory for overflow test")
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ def _ru_maxrss_bytes() -> int | None:
|
||||
|
||||
def _gpu_used_bytes() -> int:
|
||||
torch.accelerator.synchronize()
|
||||
free_bytes, total_bytes = current_platform.mem_get_info()
|
||||
free_bytes, total_bytes = torch.accelerator.get_memory_info()
|
||||
return int(total_bytes - free_bytes)
|
||||
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ def test_memory_profiling():
|
||||
weights_memory = 128 * 1024 * 1024 * 4 # 512 MiB
|
||||
|
||||
def measure_current_non_torch():
|
||||
free, total = torch.cuda.mem_get_info()
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
current_used = total - free
|
||||
current_torch = torch.accelerator.memory_reserved()
|
||||
current_non_torch = current_used - current_torch
|
||||
@@ -81,8 +81,9 @@ def test_memory_snapshot_uses_psutil_on_integrated_gpu():
|
||||
with (
|
||||
patch("vllm.utils.mem_utils.current_platform") as mock_platform,
|
||||
patch("vllm.utils.mem_utils.psutil") as mock_psutil,
|
||||
patch("torch.accelerator") as mock_accelerator,
|
||||
):
|
||||
mock_platform.mem_get_info.return_value = (
|
||||
mock_accelerator.get_memory_info.return_value = (
|
||||
mock_cuda_free,
|
||||
mock_cuda_total,
|
||||
)
|
||||
@@ -90,8 +91,8 @@ def test_memory_snapshot_uses_psutil_on_integrated_gpu():
|
||||
mock_platform.memory_stats.return_value = {
|
||||
"allocated_bytes.all.peak": 0,
|
||||
}
|
||||
mock_platform.memory_reserved.return_value = 0
|
||||
mock_platform.current_device = lambda: "cuda:0"
|
||||
mock_accelerator.memory_reserved.return_value = 0
|
||||
mock_accelerator.current_device = lambda: "cuda:0"
|
||||
|
||||
mock_vmem = MagicMock()
|
||||
mock_vmem.available = mock_psutil_available
|
||||
@@ -105,24 +106,25 @@ def test_memory_snapshot_uses_psutil_on_integrated_gpu():
|
||||
|
||||
|
||||
def test_memory_snapshot_uses_cuda_on_discrete_gpu():
|
||||
"""On discrete GPUs, free_memory should come from CUDA mem_get_info."""
|
||||
"""On discrete GPUs, free_memory should come from accelerator get_memory_info."""
|
||||
mock_cuda_free = 70 * 1024**3
|
||||
mock_cuda_total = 80 * 1024**3
|
||||
|
||||
with (
|
||||
patch("vllm.utils.mem_utils.current_platform") as mock_platform,
|
||||
patch("vllm.utils.mem_utils.psutil") as mock_psutil,
|
||||
patch("torch.accelerator") as mock_accelerator,
|
||||
):
|
||||
mock_platform.mem_get_info.return_value = (
|
||||
mock_accelerator.get_memory_info.return_value = (
|
||||
mock_cuda_free,
|
||||
mock_cuda_total,
|
||||
)
|
||||
mock_platform.is_integrated_gpu.return_value = False
|
||||
mock_platform.memory_stats.return_value = {
|
||||
mock_accelerator.memory_stats.return_value = {
|
||||
"allocated_bytes.all.peak": 0,
|
||||
}
|
||||
mock_platform.memory_reserved.return_value = 0
|
||||
mock_platform.current_device = lambda: "cuda:0"
|
||||
mock_accelerator.memory_reserved.return_value = 0
|
||||
mock_accelerator.current_device = lambda: "cuda:0"
|
||||
|
||||
snapshot = MemorySnapshot(device="cuda:0")
|
||||
|
||||
|
||||
@@ -29,9 +29,8 @@ def _gpu_snapshot(tag: str, prev_alloc: float = 0.0) -> dict:
|
||||
torch.accelerator.synchronize()
|
||||
alloc = torch.accelerator.memory_allocated()
|
||||
reserved = torch.accelerator.memory_reserved()
|
||||
# mem_get_info is not available on torch.accelerator
|
||||
try:
|
||||
drv_free, drv_total = torch.cuda.mem_get_info()
|
||||
drv_free, drv_total = torch.accelerator.get_memory_info()
|
||||
drv_used = drv_total - drv_free
|
||||
drv_pct = drv_used / drv_total * 100
|
||||
except Exception:
|
||||
|
||||
@@ -1285,7 +1285,7 @@ def test_token_logprobs_large_batch_int64_row_offset():
|
||||
batch_size = 2**31 // vocab_size + 64 # batch_size * vocab_size > 2**31
|
||||
# logits (the large input) plus small logprob/rank outputs; ~1 GB headroom.
|
||||
required_bytes = batch_size * vocab_size * 4 + (1 << 30)
|
||||
if torch.cuda.mem_get_info()[0] < required_bytes:
|
||||
if torch.accelerator.get_memory_info()[0] < required_bytes:
|
||||
pytest.skip(f"needs ~{required_bytes / 1e9:.0f} GB of free GPU memory")
|
||||
|
||||
logits = torch.randn(batch_size, vocab_size, device=device, dtype=torch.float32)
|
||||
|
||||
@@ -426,7 +426,7 @@ class TestTritonTopkTopp:
|
||||
# logits is modified in place; the only extra device memory is the
|
||||
# per-SM scratch buffer (~num_sm * vocab), so allow ~1 GB of headroom.
|
||||
required_bytes = batch_size * vocab_size * 4 + (1 << 30)
|
||||
if torch.cuda.mem_get_info()[0] < required_bytes:
|
||||
if torch.accelerator.get_memory_info()[0] < required_bytes:
|
||||
pytest.skip(f"needs ~{required_bytes / 1e9:.0f} GB of free GPU memory")
|
||||
|
||||
logits = torch.randn(
|
||||
|
||||
Reference in New Issue
Block a user