[Bugfix][ROCm] Fix cumem sleep and teardown (#46203)
Signed-off-by: pei.zhang <pei.zhang@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Cursor
Matthew Wong
Andreas Karatzas
parent
37a682d392
commit
68afd78897
@@ -3,14 +3,15 @@
|
||||
|
||||
import dataclasses
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Protocol
|
||||
from typing import Protocol, TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# py_device, py_size_or_aligned_size, py_ptr, py_handle
|
||||
HandleType = tuple[int, int, int, int]
|
||||
# py_handle has type list[int] on ROCm and int otherwise
|
||||
HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -18,6 +19,7 @@ class AllocationData:
|
||||
handle: HandleType
|
||||
tag: str
|
||||
cpu_backup_tensor: torch.Tensor | None = None
|
||||
is_asleep: bool = False
|
||||
|
||||
|
||||
class MemAllocator(Protocol):
|
||||
|
||||
Reference in New Issue
Block a user