[kv_offload+HMA][13/N]: Enable HMA support (#41445)
This is the final PR in a series to enables HMA support for the offloading connector. The connector advertises `SupportsHMA` and is validated with unit tests and e2e tests. Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -829,7 +829,6 @@ class OffloadingConnectorScheduler:
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its blocks are freed.
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
@@ -42,7 +43,7 @@ from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class OffloadingConnector(KVConnectorBase_V1):
|
||||
class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
return True
|
||||
@@ -149,7 +150,15 @@ class OffloadingConnector(KVConnectorBase_V1):
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
return self.connector_scheduler.request_finished(request)
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
assert self.connector_scheduler is not None
|
||||
|
||||
Reference in New Issue
Block a user