[kv_offload] Add req_id to ReqContext for per-request tracking (#42507)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -627,7 +627,7 @@ def _make_scheduler_with_lookup(
|
|||||||
return scheduler
|
return scheduler
|
||||||
|
|
||||||
|
|
||||||
_EMPTY_REQ_CTX = ReqContext()
|
_EMPTY_REQ_CTX = ReqContext(req_id="")
|
||||||
|
|
||||||
|
|
||||||
class TestMaximalPrefixLookup:
|
class TestMaximalPrefixLookup:
|
||||||
|
|||||||
@@ -19,9 +19,11 @@ from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
|||||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||||
|
|
||||||
|
|
||||||
def make_req_context(kv_transfer_params: dict | None = None) -> ReqContext:
|
def make_req_context(
|
||||||
|
req_id: str = "", kv_transfer_params: dict | None = None
|
||||||
|
) -> ReqContext:
|
||||||
"""Create a ReqContext as production code would, from a request's params."""
|
"""Create a ReqContext as production code would, from a request's params."""
|
||||||
return ReqContext(kv_transfer_params=kv_transfer_params)
|
return ReqContext(req_id=req_id, kv_transfer_params=kv_transfer_params)
|
||||||
|
|
||||||
|
|
||||||
_EMPTY_REQ_CTX = make_req_context()
|
_EMPTY_REQ_CTX = make_req_context()
|
||||||
|
|||||||
@@ -138,7 +138,10 @@ class RequestOffloadState:
|
|||||||
self.group_states = tuple(
|
self.group_states = tuple(
|
||||||
RequestGroupState() for _ in self.config.kv_group_configs
|
RequestGroupState() for _ in self.config.kv_group_configs
|
||||||
)
|
)
|
||||||
self.req_context = ReqContext(kv_transfer_params=self.req.kv_transfer_params)
|
self.req_context = ReqContext(
|
||||||
|
req_id=self.req.request_id,
|
||||||
|
kv_transfer_params=self.req.kv_transfer_params,
|
||||||
|
)
|
||||||
|
|
||||||
def update_offload_keys(self) -> None:
|
def update_offload_keys(self) -> None:
|
||||||
for group_config, group_state in zip(
|
for group_config, group_state in zip(
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ def get_offload_group_idx(key: OffloadKey) -> int:
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ReqContext:
|
class ReqContext:
|
||||||
|
req_id: str
|
||||||
kv_transfer_params: dict[str, Any] | None = None
|
kv_transfer_params: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user