[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)
This commit is contained in:
@@ -76,6 +76,7 @@ class Request:
|
||||
resumable: bool = False,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
abort_immediately: bool = False,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.client_index = client_index
|
||||
@@ -182,6 +183,10 @@ class Request:
|
||||
# None entry in the queue means finished.
|
||||
self.streaming_queue: deque[StreamingUpdate | None] | None = None
|
||||
|
||||
# If True, request should be aborted immediately after being added to
|
||||
# the scheduler so the connector's request_finished hook runs.
|
||||
self.abort_immediately = abort_immediately
|
||||
|
||||
@classmethod
|
||||
def from_engine_core_request(
|
||||
cls,
|
||||
@@ -206,6 +211,7 @@ class Request:
|
||||
resumable=request.resumable,
|
||||
reasoning_ended=request.reasoning_ended,
|
||||
reasoning_parser_kwargs=request.reasoning_parser_kwargs,
|
||||
abort_immediately=request.abort_immediately,
|
||||
)
|
||||
|
||||
def append_output_token_ids(
|
||||
|
||||
Reference in New Issue
Block a user