[Misc] Support passing multiple request ids at once to AsyncLLM.abort() (#22944)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -998,7 +998,7 @@ class AsyncLLMEngine(EngineClient):
|
||||
await self.abort(request_id)
|
||||
raise
|
||||
|
||||
async def abort(self, request_id: str) -> None:
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
"""Abort a request.
|
||||
|
||||
Abort a submitted request. If the request is finished or not found,
|
||||
@@ -1007,6 +1007,9 @@ class AsyncLLMEngine(EngineClient):
|
||||
Args:
|
||||
request_id: The unique id of the request.
|
||||
"""
|
||||
if not isinstance(request_id, str):
|
||||
raise RuntimeError("Only single-request abort supported in"
|
||||
" deprecated V0")
|
||||
if not self.is_running:
|
||||
raise AsyncEngineDeadError(
|
||||
"Background loop is not running. If it was running, "
|
||||
|
||||
@@ -5,8 +5,8 @@ import asyncio
|
||||
import copy
|
||||
import pickle
|
||||
from contextlib import contextmanager, suppress
|
||||
from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
|
||||
Optional, Union, cast)
|
||||
from typing import (Any, AsyncGenerator, Dict, Iterable, Iterator, List,
|
||||
Mapping, Optional, Union, cast)
|
||||
|
||||
import cloudpickle
|
||||
import psutil
|
||||
@@ -404,9 +404,13 @@ class MQLLMEngineClient(EngineClient):
|
||||
error_message="Unable to start RPC Server",
|
||||
socket=socket)
|
||||
|
||||
async def abort(self, request_id: str):
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]):
|
||||
"""Send an ABORT_REQUEST signal to the RPC Server"""
|
||||
|
||||
if not isinstance(request_id, str):
|
||||
raise RuntimeError("Only single-request abort supported in"
|
||||
" deprecated V0")
|
||||
|
||||
with suppress(MQClientClosedError):
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCAbortRequest(request_id), socket=self.input_socket)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import AsyncGenerator, Mapping, Optional
|
||||
from typing import AsyncGenerator, Iterable, Mapping, Optional, Union
|
||||
|
||||
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
|
||||
from vllm.config import DecodingConfig, ModelConfig, VllmConfig
|
||||
@@ -229,11 +229,12 @@ class EngineClient(ABC):
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def abort(self, request_id: str) -> None:
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
"""Abort a request.
|
||||
|
||||
Args:
|
||||
request_id: The unique id of the request.
|
||||
request_id: The unique id of the request,
|
||||
or an iterable of such ids.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@@ -1315,6 +1315,11 @@ def common_broadcastable_dtype(dtypes: Collection[torch.dtype]):
|
||||
)
|
||||
|
||||
|
||||
def as_list(maybe_list: Iterable[T]) -> list[T]:
|
||||
"""Convert iterable to list, unless it's already a list."""
|
||||
return maybe_list if isinstance(maybe_list, list) else list(maybe_list)
|
||||
|
||||
|
||||
# `collections` helpers
|
||||
def is_list_of(
|
||||
value: object,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping
|
||||
from copy import copy
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
@@ -27,7 +27,8 @@ from vllm.transformers_utils.config import (
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import Device, cancel_task_threadsafe, cdiv, deprecate_kwargs
|
||||
from vllm.utils import (Device, as_list, cancel_task_threadsafe, cdiv,
|
||||
deprecate_kwargs)
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
@@ -431,14 +432,16 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
self.output_handler = asyncio.create_task(output_handler())
|
||||
|
||||
async def abort(self, request_id: str) -> None:
|
||||
async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
|
||||
"""Abort RequestId in OutputProcessor and EngineCore."""
|
||||
|
||||
request_ids = self.output_processor.abort_requests((request_id, ))
|
||||
await self.engine_core.abort_requests_async(request_ids)
|
||||
request_ids = (request_id, ) if isinstance(
|
||||
request_id, str) else as_list(request_id)
|
||||
all_request_ids = self.output_processor.abort_requests(request_ids)
|
||||
await self.engine_core.abort_requests_async(all_request_ids)
|
||||
|
||||
if self.log_requests:
|
||||
logger.info("Aborted request %s.", request_id)
|
||||
logger.info("Aborted request(s) %s.", ",".join(request_ids))
|
||||
|
||||
async def encode(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user