[BugFix] Overhaul async request cancellation (#7111)
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
from typing import (AsyncIterator, List, Mapping, Optional, Protocol,
|
||||
from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
|
||||
runtime_checkable)
|
||||
|
||||
from transformers import PreTrainedTokenizer
|
||||
@@ -30,7 +30,7 @@ class AsyncEngineClient(Protocol):
|
||||
def errored(self) -> bool:
|
||||
...
|
||||
|
||||
async def generate(
|
||||
def generate(
|
||||
self,
|
||||
inputs: PromptInputs,
|
||||
sampling_params: SamplingParams,
|
||||
@@ -38,17 +38,17 @@ class AsyncEngineClient(Protocol):
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
prompt_adapter_request: Optional[PromptAdapterRequest] = None
|
||||
) -> AsyncIterator[RequestOutput]:
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""Generates outputs for a request"""
|
||||
|
||||
async def encode(
|
||||
def encode(
|
||||
self,
|
||||
inputs: PromptInputs,
|
||||
pooling_params: PoolingParams,
|
||||
request_id: str,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
trace_headers: Optional[Mapping[str, str]] = None,
|
||||
) -> AsyncIterator[EmbeddingRequestOutput]:
|
||||
) -> AsyncGenerator[EmbeddingRequestOutput, None]:
|
||||
"""Generate outputs for a request from an embedding model."""
|
||||
|
||||
async def abort(self, request_id: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user