Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
import asyncio
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional, Union, cast
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -36,14 +36,10 @@ class RequestOutputCollector:
|
||||
|
||||
def __init__(self, output_kind: RequestOutputKind):
|
||||
self.aggregate = output_kind == RequestOutputKind.DELTA
|
||||
self.output: Optional[Union[RequestOutput, PoolingRequestOutput, Exception]] = (
|
||||
None
|
||||
)
|
||||
self.output: RequestOutput | PoolingRequestOutput | Exception | None = None
|
||||
self.ready = asyncio.Event()
|
||||
|
||||
def put(
|
||||
self, output: Union[RequestOutput, PoolingRequestOutput, Exception]
|
||||
) -> None:
|
||||
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
|
||||
"""Non-blocking put operation."""
|
||||
if self.output is None or isinstance(output, Exception):
|
||||
self.output = output
|
||||
@@ -53,7 +49,7 @@ class RequestOutputCollector:
|
||||
# (if n > 1) do not override each other.
|
||||
self.output.add(output, aggregate=self.aggregate)
|
||||
|
||||
async def get(self) -> Union[RequestOutput, PoolingRequestOutput]:
|
||||
async def get(self) -> RequestOutput | PoolingRequestOutput:
|
||||
"""Get operation blocks on put event."""
|
||||
while (output := self.output) is None:
|
||||
await self.ready.wait()
|
||||
@@ -63,7 +59,7 @@ class RequestOutputCollector:
|
||||
raise output
|
||||
return output
|
||||
|
||||
def get_nowait(self) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
|
||||
"""Non-blocking get operation."""
|
||||
output = self.output
|
||||
if output is not None:
|
||||
@@ -76,7 +72,7 @@ class RequestOutputCollector:
|
||||
|
||||
@dataclass
|
||||
class OutputProcessorOutput:
|
||||
request_outputs: list[Union[RequestOutput, PoolingRequestOutput]]
|
||||
request_outputs: list[RequestOutput | PoolingRequestOutput]
|
||||
reqs_to_abort: list[str]
|
||||
|
||||
|
||||
@@ -84,22 +80,22 @@ class RequestState:
|
||||
def __init__(
|
||||
self,
|
||||
request_id: str,
|
||||
parent_req: Optional[ParentRequest],
|
||||
parent_req: ParentRequest | None,
|
||||
request_index: int,
|
||||
lora_name: Optional[str],
|
||||
lora_name: str | None,
|
||||
output_kind: RequestOutputKind,
|
||||
prompt: Optional[str],
|
||||
prompt_token_ids: Optional[list[int]],
|
||||
prompt_embeds: Optional[torch.Tensor],
|
||||
logprobs_processor: Optional[LogprobsProcessor],
|
||||
detokenizer: Optional[IncrementalDetokenizer],
|
||||
max_tokens_param: Optional[int],
|
||||
prompt: str | None,
|
||||
prompt_token_ids: list[int] | None,
|
||||
prompt_embeds: torch.Tensor | None,
|
||||
logprobs_processor: LogprobsProcessor | None,
|
||||
detokenizer: IncrementalDetokenizer | None,
|
||||
max_tokens_param: int | None,
|
||||
arrival_time: float,
|
||||
queue: Optional[RequestOutputCollector],
|
||||
queue: RequestOutputCollector | None,
|
||||
log_stats: bool,
|
||||
top_p: Optional[float] = None,
|
||||
n: Optional[int] = None,
|
||||
temperature: Optional[float] = None,
|
||||
top_p: float | None = None,
|
||||
n: int | None = None,
|
||||
temperature: float | None = None,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.parent_req = parent_req
|
||||
@@ -129,10 +125,10 @@ class RequestState:
|
||||
cls,
|
||||
tokenizer: AnyTokenizer,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest],
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None,
|
||||
request_index: int,
|
||||
queue: Optional[RequestOutputCollector],
|
||||
queue: RequestOutputCollector | None,
|
||||
log_stats: bool,
|
||||
) -> "RequestState":
|
||||
if sampling_params := request.sampling_params:
|
||||
@@ -186,11 +182,11 @@ class RequestState:
|
||||
def make_request_output(
|
||||
self,
|
||||
new_token_ids: list[int],
|
||||
pooling_output: Optional[torch.Tensor],
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Union[int, str, None],
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
|
||||
pooling_output: torch.Tensor | None,
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
|
||||
@@ -222,10 +218,10 @@ class RequestState:
|
||||
def _new_request_output(
|
||||
self,
|
||||
request_id: str,
|
||||
outputs: Union[list[CompletionOutput], list[PoolingOutput]],
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: Optional[dict[str, Any]] = None,
|
||||
) -> Union[RequestOutput, PoolingRequestOutput]:
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
first_output = outputs[0]
|
||||
if isinstance(first_output, PoolingOutput):
|
||||
assert len(outputs) == 1
|
||||
@@ -264,8 +260,8 @@ class RequestState:
|
||||
def _new_completion_output(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Union[int, str, None],
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
) -> CompletionOutput:
|
||||
assert self.detokenizer is not None
|
||||
assert self.logprobs_processor is not None
|
||||
@@ -308,7 +304,7 @@ class OutputProcessor:
|
||||
self.request_states: dict[str, RequestState] = {}
|
||||
self.parent_requests: dict[str, ParentRequest] = {}
|
||||
self.lora_states = LoRARequestStates()
|
||||
self.tracer: Optional[Tracer] = None
|
||||
self.tracer: Tracer | None = None
|
||||
|
||||
def get_num_unfinished_requests(self):
|
||||
return len(self.request_states)
|
||||
@@ -360,10 +356,10 @@ class OutputProcessor:
|
||||
def add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
prompt: Optional[str],
|
||||
parent_req: Optional[ParentRequest] = None,
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None = None,
|
||||
request_index: int = 0,
|
||||
queue: Optional[RequestOutputCollector] = None,
|
||||
queue: RequestOutputCollector | None = None,
|
||||
) -> None:
|
||||
request_id = request.request_id
|
||||
if request_id in self.request_states:
|
||||
@@ -386,8 +382,8 @@ class OutputProcessor:
|
||||
def process_outputs(
|
||||
self,
|
||||
engine_core_outputs: list[EngineCoreOutput],
|
||||
engine_core_timestamp: Optional[float] = None,
|
||||
iteration_stats: Optional[IterationStats] = None,
|
||||
engine_core_timestamp: float | None = None,
|
||||
iteration_stats: IterationStats | None = None,
|
||||
) -> OutputProcessorOutput:
|
||||
"""
|
||||
Process the EngineCoreOutputs:
|
||||
@@ -411,7 +407,7 @@ class OutputProcessor:
|
||||
within the loop below.
|
||||
"""
|
||||
|
||||
request_outputs: Union[list[RequestOutput], list[PoolingRequestOutput]] = []
|
||||
request_outputs: list[RequestOutput] | list[PoolingRequestOutput] = []
|
||||
reqs_to_abort: list[str] = []
|
||||
for engine_core_output in engine_core_outputs:
|
||||
req_id = engine_core_output.request_id
|
||||
@@ -492,7 +488,7 @@ class OutputProcessor:
|
||||
self,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
req_state: RequestState,
|
||||
iteration_stats: Optional[IterationStats],
|
||||
iteration_stats: IterationStats | None,
|
||||
) -> None:
|
||||
assert req_state.stats is not None
|
||||
assert iteration_stats is not None
|
||||
@@ -555,8 +551,8 @@ class OutputProcessor:
|
||||
self,
|
||||
req_state: RequestState,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
engine_core_timestamp: Optional[float],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
engine_core_timestamp: float | None,
|
||||
iteration_stats: IterationStats | None,
|
||||
):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
@@ -577,8 +573,8 @@ class OutputProcessor:
|
||||
def _update_stats_from_finished(
|
||||
self,
|
||||
req_state: RequestState,
|
||||
finish_reason: Optional[FinishReason],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
finish_reason: FinishReason | None,
|
||||
iteration_stats: IterationStats | None,
|
||||
):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user