Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+43 -47
View File
@@ -4,7 +4,7 @@
import asyncio
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, Optional, Union, cast
from typing import Any, cast
import torch
@@ -36,14 +36,10 @@ class RequestOutputCollector:
def __init__(self, output_kind: RequestOutputKind):
self.aggregate = output_kind == RequestOutputKind.DELTA
self.output: Optional[Union[RequestOutput, PoolingRequestOutput, Exception]] = (
None
)
self.output: RequestOutput | PoolingRequestOutput | Exception | None = None
self.ready = asyncio.Event()
def put(
self, output: Union[RequestOutput, PoolingRequestOutput, Exception]
) -> None:
def put(self, output: RequestOutput | PoolingRequestOutput | Exception) -> None:
"""Non-blocking put operation."""
if self.output is None or isinstance(output, Exception):
self.output = output
@@ -53,7 +49,7 @@ class RequestOutputCollector:
# (if n > 1) do not override each other.
self.output.add(output, aggregate=self.aggregate)
async def get(self) -> Union[RequestOutput, PoolingRequestOutput]:
async def get(self) -> RequestOutput | PoolingRequestOutput:
"""Get operation blocks on put event."""
while (output := self.output) is None:
await self.ready.wait()
@@ -63,7 +59,7 @@ class RequestOutputCollector:
raise output
return output
def get_nowait(self) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
def get_nowait(self) -> RequestOutput | PoolingRequestOutput | None:
"""Non-blocking get operation."""
output = self.output
if output is not None:
@@ -76,7 +72,7 @@ class RequestOutputCollector:
@dataclass
class OutputProcessorOutput:
request_outputs: list[Union[RequestOutput, PoolingRequestOutput]]
request_outputs: list[RequestOutput | PoolingRequestOutput]
reqs_to_abort: list[str]
@@ -84,22 +80,22 @@ class RequestState:
def __init__(
self,
request_id: str,
parent_req: Optional[ParentRequest],
parent_req: ParentRequest | None,
request_index: int,
lora_name: Optional[str],
lora_name: str | None,
output_kind: RequestOutputKind,
prompt: Optional[str],
prompt_token_ids: Optional[list[int]],
prompt_embeds: Optional[torch.Tensor],
logprobs_processor: Optional[LogprobsProcessor],
detokenizer: Optional[IncrementalDetokenizer],
max_tokens_param: Optional[int],
prompt: str | None,
prompt_token_ids: list[int] | None,
prompt_embeds: torch.Tensor | None,
logprobs_processor: LogprobsProcessor | None,
detokenizer: IncrementalDetokenizer | None,
max_tokens_param: int | None,
arrival_time: float,
queue: Optional[RequestOutputCollector],
queue: RequestOutputCollector | None,
log_stats: bool,
top_p: Optional[float] = None,
n: Optional[int] = None,
temperature: Optional[float] = None,
top_p: float | None = None,
n: int | None = None,
temperature: float | None = None,
):
self.request_id = request_id
self.parent_req = parent_req
@@ -129,10 +125,10 @@ class RequestState:
cls,
tokenizer: AnyTokenizer,
request: EngineCoreRequest,
prompt: Optional[str],
parent_req: Optional[ParentRequest],
prompt: str | None,
parent_req: ParentRequest | None,
request_index: int,
queue: Optional[RequestOutputCollector],
queue: RequestOutputCollector | None,
log_stats: bool,
) -> "RequestState":
if sampling_params := request.sampling_params:
@@ -186,11 +182,11 @@ class RequestState:
def make_request_output(
self,
new_token_ids: list[int],
pooling_output: Optional[torch.Tensor],
finish_reason: Optional[FinishReason],
stop_reason: Union[int, str, None],
kv_transfer_params: Optional[dict[str, Any]] = None,
) -> Optional[Union[RequestOutput, PoolingRequestOutput]]:
pooling_output: torch.Tensor | None,
finish_reason: FinishReason | None,
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput | None:
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
@@ -222,10 +218,10 @@ class RequestState:
def _new_request_output(
self,
request_id: str,
outputs: Union[list[CompletionOutput], list[PoolingOutput]],
outputs: list[CompletionOutput] | list[PoolingOutput],
finished: bool,
kv_transfer_params: Optional[dict[str, Any]] = None,
) -> Union[RequestOutput, PoolingRequestOutput]:
kv_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput:
first_output = outputs[0]
if isinstance(first_output, PoolingOutput):
assert len(outputs) == 1
@@ -264,8 +260,8 @@ class RequestState:
def _new_completion_output(
self,
token_ids: list[int],
finish_reason: Optional[FinishReason],
stop_reason: Union[int, str, None],
finish_reason: FinishReason | None,
stop_reason: int | str | None,
) -> CompletionOutput:
assert self.detokenizer is not None
assert self.logprobs_processor is not None
@@ -308,7 +304,7 @@ class OutputProcessor:
self.request_states: dict[str, RequestState] = {}
self.parent_requests: dict[str, ParentRequest] = {}
self.lora_states = LoRARequestStates()
self.tracer: Optional[Tracer] = None
self.tracer: Tracer | None = None
def get_num_unfinished_requests(self):
return len(self.request_states)
@@ -360,10 +356,10 @@ class OutputProcessor:
def add_request(
self,
request: EngineCoreRequest,
prompt: Optional[str],
parent_req: Optional[ParentRequest] = None,
prompt: str | None,
parent_req: ParentRequest | None = None,
request_index: int = 0,
queue: Optional[RequestOutputCollector] = None,
queue: RequestOutputCollector | None = None,
) -> None:
request_id = request.request_id
if request_id in self.request_states:
@@ -386,8 +382,8 @@ class OutputProcessor:
def process_outputs(
self,
engine_core_outputs: list[EngineCoreOutput],
engine_core_timestamp: Optional[float] = None,
iteration_stats: Optional[IterationStats] = None,
engine_core_timestamp: float | None = None,
iteration_stats: IterationStats | None = None,
) -> OutputProcessorOutput:
"""
Process the EngineCoreOutputs:
@@ -411,7 +407,7 @@ class OutputProcessor:
within the loop below.
"""
request_outputs: Union[list[RequestOutput], list[PoolingRequestOutput]] = []
request_outputs: list[RequestOutput] | list[PoolingRequestOutput] = []
reqs_to_abort: list[str] = []
for engine_core_output in engine_core_outputs:
req_id = engine_core_output.request_id
@@ -492,7 +488,7 @@ class OutputProcessor:
self,
engine_core_output: EngineCoreOutput,
req_state: RequestState,
iteration_stats: Optional[IterationStats],
iteration_stats: IterationStats | None,
) -> None:
assert req_state.stats is not None
assert iteration_stats is not None
@@ -555,8 +551,8 @@ class OutputProcessor:
self,
req_state: RequestState,
engine_core_output: EngineCoreOutput,
engine_core_timestamp: Optional[float],
iteration_stats: Optional[IterationStats],
engine_core_timestamp: float | None,
iteration_stats: IterationStats | None,
):
if iteration_stats is None:
return
@@ -577,8 +573,8 @@ class OutputProcessor:
def _update_stats_from_finished(
self,
req_state: RequestState,
finish_reason: Optional[FinishReason],
iteration_stats: Optional[IterationStats],
finish_reason: FinishReason | None,
iteration_stats: IterationStats | None,
):
if iteration_stats is None:
return