Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+41 -42
View File
@@ -8,14 +8,13 @@ from collections import defaultdict
from collections.abc import Iterator
from contextlib import contextmanager
from copy import deepcopy
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union, cast
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
import numpy as np
import torch
import torch.distributed
import torch.nn as nn
from tqdm import tqdm
from typing_extensions import TypeAlias
import vllm.envs as envs
from vllm.attention import Attention, AttentionType
@@ -162,7 +161,7 @@ logger = init_logger(__name__)
AttnMetadataDict: TypeAlias = dict[str, AttentionMetadata]
# list when ubatching is enabled
PerLayerAttnMetadata: TypeAlias = Union[list[AttnMetadataDict], AttnMetadataDict]
PerLayerAttnMetadata: TypeAlias = list[AttnMetadataDict] | AttnMetadataDict
# Wrapper for ModelRunnerOutput to support overlapped execution.
@@ -295,7 +294,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
# Sampler
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
self.eplb_state: Optional[EplbState] = None
self.eplb_state: EplbState | None = None
"""
State of the expert parallelism load balancer.
@@ -447,14 +446,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
# CUDA event to synchronize use of reused CPU tensors between steps
# when async scheduling is enabled.
self.prepare_inputs_event: Optional[torch.cuda.Event] = None
self.prepare_inputs_event: torch.cuda.Event | None = None
if self.use_async_scheduling:
self.prepare_inputs_event = torch.cuda.Event()
# Start in a completed state.
self.prepare_inputs_event.record(torch.cuda.default_stream())
# None in the first PP rank. The rest are set after load_model.
self.intermediate_tensors: Optional[IntermediateTensors] = None
self.intermediate_tensors: IntermediateTensors | None = None
# OPTIMIZATION: Cache the tensors rather than creating them every step.
# Keep in int64 to avoid overflow with long context
@@ -495,7 +494,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
else None
)
self.reorder_batch_threshold: Optional[int] = None
self.reorder_batch_threshold: int | None = None
# Attention layers that are only in the KVCacheConfig of the runner
# (e.g., KV sharing, encoder-only attention), but not in the
@@ -503,7 +502,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.runner_only_attn_layers: set[str] = set()
# Cached outputs.
self._draft_token_ids: Optional[Union[list[list[int]], torch.Tensor]] = None
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
self.transfer_event = torch.cuda.Event()
self.sampled_token_ids_pinned_cpu = torch.empty(
(self.max_model_len, 1),
@@ -527,7 +526,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
return self.positions.gpu[num_tokens]
def _make_buffer(
self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True
self, *size: int | torch.SymInt, dtype: torch.dtype, numpy: bool = True
) -> CpuGpuBuffer:
return CpuGpuBuffer(
*size,
@@ -928,7 +927,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def _get_cumsum_and_arange(
self,
num_tokens: np.ndarray,
cumsum_dtype: Optional[np.dtype] = None,
cumsum_dtype: np.dtype | None = None,
) -> tuple[np.ndarray, np.ndarray]:
"""Get the cumulative sum and batched arange of the given array.
# E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2])
@@ -1024,7 +1023,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
scheduler_output: "SchedulerOutput",
kv_cache_spec: KVCacheSpec,
num_reqs: int,
) -> Optional[np.ndarray]:
) -> np.ndarray | None:
if not isinstance(kv_cache_spec, CrossAttentionSpec):
return None
@@ -1042,12 +1041,12 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
) -> tuple[
PerLayerAttnMetadata,
torch.Tensor,
Optional[SpecDecodeMetadata],
SpecDecodeMetadata | None,
np.ndarray,
Optional[CommonAttentionMetadata],
CommonAttentionMetadata | None,
int,
Optional[UBatchSlices],
Optional[torch.Tensor],
UBatchSlices | None,
torch.Tensor | None,
bool,
]:
"""
@@ -2048,7 +2047,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
)
self._sync_device()
pooler_output: list[Optional[torch.Tensor]] = []
pooler_output: list[torch.Tensor | None] = []
for raw_output, seq_len, prompt_len in zip(
raw_pooler_output, seq_lens_cpu, pooling_metadata.prompt_lens
):
@@ -2091,13 +2090,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self,
scheduler_output: "SchedulerOutput",
num_input_tokens: int, # Padded
intermediate_tensors: Optional[IntermediateTensors] = None,
intermediate_tensors: IntermediateTensors | None = None,
) -> tuple[
int,
Optional[torch.Tensor],
Optional[torch.Tensor],
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor,
Optional[IntermediateTensors],
IntermediateTensors | None,
dict[str, Any],
]:
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
@@ -2197,8 +2196,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def _sample(
self,
logits: Optional[torch.Tensor],
spec_decode_metadata: Optional[SpecDecodeMetadata],
logits: torch.Tensor | None,
spec_decode_metadata: SpecDecodeMetadata | None,
) -> SamplerOutput:
# Sample the next token and get logprobs if needed.
sampling_metadata = self.input_batch.sampling_metadata
@@ -2243,14 +2242,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self,
scheduler_output: "SchedulerOutput",
sampler_output: SamplerOutput,
logits: Optional[torch.Tensor],
logits: torch.Tensor | None,
hidden_states: torch.Tensor,
num_scheduled_tokens: int,
) -> tuple[
dict[str, int],
Optional[LogprobsLists],
LogprobsLists | None,
list[list[int]],
dict[str, Optional[LogprobsTensors]],
dict[str, LogprobsTensors | None],
list[str],
dict[str, int],
list[int],
@@ -2377,10 +2376,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def _model_forward(
self,
input_ids: Optional[torch.Tensor] = None,
positions: Optional[torch.Tensor] = None,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**model_kwargs: dict[str, Any],
) -> Any:
"""Helper method to call the model forward pass.
@@ -2411,8 +2410,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def execute_model(
self,
scheduler_output: "SchedulerOutput",
intermediate_tensors: Optional[IntermediateTensors] = None,
) -> Union[ModelRunnerOutput, AsyncModelRunnerOutput, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors:
with record_function_or_nullcontext("Preprocess"):
with self.synchronize_input_prep():
# Update persistent batch states.
@@ -2678,7 +2677,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
return async_output
def take_draft_token_ids(self) -> Optional[DraftTokenIds]:
def take_draft_token_ids(self) -> DraftTokenIds | None:
if self._draft_token_ids is None:
return None
req_ids = self.input_batch.req_ids
@@ -2692,14 +2691,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def propose_draft_token_ids(
self,
scheduler_output: "SchedulerOutput",
sampled_token_ids: Union[torch.Tensor, list[list[int]]],
sampled_token_ids: torch.Tensor | list[list[int]],
sampling_metadata: SamplingMetadata,
hidden_states: torch.Tensor,
sample_hidden_states: torch.Tensor,
aux_hidden_states: Optional[list[torch.Tensor]],
spec_decode_metadata: Optional[SpecDecodeMetadata],
aux_hidden_states: list[torch.Tensor] | None,
spec_decode_metadata: SpecDecodeMetadata | None,
common_attn_metadata: CommonAttentionMetadata,
) -> Union[list[list[int]], torch.Tensor]:
) -> list[list[int]] | torch.Tensor:
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
if self.speculative_config.method == "ngram":
assert isinstance(sampled_token_ids, list)
@@ -2961,7 +2960,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.model, self.vllm_config, CUDAGraphMode.NONE, self.device
)
def _get_eagle3_aux_layers_from_config(self) -> Optional[tuple[int, ...]]:
def _get_eagle3_aux_layers_from_config(self) -> tuple[int, ...] | None:
"""Extract Eagle3 auxiliary layer indices from speculative config.
These indices specify which hidden states from the base model should
@@ -3007,13 +3006,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self,
hidden_states: torch.Tensor,
num_scheduled_tokens: dict[str, int],
) -> dict[str, Optional[LogprobsTensors]]:
) -> dict[str, LogprobsTensors | None]:
num_prompt_logprobs_dict = self.input_batch.num_prompt_logprobs
if not num_prompt_logprobs_dict:
return {}
in_progress_dict = self.input_batch.in_progress_prompt_logprobs_cpu
prompt_logprobs_dict: dict[str, Optional[LogprobsTensors]] = {}
prompt_logprobs_dict: dict[str, LogprobsTensors | None] = {}
# Since prompt logprobs are a rare feature, prioritize simple,
# maintainable loop over optimal performance.
@@ -3107,7 +3106,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def _get_nans_in_logits(
self,
logits: Optional[torch.Tensor],
logits: torch.Tensor | None,
) -> dict[str, int]:
try:
if logits is None:
@@ -3190,7 +3189,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
def _dummy_run(
self,
num_tokens: int,
cudagraph_runtime_mode: Optional[CUDAGraphMode] = None,
cudagraph_runtime_mode: CUDAGraphMode | None = None,
force_attention: bool = False,
uniform_decode: bool = False,
allow_microbatching: bool = True,
@@ -3294,7 +3293,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
dp_rank = self.parallel_config.data_parallel_rank
num_tokens_after_padding = int(num_tokens_across_dp[dp_rank])
attn_metadata: Optional[PerLayerAttnMetadata] = None
attn_metadata: PerLayerAttnMetadata | None = None
# If force_attention is True, we always capture attention. Otherwise,
# it only happens for cudagraph_runtime_mode=FULL.