Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -8,14 +8,13 @@ from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union, cast
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.distributed
|
||||
import torch.nn as nn
|
||||
from tqdm import tqdm
|
||||
from typing_extensions import TypeAlias
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.attention import Attention, AttentionType
|
||||
@@ -162,7 +161,7 @@ logger = init_logger(__name__)
|
||||
|
||||
AttnMetadataDict: TypeAlias = dict[str, AttentionMetadata]
|
||||
# list when ubatching is enabled
|
||||
PerLayerAttnMetadata: TypeAlias = Union[list[AttnMetadataDict], AttnMetadataDict]
|
||||
PerLayerAttnMetadata: TypeAlias = list[AttnMetadataDict] | AttnMetadataDict
|
||||
|
||||
|
||||
# Wrapper for ModelRunnerOutput to support overlapped execution.
|
||||
@@ -295,7 +294,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
# Sampler
|
||||
self.sampler = Sampler(logprobs_mode=self.model_config.logprobs_mode)
|
||||
|
||||
self.eplb_state: Optional[EplbState] = None
|
||||
self.eplb_state: EplbState | None = None
|
||||
"""
|
||||
State of the expert parallelism load balancer.
|
||||
|
||||
@@ -447,14 +446,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
# CUDA event to synchronize use of reused CPU tensors between steps
|
||||
# when async scheduling is enabled.
|
||||
self.prepare_inputs_event: Optional[torch.cuda.Event] = None
|
||||
self.prepare_inputs_event: torch.cuda.Event | None = None
|
||||
if self.use_async_scheduling:
|
||||
self.prepare_inputs_event = torch.cuda.Event()
|
||||
# Start in a completed state.
|
||||
self.prepare_inputs_event.record(torch.cuda.default_stream())
|
||||
|
||||
# None in the first PP rank. The rest are set after load_model.
|
||||
self.intermediate_tensors: Optional[IntermediateTensors] = None
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
# OPTIMIZATION: Cache the tensors rather than creating them every step.
|
||||
# Keep in int64 to avoid overflow with long context
|
||||
@@ -495,7 +494,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
else None
|
||||
)
|
||||
|
||||
self.reorder_batch_threshold: Optional[int] = None
|
||||
self.reorder_batch_threshold: int | None = None
|
||||
|
||||
# Attention layers that are only in the KVCacheConfig of the runner
|
||||
# (e.g., KV sharing, encoder-only attention), but not in the
|
||||
@@ -503,7 +502,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self.runner_only_attn_layers: set[str] = set()
|
||||
|
||||
# Cached outputs.
|
||||
self._draft_token_ids: Optional[Union[list[list[int]], torch.Tensor]] = None
|
||||
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
|
||||
self.transfer_event = torch.cuda.Event()
|
||||
self.sampled_token_ids_pinned_cpu = torch.empty(
|
||||
(self.max_model_len, 1),
|
||||
@@ -527,7 +526,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
return self.positions.gpu[num_tokens]
|
||||
|
||||
def _make_buffer(
|
||||
self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True
|
||||
self, *size: int | torch.SymInt, dtype: torch.dtype, numpy: bool = True
|
||||
) -> CpuGpuBuffer:
|
||||
return CpuGpuBuffer(
|
||||
*size,
|
||||
@@ -928,7 +927,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
def _get_cumsum_and_arange(
|
||||
self,
|
||||
num_tokens: np.ndarray,
|
||||
cumsum_dtype: Optional[np.dtype] = None,
|
||||
cumsum_dtype: np.dtype | None = None,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Get the cumulative sum and batched arange of the given array.
|
||||
# E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2])
|
||||
@@ -1024,7 +1023,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
scheduler_output: "SchedulerOutput",
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
num_reqs: int,
|
||||
) -> Optional[np.ndarray]:
|
||||
) -> np.ndarray | None:
|
||||
if not isinstance(kv_cache_spec, CrossAttentionSpec):
|
||||
return None
|
||||
|
||||
@@ -1042,12 +1041,12 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
) -> tuple[
|
||||
PerLayerAttnMetadata,
|
||||
torch.Tensor,
|
||||
Optional[SpecDecodeMetadata],
|
||||
SpecDecodeMetadata | None,
|
||||
np.ndarray,
|
||||
Optional[CommonAttentionMetadata],
|
||||
CommonAttentionMetadata | None,
|
||||
int,
|
||||
Optional[UBatchSlices],
|
||||
Optional[torch.Tensor],
|
||||
UBatchSlices | None,
|
||||
torch.Tensor | None,
|
||||
bool,
|
||||
]:
|
||||
"""
|
||||
@@ -2048,7 +2047,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
)
|
||||
self._sync_device()
|
||||
|
||||
pooler_output: list[Optional[torch.Tensor]] = []
|
||||
pooler_output: list[torch.Tensor | None] = []
|
||||
for raw_output, seq_len, prompt_len in zip(
|
||||
raw_pooler_output, seq_lens_cpu, pooling_metadata.prompt_lens
|
||||
):
|
||||
@@ -2091,13 +2090,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
num_input_tokens: int, # Padded
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> tuple[
|
||||
int,
|
||||
Optional[torch.Tensor],
|
||||
Optional[torch.Tensor],
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor,
|
||||
Optional[IntermediateTensors],
|
||||
IntermediateTensors | None,
|
||||
dict[str, Any],
|
||||
]:
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
@@ -2197,8 +2196,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
def _sample(
|
||||
self,
|
||||
logits: Optional[torch.Tensor],
|
||||
spec_decode_metadata: Optional[SpecDecodeMetadata],
|
||||
logits: torch.Tensor | None,
|
||||
spec_decode_metadata: SpecDecodeMetadata | None,
|
||||
) -> SamplerOutput:
|
||||
# Sample the next token and get logprobs if needed.
|
||||
sampling_metadata = self.input_batch.sampling_metadata
|
||||
@@ -2243,14 +2242,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
sampler_output: SamplerOutput,
|
||||
logits: Optional[torch.Tensor],
|
||||
logits: torch.Tensor | None,
|
||||
hidden_states: torch.Tensor,
|
||||
num_scheduled_tokens: int,
|
||||
) -> tuple[
|
||||
dict[str, int],
|
||||
Optional[LogprobsLists],
|
||||
LogprobsLists | None,
|
||||
list[list[int]],
|
||||
dict[str, Optional[LogprobsTensors]],
|
||||
dict[str, LogprobsTensors | None],
|
||||
list[str],
|
||||
dict[str, int],
|
||||
list[int],
|
||||
@@ -2377,10 +2376,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
def _model_forward(
|
||||
self,
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**model_kwargs: dict[str, Any],
|
||||
) -> Any:
|
||||
"""Helper method to call the model forward pass.
|
||||
@@ -2411,8 +2410,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
def execute_model(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
) -> Union[ModelRunnerOutput, AsyncModelRunnerOutput, IntermediateTensors]:
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors:
|
||||
with record_function_or_nullcontext("Preprocess"):
|
||||
with self.synchronize_input_prep():
|
||||
# Update persistent batch states.
|
||||
@@ -2678,7 +2677,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
return async_output
|
||||
|
||||
def take_draft_token_ids(self) -> Optional[DraftTokenIds]:
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
if self._draft_token_ids is None:
|
||||
return None
|
||||
req_ids = self.input_batch.req_ids
|
||||
@@ -2692,14 +2691,14 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
def propose_draft_token_ids(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
sampled_token_ids: Union[torch.Tensor, list[list[int]]],
|
||||
sampled_token_ids: torch.Tensor | list[list[int]],
|
||||
sampling_metadata: SamplingMetadata,
|
||||
hidden_states: torch.Tensor,
|
||||
sample_hidden_states: torch.Tensor,
|
||||
aux_hidden_states: Optional[list[torch.Tensor]],
|
||||
spec_decode_metadata: Optional[SpecDecodeMetadata],
|
||||
aux_hidden_states: list[torch.Tensor] | None,
|
||||
spec_decode_metadata: SpecDecodeMetadata | None,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> Union[list[list[int]], torch.Tensor]:
|
||||
) -> list[list[int]] | torch.Tensor:
|
||||
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
if self.speculative_config.method == "ngram":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
@@ -2961,7 +2960,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self.model, self.vllm_config, CUDAGraphMode.NONE, self.device
|
||||
)
|
||||
|
||||
def _get_eagle3_aux_layers_from_config(self) -> Optional[tuple[int, ...]]:
|
||||
def _get_eagle3_aux_layers_from_config(self) -> tuple[int, ...] | None:
|
||||
"""Extract Eagle3 auxiliary layer indices from speculative config.
|
||||
|
||||
These indices specify which hidden states from the base model should
|
||||
@@ -3007,13 +3006,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_scheduled_tokens: dict[str, int],
|
||||
) -> dict[str, Optional[LogprobsTensors]]:
|
||||
) -> dict[str, LogprobsTensors | None]:
|
||||
num_prompt_logprobs_dict = self.input_batch.num_prompt_logprobs
|
||||
if not num_prompt_logprobs_dict:
|
||||
return {}
|
||||
|
||||
in_progress_dict = self.input_batch.in_progress_prompt_logprobs_cpu
|
||||
prompt_logprobs_dict: dict[str, Optional[LogprobsTensors]] = {}
|
||||
prompt_logprobs_dict: dict[str, LogprobsTensors | None] = {}
|
||||
|
||||
# Since prompt logprobs are a rare feature, prioritize simple,
|
||||
# maintainable loop over optimal performance.
|
||||
@@ -3107,7 +3106,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
|
||||
def _get_nans_in_logits(
|
||||
self,
|
||||
logits: Optional[torch.Tensor],
|
||||
logits: torch.Tensor | None,
|
||||
) -> dict[str, int]:
|
||||
try:
|
||||
if logits is None:
|
||||
@@ -3190,7 +3189,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
def _dummy_run(
|
||||
self,
|
||||
num_tokens: int,
|
||||
cudagraph_runtime_mode: Optional[CUDAGraphMode] = None,
|
||||
cudagraph_runtime_mode: CUDAGraphMode | None = None,
|
||||
force_attention: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
allow_microbatching: bool = True,
|
||||
@@ -3294,7 +3293,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
dp_rank = self.parallel_config.data_parallel_rank
|
||||
num_tokens_after_padding = int(num_tokens_across_dp[dp_rank])
|
||||
|
||||
attn_metadata: Optional[PerLayerAttnMetadata] = None
|
||||
attn_metadata: PerLayerAttnMetadata | None = None
|
||||
|
||||
# If force_attention is True, we always capture attention. Otherwise,
|
||||
# it only happens for cudagraph_runtime_mode=FULL.
|
||||
|
||||
Reference in New Issue
Block a user