Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -6,7 +6,7 @@ import itertools
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -81,7 +81,7 @@ def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]:
|
||||
|
||||
|
||||
def _load_logitsprocs_by_fqcns(
|
||||
logits_processors: Optional[Sequence[Union[str, type[LogitsProcessor]]]],
|
||||
logits_processors: Sequence[str | type[LogitsProcessor]] | None,
|
||||
) -> list[type[LogitsProcessor]]:
|
||||
"""Load logit processor types, identifying them by fully-qualified class
|
||||
names (FQCNs).
|
||||
@@ -146,7 +146,7 @@ def _load_logitsprocs_by_fqcns(
|
||||
|
||||
|
||||
def _load_custom_logitsprocs(
|
||||
logits_processors: Optional[Sequence[Union[str, type[LogitsProcessor]]]],
|
||||
logits_processors: Sequence[str | type[LogitsProcessor]] | None,
|
||||
) -> list[type[LogitsProcessor]]:
|
||||
"""Load all custom logits processors.
|
||||
|
||||
@@ -176,7 +176,7 @@ def build_logitsprocs(
|
||||
device: torch.device,
|
||||
is_pin_memory: bool,
|
||||
is_pooling_model: bool,
|
||||
custom_logitsprocs: Sequence[Union[str, type[LogitsProcessor]]] = (),
|
||||
custom_logitsprocs: Sequence[str | type[LogitsProcessor]] = (),
|
||||
) -> LogitsProcessors:
|
||||
if is_pooling_model:
|
||||
if custom_logitsprocs:
|
||||
@@ -249,7 +249,7 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
def new_req_logits_processor(
|
||||
self,
|
||||
params: SamplingParams,
|
||||
) -> Optional[RequestLogitsProcessor]:
|
||||
) -> RequestLogitsProcessor | None:
|
||||
"""Consume request info; return a per-request logits processor.
|
||||
|
||||
Return None if logits processor does not need to be applied to request
|
||||
@@ -267,9 +267,9 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
def _new_state(
|
||||
self,
|
||||
params: SamplingParams,
|
||||
prompt_ids: Optional[list[int]],
|
||||
prompt_ids: list[int] | None,
|
||||
output_ids: list[int],
|
||||
) -> Optional[partial[torch.Tensor]]:
|
||||
) -> partial[torch.Tensor] | None:
|
||||
"""Return state representation for new request
|
||||
|
||||
Returns None if logits processor is not applicable to request
|
||||
@@ -292,7 +292,7 @@ class AdapterLogitsProcessor(LogitsProcessor):
|
||||
return partial(req_lp, *args)
|
||||
return None
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
process_dict_updates(
|
||||
self.req_info,
|
||||
batch_update,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Callable, Optional, TypeVar
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
@@ -49,7 +49,7 @@ class MinPLogitsProcessor(LogitsProcessor):
|
||||
def get_min_p_by_index(self, index: int) -> float:
|
||||
return float(self.min_p_cpu[index])
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
if not batch_update:
|
||||
return
|
||||
|
||||
@@ -131,7 +131,7 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
|
||||
outcome of argmax in greedy sampling."""
|
||||
return False
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
needs_update = process_dict_updates(
|
||||
self.biases, batch_update, lambda params, _, __: params.logit_bias or None
|
||||
)
|
||||
@@ -185,14 +185,14 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
@staticmethod
|
||||
def add_request(
|
||||
params: SamplingParams, _: Optional[list[int]], output_tok_ids: list[int]
|
||||
) -> Optional[tuple[int, Sequence[int], set[int]]]:
|
||||
params: SamplingParams, _: list[int] | None, output_tok_ids: list[int]
|
||||
) -> tuple[int, Sequence[int], set[int]] | None:
|
||||
min_tokens = params.min_tokens
|
||||
if not min_tokens or len(output_tok_ids) >= min_tokens:
|
||||
return None
|
||||
return min_tokens, output_tok_ids, params.all_stop_token_ids
|
||||
|
||||
def update_state(self, batch_update: Optional[BatchUpdate]):
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
needs_update = process_dict_updates(
|
||||
self.min_toks, batch_update, self.add_request
|
||||
)
|
||||
@@ -235,8 +235,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def process_dict_updates(
|
||||
req_entries: dict[int, T],
|
||||
batch_update: Optional[BatchUpdate],
|
||||
new_state: Callable[[SamplingParams, Optional[list[int]], list[int]], Optional[T]],
|
||||
batch_update: BatchUpdate | None,
|
||||
new_state: Callable[[SamplingParams, list[int] | None, list[int]], T | None],
|
||||
) -> bool:
|
||||
"""Utility function to update dict state for sparse LogitsProcessors."""
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ RemovedRequest = int
|
||||
|
||||
# (index, params, prompt_tok_ids, output_tok_ids) tuples for new
|
||||
# requests added to the batch.
|
||||
AddedRequest = tuple[int, SamplingParams, Optional[list[int]], list[int]]
|
||||
AddedRequest = tuple[int, SamplingParams, list[int] | None, list[int]]
|
||||
|
||||
# (index 1, index 2, directionality) tuples representing
|
||||
# one-way moves or two-way swaps of requests in batch
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
AddedRequest,
|
||||
@@ -43,9 +43,9 @@ class BatchUpdateBuilder:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
removed: Optional[list[RemovedRequest]] = None,
|
||||
added: Optional[list[AddedRequest]] = None,
|
||||
moved: Optional[list[MovedRequest]] = None,
|
||||
removed: list[RemovedRequest] | None = None,
|
||||
added: list[AddedRequest] | None = None,
|
||||
moved: list[MovedRequest] | None = None,
|
||||
) -> None:
|
||||
self._removed = removed or []
|
||||
self.added = added or []
|
||||
@@ -92,14 +92,14 @@ class BatchUpdateBuilder:
|
||||
def has_removed(self) -> bool:
|
||||
return bool(self._removed)
|
||||
|
||||
def peek_removed(self) -> Optional[int]:
|
||||
def peek_removed(self) -> int | None:
|
||||
"""Return lowest removed request index"""
|
||||
if self.has_removed():
|
||||
self._ensure_removed_sorted()
|
||||
return self._removed[-1]
|
||||
return None
|
||||
|
||||
def pop_removed(self) -> Optional[int]:
|
||||
def pop_removed(self) -> int | None:
|
||||
"""Pop lowest removed request index"""
|
||||
if self.has_removed():
|
||||
self._ensure_removed_sorted()
|
||||
@@ -116,7 +116,7 @@ class BatchUpdateBuilder:
|
||||
self.batch_changed = False
|
||||
return batch_changed
|
||||
|
||||
def get_and_reset(self, batch_size: int) -> Optional[BatchUpdate]:
|
||||
def get_and_reset(self, batch_size: int) -> BatchUpdate | None:
|
||||
"""Generate a logitsprocs batch update data structure and reset
|
||||
internal batch update builder state.
|
||||
|
||||
@@ -148,9 +148,7 @@ class BatchUpdateBuilder:
|
||||
class LogitsProcessors:
|
||||
"""Encapsulates initialized logitsproc objects."""
|
||||
|
||||
def __init__(
|
||||
self, logitsprocs: Optional[Iterator["LogitsProcessor"]] = None
|
||||
) -> None:
|
||||
def __init__(self, logitsprocs: Iterator["LogitsProcessor"] | None = None) -> None:
|
||||
self.argmax_invariant: list[LogitsProcessor] = []
|
||||
self.non_argmax_invariant: list[LogitsProcessor] = []
|
||||
if logitsprocs:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,20 +10,20 @@ from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
|
||||
@dataclass
|
||||
class SamplingMetadata:
|
||||
temperature: Optional[torch.Tensor]
|
||||
temperature: torch.Tensor | None
|
||||
all_greedy: bool
|
||||
all_random: bool
|
||||
|
||||
top_p: Optional[torch.Tensor]
|
||||
top_k: Optional[torch.Tensor]
|
||||
top_p: torch.Tensor | None
|
||||
top_k: torch.Tensor | None
|
||||
|
||||
generators: dict[int, torch.Generator]
|
||||
|
||||
# None means no logprobs, 0 means sampled token logprobs only
|
||||
max_num_logprobs: Optional[int]
|
||||
max_num_logprobs: int | None
|
||||
|
||||
no_penalties: bool
|
||||
prompt_token_ids: Optional[torch.Tensor]
|
||||
prompt_token_ids: torch.Tensor | None
|
||||
frequency_penalties: torch.Tensor
|
||||
presence_penalties: torch.Tensor
|
||||
repetition_penalties: torch.Tensor
|
||||
@@ -33,7 +32,7 @@ class SamplingMetadata:
|
||||
|
||||
# `allowed_token_ids_mask` is a 2D bool tensor of shape (max batch size,
|
||||
# vocab size).
|
||||
allowed_token_ids_mask: Optional[torch.Tensor]
|
||||
allowed_token_ids_mask: torch.Tensor | None
|
||||
|
||||
# req_index -> bad_words_token_ids
|
||||
bad_words_token_ids: dict[int, list[list[int]]]
|
||||
@@ -42,4 +41,4 @@ class SamplingMetadata:
|
||||
logitsprocs: LogitsProcessors
|
||||
|
||||
# Speculative token ids
|
||||
spec_token_ids: Optional[list[list[int]]] = None
|
||||
spec_token_ids: list[list[int]] | None = None
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -84,9 +83,9 @@ class TopKTopPSampler(nn.Module):
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
generators: dict[int, torch.Generator],
|
||||
k: Optional[torch.Tensor],
|
||||
p: Optional[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
PyTorch-native implementation of top-k and top-p sampling.
|
||||
|
||||
@@ -105,9 +104,9 @@ class TopKTopPSampler(nn.Module):
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
generators: dict[int, torch.Generator],
|
||||
k: Optional[torch.Tensor],
|
||||
p: Optional[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""More optimized implementation for top-k and top-p sampling."""
|
||||
# We prefer `random_sample` over `flashinfer_sample` when sorting is
|
||||
# not needed. This is because `random_sample` does not require
|
||||
@@ -132,9 +131,9 @@ class TopKTopPSampler(nn.Module):
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
generators: dict[int, torch.Generator],
|
||||
k: Optional[torch.Tensor],
|
||||
p: Optional[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
PyTorch-native implementation of top-k and top-p sampling for CPU.
|
||||
|
||||
@@ -170,8 +169,8 @@ class TopKTopPSampler(nn.Module):
|
||||
|
||||
def apply_top_k_top_p(
|
||||
logits: torch.Tensor,
|
||||
k: Optional[torch.Tensor],
|
||||
p: Optional[torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply top-k and top-p masks to the logits.
|
||||
|
||||
@@ -262,8 +261,8 @@ def random_sample(
|
||||
|
||||
def flashinfer_sample(
|
||||
logits: torch.Tensor,
|
||||
k: Optional[torch.Tensor],
|
||||
p: Optional[torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
generators: dict[int, torch.Generator],
|
||||
) -> torch.Tensor:
|
||||
"""Sample from the logits using FlashInfer.
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -49,7 +48,7 @@ class RejectionSampler(nn.Module):
|
||||
self,
|
||||
metadata: SpecDecodeMetadata,
|
||||
# [num_tokens, vocab_size]
|
||||
draft_probs: Optional[torch.Tensor],
|
||||
draft_probs: torch.Tensor | None,
|
||||
# [num_tokens, vocab_size]
|
||||
target_logits: torch.Tensor,
|
||||
# [batch_size, 1]
|
||||
@@ -216,7 +215,7 @@ class RejectionSampler(nn.Module):
|
||||
@staticmethod
|
||||
def _combine_outputs_with_spec_tokens(
|
||||
output_token_ids: list[list[int]],
|
||||
spec_token_ids: Optional[list[list[int]]] = None,
|
||||
spec_token_ids: list[list[int]] | None = None,
|
||||
) -> list[list[int]]:
|
||||
if spec_token_ids is None:
|
||||
return output_token_ids
|
||||
@@ -240,7 +239,7 @@ def rejection_sample(
|
||||
# [batch_size]
|
||||
cu_num_draft_tokens: torch.Tensor,
|
||||
# [num_tokens, vocab_size]
|
||||
draft_probs: Optional[torch.Tensor],
|
||||
draft_probs: torch.Tensor | None,
|
||||
# [num_tokens, vocab_size]
|
||||
target_probs: torch.Tensor,
|
||||
# [batch_size, 1]
|
||||
@@ -493,7 +492,7 @@ def sample_recovered_tokens(
|
||||
# [num_tokens]
|
||||
draft_token_ids: torch.Tensor,
|
||||
# [num_tokens, vocab_size]
|
||||
draft_probs: Optional[torch.Tensor],
|
||||
draft_probs: torch.Tensor | None,
|
||||
# [num_tokens, vocab_size]
|
||||
target_probs: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""A layer that samples the next tokens from the model's outputs."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -140,7 +138,7 @@ class Sampler(nn.Module):
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Sample logits based on sampling metadata.
|
||||
|
||||
The various logits processing functions called in this method
|
||||
@@ -243,7 +241,7 @@ class Sampler(nn.Module):
|
||||
@staticmethod
|
||||
def _combine_outputs_with_spec_tokens(
|
||||
output_token_ids: list[list[int]],
|
||||
spec_token_ids: Optional[list[list[int]]] = None,
|
||||
spec_token_ids: list[list[int]] | None = None,
|
||||
) -> list[list[int]]:
|
||||
if spec_token_ids is None:
|
||||
return output_token_ids
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -48,7 +47,7 @@ class TPUSupportedSamplingMetadata:
|
||||
|
||||
min_tokens = None # impl is not vectorized
|
||||
|
||||
logit_bias: list[Optional[dict[int, float]]] = field(default_factory=lambda: list())
|
||||
logit_bias: list[dict[int, float] | None] = field(default_factory=lambda: list())
|
||||
|
||||
allowed_token_ids_mask = None
|
||||
bad_words_token_ids = None
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sampler layer implementing TPU supported operations."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -166,8 +164,8 @@ class Sampler(nn.Module):
|
||||
|
||||
def apply_top_k_top_p(
|
||||
logits: torch.Tensor,
|
||||
k: Optional[torch.Tensor],
|
||||
p: Optional[torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
p: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply top-k and top-p optimized for TPU.
|
||||
|
||||
Reference in New Issue
Block a user