Update some more deprecated type hinting (#17998)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
|
||||
from array import array
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -25,10 +25,10 @@ class SequenceGroupToSample:
|
||||
# |-- query_len ---|
|
||||
|
||||
# Sequence ids for the sequence group in a previous step.
|
||||
seq_ids: List[int]
|
||||
seq_ids: list[int]
|
||||
sampling_params: SamplingParams
|
||||
# seq_id -> sequence data.
|
||||
seq_data: Dict[int, SequenceData]
|
||||
seq_data: dict[int, SequenceData]
|
||||
# The length of the sequence (all tokens seen in the past + new token to
|
||||
# compute attention) of the sequence group. None if it is in a decode
|
||||
# stage.
|
||||
@@ -44,9 +44,9 @@ class SequenceGroupToSample:
|
||||
is_prompt: bool
|
||||
# Query token indices from logits. to compute prompt logprob. Empty if
|
||||
# prompt logprob is not required.
|
||||
prompt_logprob_indices: List[int]
|
||||
prompt_logprob_indices: list[int]
|
||||
# Sample token indices from logits. Empty if sampling is not required.
|
||||
sample_indices: List[int]
|
||||
sample_indices: list[int]
|
||||
|
||||
@property
|
||||
def do_sample(self):
|
||||
@@ -78,7 +78,7 @@ class SamplingMetadataCache:
|
||||
"""Used to cache SamplingMetadata objects between scheduler iterations"""
|
||||
|
||||
def __init__(self):
|
||||
self._seq_group_to_sample_cache: Dict[int, PyObjectCache] = {}
|
||||
self._seq_group_to_sample_cache: dict[int, PyObjectCache] = {}
|
||||
|
||||
def get_cached_seq_group_to_sample(self, num_seqs):
|
||||
if num_seqs not in self._seq_group_to_sample_cache:
|
||||
@@ -130,9 +130,9 @@ class SamplingMetadata:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
seq_groups: List[SequenceGroupToSample],
|
||||
seq_groups: list[SequenceGroupToSample],
|
||||
selected_token_indices: torch.Tensor,
|
||||
categorized_sample_indices: Dict[SamplingType, torch.Tensor],
|
||||
categorized_sample_indices: dict[SamplingType, torch.Tensor],
|
||||
num_prompts: int,
|
||||
skip_sampler_cpu_output: bool = False,
|
||||
reuse_sampling_tensors: bool = False,
|
||||
@@ -146,12 +146,12 @@ class SamplingMetadata:
|
||||
|
||||
@staticmethod
|
||||
def prepare(
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
seq_lens: List[int],
|
||||
query_lens: List[int],
|
||||
seq_group_metadata_list: list[SequenceGroupMetadata],
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
device: str,
|
||||
pin_memory: bool,
|
||||
generators: Optional[Dict[str, torch.Generator]] = None,
|
||||
generators: Optional[dict[str, torch.Generator]] = None,
|
||||
cache: Optional[SamplingMetadataCache] = None,
|
||||
) -> "SamplingMetadata":
|
||||
(
|
||||
@@ -195,16 +195,16 @@ class SamplingMetadata:
|
||||
|
||||
|
||||
def _prepare_seq_groups(
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
seq_lens: List[int],
|
||||
query_lens: List[int],
|
||||
seq_group_metadata_list: list[SequenceGroupMetadata],
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
device: str,
|
||||
generators: Optional[Dict[str, torch.Generator]] = None,
|
||||
generators: Optional[dict[str, torch.Generator]] = None,
|
||||
cache: Optional[SamplingMetadataCache] = None,
|
||||
) -> Tuple[
|
||||
List[SequenceGroupToSample],
|
||||
List[int],
|
||||
Dict[SamplingType, List[int]],
|
||||
) -> tuple[
|
||||
list[SequenceGroupToSample],
|
||||
list[int],
|
||||
dict[SamplingType, list[int]],
|
||||
int,
|
||||
]:
|
||||
"""Prepare sequence groups and indices for sampling.
|
||||
@@ -227,17 +227,17 @@ def _prepare_seq_groups(
|
||||
num_prompts: Total number of prompts from `seq_group_metadata_list`.
|
||||
"""
|
||||
# Batched sequence groups for the current model forward stsep.
|
||||
seq_groups: List[SequenceGroupToSample] = []
|
||||
seq_groups: list[SequenceGroupToSample] = []
|
||||
# A list of token indices to sample/compute logprob. It is used to
|
||||
# prune the outcome logits from the model for the performance.
|
||||
selected_token_indices: List[int] = []
|
||||
selected_token_indices: list[int] = []
|
||||
# Used for selected_token_indices.
|
||||
model_output_idx = 0
|
||||
|
||||
# Sampling type -> (
|
||||
# indices to sample/prompt logprob within pruned output logits,
|
||||
# indices to sample within pruned logits)
|
||||
categorized_sample_indices: Dict[SamplingType, List[int]] = {
|
||||
categorized_sample_indices: dict[SamplingType, list[int]] = {
|
||||
t: []
|
||||
for t in SamplingType
|
||||
}
|
||||
@@ -265,9 +265,9 @@ def _prepare_seq_groups(
|
||||
# If the current seq group is in decode stage, it is None.
|
||||
seq_len: Optional[int] = None
|
||||
query_len: Optional[int] = None
|
||||
prompt_logprob_indices: List[int] = (sample_obj.prompt_logprob_indices
|
||||
prompt_logprob_indices: list[int] = (sample_obj.prompt_logprob_indices
|
||||
if cache is not None else [])
|
||||
sample_indices: List[int] = (sample_obj.sample_indices
|
||||
sample_indices: list[int] = (sample_obj.sample_indices
|
||||
if cache is not None else [])
|
||||
do_sample = seq_group_metadata.do_sample
|
||||
|
||||
@@ -389,16 +389,16 @@ class SamplingTensors:
|
||||
vocab_size: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> Tuple["SamplingTensors", bool, bool, bool]:
|
||||
prompt_tokens: List[array] = []
|
||||
output_tokens: List[array] = []
|
||||
top_ks: List[int] = []
|
||||
temperatures: List[float] = []
|
||||
top_ps: List[float] = []
|
||||
min_ps: List[float] = []
|
||||
presence_penalties: List[float] = []
|
||||
frequency_penalties: List[float] = []
|
||||
repetition_penalties: List[float] = []
|
||||
) -> tuple["SamplingTensors", bool, bool, bool]:
|
||||
prompt_tokens: list[array] = []
|
||||
output_tokens: list[array] = []
|
||||
top_ks: list[int] = []
|
||||
temperatures: list[float] = []
|
||||
top_ps: list[float] = []
|
||||
min_ps: list[float] = []
|
||||
presence_penalties: list[float] = []
|
||||
frequency_penalties: list[float] = []
|
||||
repetition_penalties: list[float] = []
|
||||
do_penalties = False
|
||||
do_top_p_top_k = False
|
||||
do_min_p = False
|
||||
@@ -496,15 +496,15 @@ class SamplingTensors:
|
||||
@classmethod
|
||||
def from_lists(
|
||||
cls,
|
||||
temperatures: List[float],
|
||||
top_ps: List[float],
|
||||
top_ks: List[int],
|
||||
min_ps: List[float],
|
||||
presence_penalties: List[float],
|
||||
frequency_penalties: List[float],
|
||||
repetition_penalties: List[float],
|
||||
prompt_tokens: List[array],
|
||||
output_tokens: List[array],
|
||||
temperatures: list[float],
|
||||
top_ps: list[float],
|
||||
top_ks: list[int],
|
||||
min_ps: list[float],
|
||||
presence_penalties: list[float],
|
||||
frequency_penalties: list[float],
|
||||
repetition_penalties: list[float],
|
||||
prompt_tokens: list[array],
|
||||
output_tokens: list[array],
|
||||
vocab_size: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
|
||||
Reference in New Issue
Block a user