diff --git a/vllm/model_executor/layers/pooler/activations.py b/vllm/model_executor/layers/pooler/activations.py index 5fd05ef682..b997d11b62 100644 --- a/vllm/model_executor/layers/pooler/activations.py +++ b/vllm/model_executor/layers/pooler/activations.py @@ -64,7 +64,7 @@ def resolve_classifier_act_fn( model_config: ModelConfig, static_num_labels: bool = True, act_fn: "PoolerActivation | None" = None, -): +) -> "PoolerActivation": if act_fn is None: return get_act_fn(model_config.hf_config, static_num_labels) @@ -78,7 +78,7 @@ _T = TypeVar("_T", torch.Tensor, list[torch.Tensor]) class PoolerActivation(nn.Module, ABC): @staticmethod - def wraps(module: nn.Module): + def wraps(module: nn.Module) -> "PoolerActivation": if isinstance(module, nn.Identity): return PoolerIdentity() if isinstance(module, (nn.Sigmoid, nn.Softmax)): @@ -140,7 +140,7 @@ class PoolerClassify(PoolerActivation): class LambdaPoolerActivation(PoolerActivation): - def __init__(self, fn: Callable[[torch.Tensor], torch.Tensor]): + def __init__(self, fn: Callable[[torch.Tensor], torch.Tensor]) -> None: super().__init__() self.fn = fn diff --git a/vllm/model_executor/layers/pooler/seqwise/methods.py b/vllm/model_executor/layers/pooler/seqwise/methods.py index 82170b5fbd..d99216fc10 100644 --- a/vllm/model_executor/layers/pooler/seqwise/methods.py +++ b/vllm/model_executor/layers/pooler/seqwise/methods.py @@ -108,7 +108,9 @@ class MeanPool(SequencePoolingMethod): return segment_sums / prompt_lens.unsqueeze(1) -def get_seq_pooling_method(pooling_type: SequencePoolingType | str): +def get_seq_pooling_method( + pooling_type: SequencePoolingType | str, +) -> SequencePoolingMethod: if pooling_type == "CLS": return CLSPool() if pooling_type == "LAST": diff --git a/vllm/model_executor/layers/pooler/special.py b/vllm/model_executor/layers/pooler/special.py index ae5926cd62..16437ac2de 100644 --- a/vllm/model_executor/layers/pooler/special.py +++ b/vllm/model_executor/layers/pooler/special.py @@ -102,6 +102,7 @@ class DispatchPooler(Pooler): # portion of the batch. Token offset is computed from the CPU # `num_scheduled_tokens_cpu` to avoid a GPU->CPU sync. group_cursor = group_metadata.pooling_cursor + assert group_cursor is not None num_group_tokens = int(group_cursor.num_scheduled_tokens_cpu.sum()) group_hidden_states = hidden_states[ token_offset : token_offset + num_group_tokens diff --git a/vllm/v1/pool/metadata.py b/vllm/v1/pool/metadata.py index 2c317873af..f772c850f0 100644 --- a/vllm/v1/pool/metadata.py +++ b/vllm/v1/pool/metadata.py @@ -20,7 +20,7 @@ class PoolingCursor: seq_lens_cpu: torch.Tensor num_scheduled_tokens_cpu: torch.Tensor - def __getitem__(self, indices: slice): + def __getitem__(self, indices: slice) -> "PoolingCursor": return PoolingCursor( first_token_indices_gpu=self.first_token_indices_gpu[indices], last_token_indices_gpu=self.last_token_indices_gpu[indices], @@ -29,19 +29,19 @@ class PoolingCursor: num_scheduled_tokens_cpu=self.num_scheduled_tokens_cpu[indices], ) - def is_partial_prefill(self): + def is_partial_prefill(self) -> bool: return not torch.all(self.prompt_lens_cpu == self.num_scheduled_tokens_cpu) - def is_finished(self): + def is_finished(self) -> torch.Tensor: return self.prompt_lens_cpu == self.seq_lens_cpu class PoolingStates: - def __init__(self): + def __init__(self) -> None: # for chunked prefill with ALL pooling self.hidden_states_cache: list[torch.Tensor] = [] - def clean(self): + def clean(self) -> None: self.hidden_states_cache.clear() @@ -72,7 +72,7 @@ class PoolingMetadata: self.tasks = tasks - def __getitem__(self, indices: slice): + def __getitem__(self, indices: slice) -> "PoolingMetadata": return PoolingMetadata( prompt_lens=self.prompt_lens[indices], prompt_token_ids=None @@ -121,7 +121,7 @@ class PoolingMetadata: seq_lens_cpu: torch.Tensor, device: torch.device, query_start_loc_gpu: torch.Tensor | None = None, - ): + ) -> None: n_seq = len(num_scheduled_tokens_np) prompt_lens = self.prompt_lens