Fix per file ruff ignores related to typing (#26254)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar, Optional, Union
|
||||
from typing import ClassVar, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -254,12 +254,12 @@ class FlashInferMetadata:
|
||||
# For cascade attention (CPU for planning).
|
||||
use_cascade: bool
|
||||
|
||||
prefill_wrapper: Optional[BatchPrefillWithPagedKVCacheWrapper] = None
|
||||
decode_wrapper: Optional[BatchDecodeWithPagedKVCacheWrapper] = None
|
||||
cascade_wrapper: Optional[MultiLevelCascadeAttentionWrapper] = None
|
||||
prefill_wrapper: BatchPrefillWithPagedKVCacheWrapper | None = None
|
||||
decode_wrapper: BatchDecodeWithPagedKVCacheWrapper | None = None
|
||||
cascade_wrapper: MultiLevelCascadeAttentionWrapper | None = None
|
||||
|
||||
qo_indptr_gpu: Optional[torch.Tensor] = None
|
||||
paged_kv_indptr_gpu: Optional[torch.Tensor] = None
|
||||
qo_indptr_gpu: torch.Tensor | None = None
|
||||
paged_kv_indptr_gpu: torch.Tensor | None = None
|
||||
|
||||
|
||||
class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
@@ -727,13 +727,13 @@ class FlashInferImpl(AttentionImpl):
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: Optional[list[float]],
|
||||
sliding_window: Optional[int],
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: Optional[float] = None,
|
||||
logits_soft_cap: float | None = None,
|
||||
attn_type: AttentionType = AttentionType.DECODER,
|
||||
kv_sharing_target_layer_name: Optional[int] = None,
|
||||
sinks: Optional[torch.Tensor] = None,
|
||||
kv_sharing_target_layer_name: int | None = None,
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
@@ -763,7 +763,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
"FlashInferImpl"
|
||||
)
|
||||
|
||||
self.sinks: Optional[torch.Tensor] = None
|
||||
self.sinks: torch.Tensor | None = None
|
||||
if sinks is not None:
|
||||
if sinks.shape[0] != num_heads:
|
||||
raise ValueError(
|
||||
@@ -776,9 +776,9 @@ class FlashInferImpl(AttentionImpl):
|
||||
self.support_trtllm_attn = (
|
||||
supports_trtllm_attention() and num_heads % num_kv_heads == 0
|
||||
)
|
||||
self.bmm1_scale: Optional[float] = None
|
||||
self.bmm2_scale: Optional[float] = None
|
||||
self.o_sf_scale: Optional[float] = None
|
||||
self.bmm1_scale: float | None = None
|
||||
self.bmm2_scale: float | None = None
|
||||
self.o_sf_scale: float | None = None
|
||||
|
||||
def fused_output_quant_supported(self, quant_key: QuantKey):
|
||||
return (
|
||||
@@ -795,9 +795,9 @@ class FlashInferImpl(AttentionImpl):
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: FlashInferMetadata,
|
||||
output: Optional[torch.Tensor] = None,
|
||||
output_scale: Optional[torch.Tensor] = None,
|
||||
output_block_scale: Optional[torch.Tensor] = None,
|
||||
output: torch.Tensor | None = None,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass with FlashInfer.
|
||||
|
||||
@@ -1093,13 +1093,13 @@ def fast_plan_decode(
|
||||
page_size: int,
|
||||
pos_encoding_mode: str = "NONE",
|
||||
window_left: int = -1,
|
||||
logits_soft_cap: Optional[float] = None,
|
||||
q_data_type: Optional[Union[str, torch.dtype]] = "float16",
|
||||
kv_data_type: Optional[Union[str, torch.dtype]] = None,
|
||||
data_type: Optional[Union[str, torch.dtype]] = None,
|
||||
sm_scale: Optional[float] = None,
|
||||
rope_scale: Optional[float] = None,
|
||||
rope_theta: Optional[float] = None,
|
||||
logits_soft_cap: float | None = None,
|
||||
q_data_type: Union[str, torch.dtype] | None = "float16",
|
||||
kv_data_type: Union[str, torch.dtype] | None = None,
|
||||
data_type: Union[str, torch.dtype] | None = None,
|
||||
sm_scale: float | None = None,
|
||||
rope_scale: float | None = None,
|
||||
rope_theta: float | None = None,
|
||||
non_blocking: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm._bc_linter import bc_linter_include
|
||||
|
||||
@@ -25,14 +25,14 @@ if TYPE_CHECKING:
|
||||
@dataclass
|
||||
class NewRequestData:
|
||||
req_id: str
|
||||
prompt_token_ids: Optional[list[int]]
|
||||
prompt_token_ids: list[int] | None
|
||||
mm_features: list[MultiModalFeatureSpec]
|
||||
sampling_params: Optional[SamplingParams]
|
||||
pooling_params: Optional[PoolingParams]
|
||||
sampling_params: SamplingParams | None
|
||||
pooling_params: PoolingParams | None
|
||||
block_ids: tuple[list[int], ...]
|
||||
num_computed_tokens: int
|
||||
lora_request: Optional[LoRARequest]
|
||||
prompt_embeds: Optional[torch.Tensor] = None
|
||||
lora_request: LoRARequest | None
|
||||
prompt_embeds: torch.Tensor | None = None
|
||||
|
||||
@classmethod
|
||||
def from_request(
|
||||
@@ -98,7 +98,7 @@ class CachedRequestData:
|
||||
# NOTE(woosuk): new_token_ids is only used for pipeline parallelism.
|
||||
# When PP is not used, new_token_ids will be empty.
|
||||
new_token_ids: list[list[int]]
|
||||
new_block_ids: list[Optional[tuple[list[int], ...]]]
|
||||
new_block_ids: list[tuple[list[int], ...] | None]
|
||||
num_computed_tokens: list[int]
|
||||
num_output_tokens: list[int]
|
||||
|
||||
@@ -160,7 +160,7 @@ class SchedulerOutput:
|
||||
# for filling the next token bitmask
|
||||
structured_output_request_ids: dict[str, int]
|
||||
# the bitmask for the whole batch
|
||||
grammar_bitmask: Optional[npt.NDArray[np.int32]]
|
||||
grammar_bitmask: npt.NDArray[np.int32] | None
|
||||
|
||||
# KV Cache Connector metadata.
|
||||
kv_connector_metadata: Optional[KVConnectorMetadata] = None
|
||||
kv_connector_metadata: KVConnectorMetadata | None = None
|
||||
|
||||
@@ -7,7 +7,7 @@ import itertools
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, Optional, Union
|
||||
from typing import Any, Union
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch
|
||||
@@ -64,7 +64,7 @@ class Scheduler(SchedulerInterface):
|
||||
# request ids should be included in the EngineCoreOutputs returned
|
||||
# by update_from_outputs(). This is currently used in the multi-engine
|
||||
# case to track request lifetimes efficiently.
|
||||
self.finished_req_ids_dict: Optional[dict[int, set[str]]] = (
|
||||
self.finished_req_ids_dict: dict[int, set[str]] | None = (
|
||||
defaultdict(set) if include_finished_set else None
|
||||
)
|
||||
|
||||
@@ -708,7 +708,7 @@ class Scheduler(SchedulerInterface):
|
||||
) -> CachedRequestData:
|
||||
req_ids: list[str] = []
|
||||
new_token_ids: list[list[int]] = []
|
||||
new_block_ids: list[Optional[tuple[list[int], ...]]] = []
|
||||
new_block_ids: list[tuple[list[int], ...] | None] = []
|
||||
num_computed_tokens: list[int] = []
|
||||
num_output_tokens: list[int] = []
|
||||
|
||||
@@ -921,7 +921,7 @@ class Scheduler(SchedulerInterface):
|
||||
kv_connector_output = model_runner_output.kv_connector_output
|
||||
|
||||
outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list)
|
||||
spec_decoding_stats: Optional[SpecDecodingStats] = None
|
||||
spec_decoding_stats: SpecDecodingStats | None = None
|
||||
kv_connector_stats = (
|
||||
kv_connector_output.kv_connector_stats if kv_connector_output else None
|
||||
)
|
||||
@@ -1212,7 +1212,7 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = finished_status
|
||||
self._free_request(request)
|
||||
|
||||
def _free_request(self, request: Request) -> Optional[dict[str, Any]]:
|
||||
def _free_request(self, request: Request) -> dict[str, Any] | None:
|
||||
assert request.is_finished()
|
||||
|
||||
delay_free_blocks, kv_xfer_params = self._connector_finished(request)
|
||||
@@ -1243,9 +1243,9 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
def make_stats(
|
||||
self,
|
||||
spec_decoding_stats: Optional[SpecDecodingStats] = None,
|
||||
kv_connector_stats: Optional[KVConnectorStats] = None,
|
||||
) -> Optional[SchedulerStats]:
|
||||
spec_decoding_stats: SpecDecodingStats | None = None,
|
||||
kv_connector_stats: KVConnectorStats | None = None,
|
||||
) -> SchedulerStats | None:
|
||||
if not self.log_stats:
|
||||
return None
|
||||
prefix_cache_stats = self.kv_cache_manager.make_prefix_cache_stats()
|
||||
@@ -1262,10 +1262,10 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
def make_spec_decoding_stats(
|
||||
self,
|
||||
spec_decoding_stats: Optional[SpecDecodingStats],
|
||||
spec_decoding_stats: SpecDecodingStats | None,
|
||||
num_draft_tokens: int,
|
||||
num_accepted_tokens: int,
|
||||
) -> Optional[SpecDecodingStats]:
|
||||
) -> SpecDecodingStats | None:
|
||||
if not self.log_stats:
|
||||
return None
|
||||
if spec_decoding_stats is None:
|
||||
@@ -1285,12 +1285,12 @@ class Scheduler(SchedulerInterface):
|
||||
# KV Connector Related Methods
|
||||
########################################################################
|
||||
|
||||
def get_kv_connector(self) -> Optional[KVConnectorBase_V1]:
|
||||
def get_kv_connector(self) -> KVConnectorBase_V1 | None:
|
||||
return self.connector
|
||||
|
||||
def _connector_finished(
|
||||
self, request: Request
|
||||
) -> tuple[bool, Optional[dict[str, Any]]]:
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Invoke the KV connector request_finished() method if applicable.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import multiprocessing
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
@@ -35,11 +35,11 @@ class StructuredOutputManager:
|
||||
"""Engine-level manager for structured output requests."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
self.backend: Optional[StructuredOutputBackend] = None
|
||||
self.reasoner: Optional[ReasoningParser] = None
|
||||
self.backend: StructuredOutputBackend | None = None
|
||||
self.reasoner: ReasoningParser | None = None
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
self._grammar_bitmask: Optional[torch.Tensor] = None
|
||||
self._grammar_bitmask: torch.Tensor | None = None
|
||||
self._full_mask = torch.tensor(-1, dtype=torch.int32)
|
||||
|
||||
max_batch_size = self.vllm_config.scheduler_config.max_num_seqs
|
||||
@@ -168,7 +168,7 @@ class StructuredOutputManager:
|
||||
requests: dict[str, Request],
|
||||
structured_output_request_ids: dict[str, int],
|
||||
scheduled_spec_decode_tokens: dict[str, list[int]],
|
||||
) -> Optional[npt.NDArray[np.int32]]:
|
||||
) -> npt.NDArray[np.int32] | None:
|
||||
# Prepare the structured output bitmask for this batch.
|
||||
if not structured_output_request_ids:
|
||||
return None
|
||||
|
||||
@@ -7,7 +7,7 @@ import copy
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -252,7 +252,7 @@ def serialize_guidance_grammar(
|
||||
|
||||
|
||||
def validate_guidance_grammar(
|
||||
sampling_params: SamplingParams, tokenizer: Optional[llguidance.LLTokenizer] = None
|
||||
sampling_params: SamplingParams, tokenizer: llguidance.LLTokenizer | None = None
|
||||
) -> None:
|
||||
tp, grm = get_structured_output_key(sampling_params)
|
||||
guidance_grm = serialize_guidance_grammar(tp, grm)
|
||||
|
||||
@@ -20,10 +20,10 @@ from vllm.v1.structured_output.backend_types import (
|
||||
@dataclasses.dataclass
|
||||
class StructuredOutputRequest:
|
||||
sampling_params: SamplingParams
|
||||
_grammar: Optional[
|
||||
Union[Future[StructuredOutputGrammar], StructuredOutputGrammar]
|
||||
] = None
|
||||
reasoning_ended: Optional[bool] = None
|
||||
_grammar: Union[Future[StructuredOutputGrammar], StructuredOutputGrammar] | None = (
|
||||
None
|
||||
)
|
||||
reasoning_ended: bool | None = None
|
||||
|
||||
def _check_grammar_completion(self) -> bool:
|
||||
# NOTE: We have to lazy import to gate circular imports
|
||||
@@ -43,7 +43,7 @@ class StructuredOutputRequest:
|
||||
return self._check_grammar_completion()
|
||||
|
||||
@property
|
||||
def grammar(self) -> Optional[StructuredOutputGrammar]:
|
||||
def grammar(self) -> StructuredOutputGrammar | None:
|
||||
completed = self._check_grammar_completion()
|
||||
return (
|
||||
cast(Optional[StructuredOutputGrammar], self._grammar)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Callable, Optional, TypeVar, Union
|
||||
from typing import Any, Callable, TypeVar, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -78,8 +78,8 @@ class WorkerBase:
|
||||
self.is_driver_worker = is_driver_worker
|
||||
|
||||
# Device and model state
|
||||
self.device: Optional[torch.device] = None
|
||||
self.model_runner: Optional[nn.Module] = None
|
||||
self.device: torch.device | None = None
|
||||
self.model_runner: nn.Module | None = None
|
||||
|
||||
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
|
||||
"""Get specifications for KV cache implementation."""
|
||||
@@ -115,8 +115,8 @@ class WorkerBase:
|
||||
raise NotImplementedError
|
||||
|
||||
def execute_model(
|
||||
self, execute_model_req: Optional[ExecuteModelRequest] = None
|
||||
) -> Optional[list[SamplerOutput]]:
|
||||
self, execute_model_req: ExecuteModelRequest | None = None
|
||||
) -> list[SamplerOutput] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
def start_worker_execution_loop(self) -> None:
|
||||
@@ -198,8 +198,8 @@ class WorkerWrapperBase:
|
||||
group.
|
||||
"""
|
||||
self.rpc_rank = rpc_rank
|
||||
self.worker: Optional[WorkerBase] = None
|
||||
self.vllm_config: Optional[VllmConfig] = None
|
||||
self.worker: WorkerBase | None = None
|
||||
self.vllm_config: VllmConfig | None = None
|
||||
# do not store this `vllm_config`, `init_worker` will set the final
|
||||
# one. TODO: investigate if we can remove this field in
|
||||
# `WorkerWrapperBase`, `init_cached_hf_modules` should be
|
||||
|
||||
Reference in New Issue
Block a user