[MyPy] Enable mypy for vllm/model_executor/layers/ (#40159)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -33,6 +33,7 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadata,
|
||||
AttentionType,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -209,6 +210,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
`self.kv_cache`.
|
||||
"""
|
||||
super().__init__()
|
||||
sliding_window: int | None
|
||||
if per_layer_sliding_window is not None:
|
||||
# per-layer sliding window
|
||||
sliding_window = per_layer_sliding_window
|
||||
@@ -335,7 +337,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
cache_config.enable_prefix_caching = False
|
||||
|
||||
impl_cls = self.attn_backend.get_impl_cls()
|
||||
self.impl = impl_cls(
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
@@ -576,7 +578,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
# Block size may get updated after model loading, refresh it
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
# Should not be called for enc-dec or encoder-only attention.
|
||||
@@ -680,9 +682,16 @@ def get_attention_context(
|
||||
extracted from the forward context.
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
attn_metadata = attn_metadata[layer_name]
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw[layer_name]
|
||||
elif isinstance(attn_metadata_raw, list):
|
||||
# list[dict[str, AttentionMetadata]]: used in speculative decoding
|
||||
# where [0] is the base-model (non-speculative) metadata dict.
|
||||
attn_metadata = attn_metadata_raw[0][layer_name]
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
attn_layer: Attention | MLAAttention = forward_context.no_compile_layers[layer_name]
|
||||
kv_cache = attn_layer.kv_cache
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
@@ -708,7 +717,7 @@ def unified_kv_cache_update(
|
||||
assert hasattr(attn_layer.impl, "do_kv_cache_update"), (
|
||||
f"{attn_layer.impl.__class__.__name__} does not support kv cache update"
|
||||
)
|
||||
attn_layer.impl.do_kv_cache_update(
|
||||
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
attn_layer,
|
||||
key,
|
||||
value,
|
||||
|
||||
@@ -29,7 +29,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
|
||||
@functools.lru_cache
|
||||
def create_chunked_local_attention_backend(
|
||||
underlying_attn_backend: AttentionBackend,
|
||||
underlying_attn_backend: type[AttentionBackend],
|
||||
attention_chunk_size: int,
|
||||
) -> type[AttentionBackend]:
|
||||
prefix = f"ChunkedLocalAttention_{attention_chunk_size}_"
|
||||
|
||||
@@ -72,7 +72,7 @@ def _get_cross_slot_mapping(
|
||||
|
||||
@functools.lru_cache
|
||||
def create_cross_attention_backend(
|
||||
underlying_attn_backend: AttentionBackend,
|
||||
underlying_attn_backend: type[AttentionBackend],
|
||||
) -> type[AttentionBackend]:
|
||||
prefix = "CrossAttention_"
|
||||
underlying_builder = underlying_attn_backend.get_builder_cls()
|
||||
@@ -87,6 +87,7 @@ def create_cross_attention_backend(
|
||||
) -> AttentionMetadata:
|
||||
new_metadata = copy(common_attn_metadata)
|
||||
new_metadata.causal = False
|
||||
assert new_metadata.encoder_seq_lens_cpu is not None
|
||||
max_encoder_len = int(new_metadata.encoder_seq_lens_cpu.max())
|
||||
new_metadata.max_seq_len = max_encoder_len
|
||||
# Any computed tokens indicated decode step>1 (no chunked prefill)
|
||||
@@ -118,7 +119,7 @@ def create_cross_attention_backend(
|
||||
self.device,
|
||||
)
|
||||
attn_metadata = super().build(common_prefix_len, new_metadata, fast_build)
|
||||
attn_metadata.slot_mapping = slot_mapping
|
||||
attn_metadata.slot_mapping = slot_mapping # type: ignore[attr-defined]
|
||||
return attn_metadata
|
||||
|
||||
# NOTE(Lucas): we need a custom impl so we can use the slot-mapping computed by
|
||||
@@ -144,8 +145,12 @@ def create_cross_attention_backend(
|
||||
and key is not None
|
||||
and value is not None
|
||||
):
|
||||
self.do_kv_cache_update(
|
||||
layer, key, value, kv_cache, attn_metadata.slot_mapping
|
||||
self.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
layer,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata.slot_mapping, # type: ignore[attr-defined]
|
||||
)
|
||||
|
||||
return super().forward(
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
@functools.lru_cache
|
||||
def create_encoder_only_attention_backend(
|
||||
underlying_attn_backend: AttentionBackend,
|
||||
underlying_attn_backend: type[AttentionBackend],
|
||||
) -> type[AttentionBackend]:
|
||||
prefix = "EncoderOnlyAttention_"
|
||||
underlying_builder = underlying_attn_backend.get_builder_cls()
|
||||
@@ -93,6 +93,6 @@ class EncoderOnlyAttention(Attention):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
# Does not need KV cache
|
||||
return None
|
||||
|
||||
@@ -389,7 +389,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
cache_config.enable_prefix_caching = False
|
||||
|
||||
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
|
||||
self.impl = impl_cls(
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
|
||||
num_heads=self.num_heads,
|
||||
head_size=self.head_size,
|
||||
scale=self.scale,
|
||||
@@ -485,16 +485,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
if self.use_direct_call:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
attn_metadata = attn_metadata[self.layer_name]
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: MLACommonMetadata
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw[self.layer_name] # type: ignore[assignment]
|
||||
elif isinstance(attn_metadata_raw, list):
|
||||
# list[dict[str, AttentionMetadata]]: used in speculative decoding
|
||||
# where [0] is the base-model (non-speculative) metadata dict.
|
||||
attn_metadata = attn_metadata_raw[0][self.layer_name] # type: ignore[assignment]
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
self_kv_cache = self.kv_cache
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
|
||||
assert isinstance(slot_mapping, dict), (
|
||||
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
|
||||
)
|
||||
self.impl.do_kv_cache_update(
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
self_kv_cache,
|
||||
@@ -612,7 +619,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
if num_mha_tokens > 0:
|
||||
self.impl.forward_mha(
|
||||
self.impl.forward_mha( # type: ignore[attr-defined]
|
||||
q[num_mqa_tokens:],
|
||||
k_c_normed[num_mqa_tokens:],
|
||||
k_pe[num_mqa_tokens:],
|
||||
@@ -695,7 +702,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# call decode attn
|
||||
if not is_sparse_impl:
|
||||
assert attn_metadata.decode is not None
|
||||
attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self)
|
||||
attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) # type: ignore[attr-defined]
|
||||
|
||||
# correct dcp attn_out with lse.
|
||||
if self.impl.dcp_world_size > 1:
|
||||
@@ -1053,9 +1060,9 @@ except ImportError:
|
||||
"AITER_MLA backends use aiter kernels instead."
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
from vllm._xpu_ops import xpu_ops
|
||||
|
||||
flash_attn_varlen_func = ops.flash_attn_varlen_func # type: ignore[no-redef]
|
||||
flash_attn_varlen_func = xpu_ops.flash_attn_varlen_func # type: ignore[no-redef,attr-defined,assignment]
|
||||
|
||||
|
||||
def dynamic_per_batched_tensor_quant(
|
||||
@@ -1988,7 +1995,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
assert isinstance(attn_metadata.prefill, FlashInferPrefillMetadata)
|
||||
self._build_fi_prefill_wrappers(attn_metadata.prefill)
|
||||
|
||||
return attn_metadata
|
||||
return attn_metadata # type: ignore[return-value]
|
||||
|
||||
|
||||
def reorg_kvcache(
|
||||
|
||||
Reference in New Issue
Block a user