[MyPy] Enable mypy for vllm/model_executor/layers/ (#40159)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -33,6 +33,7 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadata,
|
||||
AttentionType,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -209,6 +210,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
`self.kv_cache`.
|
||||
"""
|
||||
super().__init__()
|
||||
sliding_window: int | None
|
||||
if per_layer_sliding_window is not None:
|
||||
# per-layer sliding window
|
||||
sliding_window = per_layer_sliding_window
|
||||
@@ -335,7 +337,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
cache_config.enable_prefix_caching = False
|
||||
|
||||
impl_cls = self.attn_backend.get_impl_cls()
|
||||
self.impl = impl_cls(
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
@@ -576,7 +578,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
# Block size may get updated after model loading, refresh it
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
# Should not be called for enc-dec or encoder-only attention.
|
||||
@@ -680,9 +682,16 @@ def get_attention_context(
|
||||
extracted from the forward context.
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
attn_metadata = attn_metadata[layer_name]
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw[layer_name]
|
||||
elif isinstance(attn_metadata_raw, list):
|
||||
# list[dict[str, AttentionMetadata]]: used in speculative decoding
|
||||
# where [0] is the base-model (non-speculative) metadata dict.
|
||||
attn_metadata = attn_metadata_raw[0][layer_name]
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
attn_layer: Attention | MLAAttention = forward_context.no_compile_layers[layer_name]
|
||||
kv_cache = attn_layer.kv_cache
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
@@ -708,7 +717,7 @@ def unified_kv_cache_update(
|
||||
assert hasattr(attn_layer.impl, "do_kv_cache_update"), (
|
||||
f"{attn_layer.impl.__class__.__name__} does not support kv cache update"
|
||||
)
|
||||
attn_layer.impl.do_kv_cache_update(
|
||||
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
attn_layer,
|
||||
key,
|
||||
value,
|
||||
|
||||
Reference in New Issue
Block a user