[Bugfix] Avoid import AttentionMetadata explicitly in Mllama (#10593)

Signed-off-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
Isotr0py
2024-11-23 18:12:20 +00:00
committed by GitHub
parent 651f6c31ac
commit 04668ebe7a
5 changed files with 21 additions and 11 deletions
+2 -1
View File
@@ -6,7 +6,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.attention import AttentionMetadata, AttentionType
from vllm.attention.selector import get_attn_backend
from vllm.attention.selector import backend_name_to_enum, get_attn_backend
from vllm.config import CacheConfig
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.layers.quantization.base_config import (
@@ -98,6 +98,7 @@ class Attention(nn.Module):
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
alibi_slopes, sliding_window, kv_cache_dtype,
blocksparse_params, logits_soft_cap)
self.backend = backend_name_to_enum(attn_backend.get_name())
# For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
# torch.compile works by registering the attention as one giant