[Model][OpenVINO] Fix regressions from #8346 (#10045)

Signed-off-by: Peter Salas <peter@fixie.ai>
This commit is contained in:
Peter Salas
2024-11-06 04:19:15 +00:00
committed by GitHub
parent 82bfc38d07
commit ffc0f2b47a
3 changed files with 15 additions and 5 deletions
+11 -1
View File
@@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import List, Tuple, Type
from typing import Dict, List, Optional, Tuple, Type
import openvino as ov
import torch
@@ -7,6 +7,7 @@ import torch
from vllm.attention.backends.abstract import (AttentionBackend,
AttentionMetadata)
from vllm.attention.backends.utils import CommonAttentionState
from vllm.multimodal import MultiModalPlaceholderMap
def copy_cache_block(src_tensor: ov.Tensor, dst_tensor: ov.Tensor,
@@ -128,3 +129,12 @@ class OpenVINOAttentionMetadata:
# Shape: scalar
# Type: i32
max_context_len: torch.Tensor
# The index maps that relate multi-modal embeddings to the corresponding
# placeholders.
#
# N.B. These aren't really related to attention and don't belong on this
# type -- this is just a temporary solution to make them available to
# `model_executable`.
multi_modal_placeholder_index_maps: Optional[Dict[
str, MultiModalPlaceholderMap.IndexMap]]