[Hardware][CPU] Enable mrope and support Qwen2-VL on CPU backend (#8770)

This commit is contained in:
Isotr0py
2024-09-24 23:16:11 -07:00
committed by GitHub
parent e3dd0692fa
commit c23953675f
2 changed files with 99 additions and 9 deletions
+16
View File
@@ -67,6 +67,7 @@ from vllm.multimodal.image import cached_get_image_processor
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors, SequenceData
from vllm.transformers_utils.processor import get_processor
from vllm.utils import is_cpu
from .utils import (PPMissingLayer, is_pp_missing_parameter,
make_empty_intermediate_tensors_factory)
@@ -281,6 +282,21 @@ class Qwen2VisionAttention(nn.Module):
context_layer = rearrange(output,
"(b s) ... -> b s ...",
b=batch_size)
elif is_cpu():
seq_length = q.size(1)
q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
attention_mask = torch.zeros([1, seq_length, seq_length],
device=q.device,
dtype=torch.bool)
for i in range(1, len(cu_seqlens)):
attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
cu_seqlens[i - 1]:cu_seqlens[i]] = True
output = F.scaled_dot_product_attention(q,
k,
v,
attention_mask,
dropout_p=0.0)
context_layer = rearrange(output, "b h s d -> b s h d ")
else:
from xformers import ops as xops
from xformers.ops.fmha.attn_bias import BlockDiagonalMask