[Model] Pixtral: use packed multimodal encoder attention (#52185)
Signed-off-by: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c47ca4aaeb
commit
28b484e5d7
@@ -5,6 +5,7 @@ from dataclasses import asdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from mistral_common.multimodal import download_image
|
||||
from mistral_common.protocol.instruct.chunk import ImageURLChunk
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
@@ -15,7 +16,9 @@ from transformers import AutoProcessor
|
||||
from vllm import SamplingParams, TextPrompt, TokensPrompt
|
||||
from vllm.inputs import MultiModalDataBuiltins
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from ....utils import VLLM_PATH, large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
@@ -123,6 +126,37 @@ FIXTURE_LOGPROBS_CHAT = {
|
||||
OutputsLogprobs = list[tuple[list[int], str, SampleLogprobs | None]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
],
|
||||
)
|
||||
def test_packed_sequence_metadata(backend: AttentionBackendEnum) -> None:
|
||||
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
|
||||
[4, 6],
|
||||
backend,
|
||||
hidden_size=64,
|
||||
tp_size=1,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert cu_seqlens.dtype == torch.int32
|
||||
assert max_seqlen.dtype == torch.int32
|
||||
if backend == AttentionBackendEnum.FLASHINFER:
|
||||
assert max_seqlen.item() >= 6
|
||||
assert sequence_lengths is not None
|
||||
assert sequence_lengths.dtype == torch.int32
|
||||
assert len(cu_seqlens) % 2 == 0
|
||||
else:
|
||||
expected_max_seqlen = 6 if backend == AttentionBackendEnum.FLASH_ATTN else 0
|
||||
assert max_seqlen.item() == expected_max_seqlen
|
||||
assert cu_seqlens.tolist() == [0, 4, 10]
|
||||
assert sequence_lengths is None
|
||||
|
||||
|
||||
# For the test author to store golden output in JSON
|
||||
def _dump_outputs_w_logprobs(
|
||||
outputs: OutputsLogprobs,
|
||||
|
||||
Reference in New Issue
Block a user