[Bugfix] Fix the two multimodal root tests that fail on main (OpenPangu-VL embed merge, MiMo sink test fixture) (#58900)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dd3bc9c72f
commit
0da126676e
@@ -5,13 +5,7 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
EMBED_DIM = 256
|
||||
NUM_HEADS = 4
|
||||
@@ -21,20 +15,11 @@ WINDOW = 8
|
||||
SEQ_LENS = [5, 37]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vision_attn_env():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
backend="nccl",
|
||||
)
|
||||
@pytest.fixture
|
||||
def vision_attn_env(dist_init):
|
||||
default_dtype = torch.get_default_dtype()
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
yield
|
||||
yield
|
||||
torch.set_default_dtype(default_dtype)
|
||||
|
||||
|
||||
|
||||
@@ -1055,15 +1055,18 @@ class OpenPanguVLForConditionalGeneration(
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is not None:
|
||||
inputs_embeds = self.embed_input_ids( # type: ignore[call-overload]
|
||||
input_ids,
|
||||
inputs_embeds,
|
||||
multimodal_embeddings,
|
||||
[self.config.image_token_id, self.config.video_token_id],
|
||||
)
|
||||
return inputs_embeds
|
||||
if multimodal_embeddings is None:
|
||||
return self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
mm_token_ids = input_ids.new_tensor(
|
||||
[self.config.image_token_id, self.config.video_token_id]
|
||||
)
|
||||
is_multimodal = torch.isin(input_ids, mm_token_ids)
|
||||
return self.embed_input_ids(
|
||||
input_ids,
|
||||
multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: OpenPanguVLImageInputs
|
||||
|
||||
Reference in New Issue
Block a user