[Model] Use mm_features to compute mrope positions for PaddleOCR-VL (#39888)
Signed-off-by: grYe99 <guorongye99@gmail.com> Co-authored-by: grYe99 <guorongye99@gmail.com>
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.paddleocr_vl import (
|
||||
PaddleOCRVLForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 2
|
||||
patch_size: int = 14
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
image_token_id: int = 151655
|
||||
video_token_id: int = 151654
|
||||
vision_start_token_id: int = 151652
|
||||
vision_end_token_id: int = 151653
|
||||
vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
|
||||
|
||||
|
||||
def make_model(config: DummyConfig) -> PaddleOCRVLForConditionalGeneration:
|
||||
model = object.__new__(PaddleOCRVLForConditionalGeneration)
|
||||
model.config = config
|
||||
return model
|
||||
|
||||
|
||||
def make_mm_feature(
|
||||
*,
|
||||
offset: int,
|
||||
length: int,
|
||||
image_grid_thw: tuple[int, int, int],
|
||||
) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{
|
||||
"image_grid_thw": MultiModalFieldElem(
|
||||
data=torch.tensor(image_grid_thw),
|
||||
field=None,
|
||||
),
|
||||
}
|
||||
),
|
||||
modality="image",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_text_only():
|
||||
model = make_model(DummyConfig())
|
||||
input_tokens = [11, 12, 13, 14, 15]
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=[],
|
||||
)
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
[0, 1, 2, 3, 4],
|
||||
]
|
||||
)
|
||||
assert torch.equal(positions, expected)
|
||||
assert delta == 0
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_single_image():
|
||||
model = make_model(DummyConfig())
|
||||
spatial_merge_size = model.config.vision_config.spatial_merge_size
|
||||
|
||||
t, h, w = 1, 2, 2
|
||||
num_image_tokens = t * h * w
|
||||
|
||||
input_tokens = (
|
||||
[10]
|
||||
+ [model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num_image_tokens
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [30, 31]
|
||||
)
|
||||
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
offset=2, # 1 (text) + 1 (vision_start)
|
||||
length=num_image_tokens,
|
||||
image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 2, 2, 2, 4, 5, 6],
|
||||
[0, 1, 2, 2, 3, 3, 4, 5, 6],
|
||||
[0, 1, 2, 3, 2, 3, 4, 5, 6],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
expected_delta = (positions.max().item() + 1) - len(input_tokens)
|
||||
assert delta == expected_delta
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_multiple_images():
|
||||
model = make_model(DummyConfig())
|
||||
spatial_merge_size = model.config.vision_config.spatial_merge_size
|
||||
|
||||
t1, h1, w1 = 1, 2, 2
|
||||
num1 = t1 * h1 * w1
|
||||
|
||||
t2, h2, w2 = 1, 1, 3
|
||||
num2 = t2 * h2 * w2
|
||||
|
||||
input_tokens = (
|
||||
[10]
|
||||
+ [model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num1
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [20, 21]
|
||||
+ [model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num2
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [30]
|
||||
)
|
||||
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
offset=2,
|
||||
length=num1,
|
||||
image_grid_thw=(t1, h1 * spatial_merge_size, w1 * spatial_merge_size),
|
||||
),
|
||||
make_mm_feature(
|
||||
offset=2 + num1 + 1 + 2 + 1,
|
||||
length=num2,
|
||||
image_grid_thw=(t2, h2 * spatial_merge_size, w2 * spatial_merge_size),
|
||||
),
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
assert positions.shape == (3, 15)
|
||||
assert not torch.equal(positions[:, 2:6], torch.arange(4).expand(3, 4) + 2)
|
||||
assert not torch.equal(positions[:, 10:13], torch.arange(3).expand(3, 3) + 10)
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_image_at_start():
|
||||
model = make_model(DummyConfig())
|
||||
spatial_merge_size = model.config.vision_config.spatial_merge_size
|
||||
|
||||
t, h, w = 1, 2, 2
|
||||
num_tokens = t * h * w
|
||||
|
||||
input_tokens = (
|
||||
[model.config.vision_start_token_id]
|
||||
+ [model.config.image_token_id] * num_tokens
|
||||
+ [model.config.vision_end_token_id]
|
||||
+ [10, 11]
|
||||
)
|
||||
|
||||
mm_features = [
|
||||
make_mm_feature(
|
||||
offset=1, # start token at index 0
|
||||
length=num_tokens,
|
||||
image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
|
||||
)
|
||||
]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 1, 1, 1, 3, 4, 5],
|
||||
[0, 1, 1, 2, 2, 3, 4, 5],
|
||||
[0, 1, 2, 1, 2, 3, 4, 5],
|
||||
]
|
||||
)
|
||||
|
||||
assert torch.equal(positions, expected)
|
||||
Reference in New Issue
Block a user