[Multimodal] Simplify MM input definitions (#33331)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -23,18 +23,16 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
|
||||
|
||||
def _dummy_elem(modality: str, key: str, size: int):
|
||||
def _dummy_elem(size: int):
|
||||
return MultiModalFieldElem(
|
||||
modality=modality,
|
||||
key=key,
|
||||
data=torch.empty((size,), dtype=torch.int8),
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
|
||||
|
||||
def _dummy_item(modality: str, size_by_key: dict[str, int]):
|
||||
return MultiModalKwargsItem.from_elems(
|
||||
[_dummy_elem(modality, key, size) for key, size in size_by_key.items()]
|
||||
def _dummy_item(size_by_key: dict[str, int]):
|
||||
return MultiModalKwargsItem(
|
||||
{key: _dummy_elem(size) for key, size in size_by_key.items()}
|
||||
)
|
||||
|
||||
|
||||
@@ -61,7 +59,7 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
|
||||
def test_minimal_put_get_cycle(self):
|
||||
"""Test basic put and get operations."""
|
||||
key = "test_key"
|
||||
value = _dummy_item("text", {"field1": 10, "field2": 20})
|
||||
value = _dummy_item({"field1": 10, "field2": 20})
|
||||
|
||||
# Put operation
|
||||
address, monotonic_id = self.storage.put(key, value)
|
||||
|
||||
@@ -119,7 +119,11 @@ def create_batched_mm_kwargs(
|
||||
)["mm_kwargs"].require_data()
|
||||
|
||||
return group_mm_kwargs_by_modality(
|
||||
[item for modality in supported_mm_limits for item in mm_kwargs[modality]]
|
||||
[
|
||||
(modality, item)
|
||||
for modality in supported_mm_limits
|
||||
for item in mm_kwargs[modality]
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -36,8 +36,6 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _dummy_elem(
|
||||
modality: str,
|
||||
key: str,
|
||||
size: int,
|
||||
*,
|
||||
rng: np.random.RandomState | None = None,
|
||||
@@ -48,21 +46,18 @@ def _dummy_elem(
|
||||
data = torch.from_numpy(rng.randint(4, size=(size,), dtype=np.int8))
|
||||
|
||||
return MultiModalFieldElem(
|
||||
modality=modality,
|
||||
key=key,
|
||||
data=data,
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
|
||||
|
||||
def _dummy_item(
|
||||
modality: str,
|
||||
size_by_key: dict[str, int],
|
||||
*,
|
||||
rng: np.random.RandomState | None = None,
|
||||
):
|
||||
return MultiModalKwargsItem.from_elems(
|
||||
[_dummy_elem(modality, key, size, rng=rng) for key, size in size_by_key.items()]
|
||||
return MultiModalKwargsItem(
|
||||
{key: _dummy_elem(size, rng=rng) for key, size in size_by_key.items()}
|
||||
)
|
||||
|
||||
|
||||
@@ -71,19 +66,19 @@ def _dummy_items(
|
||||
*,
|
||||
rng: np.random.RandomState | None = None,
|
||||
):
|
||||
return MultiModalKwargsItems.from_seq(
|
||||
[
|
||||
_dummy_item(modality, size_by_key, rng=rng)
|
||||
return MultiModalKwargsItems(
|
||||
{
|
||||
modality: [_dummy_item(size_by_key, rng=rng)]
|
||||
for modality, size_by_key in size_by_key_modality.items()
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("item", "expected_size"),
|
||||
[
|
||||
(_dummy_item("a", {"a1": 100}), 100),
|
||||
(_dummy_item("a", {"a1": 100, "a2": 110}), 210),
|
||||
(_dummy_item({"a1": 100}), 100),
|
||||
(_dummy_item({"a1": 100, "a2": 110}), 210),
|
||||
(_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501
|
||||
],
|
||||
)
|
||||
@@ -143,7 +138,7 @@ def _compare_caches(
|
||||
|
||||
rng = np.random.RandomState(seed)
|
||||
all_items = [
|
||||
_dummy_item("item", {"key": item_size_gb}, rng=rng)
|
||||
_dummy_item({"key": item_size_gb}, rng=rng)
|
||||
for _ in range(int(item_capacity / hit_rate))
|
||||
]
|
||||
all_hashes = [
|
||||
@@ -245,13 +240,13 @@ def _run_test_cache_eviction_lru(
|
||||
"image_C",
|
||||
]
|
||||
request1_items = {
|
||||
h: MultiModalKwargsItem.dummy(h, nbytes=2 * base_item_size)
|
||||
h: MultiModalKwargsItem.dummy(nbytes=2 * base_item_size)
|
||||
for h in request1_hashes
|
||||
}
|
||||
|
||||
request2_hashes = ["image_D", "image_E", "image_A", "image_C"]
|
||||
request2_items = {
|
||||
h: MultiModalKwargsItem.dummy(h, nbytes=1 * base_item_size)
|
||||
h: MultiModalKwargsItem.dummy(nbytes=1 * base_item_size)
|
||||
for h in request2_hashes
|
||||
}
|
||||
|
||||
@@ -356,15 +351,14 @@ def _run_test_cache_eviction_shm(
|
||||
):
|
||||
request1_hashes = ["image_A", "image_B", "image_C"]
|
||||
request1_items = {
|
||||
h: MultiModalKwargsItem.dummy(h, nbytes=5 * base_item_size)
|
||||
for h in request1_hashes
|
||||
h: MultiModalKwargsItem.dummy(5 * base_item_size) for h in request1_hashes
|
||||
}
|
||||
request1_items_p0_result = []
|
||||
|
||||
request2_hashes = ["image_G", "image_A"]
|
||||
request2_items = {
|
||||
h: MultiModalKwargsItem.dummy(
|
||||
h, nbytes=(5 if h in request1_hashes else 2) * base_item_size
|
||||
(5 if h in request1_hashes else 2) * base_item_size
|
||||
)
|
||||
for h in request2_hashes
|
||||
}
|
||||
@@ -373,7 +367,7 @@ def _run_test_cache_eviction_shm(
|
||||
request3_hashes = ["image_G", "image_H", "image_I", "image_B"]
|
||||
request3_items = {
|
||||
h: MultiModalKwargsItem.dummy(
|
||||
h, nbytes=(5 if h in request1_hashes else 2) * base_item_size
|
||||
(5 if h in request1_hashes else 2) * base_item_size
|
||||
)
|
||||
for h in request3_hashes
|
||||
}
|
||||
@@ -532,7 +526,7 @@ def test_processor_cache_shared_across_loras():
|
||||
lora_a_identifier = f"12345:{base_mm_hash}"
|
||||
lora_b_identifier = f"67890:{base_mm_hash}"
|
||||
|
||||
item_data = MultiModalKwargsItem.dummy("test_image", nbytes=1024)
|
||||
item_data = MultiModalKwargsItem.dummy(1024)
|
||||
|
||||
feature_lora_a = MultiModalFeatureSpec(
|
||||
data=item_data,
|
||||
|
||||
@@ -77,7 +77,7 @@ def make_request(
|
||||
for j, position in enumerate(mm_positions):
|
||||
identifier = mm_hashes[j] if mm_hashes else f"hash_{j}"
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("dummy_m"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
mm_position=position,
|
||||
identifier=identifier,
|
||||
modality="image",
|
||||
|
||||
@@ -68,7 +68,7 @@ def make_request(
|
||||
for j, position in enumerate(mm_positions):
|
||||
identifier = mm_hashes[j] if mm_hashes else f"hash_{j}"
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("dummy_m"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
mm_position=position,
|
||||
identifier=identifier,
|
||||
modality="image",
|
||||
|
||||
@@ -56,7 +56,7 @@ def _create_random_request(
|
||||
for j, position in enumerate(mm_positions):
|
||||
identifier = f"{request_id}_hash_{j}"
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("dummy_m"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
mm_position=position,
|
||||
identifier=identifier,
|
||||
modality="image",
|
||||
|
||||
@@ -1707,7 +1707,7 @@ def create_requests_with_priority(
|
||||
# Unique dummy hash for each mm item
|
||||
identifier = f"hash{i}_{j}"
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("dummy_m"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
mm_position=position,
|
||||
identifier=identifier,
|
||||
modality="image",
|
||||
|
||||
@@ -236,7 +236,7 @@ def create_requests(
|
||||
# Unique dummy hash for each mm item
|
||||
identifier = f"hash{i}_{j}"
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("dummy_m"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
mm_position=position,
|
||||
identifier=identifier,
|
||||
modality="image",
|
||||
|
||||
@@ -131,7 +131,7 @@ def test_e2e_streaming_with_multimodal_features(mock_model_runner_with_input_bat
|
||||
|
||||
# Step 1: Create initial request state with one multimodal feature
|
||||
mm_feature_1 = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("audio"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="audio",
|
||||
identifier="audio_1",
|
||||
mm_position=PlaceholderRange(offset=2, length=10),
|
||||
@@ -158,7 +158,7 @@ def test_e2e_streaming_with_multimodal_features(mock_model_runner_with_input_bat
|
||||
# The scheduler has already set prompt_token_ids to the full sequence
|
||||
# (original prompt + intermediate outputs + new prompt with new multimodal feature)
|
||||
mm_feature_2 = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("audio"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="audio",
|
||||
identifier="audio_2",
|
||||
mm_position=PlaceholderRange(offset=15, length=5),
|
||||
|
||||
@@ -174,7 +174,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
scheduler = create_scheduler()
|
||||
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("audio"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="audio",
|
||||
identifier="",
|
||||
mm_position=PlaceholderRange(offset=1, length=1),
|
||||
@@ -187,7 +187,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy("audio"),
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
modality="audio",
|
||||
identifier="",
|
||||
mm_position=PlaceholderRange(offset=2, length=1),
|
||||
|
||||
@@ -104,14 +104,10 @@ class MyRequest(msgspec.Struct):
|
||||
|
||||
def test_multimodal_kwargs():
|
||||
e1 = MultiModalFieldElem(
|
||||
"audio",
|
||||
"a0",
|
||||
torch.zeros(1000, dtype=torch.bfloat16),
|
||||
MultiModalBatchedField(),
|
||||
)
|
||||
e2 = MultiModalFieldElem(
|
||||
"video",
|
||||
"v0",
|
||||
[torch.zeros(1000, dtype=torch.int8) for _ in range(4)],
|
||||
MultiModalFlatField(
|
||||
slices=[[slice(1, 2, 3), slice(4, 5, 6)], [slice(None, 2)]],
|
||||
@@ -119,21 +115,20 @@ def test_multimodal_kwargs():
|
||||
),
|
||||
)
|
||||
e3 = MultiModalFieldElem(
|
||||
"image",
|
||||
"i0",
|
||||
torch.zeros(1000, dtype=torch.int32),
|
||||
MultiModalSharedField(batch_size=4),
|
||||
)
|
||||
e4 = MultiModalFieldElem(
|
||||
"image",
|
||||
"i1",
|
||||
torch.zeros(1000, dtype=torch.int32),
|
||||
MultiModalFlatField(slices=[slice(1, 2, 3), slice(4, 5, 6)], dim=2),
|
||||
)
|
||||
audio = MultiModalKwargsItem.from_elems([e1])
|
||||
video = MultiModalKwargsItem.from_elems([e2])
|
||||
image = MultiModalKwargsItem.from_elems([e3, e4])
|
||||
mm = MultiModalKwargsItems.from_seq([audio, video, image])
|
||||
mm = MultiModalKwargsItems(
|
||||
{
|
||||
"audio": [MultiModalKwargsItem({"a0": e1})],
|
||||
"video": [MultiModalKwargsItem({"v0": e2})],
|
||||
"image": [MultiModalKwargsItem({"i0": e3, "i1": e4})],
|
||||
}
|
||||
)
|
||||
|
||||
# pack mm kwargs into a mock request so that it can be decoded properly
|
||||
req = MyRequest([mm])
|
||||
@@ -147,8 +142,8 @@ def test_multimodal_kwargs():
|
||||
|
||||
total_len = sum(memoryview(x).cast("B").nbytes for x in encoded)
|
||||
|
||||
# expected total encoding length, should be 14395, +-20 for minor changes
|
||||
assert 14375 <= total_len <= 14425
|
||||
# expected total encoding length, should be 14319, +-20 for minor changes
|
||||
assert 14300 <= total_len <= 14340
|
||||
decoded = decoder.decode(encoded).mm[0]
|
||||
assert isinstance(decoded, MultiModalKwargsItems)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user