[Core] Skip broadcasting mm tensor data to workers for prefix-cache-covered items (#52041)
Signed-off-by: Xiaoan (Sean) Liu <shawnliu0327@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
2b7fcbf527
commit
db92053e97
@@ -2,6 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalBatchedField,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.utils import strip_covered_mm_data
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
|
||||
|
||||
@@ -34,3 +42,82 @@ def test_repr_with_multi_element_tensor() -> None:
|
||||
|
||||
assert "prompt_embeds_shape=torch.Size([10, 768])" in repr(new_requests_data)
|
||||
assert "prompt_embeds_shape=torch.Size([10, 768])" in new_requests_data.anon_repr()
|
||||
|
||||
|
||||
def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem.dummy(),
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
identifier=f"hash_{offset}",
|
||||
modality="image",
|
||||
)
|
||||
|
||||
|
||||
def test_strip_covered_mm_data() -> None:
|
||||
"""Items fully inside the computed prefix lose their data; items touching
|
||||
the uncomputed region keep it; already-None data stays None."""
|
||||
from dataclasses import replace
|
||||
|
||||
covered = _mm_feature(offset=0, length=100)
|
||||
boundary = _mm_feature(offset=150, length=100) # ends exactly at 250
|
||||
uncovered = _mm_feature(offset=300, length=100)
|
||||
already_none = replace(_mm_feature(offset=100, length=50), data=None)
|
||||
|
||||
stripped = strip_covered_mm_data(
|
||||
[covered, boundary, uncovered, already_none], num_computed_tokens=250
|
||||
)
|
||||
|
||||
assert stripped[0].data is None # fully covered -> stripped
|
||||
assert stripped[1].data is None # span end == computed -> covered -> stripped
|
||||
assert stripped[2].data is not None # extends past prefix -> kept
|
||||
assert stripped[3].data is None # was already None
|
||||
# non-data fields are preserved
|
||||
assert stripped[0].identifier == covered.identifier
|
||||
assert stripped[0].mm_position == covered.mm_position
|
||||
# original list is not mutated
|
||||
assert covered.data is not None
|
||||
|
||||
|
||||
def test_strip_covered_mm_data_zero_computed() -> None:
|
||||
"""With no prefix hit nothing is stripped."""
|
||||
features = [_mm_feature(offset=0, length=100)]
|
||||
stripped = strip_covered_mm_data(features, num_computed_tokens=0)
|
||||
assert stripped[0].data is not None
|
||||
|
||||
|
||||
def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
data = MultiModalKwargsItem(
|
||||
{
|
||||
"pixel_values": MultiModalFieldElem(
|
||||
data=torch.empty(4), field=MultiModalBatchedField()
|
||||
),
|
||||
"image_grid_thw": MultiModalFieldElem(
|
||||
data=torch.ones(1, 3, dtype=torch.long),
|
||||
field=MultiModalBatchedField(keep_on_cpu=True),
|
||||
),
|
||||
}
|
||||
)
|
||||
return MultiModalFeatureSpec(
|
||||
data=data,
|
||||
mm_position=PlaceholderRange(offset=offset, length=length),
|
||||
identifier=f"hash_{offset}",
|
||||
modality="image",
|
||||
)
|
||||
|
||||
|
||||
def test_strip_covered_mm_data_mrope() -> None:
|
||||
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
|
||||
(the worker needs them to compute positions); payload fields are dropped."""
|
||||
covered = _mm_feature_mixed(offset=0, length=100)
|
||||
uncovered = _mm_feature_mixed(offset=300, length=100)
|
||||
|
||||
stripped = strip_covered_mm_data(
|
||||
[covered, uncovered], num_computed_tokens=250, uses_mrope=True
|
||||
)
|
||||
|
||||
assert stripped[0].data is not None
|
||||
assert list(stripped[0].data.keys()) == ["image_grid_thw"]
|
||||
assert stripped[1].data is not None
|
||||
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
|
||||
# original list is not mutated
|
||||
assert set(covered.data.keys()) == {"pixel_values", "image_grid_thw"}
|
||||
|
||||
@@ -5,6 +5,7 @@ import bisect
|
||||
import mimetypes
|
||||
from collections import defaultdict
|
||||
from collections.abc import Generator, Sequence
|
||||
from dataclasses import replace
|
||||
from itertools import groupby
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -212,6 +213,36 @@ def _batch_mm_items(
|
||||
}
|
||||
|
||||
|
||||
def strip_covered_mm_data(
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
num_computed_tokens: int,
|
||||
uses_mrope: bool = False,
|
||||
) -> list[MultiModalFeatureSpec]:
|
||||
"""Drop the tensor data of mm items whose placeholder span is fully inside
|
||||
a prefix-cache-covered region: no encoder run can be scheduled for them,
|
||||
so the workers never consume the payload fields. M-RoPE models are the
|
||||
exception: the worker computes positions for the whole prompt from the
|
||||
CPU-side metadata fields (e.g. grid dims), so those are kept. The
|
||||
scheduler-side ``Request`` keeps the full features."""
|
||||
if not mm_features or num_computed_tokens == 0:
|
||||
return mm_features
|
||||
|
||||
def maybe_strip(f: MultiModalFeatureSpec) -> MultiModalFeatureSpec:
|
||||
if f.data is None or (
|
||||
f.mm_position.offset + f.mm_position.length > num_computed_tokens
|
||||
):
|
||||
return f
|
||||
|
||||
data = None
|
||||
if uses_mrope:
|
||||
data = MultiModalKwargsItem(
|
||||
{k: elem for k, elem in f.data.items() if elem.field.keep_on_cpu}
|
||||
)
|
||||
return replace(f, data=data)
|
||||
|
||||
return [maybe_strip(f) for f in mm_features]
|
||||
|
||||
|
||||
def group_and_batch_mm_items(
|
||||
items: Sequence[MultiModalKwargsItem],
|
||||
*,
|
||||
|
||||
@@ -6,6 +6,7 @@ from functools import cached_property
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
||||
from vllm.multimodal.utils import strip_covered_mm_data
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
@@ -53,11 +54,16 @@ class NewRequestData:
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
prefill_token_ids: list[int] | None = None,
|
||||
uses_mrope: bool = False,
|
||||
) -> "NewRequestData":
|
||||
return cls(
|
||||
req_id=request.request_id,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
mm_features=request.mm_features,
|
||||
mm_features=strip_covered_mm_data(
|
||||
request.mm_features,
|
||||
request.num_computed_tokens,
|
||||
uses_mrope=uses_mrope,
|
||||
),
|
||||
sampling_params=request.sampling_params,
|
||||
pooling_params=request.pooling_params,
|
||||
block_ids=block_ids,
|
||||
|
||||
@@ -82,6 +82,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.lora_config = vllm_config.lora_config
|
||||
self.model_uses_mrope = vllm_config.model_config.uses_mrope
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.kv_events_config = vllm_config.kv_events_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
@@ -1199,13 +1200,16 @@ class Scheduler(SchedulerInterface):
|
||||
req,
|
||||
req_to_new_blocks[req.request_id].get_block_ids(),
|
||||
req._all_token_ids,
|
||||
uses_mrope=self.model_uses_mrope,
|
||||
)
|
||||
for req in scheduled_new_reqs
|
||||
]
|
||||
else:
|
||||
new_reqs_data = [
|
||||
NewRequestData.from_request(
|
||||
req, req_to_new_blocks[req.request_id].get_block_ids()
|
||||
req,
|
||||
req_to_new_blocks[req.request_id].get_block_ids(),
|
||||
uses_mrope=self.model_uses_mrope,
|
||||
)
|
||||
for req in scheduled_new_reqs
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user