diff --git a/tests/v1/core/test_output.py b/tests/v1/core/test_output.py index 9dea19320e..bac225f5f0 100644 --- a/tests/v1/core/test_output.py +++ b/tests/v1/core/test_output.py @@ -2,6 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch +from vllm.multimodal.inputs import ( + MultiModalBatchedField, + MultiModalFeatureSpec, + MultiModalFieldElem, + MultiModalKwargsItem, + PlaceholderRange, +) +from vllm.multimodal.utils import strip_covered_mm_data from vllm.v1.core.sched.output import NewRequestData @@ -34,3 +42,82 @@ def test_repr_with_multi_element_tensor() -> None: assert "prompt_embeds_shape=torch.Size([10, 768])" in repr(new_requests_data) assert "prompt_embeds_shape=torch.Size([10, 768])" in new_requests_data.anon_repr() + + +def _mm_feature(offset: int, length: int) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=MultiModalKwargsItem.dummy(), + mm_position=PlaceholderRange(offset=offset, length=length), + identifier=f"hash_{offset}", + modality="image", + ) + + +def test_strip_covered_mm_data() -> None: + """Items fully inside the computed prefix lose their data; items touching + the uncomputed region keep it; already-None data stays None.""" + from dataclasses import replace + + covered = _mm_feature(offset=0, length=100) + boundary = _mm_feature(offset=150, length=100) # ends exactly at 250 + uncovered = _mm_feature(offset=300, length=100) + already_none = replace(_mm_feature(offset=100, length=50), data=None) + + stripped = strip_covered_mm_data( + [covered, boundary, uncovered, already_none], num_computed_tokens=250 + ) + + assert stripped[0].data is None # fully covered -> stripped + assert stripped[1].data is None # span end == computed -> covered -> stripped + assert stripped[2].data is not None # extends past prefix -> kept + assert stripped[3].data is None # was already None + # non-data fields are preserved + assert stripped[0].identifier == covered.identifier + assert stripped[0].mm_position == covered.mm_position + # original list is not mutated + assert covered.data is not None + + +def test_strip_covered_mm_data_zero_computed() -> None: + """With no prefix hit nothing is stripped.""" + features = [_mm_feature(offset=0, length=100)] + stripped = strip_covered_mm_data(features, num_computed_tokens=0) + assert stripped[0].data is not None + + +def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec: + data = MultiModalKwargsItem( + { + "pixel_values": MultiModalFieldElem( + data=torch.empty(4), field=MultiModalBatchedField() + ), + "image_grid_thw": MultiModalFieldElem( + data=torch.ones(1, 3, dtype=torch.long), + field=MultiModalBatchedField(keep_on_cpu=True), + ), + } + ) + return MultiModalFeatureSpec( + data=data, + mm_position=PlaceholderRange(offset=offset, length=length), + identifier=f"hash_{offset}", + modality="image", + ) + + +def test_strip_covered_mm_data_mrope() -> None: + """For M-RoPE models, covered items keep their keep_on_cpu metadata fields + (the worker needs them to compute positions); payload fields are dropped.""" + covered = _mm_feature_mixed(offset=0, length=100) + uncovered = _mm_feature_mixed(offset=300, length=100) + + stripped = strip_covered_mm_data( + [covered, uncovered], num_computed_tokens=250, uses_mrope=True + ) + + assert stripped[0].data is not None + assert list(stripped[0].data.keys()) == ["image_grid_thw"] + assert stripped[1].data is not None + assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"} + # original list is not mutated + assert set(covered.data.keys()) == {"pixel_values", "image_grid_thw"} diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index fc400710f0..ac9b028b46 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -5,6 +5,7 @@ import bisect import mimetypes from collections import defaultdict from collections.abc import Generator, Sequence +from dataclasses import replace from itertools import groupby from typing import TYPE_CHECKING, Any @@ -212,6 +213,36 @@ def _batch_mm_items( } +def strip_covered_mm_data( + mm_features: list[MultiModalFeatureSpec], + num_computed_tokens: int, + uses_mrope: bool = False, +) -> list[MultiModalFeatureSpec]: + """Drop the tensor data of mm items whose placeholder span is fully inside + a prefix-cache-covered region: no encoder run can be scheduled for them, + so the workers never consume the payload fields. M-RoPE models are the + exception: the worker computes positions for the whole prompt from the + CPU-side metadata fields (e.g. grid dims), so those are kept. The + scheduler-side ``Request`` keeps the full features.""" + if not mm_features or num_computed_tokens == 0: + return mm_features + + def maybe_strip(f: MultiModalFeatureSpec) -> MultiModalFeatureSpec: + if f.data is None or ( + f.mm_position.offset + f.mm_position.length > num_computed_tokens + ): + return f + + data = None + if uses_mrope: + data = MultiModalKwargsItem( + {k: elem for k, elem in f.data.items() if elem.field.keep_on_cpu} + ) + return replace(f, data=data) + + return [maybe_strip(f) for f in mm_features] + + def group_and_batch_mm_items( items: Sequence[MultiModalKwargsItem], *, diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 4d11be021e..b26a729264 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -6,6 +6,7 @@ from functools import cached_property from typing import TYPE_CHECKING from vllm.config.ec_manager_config import EncoderCacheManagerMetadata +from vllm.multimodal.utils import strip_covered_mm_data if TYPE_CHECKING: import numpy as np @@ -53,11 +54,16 @@ class NewRequestData: request: Request, block_ids: tuple[list[int], ...], prefill_token_ids: list[int] | None = None, + uses_mrope: bool = False, ) -> "NewRequestData": return cls( req_id=request.request_id, prompt_token_ids=request.prompt_token_ids, - mm_features=request.mm_features, + mm_features=strip_covered_mm_data( + request.mm_features, + request.num_computed_tokens, + uses_mrope=uses_mrope, + ), sampling_params=request.sampling_params, pooling_params=request.pooling_params, block_ids=block_ids, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index e4a2132866..2115ca6734 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -82,6 +82,7 @@ class Scheduler(SchedulerInterface): self.scheduler_config = vllm_config.scheduler_config self.cache_config = vllm_config.cache_config self.lora_config = vllm_config.lora_config + self.model_uses_mrope = vllm_config.model_config.uses_mrope self.kv_cache_config = kv_cache_config self.kv_events_config = vllm_config.kv_events_config self.parallel_config = vllm_config.parallel_config @@ -1199,13 +1200,16 @@ class Scheduler(SchedulerInterface): req, req_to_new_blocks[req.request_id].get_block_ids(), req._all_token_ids, + uses_mrope=self.model_uses_mrope, ) for req in scheduled_new_reqs ] else: new_reqs_data = [ NewRequestData.from_request( - req, req_to_new_blocks[req.request_id].get_block_ids() + req, + req_to_new_blocks[req.request_id].get_block_ids(), + uses_mrope=self.model_uses_mrope, ) for req in scheduled_new_reqs ]