[Bugfix] Keep grid dims for XD-RoPE models on a prefix-cache hit (#53456)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
drakosha
2026-08-25 01:14:32 +00:00
committed by GitHub
co-authored by Claude
parent 09fddeb4ce
commit 9bf0d4717e
5 changed files with 32 additions and 4 deletions
+18
View File
@@ -105,6 +105,24 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
)
def test_strip_covered_mm_data_xdrope() -> None:
"""XD-RoPE models (e.g. HunyuanOCR) compute positions the same way, so a
covered item must keep its grid dims: stripping them made the worker index
an empty ``image_grid_thw`` and crash the engine on the second request with
an identical prompt."""
covered = _mm_feature_mixed(offset=0, length=100)
uncovered = _mm_feature_mixed(offset=300, length=100)
stripped = strip_covered_mm_data(
[covered, uncovered], num_computed_tokens=250, uses_xdrope=True
)
assert stripped[0].data is not None
assert list(stripped[0].data.keys()) == ["image_grid_thw"]
assert stripped[1].data is not None
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
def test_strip_covered_mm_data_mrope() -> None:
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
(the worker needs them to compute positions); payload fields are dropped."""
+4
View File
@@ -1843,6 +1843,10 @@ class ModelConfig:
def uses_xdrope_dim(self) -> int:
return uses_xdrope_dim(self.hf_config)
@property
def uses_xdrope(self) -> bool:
return self.uses_xdrope_dim > 0
@property
def is_multimodal_model(self) -> bool:
return self.multimodal_config is not None
+5 -4
View File
@@ -216,12 +216,13 @@ def strip_covered_mm_data(
mm_features: list[MultiModalFeatureSpec],
num_computed_tokens: int,
uses_mrope: bool = False,
uses_xdrope: bool = False,
) -> list[MultiModalFeatureSpec]:
"""Drop the tensor data of mm items whose placeholder span is fully inside
a prefix-cache-covered region: no encoder run can be scheduled for them,
so the workers never consume the payload fields. M-RoPE models are the
exception: the worker computes positions for the whole prompt from the
CPU-side metadata fields (e.g. grid dims), so those are kept. The
so the workers never consume the payload fields. M-RoPE and XD-RoPE models
are the exception: the worker computes positions for the whole prompt from
the CPU-side metadata fields (e.g. grid dims), so those are kept. The
scheduler-side ``Request`` keeps the full features."""
if not mm_features or num_computed_tokens == 0:
return mm_features
@@ -233,7 +234,7 @@ def strip_covered_mm_data(
return f
data = None
if uses_mrope:
if uses_mrope or uses_xdrope:
data = MultiModalKwargsItem(
{k: elem for k, elem in f.data.items() if elem.field.keep_on_cpu}
)
+2
View File
@@ -55,6 +55,7 @@ class NewRequestData:
block_ids: tuple[list[int], ...],
prefill_token_ids: list[int] | None = None,
uses_mrope: bool = False,
uses_xdrope: bool = False,
) -> "NewRequestData":
return cls(
req_id=request.request_id,
@@ -63,6 +64,7 @@ class NewRequestData:
request.mm_features,
request.num_computed_tokens,
uses_mrope=uses_mrope,
uses_xdrope=uses_xdrope,
),
sampling_params=request.sampling_params,
pooling_params=request.pooling_params,
+3
View File
@@ -87,6 +87,7 @@ class Scheduler(SchedulerInterface):
self.cache_config = vllm_config.cache_config
self.lora_config = vllm_config.lora_config
self.model_uses_mrope = vllm_config.model_config.uses_mrope
self.model_uses_xdrope = vllm_config.model_config.uses_xdrope
self.kv_cache_config = kv_cache_config
self.kv_events_config = vllm_config.kv_events_config
self.parallel_config = vllm_config.parallel_config
@@ -1223,6 +1224,7 @@ class Scheduler(SchedulerInterface):
req_to_new_blocks[req.request_id].get_block_ids(),
req._all_token_ids,
uses_mrope=self.model_uses_mrope,
uses_xdrope=self.model_uses_xdrope,
)
for req in scheduled_new_reqs
]
@@ -1232,6 +1234,7 @@ class Scheduler(SchedulerInterface):
req,
req_to_new_blocks[req.request_id].get_block_ids(),
uses_mrope=self.model_uses_mrope,
uses_xdrope=self.model_uses_xdrope,
)
for req in scheduled_new_reqs
]