[Bugfix] Keep grid dims for XD-RoPE models on a prefix-cache hit (#53456)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -105,6 +105,24 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
|
||||
)
|
||||
|
||||
|
||||
def test_strip_covered_mm_data_xdrope() -> None:
|
||||
"""XD-RoPE models (e.g. HunyuanOCR) compute positions the same way, so a
|
||||
covered item must keep its grid dims: stripping them made the worker index
|
||||
an empty ``image_grid_thw`` and crash the engine on the second request with
|
||||
an identical prompt."""
|
||||
covered = _mm_feature_mixed(offset=0, length=100)
|
||||
uncovered = _mm_feature_mixed(offset=300, length=100)
|
||||
|
||||
stripped = strip_covered_mm_data(
|
||||
[covered, uncovered], num_computed_tokens=250, uses_xdrope=True
|
||||
)
|
||||
|
||||
assert stripped[0].data is not None
|
||||
assert list(stripped[0].data.keys()) == ["image_grid_thw"]
|
||||
assert stripped[1].data is not None
|
||||
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
|
||||
|
||||
|
||||
def test_strip_covered_mm_data_mrope() -> None:
|
||||
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
|
||||
(the worker needs them to compute positions); payload fields are dropped."""
|
||||
|
||||
@@ -1843,6 +1843,10 @@ class ModelConfig:
|
||||
def uses_xdrope_dim(self) -> int:
|
||||
return uses_xdrope_dim(self.hf_config)
|
||||
|
||||
@property
|
||||
def uses_xdrope(self) -> bool:
|
||||
return self.uses_xdrope_dim > 0
|
||||
|
||||
@property
|
||||
def is_multimodal_model(self) -> bool:
|
||||
return self.multimodal_config is not None
|
||||
|
||||
@@ -216,12 +216,13 @@ def strip_covered_mm_data(
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
num_computed_tokens: int,
|
||||
uses_mrope: bool = False,
|
||||
uses_xdrope: bool = False,
|
||||
) -> list[MultiModalFeatureSpec]:
|
||||
"""Drop the tensor data of mm items whose placeholder span is fully inside
|
||||
a prefix-cache-covered region: no encoder run can be scheduled for them,
|
||||
so the workers never consume the payload fields. M-RoPE models are the
|
||||
exception: the worker computes positions for the whole prompt from the
|
||||
CPU-side metadata fields (e.g. grid dims), so those are kept. The
|
||||
so the workers never consume the payload fields. M-RoPE and XD-RoPE models
|
||||
are the exception: the worker computes positions for the whole prompt from
|
||||
the CPU-side metadata fields (e.g. grid dims), so those are kept. The
|
||||
scheduler-side ``Request`` keeps the full features."""
|
||||
if not mm_features or num_computed_tokens == 0:
|
||||
return mm_features
|
||||
@@ -233,7 +234,7 @@ def strip_covered_mm_data(
|
||||
return f
|
||||
|
||||
data = None
|
||||
if uses_mrope:
|
||||
if uses_mrope or uses_xdrope:
|
||||
data = MultiModalKwargsItem(
|
||||
{k: elem for k, elem in f.data.items() if elem.field.keep_on_cpu}
|
||||
)
|
||||
|
||||
@@ -55,6 +55,7 @@ class NewRequestData:
|
||||
block_ids: tuple[list[int], ...],
|
||||
prefill_token_ids: list[int] | None = None,
|
||||
uses_mrope: bool = False,
|
||||
uses_xdrope: bool = False,
|
||||
) -> "NewRequestData":
|
||||
return cls(
|
||||
req_id=request.request_id,
|
||||
@@ -63,6 +64,7 @@ class NewRequestData:
|
||||
request.mm_features,
|
||||
request.num_computed_tokens,
|
||||
uses_mrope=uses_mrope,
|
||||
uses_xdrope=uses_xdrope,
|
||||
),
|
||||
sampling_params=request.sampling_params,
|
||||
pooling_params=request.pooling_params,
|
||||
|
||||
@@ -87,6 +87,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.lora_config = vllm_config.lora_config
|
||||
self.model_uses_mrope = vllm_config.model_config.uses_mrope
|
||||
self.model_uses_xdrope = vllm_config.model_config.uses_xdrope
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.kv_events_config = vllm_config.kv_events_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
@@ -1223,6 +1224,7 @@ class Scheduler(SchedulerInterface):
|
||||
req_to_new_blocks[req.request_id].get_block_ids(),
|
||||
req._all_token_ids,
|
||||
uses_mrope=self.model_uses_mrope,
|
||||
uses_xdrope=self.model_uses_xdrope,
|
||||
)
|
||||
for req in scheduled_new_reqs
|
||||
]
|
||||
@@ -1232,6 +1234,7 @@ class Scheduler(SchedulerInterface):
|
||||
req,
|
||||
req_to_new_blocks[req.request_id].get_block_ids(),
|
||||
uses_mrope=self.model_uses_mrope,
|
||||
uses_xdrope=self.model_uses_xdrope,
|
||||
)
|
||||
for req in scheduled_new_reqs
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user