[MRV2] Enable mm prefix bidi attention support on MRV2 (#46942)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Isotr0py
2026-07-05 14:45:29 +00:00
committed by GitHub
co-authored by Nick Hill
parent 8974ed89cd
commit cc1d020d01
5 changed files with 165 additions and 3 deletions
+1 -1
View File
@@ -1252,7 +1252,7 @@ class ModelConfig:
def is_deepseek_mla(self) -> bool:
return self.model_arch_config.is_deepseek_mla
@property
@cached_property
def is_mm_prefix_lm(self) -> bool:
return self.model_arch_config.is_mm_prefix_lm
+27
View File
@@ -14,6 +14,7 @@ from vllm.config import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backend import (
AttentionCGSupport,
@@ -487,6 +488,7 @@ def build_attn_metadata(
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
dcp_local_seq_lens: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
for_cudagraph_capture: bool = False,
causal: bool = True,
@@ -523,6 +525,7 @@ def build_attn_metadata(
causal=causal,
dcp_local_seq_lens=dcp_local_seq_lens,
positions=positions,
mm_req_doc_ranges=mm_req_doc_ranges,
rswa_prefix_lens=rswa_prefix_lens,
**common_attn_metadata_extra_kwargs,
)
@@ -550,3 +553,27 @@ def build_attn_metadata(
for layer_name in attn_group.layer_names:
attn_metadata[layer_name] = metadata
return attn_metadata
def compute_mm_prefix_ranges(
req_ids: list[str],
mm_features: dict[str, list[MultiModalFeatureSpec]],
sliding_window: int | None = None,
) -> dict[int, list[tuple[int, int]]]:
"""Compute PrefixLM bidirectional ranges for multimodal tokens.
Ranges exceeding sliding_window are skipped to prevent early tokens
from attending across the entire image span.
"""
req_doc_ranges: dict[int, list[tuple[int, int]]] = {}
for req_idx, req_id in enumerate(req_ids):
image_doc_ranges = []
for mm_feature in mm_features.get(req_id, ()):
if mm_feature.modality not in ("image", "video"):
continue
for r in mm_feature.mm_position.extract_embeds_range():
if sliding_window is not None and (r[1] - r[0] + 1) > sliding_window:
continue
image_doc_ranges.append(r)
req_doc_ranges[req_idx] = image_doc_ranges
return req_doc_ranges
+16 -1
View File
@@ -9,7 +9,10 @@ from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
compute_mm_prefix_ranges,
)
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.rope import get_rope_state
@@ -152,6 +155,17 @@ class DefaultModelState(ModelState):
max_seq_len = self.max_model_len
else:
max_seq_len = seq_lens_cpu_upper_bound[:num_reqs].max().item()
req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None
if (
self.supports_mm_inputs
and self.encoder_cache is not None
and self.model_config.is_mm_prefix_lm
):
req_doc_ranges = compute_mm_prefix_ranges(
req_ids=input_batch.req_ids,
mm_features=self.encoder_cache.mm_features,
sliding_window=self.model_config.get_sliding_window(),
)
attn_metadata = build_attn_metadata(
attn_groups=attn_groups,
num_reqs=num_reqs,
@@ -167,6 +181,7 @@ class DefaultModelState(ModelState):
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
positions=input_batch.positions,
mm_req_doc_ranges=req_doc_ranges,
for_cudagraph_capture=for_capture,
rswa_prefix_lens=input_batch.prompt_lens,
)