[v1] Add PrefixLM support to FlexAttention backend (#27938)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2025-12-07 15:51:36 +00:00
committed by GitHub
parent 541a2ef892
commit b952f4d3c3
16 changed files with 173 additions and 25 deletions
+25 -1
View File
@@ -48,7 +48,10 @@ from vllm.distributed.parallel_state import (
is_global_first_rank,
prepare_communication_buffer_for_model,
)
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.forward_context import (
BatchDescriptor,
set_forward_context,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.rotary_embedding import (
@@ -329,6 +332,7 @@ class GPUModelRunner(
self.use_alibi = model_config.uses_alibi
self.cascade_attn_enabled = not self.model_config.disable_cascade_attn
self.is_mm_prefix_lm = self.model_config.is_mm_prefix_lm
# Multi-modal data support
self.mm_registry = MULTIMODAL_REGISTRY
@@ -1700,6 +1704,26 @@ class GPUModelRunner(
for layer_name in attn_group.layer_names:
attn_metadata[layer_name] = attn_metadata_i
if self.is_mm_prefix_lm:
req_doc_ranges = {}
for req_id in self.input_batch.req_ids:
image_doc_ranges = []
req_state = self.requests[req_id]
for mm_feature in req_state.mm_features:
pos_info = mm_feature.mm_position
img_doc_range = pos_info.extract_embeds_range()
image_doc_ranges.extend(img_doc_range)
req_idx = self.input_batch.req_id_to_index[req_id]
req_doc_ranges[req_idx] = image_doc_ranges
if isinstance(attn_metadata, list):
for ub_metadata in attn_metadata:
for _metadata in ub_metadata.values():
_metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined]
else:
for _metadata in attn_metadata.values():
_metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined]
if spec_decode_common_attn_metadata is not None and (
num_reqs != num_reqs_padded or num_tokens != num_tokens_padded
):