[v1] Add PrefixLM support to FlexAttention backend (#27938)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -48,7 +48,10 @@ from vllm.distributed.parallel_state import (
|
||||
is_global_first_rank,
|
||||
prepare_communication_buffer_for_model,
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
set_forward_context,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.rotary_embedding import (
|
||||
@@ -329,6 +332,7 @@ class GPUModelRunner(
|
||||
self.use_alibi = model_config.uses_alibi
|
||||
|
||||
self.cascade_attn_enabled = not self.model_config.disable_cascade_attn
|
||||
self.is_mm_prefix_lm = self.model_config.is_mm_prefix_lm
|
||||
|
||||
# Multi-modal data support
|
||||
self.mm_registry = MULTIMODAL_REGISTRY
|
||||
@@ -1700,6 +1704,26 @@ class GPUModelRunner(
|
||||
for layer_name in attn_group.layer_names:
|
||||
attn_metadata[layer_name] = attn_metadata_i
|
||||
|
||||
if self.is_mm_prefix_lm:
|
||||
req_doc_ranges = {}
|
||||
for req_id in self.input_batch.req_ids:
|
||||
image_doc_ranges = []
|
||||
req_state = self.requests[req_id]
|
||||
for mm_feature in req_state.mm_features:
|
||||
pos_info = mm_feature.mm_position
|
||||
img_doc_range = pos_info.extract_embeds_range()
|
||||
image_doc_ranges.extend(img_doc_range)
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
req_doc_ranges[req_idx] = image_doc_ranges
|
||||
|
||||
if isinstance(attn_metadata, list):
|
||||
for ub_metadata in attn_metadata:
|
||||
for _metadata in ub_metadata.values():
|
||||
_metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined]
|
||||
else:
|
||||
for _metadata in attn_metadata.values():
|
||||
_metadata.mm_prefix_range = req_doc_ranges # type: ignore[attr-defined]
|
||||
|
||||
if spec_decode_common_attn_metadata is not None and (
|
||||
num_reqs != num_reqs_padded or num_tokens != num_tokens_padded
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user