[Encoder Decoder] Update Mllama to run with both FlashAttention and XFormers (#9982)
Signed-off-by: Sourashis Roy <sroy@roblox.com>
This commit is contained in:
@@ -32,6 +32,8 @@ from transformers.models.mllama.processing_mllama import (
|
||||
|
||||
import vllm.distributed.parallel_state as ps
|
||||
from vllm.attention import Attention, AttentionMetadata, AttentionType
|
||||
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.attention.backends.xformers import XFormersMetadata
|
||||
from vllm.attention.ops.paged_attn import PagedAttention
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
@@ -799,12 +801,13 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
q = self.q_norm(q)
|
||||
|
||||
if attention_mask is not None:
|
||||
output = self.attention_with_mask(q, k, v, kv_cache,
|
||||
attention_mask,
|
||||
kv_range_for_decode,
|
||||
attn_metadata)
|
||||
output = self._attention_with_mask(q, k, v, kv_cache,
|
||||
attention_mask,
|
||||
kv_range_for_decode,
|
||||
attn_metadata)
|
||||
else:
|
||||
output = self.attn(q,
|
||||
output = self.attn(q.view(-1,
|
||||
self.num_local_heads * self.head_dim),
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
@@ -813,7 +816,7 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
out, _ = self.o_proj(output)
|
||||
return out
|
||||
|
||||
def attention_with_mask(
|
||||
def _attention_with_mask(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -824,14 +827,35 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
attn_metadata: AttentionMetadata,
|
||||
) -> torch.Tensor:
|
||||
# Skip writing kv-cache for the initial profiling run.
|
||||
if len(kv_cache.shape) == 3:
|
||||
key_cache, value_cache = PagedAttention.split_kv_cache(
|
||||
kv_cache, self.num_local_key_value_heads, self.head_dim)
|
||||
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
|
||||
cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
|
||||
PagedAttention.write_to_paged_cache(
|
||||
cached_k, cached_v, key_cache, value_cache,
|
||||
attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
|
||||
if len(kv_cache.shape) > 1:
|
||||
if isinstance(attn_metadata, FlashAttentionMetadata):
|
||||
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
|
||||
cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
cached_k,
|
||||
cached_v,
|
||||
kv_cache[0],
|
||||
kv_cache[1],
|
||||
attn_metadata.
|
||||
cross_slot_mapping, # type: ignore[union-attr]
|
||||
"auto",
|
||||
1.0,
|
||||
1.0,
|
||||
)
|
||||
elif isinstance(attn_metadata, XFormersMetadata):
|
||||
key_cache, value_cache = PagedAttention.split_kv_cache(
|
||||
kv_cache, self.num_local_key_value_heads, self.head_dim)
|
||||
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
|
||||
cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
|
||||
PagedAttention.write_to_paged_cache(
|
||||
cached_k, cached_v, key_cache, value_cache,
|
||||
attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported AttentionMetadata {type(attn_metadata)} "
|
||||
f"class found. Expected the AttentionMetadata to "
|
||||
f"be either XFormersMetadata or FlashAttentionMetadata.")
|
||||
|
||||
# We have to call torch.sdpa for prefill when using a
|
||||
# custom cross-attention mask. Because the mask is not a
|
||||
# standard causal mask, neither a block diagonal mask which
|
||||
|
||||
@@ -9,15 +9,13 @@ from vllm.attention.backends.abstract import (AttentionBackend,
|
||||
AttentionMetadata)
|
||||
from vllm.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
|
||||
get_global_forced_attn_backend,
|
||||
global_force_attn_backend)
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
get_global_forced_attn_backend)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.inputs import INPUT_REGISTRY, InputRegistry
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.model_loader.utils import get_architecture_class_name
|
||||
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
|
||||
MultiModalRegistry)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -35,11 +33,6 @@ from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The Mllama model has PagedAttention specific logic because of which it
|
||||
# can only be run with the XFORMERS backend
|
||||
# TODO Make Mllama model work with Flash Attention backend.
|
||||
_XFORMERS_ONLY_ENCODER_DECODER_ARCHS = ["MllamaForConditionalGeneration"]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
|
||||
@@ -97,7 +90,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
|
||||
models) but these arguments are present here for compatibility with
|
||||
the base-class constructor.
|
||||
'''
|
||||
self._maybe_force_supported_attention_backend(vllm_config.model_config)
|
||||
self._maybe_force_supported_attention_backend()
|
||||
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
@@ -108,12 +101,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
|
||||
# Crash for unsupported encoder/scenarios
|
||||
assert_enc_dec_mr_supported_scenario(self)
|
||||
|
||||
def _is_xformers_only_encoder_decoder_model(self,
|
||||
model: ModelConfig) -> bool:
|
||||
return get_architecture_class_name(
|
||||
model) in _XFORMERS_ONLY_ENCODER_DECODER_ARCHS
|
||||
|
||||
def _maybe_force_supported_attention_backend(self, model: ModelConfig):
|
||||
def _maybe_force_supported_attention_backend(self):
|
||||
'''
|
||||
Force vLLM to use the XFormers attention backend,
|
||||
which is currently the only supported option.
|
||||
@@ -128,23 +116,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
|
||||
maybe_global_forced_backend = get_global_forced_attn_backend()
|
||||
is_forced_by_global = maybe_global_forced_backend is not None
|
||||
is_forced_by_env_var = maybe_env_var_forced_backend is not None
|
||||
|
||||
if not (is_forced_by_global or is_forced_by_env_var) \
|
||||
and self._is_xformers_only_encoder_decoder_model(model):
|
||||
# The user has not already specified an attention backend
|
||||
# override
|
||||
logger.info(
|
||||
"Encoder-Decoder Model Architecture %s requires XFormers "
|
||||
"backend; overriding backend auto-selection and "
|
||||
"forcing XFormers.", get_architecture_class_name(model))
|
||||
global_force_attn_backend(_Backend.XFORMERS)
|
||||
elif is_forced_by_global:
|
||||
if is_forced_by_global: # noqa: SIM102
|
||||
# Backend override enforced by global variable takes
|
||||
# precedence over vLLM backend environment variable.
|
||||
if maybe_global_forced_backend not in\
|
||||
[_Backend.XFORMERS, _Backend.FLASH_ATTN]:
|
||||
raise_backend_err()
|
||||
elif is_forced_by_env_var:
|
||||
elif is_forced_by_env_var: # noqa: SIM102
|
||||
# Backend override enforced by vLLM backend
|
||||
# environment variable
|
||||
if maybe_env_var_forced_backend not in\
|
||||
|
||||
Reference in New Issue
Block a user