[FlexAttention] make custom mask mods fully cudagraphable (#45232)

Signed-off-by: Angel Li <liangel@meta.com>
This commit is contained in:
liangel-02
2026-06-17 11:53:12 +08:00
committed by GitHub
parent 7b5d60cc37
commit 20a5f8b43b
2 changed files with 98 additions and 1 deletions
+67
View File
@@ -13,6 +13,7 @@ from tests.v1.attention.utils import (
create_standard_kv_cache_spec,
create_vllm_config,
)
from vllm.model_executor.layers.attention import Attention
from vllm.v1.attention.backends.flex_attention import (
BlockSparsityHint,
FlexAttentionMetadataBuilder,
@@ -79,6 +80,72 @@ def test_flex_attention_full_cudagraphs(vllm_runner):
)
def windowed_causal_mask_mod(b, h, q_idx, kv_idx):
return (kv_idx <= q_idx) & (q_idx - kv_idx < 4)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
)
def test_flex_attention_custom_mask_full_cudagraphs(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setattr(
Attention,
"logical_mask_mod",
staticmethod(windowed_causal_mask_mod),
raising=False,
)
model_name = "Qwen/Qwen2.5-1.5B-Instruct"
seed = 42
max_tokens = 24
num_logprobs = 5
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
]
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=True,
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_eager:
output_eager = llm_eager.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
set_random_seed(seed)
with vllm_runner(
model_name,
runner="generate",
tensor_parallel_size=1,
num_gpu_blocks_override=128,
enforce_eager=False,
gpu_memory_utilization=0.85,
compilation_config={
"cudagraph_mode": "FULL",
"cudagraph_capture_sizes": [4],
},
attention_config={"backend": "FLEX_ATTENTION"},
) as llm_cudagraph:
output_cudagraph = llm_cudagraph.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=output_eager,
outputs_1_lst=output_cudagraph,
name_0="eager",
name_1="cudagraph",
)
@pytest.mark.skipif(
not torch.cuda.is_available() or TORCH_VERSION < MINIMUM_TORCH_VERSION,
reason="CUDA not available or PyTorch version < 2.7",
+31 -1
View File
@@ -22,9 +22,10 @@ from torch.nn.attention.flex_attention import (
)
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import is_quantized_kv_cache, is_torch_equal_or_newer
@@ -807,6 +808,13 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
self.persistent_physical_to_logical = None
self.persistent_kv_indices = None
self.custom_logical_mask_mod: _mask_mod_signature | None = None
if self._uses_full_cudagraphs():
layers = get_layers_from_vllm_config(
vllm_config, Attention, self.layer_names
)
self.custom_logical_mask_mod = self._maybe_get_custom_mask_mod(layers)
@staticmethod
def _get_block_sizes(
attn_cfg,
@@ -853,6 +861,21 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
def _maybe_get_custom_mask_mod(self, layers) -> _mask_mod_signature | None:
mask_mods = {
getattr(layer, "logical_mask_mod", None) for layer in layers.values()
}
if len(mask_mods) > 1:
raise ValueError(
f"Found differing mask mods {mask_mods}, "
"cannot use alternating mask mods w/ full CUDA graphs"
)
return next(iter(mask_mods), None)
def _uses_full_cudagraphs(self) -> bool:
mode = self.vllm_config.compilation_config.cudagraph_mode
return mode is not None and mode.has_full_cudagraphs()
def build(
self,
common_prefix_len: int,
@@ -924,9 +947,16 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat
else causal_mask_mod
)
sliding_window = None
if self._uses_full_cudagraphs():
if self.custom_logical_mask_mod is not None:
logical_mask_mod = self.custom_logical_mask_mod
sliding_window = getattr(self.kv_cache_spec, "sliding_window", None)
out = FlexAttentionMetadata(
causal=common_attn_metadata.causal,
logical_mask_mod=logical_mask_mod,
sliding_window=sliding_window,
num_actual_tokens=num_actual_tokens,
max_query_len=max_query_len,
query_start_loc=query_start_loc,