From d6aca837c3c6b773a557448d526db219a5a5718b Mon Sep 17 00:00:00 2001 From: Maroon Ayoub Date: Mon, 12 Oct 2026 21:56:51 +0300 Subject: [PATCH] [Attention] Make the sparse MLA prefill workspace size configurable (#61023) Signed-off-by: Maroon Ayoub Co-authored-by: Claude --- vllm/config/attention.py | 9 +++++++++ vllm/config/vllm.py | 10 ++++++++++ .../attention/backends/mla/flashmla_sparse.py | 20 ++++++++----------- vllm/v1/attention/backends/mla/indexer.py | 11 +++++----- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/vllm/config/attention.py b/vllm/config/attention.py index 80548f58f2..67605f3c7e 100644 --- a/vllm/config/attention.py +++ b/vllm/config/attention.py @@ -117,6 +117,15 @@ class AttentionConfig: When False (default), pure prefill batches use forward_mha when implemented. Set to True to always use the MQA path.""" + sparse_mla_prefill_workspace_factor: int | None = Field(default=None, ge=2) + """Sparse MLA prefill workspace size, in multiples of `max_model_len`. + FlashMLA sparse gathers a prefill step's context KV into this workspace and + runs attention in more chunks when the step holds more. The sparse indexer's + prefill K buffer is sized to fit in the same memory. Lower values return + memory to the KV cache at long `max_model_len`; the minimum of 2 leaves room + for a full-length request's context-parallel shard. If None, 5 when + `max_model_len` is at most 256K tokens, else 2.""" + flex_attn_block_m: int | None = None """Triton kernel BLOCK_M tile size for flex attention. Must be a power of 2 >= 16. If None and VLLM_BATCH_INVARIANT=1, diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index af4c99113f..f27c8418ab 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1787,6 +1787,16 @@ class VllmConfig: self._maybe_disable_dynamic_sd_for_data_parallel() self._maybe_override_dynamic_sd_cudagraph_mode() + if ( + self.attention_config.sparse_mla_prefill_workspace_factor is None + and self.model_config is not None + ): + # Resolved once: KV cache auto-fit can lower max_model_len later, and + # factor * max_model_len must stay within the workspace reserved at init. + self.attention_config.sparse_mla_prefill_workspace_factor = ( + 5 if self.model_config.max_model_len <= 256 * 1024 else 2 + ) + if ( self.attention_config.hisparse_config is None and self.kv_transfer_config is not None diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 4fe0be0b1f..3cf0c33c53 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -334,14 +334,14 @@ class FlashMLASparseMetadata(SparseMLACommonMetadata): fp8_use_mixed_batch: bool = False -def get_prefill_workspace_size(max_model_len: int): - # NOTE(Lucas): 5 is a magic number for controlling the prefill buffer size. - # May be tuned later. - # Memory usage: 5 * max_model_len * 576 * 2 bytes - # Example: DeepSeek-V3.2 with max_model_len=163840 -> +def get_prefill_workspace_size(vllm_config: VllmConfig) -> int: + # Memory usage: factor * max_model_len * 576 * 2 bytes + # Example: DeepSeek-V3.2 with max_model_len=163840 and the default factor -> # 5 * 163840 * 576 * 2 = ~900 MB # This fits nicely below the typical MoE workspace size of >2GB so this is "free" - return max_model_len * 5 + factor = vllm_config.attention_config.sparse_mla_prefill_workspace_factor + assert factor is not None + return vllm_config.model_config.max_model_len * factor class FlashMLASparseMetadataBuilder( @@ -562,9 +562,7 @@ class FlashMLASparseMetadataBuilder( prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:] # Chunk prefill requests to fit within workspace size - max_prefill_buffer_size = get_prefill_workspace_size( - self.vllm_config.model_config.max_model_len - ) + max_prefill_buffer_size = get_prefill_workspace_size(self.vllm_config) # Workspace rows per entry, and the prefill rows each entry covers. workspace_rows = prefill_seq_lens_cpu row_bounds = np.arange(num_prefills + 1) @@ -804,9 +802,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS: # Reserve workspace during initialization assert vllm_config is not None and vllm_config.model_config is not None - prefill_workspace_size = get_prefill_workspace_size( - vllm_config.model_config.max_model_len - ) + prefill_workspace_size = get_prefill_workspace_size(vllm_config) parallel_config = vllm_config.parallel_config self.pcp_dcp_kv_gather = ( parallel_config.prefill_context_parallel_size > 1 diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 97de9ffa9b..2d797ae662 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -917,15 +917,16 @@ class KpoolTailMetadataBuilder(AttentionMetadataBuilder): def get_max_prefill_buffer_size(vllm_config: VllmConfig): max_model_len = vllm_config.model_config.max_model_len - # NOTE(Chen): 40 is a magic number for controlling the prefill buffer size. + factor = vllm_config.attention_config.sparse_mla_prefill_workspace_factor + assert factor is not None # Each entry is 128 fp8 bytes and 4 scale bytes for a total of 132 bytes. - # The flashmla_sparse backend uses a workspace size of 5 * max_model_len. + # The flashmla_sparse backend uses a workspace size of factor * max_model_len. # The memory usage of the workspace there is 576 * 2 bytes; so we size this as - # (576 * 2 // 132) * 5 = 40 to maximize this workspace size while still fitting - # within the flashmla_sparse workspace. + # (576 * 2 // 132) * factor (40 at factor 5) to maximize this workspace + # size while still fitting within the flashmla_sparse workspace. # For DeepSeek-V3.2, the max_model_len is 163840. # 40 * 163840 * 132 = 865075200 bytes = 825 MB - return max_model_len * 40 + return max_model_len * (576 * 2 // 132) * factor def _supports_varlen_paged_mqa_logits() -> bool: