[Attention] Make the sparse MLA prefill workspace size configurable (#61023)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Maroon Ayoub
2026-10-12 14:56:51 -04:00
committed by GitHub
co-authored by Claude
parent 56e3b081ae
commit d6aca837c3
4 changed files with 33 additions and 17 deletions
+9
View File
@@ -117,6 +117,15 @@ class AttentionConfig:
When False (default), pure prefill batches use forward_mha when implemented.
Set to True to always use the MQA path."""
sparse_mla_prefill_workspace_factor: int | None = Field(default=None, ge=2)
"""Sparse MLA prefill workspace size, in multiples of `max_model_len`.
FlashMLA sparse gathers a prefill step's context KV into this workspace and
runs attention in more chunks when the step holds more. The sparse indexer's
prefill K buffer is sized to fit in the same memory. Lower values return
memory to the KV cache at long `max_model_len`; the minimum of 2 leaves room
for a full-length request's context-parallel shard. If None, 5 when
`max_model_len` is at most 256K tokens, else 2."""
flex_attn_block_m: int | None = None
"""Triton kernel BLOCK_M tile size for flex attention.
Must be a power of 2 >= 16. If None and VLLM_BATCH_INVARIANT=1,
+10
View File
@@ -1787,6 +1787,16 @@ class VllmConfig:
self._maybe_disable_dynamic_sd_for_data_parallel()
self._maybe_override_dynamic_sd_cudagraph_mode()
if (
self.attention_config.sparse_mla_prefill_workspace_factor is None
and self.model_config is not None
):
# Resolved once: KV cache auto-fit can lower max_model_len later, and
# factor * max_model_len must stay within the workspace reserved at init.
self.attention_config.sparse_mla_prefill_workspace_factor = (
5 if self.model_config.max_model_len <= 256 * 1024 else 2
)
if (
self.attention_config.hisparse_config is None
and self.kv_transfer_config is not None
@@ -334,14 +334,14 @@ class FlashMLASparseMetadata(SparseMLACommonMetadata):
fp8_use_mixed_batch: bool = False
def get_prefill_workspace_size(max_model_len: int):
# NOTE(Lucas): 5 is a magic number for controlling the prefill buffer size.
# May be tuned later.
# Memory usage: 5 * max_model_len * 576 * 2 bytes
# Example: DeepSeek-V3.2 with max_model_len=163840 ->
def get_prefill_workspace_size(vllm_config: VllmConfig) -> int:
# Memory usage: factor * max_model_len * 576 * 2 bytes
# Example: DeepSeek-V3.2 with max_model_len=163840 and the default factor ->
# 5 * 163840 * 576 * 2 = ~900 MB
# This fits nicely below the typical MoE workspace size of >2GB so this is "free"
return max_model_len * 5
factor = vllm_config.attention_config.sparse_mla_prefill_workspace_factor
assert factor is not None
return vllm_config.model_config.max_model_len * factor
class FlashMLASparseMetadataBuilder(
@@ -562,9 +562,7 @@ class FlashMLASparseMetadataBuilder(
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
# Chunk prefill requests to fit within workspace size
max_prefill_buffer_size = get_prefill_workspace_size(
self.vllm_config.model_config.max_model_len
)
max_prefill_buffer_size = get_prefill_workspace_size(self.vllm_config)
# Workspace rows per entry, and the prefill rows each entry covers.
workspace_rows = prefill_seq_lens_cpu
row_bounds = np.arange(num_prefills + 1)
@@ -804,9 +802,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS:
# Reserve workspace during initialization
assert vllm_config is not None and vllm_config.model_config is not None
prefill_workspace_size = get_prefill_workspace_size(
vllm_config.model_config.max_model_len
)
prefill_workspace_size = get_prefill_workspace_size(vllm_config)
parallel_config = vllm_config.parallel_config
self.pcp_dcp_kv_gather = (
parallel_config.prefill_context_parallel_size > 1
+6 -5
View File
@@ -917,15 +917,16 @@ class KpoolTailMetadataBuilder(AttentionMetadataBuilder):
def get_max_prefill_buffer_size(vllm_config: VllmConfig):
max_model_len = vllm_config.model_config.max_model_len
# NOTE(Chen): 40 is a magic number for controlling the prefill buffer size.
factor = vllm_config.attention_config.sparse_mla_prefill_workspace_factor
assert factor is not None
# Each entry is 128 fp8 bytes and 4 scale bytes for a total of 132 bytes.
# The flashmla_sparse backend uses a workspace size of 5 * max_model_len.
# The flashmla_sparse backend uses a workspace size of factor * max_model_len.
# The memory usage of the workspace there is 576 * 2 bytes; so we size this as
# (576 * 2 // 132) * 5 = 40 to maximize this workspace size while still fitting
# within the flashmla_sparse workspace.
# (576 * 2 // 132) * factor (40 at factor 5) to maximize this workspace
# size while still fitting within the flashmla_sparse workspace.
# For DeepSeek-V3.2, the max_model_len is 163840.
# 40 * 163840 * 132 = 865075200 bytes = 825 MB
return max_model_len * 40
return max_model_len * (576 * 2 // 132) * factor
def _supports_varlen_paged_mqa_logits() -> bool: