[Attention] Make the sparse MLA prefill workspace size configurable (#61023)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -117,6 +117,15 @@ class AttentionConfig:
|
||||
When False (default), pure prefill batches use forward_mha when implemented.
|
||||
Set to True to always use the MQA path."""
|
||||
|
||||
sparse_mla_prefill_workspace_factor: int | None = Field(default=None, ge=2)
|
||||
"""Sparse MLA prefill workspace size, in multiples of `max_model_len`.
|
||||
FlashMLA sparse gathers a prefill step's context KV into this workspace and
|
||||
runs attention in more chunks when the step holds more. The sparse indexer's
|
||||
prefill K buffer is sized to fit in the same memory. Lower values return
|
||||
memory to the KV cache at long `max_model_len`; the minimum of 2 leaves room
|
||||
for a full-length request's context-parallel shard. If None, 5 when
|
||||
`max_model_len` is at most 256K tokens, else 2."""
|
||||
|
||||
flex_attn_block_m: int | None = None
|
||||
"""Triton kernel BLOCK_M tile size for flex attention.
|
||||
Must be a power of 2 >= 16. If None and VLLM_BATCH_INVARIANT=1,
|
||||
|
||||
@@ -1787,6 +1787,16 @@ class VllmConfig:
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if (
|
||||
self.attention_config.sparse_mla_prefill_workspace_factor is None
|
||||
and self.model_config is not None
|
||||
):
|
||||
# Resolved once: KV cache auto-fit can lower max_model_len later, and
|
||||
# factor * max_model_len must stay within the workspace reserved at init.
|
||||
self.attention_config.sparse_mla_prefill_workspace_factor = (
|
||||
5 if self.model_config.max_model_len <= 256 * 1024 else 2
|
||||
)
|
||||
|
||||
if (
|
||||
self.attention_config.hisparse_config is None
|
||||
and self.kv_transfer_config is not None
|
||||
|
||||
@@ -334,14 +334,14 @@ class FlashMLASparseMetadata(SparseMLACommonMetadata):
|
||||
fp8_use_mixed_batch: bool = False
|
||||
|
||||
|
||||
def get_prefill_workspace_size(max_model_len: int):
|
||||
# NOTE(Lucas): 5 is a magic number for controlling the prefill buffer size.
|
||||
# May be tuned later.
|
||||
# Memory usage: 5 * max_model_len * 576 * 2 bytes
|
||||
# Example: DeepSeek-V3.2 with max_model_len=163840 ->
|
||||
def get_prefill_workspace_size(vllm_config: VllmConfig) -> int:
|
||||
# Memory usage: factor * max_model_len * 576 * 2 bytes
|
||||
# Example: DeepSeek-V3.2 with max_model_len=163840 and the default factor ->
|
||||
# 5 * 163840 * 576 * 2 = ~900 MB
|
||||
# This fits nicely below the typical MoE workspace size of >2GB so this is "free"
|
||||
return max_model_len * 5
|
||||
factor = vllm_config.attention_config.sparse_mla_prefill_workspace_factor
|
||||
assert factor is not None
|
||||
return vllm_config.model_config.max_model_len * factor
|
||||
|
||||
|
||||
class FlashMLASparseMetadataBuilder(
|
||||
@@ -562,9 +562,7 @@ class FlashMLASparseMetadataBuilder(
|
||||
|
||||
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
|
||||
# Chunk prefill requests to fit within workspace size
|
||||
max_prefill_buffer_size = get_prefill_workspace_size(
|
||||
self.vllm_config.model_config.max_model_len
|
||||
)
|
||||
max_prefill_buffer_size = get_prefill_workspace_size(self.vllm_config)
|
||||
# Workspace rows per entry, and the prefill rows each entry covers.
|
||||
workspace_rows = prefill_seq_lens_cpu
|
||||
row_bounds = np.arange(num_prefills + 1)
|
||||
@@ -804,9 +802,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS:
|
||||
# Reserve workspace during initialization
|
||||
assert vllm_config is not None and vllm_config.model_config is not None
|
||||
prefill_workspace_size = get_prefill_workspace_size(
|
||||
vllm_config.model_config.max_model_len
|
||||
)
|
||||
prefill_workspace_size = get_prefill_workspace_size(vllm_config)
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.pcp_dcp_kv_gather = (
|
||||
parallel_config.prefill_context_parallel_size > 1
|
||||
|
||||
@@ -917,15 +917,16 @@ class KpoolTailMetadataBuilder(AttentionMetadataBuilder):
|
||||
|
||||
def get_max_prefill_buffer_size(vllm_config: VllmConfig):
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
# NOTE(Chen): 40 is a magic number for controlling the prefill buffer size.
|
||||
factor = vllm_config.attention_config.sparse_mla_prefill_workspace_factor
|
||||
assert factor is not None
|
||||
# Each entry is 128 fp8 bytes and 4 scale bytes for a total of 132 bytes.
|
||||
# The flashmla_sparse backend uses a workspace size of 5 * max_model_len.
|
||||
# The flashmla_sparse backend uses a workspace size of factor * max_model_len.
|
||||
# The memory usage of the workspace there is 576 * 2 bytes; so we size this as
|
||||
# (576 * 2 // 132) * 5 = 40 to maximize this workspace size while still fitting
|
||||
# within the flashmla_sparse workspace.
|
||||
# (576 * 2 // 132) * factor (40 at factor 5) to maximize this workspace
|
||||
# size while still fitting within the flashmla_sparse workspace.
|
||||
# For DeepSeek-V3.2, the max_model_len is 163840.
|
||||
# 40 * 163840 * 132 = 865075200 bytes = 825 MB
|
||||
return max_model_len * 40
|
||||
return max_model_len * (576 * 2 // 132) * factor
|
||||
|
||||
|
||||
def _supports_varlen_paged_mqa_logits() -> bool:
|
||||
|
||||
Reference in New Issue
Block a user