From 7e950521b39fd201979a888eda82ff0f7ecc4731 Mon Sep 17 00:00:00 2001 From: Joe Rowell Date: Wed, 15 Jul 2026 02:18:41 +0200 Subject: [PATCH] fix: size FlashInfer prefill workspace to batch head footprint (#48428) Signed-off-by: Joe Rowell --- vllm/v1/attention/backends/flashinfer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 7342823943..da4e59d4ea 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -82,6 +82,7 @@ from vllm.v1.kv_cache_interface import ( from vllm.v1.utils import CpuGpuBuffer FLASHINFER_WORKSPACE_BUFFER_SIZE_BATCH_INVARIANT = 2048 * 1024 * 1024 +FLASHINFER_PREFILL_WORKSPACE_BYTES_PER_ELEM = 16 FP8_DTYPE = current_platform.fp8_dtype() FP4_DTYPE = torch.uint8 @@ -648,6 +649,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.disable_split_kv = False self.compilation_config = vllm_config.compilation_config + self.max_num_batched_tokens = ( + vllm_config.scheduler_config.max_num_batched_tokens + ) max_num_pages_per_req = cdiv( self.model_config.max_model_len, self.kv_cache_spec.block_size ) @@ -933,6 +937,21 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): buffer_size = envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE if envs.VLLM_BATCH_INVARIANT: buffer_size = FLASHINFER_WORKSPACE_BUFFER_SIZE_BATCH_INVARIANT + else: + # FlashInfer prefill temp buffers (batch_prefill_tmp_v, ...) + # scale with the prefill chunk and query-head footprint, NOT + # context length. The fixed ~394 MiB default is too small for + # wide-head models at the default 8192-token chunk on some + # archs (e.g. sm_120), where FlashInfer hard-errors instead of + # growing. Size to the batch's head footprint; never shrink + # below the configured default. + est = ( + self.max_num_batched_tokens + * self.num_qo_heads + * self.head_dim + * FLASHINFER_PREFILL_WORKSPACE_BYTES_PER_ELEM + ) + buffer_size = max(buffer_size, est) self._workspace_buffer = torch.zeros( buffer_size, dtype=torch.uint8, device=self.device )