diff --git a/vllm/v1/attention/backends/mla/prefill/flashinfer.py b/vllm/v1/attention/backends/mla/prefill/flashinfer.py index 77199fb523..0204f6ee1a 100644 --- a/vllm/v1/attention/backends/mla/prefill/flashinfer.py +++ b/vllm/v1/attention/backends/mla/prefill/flashinfer.py @@ -77,6 +77,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend): self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = [] self._global_hyperparameters: PerLayerParameters | None = None + (self._workspace_buffer,) = current_workspace_manager().get_simultaneous( + ((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8), + ) def _ensure_chunks( self, @@ -123,21 +126,17 @@ class FlashInferPrefillBackend(MLAPrefillBackend): global_hyperparameters = self._resolve_global_hyperparameters() qo_indptr = prefill_metadata.query_start_loc has_context = prefill_metadata.chunked_context is not None - (workspace_buffer,) = current_workspace_manager().get_simultaneous( - ((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8), - ) - if self._prefill_main is None: self._prefill_main = BatchPrefillWithRaggedKVCacheWrapper( - workspace_buffer, "NHD", backend="cutlass" + self._workspace_buffer, "NHD", backend="cutlass" ) - self._ensure_chunks(_DEFAULT_NUM_CHUNKS, workspace_buffer) + self._ensure_chunks(_DEFAULT_NUM_CHUNKS, self._workspace_buffer) if has_context: chunked_context = prefill_metadata.chunked_context assert chunked_context is not None num_chunks = chunked_context.cu_seq_lens.shape[0] - self._ensure_chunks(num_chunks, workspace_buffer) + self._ensure_chunks(num_chunks, self._workspace_buffer) num_qo_heads = self.num_heads num_kv_heads = num_qo_heads diff --git a/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py b/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py index 7462dc39e7..afb0444a31 100644 --- a/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py +++ b/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py @@ -61,15 +61,12 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): v_head_dim=v_head_dim, vllm_config=vllm_config, ) - - def _get_workspace_buffer(self) -> torch.Tensor: - (workspace_buffer,) = current_workspace_manager().get_simultaneous( + (self._workspace_buffer,) = current_workspace_manager().get_simultaneous( ( (envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8, ), ) - return workspace_buffer def prepare_metadata( self, @@ -89,7 +86,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: from flashinfer.prefill import trtllm_ragged_attention_deepseek - workspace_buffer = self._get_workspace_buffer() out = torch.empty( q.shape[0], q.shape[1], @@ -102,7 +98,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): query=q, key=k, value=v, - workspace_buffer=workspace_buffer, + workspace_buffer=self._workspace_buffer, seq_lens=self._query_seq_lens, max_q_len=self._prefill_metadata.max_query_len, max_kv_len=self._prefill_metadata.max_query_len, @@ -135,7 +131,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): assert self._prefill_metadata.chunked_context is not None assert self._prefill_metadata.chunked_context.seq_lens[chunk_idx] is not None - workspace_buffer = self._get_workspace_buffer() out = torch.empty( q.shape[0], @@ -149,7 +144,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): query=q, key=k, value=v, - workspace_buffer=workspace_buffer, + workspace_buffer=self._workspace_buffer, seq_lens=self._prefill_metadata.chunked_context.seq_lens[chunk_idx], max_q_len=self._prefill_metadata.max_query_len, max_kv_len=self._prefill_metadata.chunked_context.max_seq_lens[chunk_idx],