[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup (#42112)

Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
Mohammad Miadh Angkad
2026-05-14 09:48:56 -04:00
committed by GitHub
parent 5bd8c71e79
commit 2317682f95
2 changed files with 9 additions and 15 deletions
@@ -77,6 +77,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None
self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = []
self._global_hyperparameters: PerLayerParameters | None = None
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8),
)
def _ensure_chunks(
self,
@@ -123,21 +126,17 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
global_hyperparameters = self._resolve_global_hyperparameters()
qo_indptr = prefill_metadata.query_start_loc
has_context = prefill_metadata.chunked_context is not None
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8),
)
if self._prefill_main is None:
self._prefill_main = BatchPrefillWithRaggedKVCacheWrapper(
workspace_buffer, "NHD", backend="cutlass"
self._workspace_buffer, "NHD", backend="cutlass"
)
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, workspace_buffer)
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, self._workspace_buffer)
if has_context:
chunked_context = prefill_metadata.chunked_context
assert chunked_context is not None
num_chunks = chunked_context.cu_seq_lens.shape[0]
self._ensure_chunks(num_chunks, workspace_buffer)
self._ensure_chunks(num_chunks, self._workspace_buffer)
num_qo_heads = self.num_heads
num_kv_heads = num_qo_heads
@@ -61,15 +61,12 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
def _get_workspace_buffer(self) -> torch.Tensor:
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
(
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
torch.uint8,
),
)
return workspace_buffer
def prepare_metadata(
self,
@@ -89,7 +86,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
from flashinfer.prefill import trtllm_ragged_attention_deepseek
workspace_buffer = self._get_workspace_buffer()
out = torch.empty(
q.shape[0],
q.shape[1],
@@ -102,7 +98,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
query=q,
key=k,
value=v,
workspace_buffer=workspace_buffer,
workspace_buffer=self._workspace_buffer,
seq_lens=self._query_seq_lens,
max_q_len=self._prefill_metadata.max_query_len,
max_kv_len=self._prefill_metadata.max_query_len,
@@ -135,7 +131,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
assert self._prefill_metadata.chunked_context is not None
assert self._prefill_metadata.chunked_context.seq_lens[chunk_idx] is not None
workspace_buffer = self._get_workspace_buffer()
out = torch.empty(
q.shape[0],
@@ -149,7 +144,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
query=q,
key=k,
value=v,
workspace_buffer=workspace_buffer,
workspace_buffer=self._workspace_buffer,
seq_lens=self._prefill_metadata.chunked_context.seq_lens[chunk_idx],
max_q_len=self._prefill_metadata.max_query_len,
max_kv_len=self._prefill_metadata.chunked_context.max_seq_lens[chunk_idx],