[Bugfix] Fix TRTLLM ragged MLA prefill workspace warmup (#42112)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
@@ -77,6 +77,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None
|
||||
self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = []
|
||||
self._global_hyperparameters: PerLayerParameters | None = None
|
||||
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8),
|
||||
)
|
||||
|
||||
def _ensure_chunks(
|
||||
self,
|
||||
@@ -123,21 +126,17 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
global_hyperparameters = self._resolve_global_hyperparameters()
|
||||
qo_indptr = prefill_metadata.query_start_loc
|
||||
has_context = prefill_metadata.chunked_context is not None
|
||||
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8),
|
||||
)
|
||||
|
||||
if self._prefill_main is None:
|
||||
self._prefill_main = BatchPrefillWithRaggedKVCacheWrapper(
|
||||
workspace_buffer, "NHD", backend="cutlass"
|
||||
self._workspace_buffer, "NHD", backend="cutlass"
|
||||
)
|
||||
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, workspace_buffer)
|
||||
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, self._workspace_buffer)
|
||||
|
||||
if has_context:
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
num_chunks = chunked_context.cu_seq_lens.shape[0]
|
||||
self._ensure_chunks(num_chunks, workspace_buffer)
|
||||
self._ensure_chunks(num_chunks, self._workspace_buffer)
|
||||
|
||||
num_qo_heads = self.num_heads
|
||||
num_kv_heads = num_qo_heads
|
||||
|
||||
@@ -61,15 +61,12 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
def _get_workspace_buffer(self) -> torch.Tensor:
|
||||
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
(
|
||||
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
|
||||
torch.uint8,
|
||||
),
|
||||
)
|
||||
return workspace_buffer
|
||||
|
||||
def prepare_metadata(
|
||||
self,
|
||||
@@ -89,7 +86,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
from flashinfer.prefill import trtllm_ragged_attention_deepseek
|
||||
|
||||
workspace_buffer = self._get_workspace_buffer()
|
||||
out = torch.empty(
|
||||
q.shape[0],
|
||||
q.shape[1],
|
||||
@@ -102,7 +98,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
workspace_buffer=workspace_buffer,
|
||||
workspace_buffer=self._workspace_buffer,
|
||||
seq_lens=self._query_seq_lens,
|
||||
max_q_len=self._prefill_metadata.max_query_len,
|
||||
max_kv_len=self._prefill_metadata.max_query_len,
|
||||
@@ -135,7 +131,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
|
||||
assert self._prefill_metadata.chunked_context is not None
|
||||
assert self._prefill_metadata.chunked_context.seq_lens[chunk_idx] is not None
|
||||
workspace_buffer = self._get_workspace_buffer()
|
||||
|
||||
out = torch.empty(
|
||||
q.shape[0],
|
||||
@@ -149,7 +144,7 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
workspace_buffer=workspace_buffer,
|
||||
workspace_buffer=self._workspace_buffer,
|
||||
seq_lens=self._prefill_metadata.chunked_context.seq_lens[chunk_idx],
|
||||
max_q_len=self._prefill_metadata.max_query_len,
|
||||
max_kv_len=self._prefill_metadata.chunked_context.max_seq_lens[chunk_idx],
|
||||
|
||||
Reference in New Issue
Block a user