[Attention] Fetch MLA prefill workspace at use instead of init (#60654)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-13 17:25:03 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent e83222396f
commit 624a5ddafd
3 changed files with 72 additions and 11 deletions
@@ -91,9 +91,14 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None
self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = []
self._global_hyperparameters: PerLayerParameters | None = None
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8),
self._wrapper_workspace_ptr = 0
self._workspace_spec = (
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
torch.uint8,
)
# Reserve capacity without retaining a view: a retained view would pin
# the old buffer when another user grows the shared workspace.
current_workspace_manager().get_simultaneous(self._workspace_spec)
def _ensure_chunks(
self,
@@ -146,16 +151,24 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
global_hyperparameters = self._resolve_global_hyperparameters()
qo_indptr = prefill_metadata.query_start_loc
has_context = prefill_metadata.chunked_context is not None
if self._prefill_main is None:
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
self._workspace_spec
)
if (
self._prefill_main is None
or self._wrapper_workspace_ptr != workspace_buffer.data_ptr()
):
self._prefill_main = BatchPrefillWithRaggedKVCacheWrapper(
self._workspace_buffer, "NHD", backend="cutlass"
workspace_buffer, "NHD", backend="cutlass"
)
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, self._workspace_buffer)
self._prefill_chunks = []
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, workspace_buffer)
self._wrapper_workspace_ptr = workspace_buffer.data_ptr()
if has_context:
chunked_context = prefill_metadata.chunked_context
assert chunked_context is not None
self._ensure_chunks(len(chunked_context.chunks), self._workspace_buffer)
self._ensure_chunks(len(chunked_context.chunks), workspace_buffer)
num_qo_heads = self.num_heads
num_kv_heads = num_qo_heads
@@ -79,18 +79,22 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
(
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
torch.uint8,
),
self._workspace_spec = (
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
torch.uint8,
)
# Reserve capacity without retaining a view: a retained view would pin
# the old buffer when another user grows the shared workspace.
current_workspace_manager().get_simultaneous(self._workspace_spec)
def prepare_metadata(
self,
prefill_metadata: "MLACommonPrefillMetadata",
) -> None:
super().prepare_metadata(prefill_metadata)
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
self._workspace_spec
)
self._query_seq_lens = (
prefill_metadata.query_start_loc[1:] - prefill_metadata.query_start_loc[:-1]
)