[Attention] Fetch MLA prefill workspace at use instead of init (#60654)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e83222396f
commit
624a5ddafd
@@ -91,9 +91,14 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None
|
||||
self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = []
|
||||
self._global_hyperparameters: PerLayerParameters | None = None
|
||||
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
((envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,), torch.uint8),
|
||||
self._wrapper_workspace_ptr = 0
|
||||
self._workspace_spec = (
|
||||
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
|
||||
torch.uint8,
|
||||
)
|
||||
# Reserve capacity without retaining a view: a retained view would pin
|
||||
# the old buffer when another user grows the shared workspace.
|
||||
current_workspace_manager().get_simultaneous(self._workspace_spec)
|
||||
|
||||
def _ensure_chunks(
|
||||
self,
|
||||
@@ -146,16 +151,24 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
global_hyperparameters = self._resolve_global_hyperparameters()
|
||||
qo_indptr = prefill_metadata.query_start_loc
|
||||
has_context = prefill_metadata.chunked_context is not None
|
||||
if self._prefill_main is None:
|
||||
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
self._workspace_spec
|
||||
)
|
||||
if (
|
||||
self._prefill_main is None
|
||||
or self._wrapper_workspace_ptr != workspace_buffer.data_ptr()
|
||||
):
|
||||
self._prefill_main = BatchPrefillWithRaggedKVCacheWrapper(
|
||||
self._workspace_buffer, "NHD", backend="cutlass"
|
||||
workspace_buffer, "NHD", backend="cutlass"
|
||||
)
|
||||
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, self._workspace_buffer)
|
||||
self._prefill_chunks = []
|
||||
self._ensure_chunks(_DEFAULT_NUM_CHUNKS, workspace_buffer)
|
||||
self._wrapper_workspace_ptr = workspace_buffer.data_ptr()
|
||||
|
||||
if has_context:
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
self._ensure_chunks(len(chunked_context.chunks), self._workspace_buffer)
|
||||
self._ensure_chunks(len(chunked_context.chunks), workspace_buffer)
|
||||
|
||||
num_qo_heads = self.num_heads
|
||||
num_kv_heads = num_qo_heads
|
||||
|
||||
@@ -79,18 +79,22 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
(
|
||||
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
|
||||
torch.uint8,
|
||||
),
|
||||
self._workspace_spec = (
|
||||
(envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,),
|
||||
torch.uint8,
|
||||
)
|
||||
# Reserve capacity without retaining a view: a retained view would pin
|
||||
# the old buffer when another user grows the shared workspace.
|
||||
current_workspace_manager().get_simultaneous(self._workspace_spec)
|
||||
|
||||
def prepare_metadata(
|
||||
self,
|
||||
prefill_metadata: "MLACommonPrefillMetadata",
|
||||
) -> None:
|
||||
super().prepare_metadata(prefill_metadata)
|
||||
(self._workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
self._workspace_spec
|
||||
)
|
||||
self._query_seq_lens = (
|
||||
prefill_metadata.query_start_loc[1:] - prefill_metadata.query_start_loc[:-1]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user