[Bugfix] Release stale FlashMLA workspace views after growth (#56902)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Maroon Ayoub
2026-09-17 16:23:15 -04:00
committed by GitHub
co-authored by Robert Shaw
parent acc2ed2a5f
commit 9612f77077
@@ -720,7 +720,9 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
)
self.pcp_dcp_kv_gather = False
self.gathered_kv_workspace: torch.Tensor | None = None
self.workspace_specs: list[tuple[tuple[int, ...], torch.dtype]] = [
(q_concat_shape, torch.bfloat16)
]
if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS:
# Reserve workspace during initialization
assert vllm_config is not None and vllm_config.model_config is not None
@@ -738,20 +740,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
# shards into a workspace of the full prefill size.
shard_rows //= parallel_config.decode_context_parallel_size
self.prefill_workspace_shape = (shard_rows, head_size)
specs = [
(q_concat_shape, torch.bfloat16),
(self.prefill_workspace_shape, torch.bfloat16),
]
self.workspace_specs.append((self.prefill_workspace_shape, torch.bfloat16))
if self.pcp_dcp_kv_gather:
specs.append(((prefill_workspace_size, head_size), torch.bfloat16))
buffers = current_workspace_manager().get_simultaneous(*specs)
self.q_concat_buffer, self.prefill_bf16_workspace = buffers[:2]
if self.pcp_dcp_kv_gather:
self.gathered_kv_workspace = buffers[2]
else:
(self.q_concat_buffer,) = current_workspace_manager().get_simultaneous(
(q_concat_shape, torch.bfloat16),
)
self.workspace_specs.append(
((prefill_workspace_size, head_size), torch.bfloat16)
)
# Reserve capacity without retaining views that prevent old storage
# from being released when another layer grows the shared workspace.
current_workspace_manager().get_simultaneous(*self.workspace_specs)
def _forward_bf16_kv(
self,
@@ -849,8 +845,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
"""All-gather this rank's upconverted KV shard so the chunk's rows attend
the whole context, and map their top-k onto the rank-major result."""
shard_rows = int(chunk.chunk_tot_seqlen)
assert self.gathered_kv_workspace is not None
gathered_kv = self.gathered_kv_workspace[: self.dcp_world_size * shard_rows]
_, _, gathered_kv_workspace = current_workspace_manager().get_simultaneous(
*self.workspace_specs
)
gathered_kv = gathered_kv_workspace[: self.dcp_world_size * shard_rows]
dist.all_gather_into_tensor(
gathered_kv, shard, group=get_dcp_group().device_group
)
@@ -902,10 +900,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
assert fp8_metadata.prefill is not None
first_chunk = fp8_metadata.prefill.chunks[0]
assert isinstance(index_group, HiSparseMLAIndexGroup)
_, prefill_bf16_workspace, *_ = (
current_workspace_manager().get_simultaneous(*self.workspace_specs)
)
prefill_ready = index_group.gather_fp8_prefill(
self.index_group_index,
kv_c_and_k_pe_cache,
self.prefill_bf16_workspace[: first_chunk.chunk_tot_seqlen],
prefill_bf16_workspace[: first_chunk.chunk_tot_seqlen],
first_chunk.block_table,
first_chunk.workspace_starts,
len(first_chunk.block_table),
@@ -1022,8 +1023,11 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
)
assert fp8_metadata.prefill is not None
_, prefill_bf16_workspace, *_ = (
current_workspace_manager().get_simultaneous(*self.workspace_specs)
)
for chunk_index, chunk in enumerate(fp8_metadata.prefill.chunks):
chunk_workspace = self.prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
chunk_workspace = prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
if uses_host_cache and chunk_index > 0:
assert isinstance(index_group, HiSparseMLAIndexGroup)
prefill_ready = index_group.gather_fp8_prefill(
@@ -1302,7 +1306,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
actual_num_heads = self.num_heads
if isinstance(q, tuple):
ql_nope, q_pe = q
q = self.q_concat_buffer[: ql_nope.shape[0]]
q_concat_buffer, *_ = current_workspace_manager().get_simultaneous(
*self.workspace_specs
)
q = q_concat_buffer[: ql_nope.shape[0]]
ops.concat_mla_q(ql_nope, q_pe, q)
else:
actual_num_heads = q.shape[1]