[Bugfix] Release stale FlashMLA workspace views after growth (#56902)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
acc2ed2a5f
commit
9612f77077
@@ -720,7 +720,9 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
)
|
||||
|
||||
self.pcp_dcp_kv_gather = False
|
||||
self.gathered_kv_workspace: torch.Tensor | None = None
|
||||
self.workspace_specs: list[tuple[tuple[int, ...], torch.dtype]] = [
|
||||
(q_concat_shape, torch.bfloat16)
|
||||
]
|
||||
if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS:
|
||||
# Reserve workspace during initialization
|
||||
assert vllm_config is not None and vllm_config.model_config is not None
|
||||
@@ -738,20 +740,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# shards into a workspace of the full prefill size.
|
||||
shard_rows //= parallel_config.decode_context_parallel_size
|
||||
self.prefill_workspace_shape = (shard_rows, head_size)
|
||||
specs = [
|
||||
(q_concat_shape, torch.bfloat16),
|
||||
(self.prefill_workspace_shape, torch.bfloat16),
|
||||
]
|
||||
self.workspace_specs.append((self.prefill_workspace_shape, torch.bfloat16))
|
||||
if self.pcp_dcp_kv_gather:
|
||||
specs.append(((prefill_workspace_size, head_size), torch.bfloat16))
|
||||
buffers = current_workspace_manager().get_simultaneous(*specs)
|
||||
self.q_concat_buffer, self.prefill_bf16_workspace = buffers[:2]
|
||||
if self.pcp_dcp_kv_gather:
|
||||
self.gathered_kv_workspace = buffers[2]
|
||||
else:
|
||||
(self.q_concat_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
(q_concat_shape, torch.bfloat16),
|
||||
)
|
||||
self.workspace_specs.append(
|
||||
((prefill_workspace_size, head_size), torch.bfloat16)
|
||||
)
|
||||
# Reserve capacity without retaining views that prevent old storage
|
||||
# from being released when another layer grows the shared workspace.
|
||||
current_workspace_manager().get_simultaneous(*self.workspace_specs)
|
||||
|
||||
def _forward_bf16_kv(
|
||||
self,
|
||||
@@ -849,8 +845,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
"""All-gather this rank's upconverted KV shard so the chunk's rows attend
|
||||
the whole context, and map their top-k onto the rank-major result."""
|
||||
shard_rows = int(chunk.chunk_tot_seqlen)
|
||||
assert self.gathered_kv_workspace is not None
|
||||
gathered_kv = self.gathered_kv_workspace[: self.dcp_world_size * shard_rows]
|
||||
_, _, gathered_kv_workspace = current_workspace_manager().get_simultaneous(
|
||||
*self.workspace_specs
|
||||
)
|
||||
gathered_kv = gathered_kv_workspace[: self.dcp_world_size * shard_rows]
|
||||
dist.all_gather_into_tensor(
|
||||
gathered_kv, shard, group=get_dcp_group().device_group
|
||||
)
|
||||
@@ -902,10 +900,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
assert fp8_metadata.prefill is not None
|
||||
first_chunk = fp8_metadata.prefill.chunks[0]
|
||||
assert isinstance(index_group, HiSparseMLAIndexGroup)
|
||||
_, prefill_bf16_workspace, *_ = (
|
||||
current_workspace_manager().get_simultaneous(*self.workspace_specs)
|
||||
)
|
||||
prefill_ready = index_group.gather_fp8_prefill(
|
||||
self.index_group_index,
|
||||
kv_c_and_k_pe_cache,
|
||||
self.prefill_bf16_workspace[: first_chunk.chunk_tot_seqlen],
|
||||
prefill_bf16_workspace[: first_chunk.chunk_tot_seqlen],
|
||||
first_chunk.block_table,
|
||||
first_chunk.workspace_starts,
|
||||
len(first_chunk.block_table),
|
||||
@@ -1022,8 +1023,11 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
)
|
||||
|
||||
assert fp8_metadata.prefill is not None
|
||||
_, prefill_bf16_workspace, *_ = (
|
||||
current_workspace_manager().get_simultaneous(*self.workspace_specs)
|
||||
)
|
||||
for chunk_index, chunk in enumerate(fp8_metadata.prefill.chunks):
|
||||
chunk_workspace = self.prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
|
||||
chunk_workspace = prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
|
||||
if uses_host_cache and chunk_index > 0:
|
||||
assert isinstance(index_group, HiSparseMLAIndexGroup)
|
||||
prefill_ready = index_group.gather_fp8_prefill(
|
||||
@@ -1302,7 +1306,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
actual_num_heads = self.num_heads
|
||||
if isinstance(q, tuple):
|
||||
ql_nope, q_pe = q
|
||||
q = self.q_concat_buffer[: ql_nope.shape[0]]
|
||||
q_concat_buffer, *_ = current_workspace_manager().get_simultaneous(
|
||||
*self.workspace_specs
|
||||
)
|
||||
q = q_concat_buffer[: ql_nope.shape[0]]
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
else:
|
||||
actual_num_heads = q.shape[1]
|
||||
|
||||
Reference in New Issue
Block a user