[Bugfix][ROCm] Fix race on AITER MLA FP8 prefill scheduling metadata under async scheduling (#58887)

Signed-off-by: Mehdi Ghanimifard <mehdi.ghanimifard@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Mehdi Ghanimifard
2026-10-01 15:02:52 +08:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Cyrus Leung
parent 87a4bf664f
commit aee8fe202a
@@ -948,6 +948,31 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device=device,
)
# get_ps_metadata_v1 builds the plan on the host and writes its outputs with
# blocking hipMemcpy that is not ordered on the current stream. Writing the
# device buffers directly lets step N+1's build (async scheduling) overwrite a
# plan that step N's prefill kernels are still reading. Build into pinned host
# staging instead and copy on the current stream. Two staging slots, each
# reused only after its previous H2D copy has completed.
self._fp8_ps_device_outputs = (
self.fp8_ps_work_indptr,
self.fp8_ps_work_info,
self.fp8_ps_reduce_indptr,
self.fp8_ps_reduce_final_map,
self.fp8_ps_reduce_partial_map,
)
self._fp8_ps_staging = [
tuple(
torch.empty(t.shape, dtype=t.dtype, device="cpu", pin_memory=True)
for t in self._fp8_ps_device_outputs
)
for _ in range(2)
]
# One event per slot, recorded after that slot's H2D copy. synchronize()
# on an event that has not been recorded yet returns immediately.
self._fp8_ps_staging_free = [torch.cuda.Event(), torch.cuda.Event()]
self._fp8_ps_slot = 0
from vllm.platforms import current_platform
from vllm.v1.worker.workspace import current_workspace_manager
@@ -1043,6 +1068,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
kvlen_granularity = 128
block_size = 1 # non-paged: each "page" is one token
slot = self._fp8_ps_slot
self._fp8_ps_slot ^= 1
staging_free = self._fp8_ps_staging_free[slot]
with gpu_sync_allowed():
staging_free.synchronize()
(
work_indptr_host,
work_info_host,
reduce_indptr_host,
reduce_final_map_host,
reduce_partial_map_host,
) = self._fp8_ps_staging[slot]
# work_metadata is not written by the host planner; it stays on device.
get_ps_metadata_v1(
qo_indptr_cpu,
kv_indptr_cpu,
@@ -1050,17 +1089,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
gqa_ratio,
num_head_k,
self.fp8_ps_work_metadata,
self.fp8_ps_work_indptr,
self.fp8_ps_work_info,
self.fp8_ps_reduce_indptr,
self.fp8_ps_reduce_final_map,
self.fp8_ps_reduce_partial_map,
work_indptr_host,
work_info_host,
reduce_indptr_host,
reduce_final_map_host,
reduce_partial_map_host,
qhead_granularity=qhead_granularity,
qlen_granularity=qlen_granularity,
kvlen_granularity=kvlen_granularity,
block_size=block_size,
is_causal=True,
)
for dst, src in zip(self._fp8_ps_device_outputs, self._fp8_ps_staging[slot]):
dst.copy_(src, non_blocking=True)
staging_free.record()
total_prefill_tokens = int(qo_indptr_cpu[-1].item())
kv_indices = torch.arange(
@@ -1068,12 +1110,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
)
# The actual number of active partial tiles for this batch is the
# final value of reduce_indptr. Resolving it here (during metadata
# build) keeps it off the per-layer forward path where a sync would
# break CUDA Graph capture. Using the device-side reduce_indptr is
# acceptable since build is allowed to incur an occasional sync.
with gpu_sync_allowed():
num_partial_tiles = int(self.fp8_ps_reduce_indptr[-1].item())
# final value of reduce_indptr, read from the host plan (no GPU sync).
num_partial_tiles = int(reduce_indptr_host[-1])
# Attach PS metadata to the metadata object so forward_mha can read it.
metadata.fp8_prefill_qo_indptr = qo_indptr