[Bugfix][ROCm] Fix race on AITER MLA FP8 prefill scheduling metadata under async scheduling (#58887)
Signed-off-by: Mehdi Ghanimifard <mehdi.ghanimifard@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Cyrus Leung
parent
87a4bf664f
commit
aee8fe202a
@@ -948,6 +948,31 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
|||||||
device=device,
|
device=device,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# get_ps_metadata_v1 builds the plan on the host and writes its outputs with
|
||||||
|
# blocking hipMemcpy that is not ordered on the current stream. Writing the
|
||||||
|
# device buffers directly lets step N+1's build (async scheduling) overwrite a
|
||||||
|
# plan that step N's prefill kernels are still reading. Build into pinned host
|
||||||
|
# staging instead and copy on the current stream. Two staging slots, each
|
||||||
|
# reused only after its previous H2D copy has completed.
|
||||||
|
self._fp8_ps_device_outputs = (
|
||||||
|
self.fp8_ps_work_indptr,
|
||||||
|
self.fp8_ps_work_info,
|
||||||
|
self.fp8_ps_reduce_indptr,
|
||||||
|
self.fp8_ps_reduce_final_map,
|
||||||
|
self.fp8_ps_reduce_partial_map,
|
||||||
|
)
|
||||||
|
self._fp8_ps_staging = [
|
||||||
|
tuple(
|
||||||
|
torch.empty(t.shape, dtype=t.dtype, device="cpu", pin_memory=True)
|
||||||
|
for t in self._fp8_ps_device_outputs
|
||||||
|
)
|
||||||
|
for _ in range(2)
|
||||||
|
]
|
||||||
|
# One event per slot, recorded after that slot's H2D copy. synchronize()
|
||||||
|
# on an event that has not been recorded yet returns immediately.
|
||||||
|
self._fp8_ps_staging_free = [torch.cuda.Event(), torch.cuda.Event()]
|
||||||
|
self._fp8_ps_slot = 0
|
||||||
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.v1.worker.workspace import current_workspace_manager
|
from vllm.v1.worker.workspace import current_workspace_manager
|
||||||
|
|
||||||
@@ -1043,6 +1068,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
|||||||
kvlen_granularity = 128
|
kvlen_granularity = 128
|
||||||
block_size = 1 # non-paged: each "page" is one token
|
block_size = 1 # non-paged: each "page" is one token
|
||||||
|
|
||||||
|
slot = self._fp8_ps_slot
|
||||||
|
self._fp8_ps_slot ^= 1
|
||||||
|
staging_free = self._fp8_ps_staging_free[slot]
|
||||||
|
with gpu_sync_allowed():
|
||||||
|
staging_free.synchronize()
|
||||||
|
(
|
||||||
|
work_indptr_host,
|
||||||
|
work_info_host,
|
||||||
|
reduce_indptr_host,
|
||||||
|
reduce_final_map_host,
|
||||||
|
reduce_partial_map_host,
|
||||||
|
) = self._fp8_ps_staging[slot]
|
||||||
|
|
||||||
|
# work_metadata is not written by the host planner; it stays on device.
|
||||||
get_ps_metadata_v1(
|
get_ps_metadata_v1(
|
||||||
qo_indptr_cpu,
|
qo_indptr_cpu,
|
||||||
kv_indptr_cpu,
|
kv_indptr_cpu,
|
||||||
@@ -1050,17 +1089,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
|||||||
gqa_ratio,
|
gqa_ratio,
|
||||||
num_head_k,
|
num_head_k,
|
||||||
self.fp8_ps_work_metadata,
|
self.fp8_ps_work_metadata,
|
||||||
self.fp8_ps_work_indptr,
|
work_indptr_host,
|
||||||
self.fp8_ps_work_info,
|
work_info_host,
|
||||||
self.fp8_ps_reduce_indptr,
|
reduce_indptr_host,
|
||||||
self.fp8_ps_reduce_final_map,
|
reduce_final_map_host,
|
||||||
self.fp8_ps_reduce_partial_map,
|
reduce_partial_map_host,
|
||||||
qhead_granularity=qhead_granularity,
|
qhead_granularity=qhead_granularity,
|
||||||
qlen_granularity=qlen_granularity,
|
qlen_granularity=qlen_granularity,
|
||||||
kvlen_granularity=kvlen_granularity,
|
kvlen_granularity=kvlen_granularity,
|
||||||
block_size=block_size,
|
block_size=block_size,
|
||||||
is_causal=True,
|
is_causal=True,
|
||||||
)
|
)
|
||||||
|
for dst, src in zip(self._fp8_ps_device_outputs, self._fp8_ps_staging[slot]):
|
||||||
|
dst.copy_(src, non_blocking=True)
|
||||||
|
staging_free.record()
|
||||||
|
|
||||||
total_prefill_tokens = int(qo_indptr_cpu[-1].item())
|
total_prefill_tokens = int(qo_indptr_cpu[-1].item())
|
||||||
kv_indices = torch.arange(
|
kv_indices = torch.arange(
|
||||||
@@ -1068,12 +1110,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# The actual number of active partial tiles for this batch is the
|
# The actual number of active partial tiles for this batch is the
|
||||||
# final value of reduce_indptr. Resolving it here (during metadata
|
# final value of reduce_indptr, read from the host plan (no GPU sync).
|
||||||
# build) keeps it off the per-layer forward path where a sync would
|
num_partial_tiles = int(reduce_indptr_host[-1])
|
||||||
# break CUDA Graph capture. Using the device-side reduce_indptr is
|
|
||||||
# acceptable since build is allowed to incur an occasional sync.
|
|
||||||
with gpu_sync_allowed():
|
|
||||||
num_partial_tiles = int(self.fp8_ps_reduce_indptr[-1].item())
|
|
||||||
|
|
||||||
# Attach PS metadata to the metadata object so forward_mha can read it.
|
# Attach PS metadata to the metadata object so forward_mha can read it.
|
||||||
metadata.fp8_prefill_qo_indptr = qo_indptr
|
metadata.fp8_prefill_qo_indptr = qo_indptr
|
||||||
|
|||||||
Reference in New Issue
Block a user