[ROCm][Refactor] Move DeepSeek-V4/V4.1 multi-stream overlap gate to ROCm platform (#58983)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-09-28 05:46:28 -07:00
committed by GitHub
parent 20b52e9f5b
commit 7ba3df63cb
4 changed files with 40 additions and 37 deletions
+6 -19
View File
@@ -649,23 +649,6 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
# Disable indexer inner overlap.
self.indexer.aux_stream = None
def _enable_multi_stream_overlap(self) -> bool:
"""ROCm multi-stream gates: streams and capture region.
Dict metadata marks piecewise cudagraph, whose eager breaks rebuild
the attention inputs on the owning stream. Forking side streams
there would rely on runtime HIP event sync, which is unreliable in
this overlap on ROCm (event waits can hang), so multi-stream only
runs where the fork/join becomes static graph edges: inside capture,
or with non-dict metadata (full cudagraph or the profile run), which
has no eager breaks. Covers both the HCA and CSA forks.
"""
attn_metadata = get_forward_context().attn_metadata
return self.aux_stream_list is not None and (
torch.cuda.is_current_stream_capturing()
or not isinstance(attn_metadata, dict)
)
def _run_sequential_pipeline(
self,
hidden_states: torch.Tensor,
@@ -709,7 +692,9 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
device=hidden_states.device,
)
if self._enable_multi_stream_overlap():
if current_platform.enable_multi_stream_overlap(
self.aux_stream_list, get_forward_context().attn_metadata
):
# The ROCm override consumes these sentinels inside the capture
# boundary, moving the stream fan-out ahead of the projections.
self._prepare_and_attn_fn(
@@ -769,7 +754,9 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
# Re-check: forward's gate ran inside a captured segment that
# _prepare_and_attn_eager (MRV1) then broke, making this region eager.
if not self._enable_multi_stream_overlap():
if not current_platform.enable_multi_stream_overlap(
self.aux_stream_list, get_forward_context().attn_metadata
):
self._run_sequential_pipeline(hidden_states, positions, o_padded)
return
+3 -18
View File
@@ -615,23 +615,6 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
self.aux_stream_list = aux_streams
return super()._run_parallel_input_projections(hidden_states)
def _enable_multi_stream_overlap(self) -> bool:
"""ROCm multi-stream gates: streams and capture region.
Dict metadata marks piecewise cudagraph, whose eager breaks rebuild
the attention inputs on the owning stream. Forking side streams
there would rely on runtime HIP event sync, which is unreliable in
this overlap on ROCm (event waits can hang), so multi-stream only
runs where the fork/join becomes static graph edges: inside capture,
or with non-dict metadata (full cudagraph or the profile run), which
has no eager breaks.
"""
attn_metadata = get_forward_context().attn_metadata
return self.aux_stream_list is not None and (
torch.cuda.is_current_stream_capturing()
or not isinstance(attn_metadata, dict)
)
def forward(
self,
positions: torch.Tensor,
@@ -641,7 +624,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
if (
self.indexer is None
or self._prepare_and_attn_fn == self._prepare_and_attn_eager
or not self._enable_multi_stream_overlap()
or not current_platform.enable_multi_stream_overlap(
self.aux_stream_list, get_forward_context().attn_metadata
)
):
# Sequential fallback: no forks outside the capture region, where
# HIP event sync is unreliable; MRV1 keeps the input prep in its
+10
View File
@@ -1120,6 +1120,16 @@ class Platform:
"""
return False
@classmethod
def enable_multi_stream_overlap(
cls,
aux_stream_list: list[torch.cuda.Stream] | None,
attn_metadata: object,
) -> bool:
"""Whether the current platform should enable multi-stream overlap
for the given aux streams and attention metadata."""
return False
@classmethod
def supports_mx(cls) -> bool:
"""Returns whether the current platform supports MX types."""
+21
View File
@@ -1106,6 +1106,27 @@ class RocmPlatform(Platform):
def is_navi(cls) -> bool:
return "gfx1" in _GCN_ARCH
@classmethod
def enable_multi_stream_overlap(
cls,
aux_stream_list: list[torch.cuda.Stream] | None,
attn_metadata: object,
) -> bool:
"""ROCm multi-stream gates: streams and capture region.
Dict metadata marks piecewise cudagraph, whose eager breaks rebuild
the attention inputs on the owning stream. Forking side streams
there would rely on runtime HIP event sync, which is unreliable in
this overlap on ROCm (event waits can hang), so multi-stream only
runs where the fork/join becomes static graph edges: inside capture,
or with non-dict metadata (full cudagraph or the profile run), which
has no eager breaks.
"""
return aux_stream_list is not None and (
torch.cuda.is_current_stream_capturing()
or not isinstance(attn_metadata, dict)
)
@classmethod
def get_static_graph_wrapper_cls(cls) -> str:
return "vllm.compilation.cuda_graph.CUDAGraphWrapper"