[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -82,6 +82,7 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_token_offset
|
||||
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
@@ -1228,7 +1229,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if self._mtp_hidden_buffer is not None:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1))
|
||||
# Each DBO ubatch owns a different slice of the batch. Writing
|
||||
# from row 0 in both threads would let one ubatch clobber the
|
||||
# other's rows in the buffer the MTP drafter reads after the step.
|
||||
offset = dbo_current_ubatch_token_offset()
|
||||
self._mtp_hidden_buffer[offset : offset + num_tokens].copy_(
|
||||
hidden_states.flatten(1)
|
||||
)
|
||||
|
||||
hidden_states = self.hc_head_op(
|
||||
hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user