[New Model][ROCm] Add AMD support for DeepSeek V4 (#40871)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: tjtanaavllm <tunjian.tan@amd.com> Co-authored-by: ganyi <ygan@amd.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: tjtanaavllm <tunjian.tan@amd.com>
This commit is contained in:
co-authored by
ganyi
tjtanaa
tjtanaavllm
parent
2228fe6868
commit
628c436301
@@ -1245,7 +1245,12 @@ class DeepseekV4Model(nn.Module):
|
||||
# DeepseekV4MultiHeadLatentAttentionWrapper.attn_gemm_parallel_execute
|
||||
# (compressor kv_score, indexer.weights_proj, indexer.compressor
|
||||
# kv_score). fused_wqa_wkv stays on the default stream.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# Disable them on ROCm because of hang issues.
|
||||
aux_stream_list = (
|
||||
None
|
||||
if current_platform.is_rocm()
|
||||
else [torch.cuda.Stream() for _ in range(3)]
|
||||
)
|
||||
|
||||
self.device = current_platform.device_type
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
|
||||
@@ -167,8 +167,12 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
)
|
||||
|
||||
# Three aux streams shared across all MTP layers, mirroring
|
||||
# DeepseekV4Model.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# DeepseekV4Model. ROCm runs the same work serially for now.
|
||||
aux_stream_list = (
|
||||
None
|
||||
if current_platform.is_rocm()
|
||||
else [torch.cuda.Stream() for _ in range(3)]
|
||||
)
|
||||
|
||||
# to map the exact layer index from weights
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
|
||||
Reference in New Issue
Block a user