[ROCm] Enable DeepSeek-V4 on gfx11 (#47017)
This PR enables DeepSeek-V4 checkpoints on ROCm gfx11/RDNA devices. Signed-off-by: Letian Kang <Letian.Kang@amd.com>
This commit is contained in:
+16
-1
@@ -2025,12 +2025,27 @@ class rocm_aiter_ops:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def register_ops_once() -> None:
|
def register_ops_once() -> None:
|
||||||
|
global _OPS_REGISTERED
|
||||||
|
|
||||||
if not (
|
if not (
|
||||||
is_aiter_found_and_supported() or is_aiter_found_and_supported_on_rdna4()
|
is_aiter_found_and_supported() or is_aiter_found_and_supported_on_rdna4()
|
||||||
):
|
):
|
||||||
|
if not current_platform.is_rocm():
|
||||||
|
return
|
||||||
|
|
||||||
|
from vllm.platforms.rocm import on_gfx11
|
||||||
|
|
||||||
|
if on_gfx11() and not _OPS_REGISTERED:
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="rocm_aiter_sparse_attn_indexer",
|
||||||
|
op_func=rocm_aiter_sparse_attn_indexer,
|
||||||
|
mutates_args=["topk_indices_buffer"],
|
||||||
|
fake_impl=rocm_aiter_sparse_attn_indexer_fake,
|
||||||
|
dispatch_key=current_platform.dispatch_key,
|
||||||
|
)
|
||||||
|
_OPS_REGISTERED = True
|
||||||
return
|
return
|
||||||
|
|
||||||
global _OPS_REGISTERED
|
|
||||||
if not _OPS_REGISTERED:
|
if not _OPS_REGISTERED:
|
||||||
# register all the custom ops here
|
# register all the custom ops here
|
||||||
direct_register_custom_op(
|
direct_register_custom_op(
|
||||||
|
|||||||
@@ -850,7 +850,13 @@ class SparseAttnIndexer(CustomOp):
|
|||||||
assert isinstance(q_quant, torch.Tensor), (
|
assert isinstance(q_quant, torch.Tensor), (
|
||||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||||
)
|
)
|
||||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
from vllm.platforms.rocm import on_gfx11
|
||||||
|
|
||||||
|
if (
|
||||||
|
rocm_aiter_ops.is_enabled()
|
||||||
|
or rocm_aiter_ops.is_rdna_aiter_enabled()
|
||||||
|
or on_gfx11()
|
||||||
|
):
|
||||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||||
hidden_states,
|
hidden_states,
|
||||||
_encode_layer_name(self.k_cache.prefix),
|
_encode_layer_name(self.k_cache.prefix),
|
||||||
|
|||||||
@@ -520,6 +520,7 @@ class RocmPlatform(Platform):
|
|||||||
"deepseek_v4_fp8",
|
"deepseek_v4_fp8",
|
||||||
"compressed-tensors",
|
"compressed-tensors",
|
||||||
"fbgemm_fp8",
|
"fbgemm_fp8",
|
||||||
|
"inc",
|
||||||
"quark",
|
"quark",
|
||||||
"mxfp4",
|
"mxfp4",
|
||||||
"mxfp8",
|
"mxfp8",
|
||||||
|
|||||||
Reference in New Issue
Block a user