[ROCm] [V1] [SpecDec] Enable Speculative Decoding on ROCm V1 Engine (#21496)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2025-08-07 19:13:17 -07:00
committed by GitHub
parent acf8aeb79e
commit 1ee5ead5f8
6 changed files with 128 additions and 41 deletions
+16
View File
@@ -986,3 +986,19 @@ def has_module_attribute(module_name, attribute_name):
return hasattr(module, attribute_name)
except ImportError:
return False
def get_attn_backend_list_based_on_platform() -> list[str]:
if current_platform.is_cuda():
return ["FLASH_ATTN_VLLM_V1", "TRITON_ATTN_VLLM_V1", "TREE_ATTN"]
elif current_platform.is_rocm():
attn_backend_list = ["TRITON_ATTN_VLLM_V1"]
try:
import aiter # noqa: F401
attn_backend_list.append("FLASH_ATTN_VLLM_V1")
except Exception:
print("Skip FLASH_ATTN_VLLM_V1 on ROCm as aiter is not installed")
return attn_backend_list
else:
raise ValueError("Unsupported platform")