[ROCm][DSv4.1] Keep the Engram tables in host memory on ROCm (#57491)
This commit is contained in:
@@ -74,7 +74,7 @@ class EngramConfig:
|
||||
if (
|
||||
model_config is None
|
||||
or field is None
|
||||
or not current_platform.is_cuda()
|
||||
or not current_platform.is_cuda_alike()
|
||||
or not getattr(model_config.hf_text_config, field, None)
|
||||
):
|
||||
raise ValueError(
|
||||
|
||||
+1
-1
@@ -1260,7 +1260,7 @@ class VllmConfig:
|
||||
"Disable --enable-dbo and set --ubatch-size to 0."
|
||||
)
|
||||
if self.engram_config is None:
|
||||
if not current_platform.is_cuda() or not model_has_engram_layers(
|
||||
if not current_platform.is_cuda_alike() or not model_has_engram_layers(
|
||||
model_config
|
||||
):
|
||||
return
|
||||
|
||||
@@ -61,9 +61,13 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.engram import Engram, EngramLayout, NgramHashState
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
|
||||
# Engram host offload and its prefetch stream are neither ROCm- nor
|
||||
# NVIDIA-specific, so they are imported rather than duplicated.
|
||||
from ..nvidia.engram import Engram
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
|
||||
Reference in New Issue
Block a user