[ROCm][DSv4.1] Keep the Engram tables in host memory on ROCm (#57491)

This commit is contained in:
JohnQinAMD
2026-09-21 10:17:11 +08:00
committed by GitHub
parent 3e67044972
commit 6dfd87f596
5 changed files with 20 additions and 10 deletions
+6 -2
View File
@@ -599,7 +599,9 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads) ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required") @pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="CUDA or ROCm required"
)
@pytest.mark.parametrize("cpu_offload", [False, True]) @pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8]) @pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch): def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch):
@@ -701,7 +703,9 @@ def test_engram_lookup_reuses_jit_across_token_shapes():
assert cache_sizes == [1] * len(cache_sizes) assert cache_sizes == [1] * len(cache_sizes)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required") @pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="CUDA or ROCm required"
)
@pytest.mark.parametrize("cpu_offload", [False, True]) @pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize("background", [False, True]) @pytest.mark.parametrize("background", [False, True])
@pytest.mark.parametrize("num_tokens", [1, 7, 256]) @pytest.mark.parametrize("num_tokens", [1, 7, 256])
+7 -5
View File
@@ -1299,7 +1299,7 @@ def test_engram_dp_shared_memory_config_validation(
monkeypatch, dp_size, load_format, multithread, error monkeypatch, dp_size, load_format, multithread, error
): ):
"""Reject invalid shared configs before distributed init; allow threaded loads.""" """Reject invalid shared configs before distributed init; allow threaded loads."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True) monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
config = cast( config = cast(
VllmConfig, VllmConfig,
SimpleNamespace( SimpleNamespace(
@@ -1324,7 +1324,7 @@ def test_engram_dp_shared_memory_config_validation(
@pytest.mark.parametrize( @pytest.mark.parametrize(
"architecture, ple_layers, cuda, supported", "architecture, ple_layers, accelerator, supported",
[ [
("DeepseekV41ForCausalLM", [1], True, True), ("DeepseekV41ForCausalLM", [1], True, True),
("DeepseekV41ForCausalLM", [], True, False), ("DeepseekV41ForCausalLM", [], True, False),
@@ -1339,9 +1339,11 @@ def test_engram_dp_shared_memory_config_validation(
(None, None, True, False), (None, None, True, False),
], ],
) )
def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, supported): def test_engram_model_support(
monkeypatch, architecture, ple_layers, accelerator, supported
):
"""A similarly named HF field must not enable unsupported implementations.""" """A similarly named HF field must not enable unsupported implementations."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda) monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: accelerator)
model = ( model = (
cast( cast(
ModelConfig, ModelConfig,
@@ -1430,7 +1432,7 @@ def test_engram_explicit_config_requires_supported_model():
@pytest.mark.parametrize("explicit", [False, True]) @pytest.mark.parametrize("explicit", [False, True])
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, explicit): def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, explicit):
"""MTP may inherit cross-DP sharding without having its own PLE layers.""" """MTP may inherit cross-DP sharding without having its own PLE layers."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True) monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
target = SimpleNamespace( target = SimpleNamespace(
architecture="Qwen4ExpForCausalLM", architecture="Qwen4ExpForCausalLM",
hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []), hf_text_config=SimpleNamespace(ple_layer_ids=[1] if target_has_ple else []),
+1 -1
View File
@@ -74,7 +74,7 @@ class EngramConfig:
if ( if (
model_config is None model_config is None
or field is None or field is None
or not current_platform.is_cuda() or not current_platform.is_cuda_alike()
or not getattr(model_config.hf_text_config, field, None) or not getattr(model_config.hf_text_config, field, None)
): ):
raise ValueError( raise ValueError(
+1 -1
View File
@@ -1260,7 +1260,7 @@ class VllmConfig:
"Disable --enable-dbo and set --ubatch-size to 0." "Disable --enable-dbo and set --ubatch-size to 0."
) )
if self.engram_config is None: if self.engram_config is None:
if not current_platform.is_cuda() or not model_has_engram_layers( if not current_platform.is_cuda_alike() or not model_has_engram_layers(
model_config model_config
): ):
return return
+5 -1
View File
@@ -61,9 +61,13 @@ from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.ubatching import dbo_current_ubatch_id from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import Engram, EngramLayout, NgramHashState from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
# Engram host offload and its prefetch stream are neither ROCm- nor
# NVIDIA-specific, so they are imported rather than duplicated.
from ..nvidia.engram import Engram
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata