[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: ywang96 <>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Juntian Liu
2026-09-13 02:44:23 -07:00
committed by GitHub
co-authored by Roger Wang ywang96 <> Woosuk Kwon Codex
parent e7a3963339
commit d2d649e674
12 changed files with 1821 additions and 195 deletions
+110 -48
View File
@@ -1158,31 +1158,68 @@ def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: in
assert config.get_parallel_size(parallel) == expected
def test_engram_rejects_elastic_cross_dp():
@pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"])
def test_engram_rejects_elastic_cross_dp(option):
parallel = ParallelConfig(
tensor_parallel_size=4,
data_parallel_size=2,
distributed_executor_backend="mp",
)
parallel.enable_elastic_ep = True
with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"):
EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel)
with pytest.raises(ValueError, match=f"{option}.*elastic EP"):
EngramConfig(**{option: True}).verify_parallel_config(parallel)
@pytest.mark.parametrize("legacy", [None, "0", "1"])
def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy):
"""Explicit settings must override the legacy environment fallback."""
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
if legacy is not None:
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
assert EngramConfig().cpu_offload == (legacy == "1")
assert EngramConfig(cpu_offload=False).cpu_offload is False
assert EngramConfig(cpu_offload=True).cpu_offload is True
def test_engram_dp_shared_memory_requires_cpu_offload():
with pytest.raises(ValueError, match="requires cpu_offload"):
EngramConfig(cpu_offload=False, dp_shared_memory=True)
@pytest.mark.parametrize(
"dp_size,load_format,multithread,error",
[
(1, "auto", False, "requires data_parallel_size > 1"),
(2, "dummy", False, "requires load_format"),
(2, "sharded_state", False, "requires load_format"),
(2, "auto", False, None),
(2, "safetensors", True, None),
(2, "pt", True, None),
],
)
def test_engram_dp_shared_memory_config_validation(
monkeypatch, dp_size, load_format, multithread, error
):
"""Reject invalid shared configs before distributed init; allow threaded loads."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
config = cast(
VllmConfig,
SimpleNamespace(
model_config=SimpleNamespace(
architecture="DeepseekV41ForCausalLM",
hf_text_config=SimpleNamespace(engram_layer_ids=[1]),
),
speculative_config=None,
engram_config=EngramConfig(cpu_offload=True, dp_shared_memory=True),
parallel_config=ParallelConfig(data_parallel_size=dp_size),
load_config=LoadConfig(
load_format=load_format,
model_loader_extra_config={"enable_multithread_load": multithread},
),
),
)
if error:
with pytest.raises(ValueError, match=error):
VllmConfig._resolve_and_verify_engram_config(config)
else:
VllmConfig._resolve_and_verify_engram_config(config)
@pytest.mark.parametrize(
"architecture, ple_layers, cuda, supported",
[
("DeepseekV41ForCausalLM", [1], True, True),
("DeepseekV41ForCausalLM", [], True, False),
("DeepseekV41ForCausalLM", [1], False, False),
("Qwen4ExpForCausalLM", [1], True, True),
("Qwen4ExpForConditionalGeneration", [1], True, True),
("Qwen4ExpForCausalLM", [], True, False),
@@ -1201,7 +1238,9 @@ def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, suppo
ModelConfig,
SimpleNamespace(
architecture=architecture,
hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers),
hf_text_config=SimpleNamespace(
ple_layer_ids=ple_layers, engram_layer_ids=ple_layers
),
),
)
if architecture is not None
@@ -1214,55 +1253,73 @@ def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, suppo
with pytest.raises(ValueError, match="requires a model with supported Engram"):
config.verify_model_config(model)
resolved = cast(
VllmConfig,
SimpleNamespace(
model_config=model,
speculative_config=None,
parallel_config=ParallelConfig(),
load_config=LoadConfig(load_format="dummy"),
engram_config=None,
),
)
VllmConfig._resolve_and_verify_engram_config(resolved)
assert (resolved.engram_config is not None) == supported
if supported:
assert resolved.engram_config.cpu_offload is True
def test_engram_config_defaults_to_none(monkeypatch):
@pytest.mark.parametrize(
("value", "expected"), [(None, True), ("0", False), ("1", True)]
)
def test_engram_cpu_offload_environment_default(monkeypatch, value, expected):
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
if value is not None:
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", value)
assert EngramConfig().cpu_offload is expected
assert EngramConfig(cpu_offload=False).cpu_offload is False
assert EngramConfig(cpu_offload=True).cpu_offload is True
def test_engram_config_defaults_to_none():
config = VllmConfig()
assert config.engram_config is None
assert config.compute_hash()
@pytest.mark.parametrize("legacy", ["0", "1"])
def test_engram_none_resolves_legacy_offload(monkeypatch, legacy):
"""Legacy enablement materializes a config before model validation."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
@pytest.mark.parametrize("explicit", [False, True])
@pytest.mark.parametrize("enable_dbo,ubatch_size", [(True, 0), (False, 2)])
def test_deepseek_engram_rejects_microbatching(explicit, enable_dbo, ubatch_size):
"""Reject overlapping Engram staging even without an explicit EngramConfig."""
config = cast(
VllmConfig,
SimpleNamespace(
model_config=SimpleNamespace(
architecture="Qwen4ExpForCausalLM",
hf_text_config=SimpleNamespace(ple_layer_ids=[1]),
architecture="DeepseekV41ForCausalLM",
hf_text_config=SimpleNamespace(engram_layer_ids=[1]),
),
speculative_config=None,
engram_config=None,
parallel_config=ParallelConfig(),
engram_config=EngramConfig(cpu_offload=False) if explicit else None,
parallel_config=ParallelConfig(
enable_dbo=enable_dbo, ubatch_size=ubatch_size
),
),
)
VllmConfig._resolve_and_verify_engram_config(config)
if legacy == "1":
assert config.engram_config is not None
assert config.engram_config.cpu_offload is True
else:
assert config.engram_config is None
with pytest.raises(
ValueError, match="Engram does not support DBO or microbatching"
):
VllmConfig._resolve_and_verify_engram_config(config)
@pytest.mark.parametrize("legacy", ["0", "1"])
def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy):
def test_engram_explicit_config_requires_supported_model():
"""Explicit all-false settings still opt into model validation."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig(engram_config=EngramConfig(cpu_offload=False))
def test_engram_legacy_offload_requires_supported_model(monkeypatch):
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig()
@pytest.mark.parametrize("target_has_ple", [False, True])
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
@pytest.mark.parametrize("explicit", [False, True])
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, explicit):
"""MTP may inherit cross-DP sharding without having its own PLE layers."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
target = SimpleNamespace(
@@ -1277,24 +1334,29 @@ def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
speculative_config=SimpleNamespace(
draft_model_config=draft, target_model_config=target
),
engram_config=EngramConfig(embedding_across_dp=True),
engram_config=EngramConfig(embedding_across_dp=True) if explicit else None,
parallel_config=ParallelConfig(),
load_config=LoadConfig(load_format="dummy"),
),
)
if target_has_ple:
VllmConfig._resolve_and_verify_engram_config(config)
else:
if explicit and not target_has_ple:
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig._resolve_and_verify_engram_config(config)
else:
VllmConfig._resolve_and_verify_engram_config(config)
assert (config.engram_config is not None) == target_has_ple
if target_has_ple:
assert config.engram_config.cpu_offload is True
def test_engram_hash_tracks_storage_and_sharding():
def test_engram_hash_tracks_execution_options():
configs = [
EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp)
for offload in (False, True)
for across_dp in (False, True)
EngramConfig(cpu_offload=True),
EngramConfig(cpu_offload=False),
EngramConfig(cpu_offload=True, embedding_across_dp=True),
EngramConfig(cpu_offload=True, dp_shared_memory=True),
]
assert len({config.compute_hash() for config in configs}) == 4
assert len({config.compute_hash() for config in configs}) == len(configs)
@pytest.mark.parametrize("port", [1, 29550, 65535])