[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: ywang96 <> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Roger Wang
ywang96 <>
Woosuk Kwon
Codex
parent
e7a3963339
commit
d2d649e674
+110
-48
@@ -1158,31 +1158,68 @@ def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: in
|
||||
assert config.get_parallel_size(parallel) == expected
|
||||
|
||||
|
||||
def test_engram_rejects_elastic_cross_dp():
|
||||
@pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"])
|
||||
def test_engram_rejects_elastic_cross_dp(option):
|
||||
parallel = ParallelConfig(
|
||||
tensor_parallel_size=4,
|
||||
data_parallel_size=2,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
parallel.enable_elastic_ep = True
|
||||
with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"):
|
||||
EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel)
|
||||
with pytest.raises(ValueError, match=f"{option}.*elastic EP"):
|
||||
EngramConfig(**{option: True}).verify_parallel_config(parallel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", [None, "0", "1"])
|
||||
def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy):
|
||||
"""Explicit settings must override the legacy environment fallback."""
|
||||
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
|
||||
if legacy is not None:
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
assert EngramConfig().cpu_offload == (legacy == "1")
|
||||
assert EngramConfig(cpu_offload=False).cpu_offload is False
|
||||
assert EngramConfig(cpu_offload=True).cpu_offload is True
|
||||
def test_engram_dp_shared_memory_requires_cpu_offload():
|
||||
with pytest.raises(ValueError, match="requires cpu_offload"):
|
||||
EngramConfig(cpu_offload=False, dp_shared_memory=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dp_size,load_format,multithread,error",
|
||||
[
|
||||
(1, "auto", False, "requires data_parallel_size > 1"),
|
||||
(2, "dummy", False, "requires load_format"),
|
||||
(2, "sharded_state", False, "requires load_format"),
|
||||
(2, "auto", False, None),
|
||||
(2, "safetensors", True, None),
|
||||
(2, "pt", True, None),
|
||||
],
|
||||
)
|
||||
def test_engram_dp_shared_memory_config_validation(
|
||||
monkeypatch, dp_size, load_format, multithread, error
|
||||
):
|
||||
"""Reject invalid shared configs before distributed init; allow threaded loads."""
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
config = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
hf_text_config=SimpleNamespace(engram_layer_ids=[1]),
|
||||
),
|
||||
speculative_config=None,
|
||||
engram_config=EngramConfig(cpu_offload=True, dp_shared_memory=True),
|
||||
parallel_config=ParallelConfig(data_parallel_size=dp_size),
|
||||
load_config=LoadConfig(
|
||||
load_format=load_format,
|
||||
model_loader_extra_config={"enable_multithread_load": multithread},
|
||||
),
|
||||
),
|
||||
)
|
||||
if error:
|
||||
with pytest.raises(ValueError, match=error):
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
else:
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"architecture, ple_layers, cuda, supported",
|
||||
[
|
||||
("DeepseekV41ForCausalLM", [1], True, True),
|
||||
("DeepseekV41ForCausalLM", [], True, False),
|
||||
("DeepseekV41ForCausalLM", [1], False, False),
|
||||
("Qwen4ExpForCausalLM", [1], True, True),
|
||||
("Qwen4ExpForConditionalGeneration", [1], True, True),
|
||||
("Qwen4ExpForCausalLM", [], True, False),
|
||||
@@ -1201,7 +1238,9 @@ def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, suppo
|
||||
ModelConfig,
|
||||
SimpleNamespace(
|
||||
architecture=architecture,
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers),
|
||||
hf_text_config=SimpleNamespace(
|
||||
ple_layer_ids=ple_layers, engram_layer_ids=ple_layers
|
||||
),
|
||||
),
|
||||
)
|
||||
if architecture is not None
|
||||
@@ -1214,55 +1253,73 @@ def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, suppo
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
config.verify_model_config(model)
|
||||
|
||||
resolved = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=model,
|
||||
speculative_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
load_config=LoadConfig(load_format="dummy"),
|
||||
engram_config=None,
|
||||
),
|
||||
)
|
||||
VllmConfig._resolve_and_verify_engram_config(resolved)
|
||||
assert (resolved.engram_config is not None) == supported
|
||||
if supported:
|
||||
assert resolved.engram_config.cpu_offload is True
|
||||
|
||||
def test_engram_config_defaults_to_none(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"), [(None, True), ("0", False), ("1", True)]
|
||||
)
|
||||
def test_engram_cpu_offload_environment_default(monkeypatch, value, expected):
|
||||
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
|
||||
if value is not None:
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", value)
|
||||
assert EngramConfig().cpu_offload is expected
|
||||
assert EngramConfig(cpu_offload=False).cpu_offload is False
|
||||
assert EngramConfig(cpu_offload=True).cpu_offload is True
|
||||
|
||||
|
||||
def test_engram_config_defaults_to_none():
|
||||
config = VllmConfig()
|
||||
assert config.engram_config is None
|
||||
assert config.compute_hash()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", ["0", "1"])
|
||||
def test_engram_none_resolves_legacy_offload(monkeypatch, legacy):
|
||||
"""Legacy enablement materializes a config before model validation."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
@pytest.mark.parametrize("explicit", [False, True])
|
||||
@pytest.mark.parametrize("enable_dbo,ubatch_size", [(True, 0), (False, 2)])
|
||||
def test_deepseek_engram_rejects_microbatching(explicit, enable_dbo, ubatch_size):
|
||||
"""Reject overlapping Engram staging even without an explicit EngramConfig."""
|
||||
config = cast(
|
||||
VllmConfig,
|
||||
SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
hf_text_config=SimpleNamespace(ple_layer_ids=[1]),
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
hf_text_config=SimpleNamespace(engram_layer_ids=[1]),
|
||||
),
|
||||
speculative_config=None,
|
||||
engram_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
engram_config=EngramConfig(cpu_offload=False) if explicit else None,
|
||||
parallel_config=ParallelConfig(
|
||||
enable_dbo=enable_dbo, ubatch_size=ubatch_size
|
||||
),
|
||||
),
|
||||
)
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
if legacy == "1":
|
||||
assert config.engram_config is not None
|
||||
assert config.engram_config.cpu_offload is True
|
||||
else:
|
||||
assert config.engram_config is None
|
||||
with pytest.raises(
|
||||
ValueError, match="Engram does not support DBO or microbatching"
|
||||
):
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", ["0", "1"])
|
||||
def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy):
|
||||
def test_engram_explicit_config_requires_supported_model():
|
||||
"""Explicit all-false settings still opt into model validation."""
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig(engram_config=EngramConfig(cpu_offload=False))
|
||||
|
||||
|
||||
def test_engram_legacy_offload_requires_supported_model(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_has_ple", [False, True])
|
||||
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
|
||||
@pytest.mark.parametrize("explicit", [False, True])
|
||||
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, explicit):
|
||||
"""MTP may inherit cross-DP sharding without having its own PLE layers."""
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
target = SimpleNamespace(
|
||||
@@ -1277,24 +1334,29 @@ def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=draft, target_model_config=target
|
||||
),
|
||||
engram_config=EngramConfig(embedding_across_dp=True),
|
||||
engram_config=EngramConfig(embedding_across_dp=True) if explicit else None,
|
||||
parallel_config=ParallelConfig(),
|
||||
load_config=LoadConfig(load_format="dummy"),
|
||||
),
|
||||
)
|
||||
if target_has_ple:
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
else:
|
||||
if explicit and not target_has_ple:
|
||||
with pytest.raises(ValueError, match="requires a model with supported Engram"):
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
else:
|
||||
VllmConfig._resolve_and_verify_engram_config(config)
|
||||
assert (config.engram_config is not None) == target_has_ple
|
||||
if target_has_ple:
|
||||
assert config.engram_config.cpu_offload is True
|
||||
|
||||
|
||||
def test_engram_hash_tracks_storage_and_sharding():
|
||||
def test_engram_hash_tracks_execution_options():
|
||||
configs = [
|
||||
EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp)
|
||||
for offload in (False, True)
|
||||
for across_dp in (False, True)
|
||||
EngramConfig(cpu_offload=True),
|
||||
EngramConfig(cpu_offload=False),
|
||||
EngramConfig(cpu_offload=True, embedding_across_dp=True),
|
||||
EngramConfig(cpu_offload=True, dp_shared_memory=True),
|
||||
]
|
||||
assert len({config.compute_hash() for config in configs}) == 4
|
||||
assert len({config.compute_hash() for config in configs}) == len(configs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("port", [1, 29550, 65535])
|
||||
|
||||
Reference in New Issue
Block a user