[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: ywang96 <>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Juntian Liu
2026-09-13 02:44:23 -07:00
committed by GitHub
co-authored by Roger Wang ywang96 <> Woosuk Kwon Codex
parent e7a3963339
commit d2d649e674
12 changed files with 1821 additions and 195 deletions
+22 -8
View File
@@ -89,7 +89,10 @@ def test_watermark_config_cli():
@pytest.mark.parametrize(
"options",
[
["--engram-config", '{"cpu_offload": false, "embedding_across_dp": true}'],
[
"--engram-config",
'{"cpu_offload": false, "embedding_across_dp": true}',
],
[
"--engram-config.cpu_offload",
"false",
@@ -98,9 +101,8 @@ def test_watermark_config_cli():
],
],
)
def test_engram_config_cli(options, monkeypatch):
"""CLI settings take precedence over the legacy offload environment."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
def test_engram_config_cli(options):
"""JSON and dotted CLI options independently control Engram settings."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = EngineArgs.from_cli_args(parser.parse_args(options))
assert args.engram_config is not None
@@ -109,14 +111,26 @@ def test_engram_config_cli(options, monkeypatch):
@pytest.mark.parametrize(
"options, provided",
[([], False), (["--engram-config", "{}"], True)],
"options,provided,dp_shared_memory",
[
([], False, False),
(["--engram-config", "{}"], True, False),
(
["--engram-config", '{"dp_shared_memory": true}'],
True,
True,
),
(["--engram-config.dp_shared_memory", "true"], True, True),
],
)
def test_engram_config_cli_optional(options, provided):
"""An explicit empty config must remain distinct from an omitted config."""
def test_engram_config_cli_optional(options, provided, dp_shared_memory):
"""Explicit configs honor defaults and the JSON/dotted DP shared-memory flag."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = EngineArgs.from_cli_args(parser.parse_args(options))
assert (args.engram_config is not None) == provided
if provided:
assert args.engram_config.cpu_offload is True
assert args.engram_config.dp_shared_memory is dp_shared_memory
@pytest.mark.parametrize(