[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: ywang96 <>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Juntian Liu
2026-09-13 02:44:23 -07:00
committed by GitHub
co-authored by Roger Wang ywang96 <> Woosuk Kwon Codex
parent e7a3963339
commit d2d649e674
12 changed files with 1821 additions and 195 deletions
+776
View File
@@ -0,0 +1,776 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Engram tables shared by co-located DP replicas.
Four GPUs on one node split the hash heads four ways, whether as DEP
(TP1 x DP4, the single-node MoE layout) or TP2 x DP2. A lookup only returns
the right rows if the ids of every replica are gathered, each rank looks up
its own heads, and each replica trades those tokens back for the heads its
peers own.
With shared host storage, DP replicas instead map the same TP slice and
prefetch only their own tokens without DP gathers.
"""
from itertools import product
from pathlib import Path
from tempfile import TemporaryDirectory
from threading import Event
from types import SimpleNamespace
import pytest
import torch
import torch.multiprocessing as mp
from vllm.config import (
EngramConfig,
VllmConfig,
get_current_vllm_config_or_none,
set_current_vllm_config,
)
from vllm.config.load import LoadConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.distributed import cleanup_dist_env_and_memory, parallel_state
from vllm.distributed.parallel_state import (
get_engram_dp_group,
get_engram_dp_size,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.forward_context import set_forward_context
from vllm.models.deepseek_v4_1.common.engram import EngramLayout
from vllm.models.deepseek_v4_1.nvidia import engram as engram_ops
from vllm.models.deepseek_v4_1.nvidia.engram import (
Engram,
ParallelEngramEmbedding,
engram_head_shard_rank,
gather_engram_hashes,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
WORLD_SIZE = 4
DIM, BLOCK = 64, 32
# Prime-sized buckets, as the model's layout builds them.
HEAD_SIZES = (97, 101, 103, 107, 109, 113, 127, 131)
def _token_counts(dp_size: int) -> tuple[tuple[int, ...], ...]:
"""Even loads, then uneven ones with an idle replica to pad around."""
return ((8,) * dp_size, (7, 0, 5, 1)[:dp_size], (7,) * dp_size)
def _full_table(rows: int) -> tuple[torch.Tensor, torch.Tensor]:
"""The unsharded table, identical on every rank."""
torch.manual_seed(0)
weight = (torch.randn(rows, DIM) * 4).to(torch.float8_e4m3fn)
scale_inv = torch.randint(120, 134, (rows, DIM // BLOCK), dtype=torch.uint8)
return weight, scale_inv
def _make_ids(head_sizes: tuple[int, ...], num_tokens: int, seed: int) -> torch.Tensor:
"""Ids of each head, drawn from that head's own bucket range."""
torch.manual_seed(seed)
offset = 0
columns = []
for size in head_sizes:
columns.append(torch.randint(offset, offset + size, (num_tokens, 1)))
offset += size
return torch.cat(columns, dim=1).to(torch.int32).cuda()
def _reference(weight, scale_inv, ids) -> torch.Tensor:
"""Dequantized lookup against the whole table."""
valid = ids >= 0
safe_ids = ids.clamp_min(0).long()
values = torch.nn.functional.embedding(safe_ids, weight).float()
scales = torch.nn.functional.embedding(safe_ids, scale_inv)
scales = (scales.to(torch.int32) << 23).view(torch.float32)
values = values.unflatten(-1, (-1, BLOCK)) * scales.unsqueeze(-1)
return values.flatten(-2).to(torch.bfloat16).masked_fill(~valid.unsqueeze(-1), 0)
@pytest.mark.parametrize(
"architecture,options,edp_ranks,etp_ranks",
[
("DeepseekV41ForCausalLM", {}, [0, 2], [0, 1]),
("Qwen4ExpForCausalLM", {}, None, [0, 1]),
(
"Qwen4ExpForCausalLM",
{"embedding_across_dp": True},
None,
[0, 1, 2, 3],
),
("Qwen4ExpForConditionalGeneration", {}, None, [0, 1]),
],
)
def test_engram_group_creation_honors_model_and_options(
monkeypatch, architecture, options, edp_ranks, etp_ranks
):
config = SimpleNamespace(
model_config=SimpleNamespace(architecture=architecture, is_moe=False),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=2,
distributed_executor_backend="mp",
),
engram_config=EngramConfig(**options),
)
for name in (
"_TP",
"_ETP",
"_ENGRAM_DP",
"_DCP",
"_PCP",
"_PP",
"_DP",
"_EP",
"_EPLB",
):
monkeypatch.setattr(parallel_state, name, None)
monkeypatch.setattr(parallel_state, "_WORLD", SimpleNamespace(local_rank=0))
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 4)
monkeypatch.setattr(torch.distributed, "get_rank", lambda: 0)
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
def make_group(group_ranks, *args, **kwargs):
ranks = next(ranks for ranks in group_ranks if 0 in ranks)
return SimpleNamespace(ranks=ranks, rank_in_group=0, world_size=len(ranks))
monkeypatch.setattr(parallel_state, "init_model_parallel_group", make_group)
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=2, backend="nccl")
group = get_engram_dp_group()
assert (group.ranks if group is not None else None) == edp_ranks
assert parallel_state.get_etp_group().ranks == etp_ranks
def _worker(rank: int, tp_size: int, port: int) -> None:
torch.accelerator.set_device_index(rank)
update_environment_variables(
{
"RANK": str(rank),
"LOCAL_RANK": str(rank),
"WORLD_SIZE": str(WORLD_SIZE),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(port),
}
)
dp_size = WORLD_SIZE // tp_size
dp_rank, tp_rank = divmod(rank, tp_size)
vllm_config = VllmConfig(
parallel_config=ParallelConfig(
tensor_parallel_size=tp_size,
data_parallel_size=dp_size,
data_parallel_rank=dp_rank,
),
scheduler_config=SchedulerConfig(
max_model_len=8,
is_encoder_decoder=False,
max_num_batched_tokens=8,
max_num_seqs=8,
),
)
# Supply the resolved Engram settings without constructing a full model.
vllm_config.engram_config = EngramConfig()
vllm_config.model_config = SimpleNamespace(
architecture="DeepseekV41ForCausalLM", is_moe=True
)
try:
init_distributed_environment()
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=tp_size)
assert get_engram_dp_size() == dp_size
assert get_engram_dp_group().ranks == [
tp_rank + replica * tp_size for replica in range(dp_size)
]
assert engram_head_shard_rank() == tp_rank * dp_size + dp_rank
for failure in ("create", "open", "mmap", "register", "pinned", None):
_check_shared_storage_lifetime_and_failures(failure)
# Reuse the processes and communication groups across storage modes.
for (cpu_offload, dp_shared_memory), n_heads, sequence_parallel in product(
[(False, False), (True, False), (True, True)],
[8, 7],
[False, True] if tp_size > 1 else [False],
):
_check_table(
vllm_config,
cpu_offload,
dp_shared_memory,
n_heads,
sequence_parallel,
)
finally:
cleanup_dist_env_and_memory()
def _check_shared_storage_lifetime_and_failures(failure):
"""One peer's failure must roll back every mapping/registration, without hangs."""
import gc
import weakref
group = get_engram_dp_group()
runtime = torch.cuda.cudart()
make_file = engram_ops.tempfile.NamedTemporaryFile
map_file = engram_ops.mmap.mmap
paths, mappings, registrations, unregistrations = [], [], [], []
failed_rank = 0 if failure == "create" else 1
def fail():
raise OSError(f"injected {failure} failure")
def temporary_file(*args, **kwargs):
if failure == "create":
fail()
file = make_file(*args, **kwargs)
paths.append(Path(file.name))
return file
def open_file(*args, **kwargs):
if failure == "open" and group.rank_in_group == failed_rank:
fail()
return open(*args, **kwargs)
def mapping(*args, **kwargs):
if failure == "mmap" and group.rank_in_group == failed_rank:
fail()
result = map_file(*args, **kwargs)
mappings.append(weakref.ref(result))
return result
def register(pointer, size, flags):
if failure == "register" and group.rank_in_group == failed_rank:
return SimpleNamespace(value=1)
result = runtime.cudaHostRegister(pointer, size, flags)
assert result.value == 0
registrations.append(pointer)
return result
def unregister(pointer):
unregistrations.append(pointer)
result = runtime.cudaHostUnregister(pointer)
assert result.value == 0
return result
with pytest.MonkeyPatch.context() as patch:
patch.setattr(
engram_ops,
"tempfile",
SimpleNamespace(NamedTemporaryFile=temporary_file),
)
patch.setattr(engram_ops, "open", open_file, raising=False)
patch.setattr(
engram_ops,
"mmap",
SimpleNamespace(mmap=mapping, MAP_SHARED=engram_ops.mmap.MAP_SHARED),
)
patch.setattr(
torch.cuda,
"cudart",
lambda: SimpleNamespace(
cudaHostRegister=register,
cudaHostUnregister=unregister,
),
)
if failure == "pinned" and group.rank_in_group == failed_rank:
patch.setattr(torch.Tensor, "is_pinned", lambda self: False)
if failure is not None:
stage = "cudaHostRegister" if failure in ("register", "pinned") else failure
with pytest.raises(RuntimeError, match=f"EDP rank {failed_rank}.*{stage}"):
engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
assert unregistrations == registrations
else:
storage = engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
assert all(not path.exists() for path in paths)
storage_ref = weakref.ref(storage)
parameter = torch.nn.Parameter(storage.weight, requires_grad=False)
views = storage.get_views(parameter, storage.weight_scale_inv)
del storage
gc.collect()
assert storage_ref() is None
assert not unregistrations
del parameter
gc.collect()
assert not unregistrations # UVA views still retain the CPU allocation.
del views
gc.collect()
assert unregistrations == registrations
for ref in mappings:
mapped = ref()
assert mapped is None or mapped.closed
assert all(not path.exists() for path in paths)
def _check_table(
vllm_config, cpu_offload, dp_shared_memory, n_heads, sequence_parallel
):
parallel = vllm_config.parallel_config
dp_size, tp_size = parallel.data_parallel_size, parallel.tensor_parallel_size
dp_rank = parallel.data_parallel_rank
tp_rank = engram_ops.get_tensor_model_parallel_rank()
multithread = (
dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1)
)
load_config = (
LoadConfig(
load_format="safetensors",
use_tqdm_on_load=False,
model_loader_extra_config={
"enable_multithread_load": True,
"num_threads": 2,
},
)
if multithread
else vllm_config.load_config
)
with pytest.MonkeyPatch.context() as m:
head_sizes = HEAD_SIZES[:n_heads]
rows = sum(head_sizes)
config = SimpleNamespace(
hidden_size=DIM,
hc_mult=2,
rms_norm_eps=1e-6,
engram_layer_ids=[0],
engram_num_embeddings=[rows],
engram_max_ngram_size=2,
engram_n_heads=n_heads,
engram_head_dim=DIM,
engram_compressed_vocab_size=32,
engram_pad_token_id=0,
engram_vocab_size=HEAD_SIZES[0],
)
with set_current_vllm_config(vllm_config):
weight, scale_inv = _full_table(rows)
layout = EngramLayout(config)
# Stub configuration inputs, not Engram's initialization or execution.
component_config = SimpleNamespace(
engram_config=EngramConfig(
cpu_offload=cpu_offload,
dp_shared_memory=dp_shared_memory,
),
scheduler_config=vllm_config.scheduler_config,
load_config=load_config,
)
with m.context() as init_patch, torch.device("cuda"):
init_patch.setattr(
engram_ops, "get_current_vllm_config", lambda: component_config
)
engram = Engram(
config,
quant_config=None,
layout=layout,
layer_hash_index=0,
use_sequence_parallel=sequence_parallel,
prefix="model.layers.0.engram",
)
layer = engram.embed_tokens
if multithread:
_load_shared_multithread(layer, weight, scale_inv, load_config)
else:
# Only the leader has valid checkpoint payload in shared mode.
loader_weight = weight if not dp_shared_memory or dp_rank == 0 else None
loader_scale = (
scale_inv if not dp_shared_memory or dp_rank == 0 else None
)
layer.weight.weight_loader(layer.weight, loader_weight)
layer.weight_scale_inv.weight_loader(
layer.weight_scale_inv, loader_scale
)
if dp_shared_memory:
assert layer.dp_size == 1
assert layer.head_start == tp_rank * layer.part_n_hash_cols
assert layer.weight.is_pinned() and layer.weight_scale_inv.is_pinned()
def unexpected_collective(*args, **kwargs):
raise AssertionError("shared lookup must not use DP collectives")
m.setattr(get_engram_dp_group(), "all_gather", unexpected_collective)
token_counts = _token_counts(dp_size)
# Production forward runs after the construction-time config has exited.
assert get_current_vllm_config_or_none() is None
for batch_idx, counts in enumerate(token_counts):
num_tokens = counts[dp_rank]
# Ids differ per replica but must agree across its TP ranks.
ids = _make_ids(head_sizes, num_tokens, seed=100 + dp_rank)
if batch_idx == 2 and dp_rank == 1:
ids.fill_(engram_ops.DEAD_ID)
with set_forward_context(
None,
vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
):
gathered = gather_engram_hashes(
ids, dp_shared_memory=layer.dp_shared_memory
)
expected_tokens = (
num_tokens if dp_shared_memory else max(counts) * dp_size
)
assert gathered.shape[0] == expected_tokens
engram.prepare_embeddings(gathered)
looked_up = engram.embed(ids)
standalone = layer(ids)
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
torch.testing.assert_close(standalone, expected, atol=0, rtol=0)
if sequence_parallel:
# SP hands back only this rank's token shard, zero-padded.
chunk = -(-num_tokens // tp_size)
expected = torch.nn.functional.pad(
expected, (0, 0, 0, 0, 0, chunk * tp_size - num_tokens)
)
expected = expected[tp_rank * chunk : (tp_rank + 1) * chunk]
torch.testing.assert_close(looked_up, expected, atol=0, rtol=0)
if (
cpu_offload
and n_heads == 7
and not sequence_parallel
and (not dp_shared_memory or tp_size == 1)
):
_check_dummy_hash_model_forward(
vllm_config,
engram,
head_sizes,
weight,
scale_inv,
dp_shared_memory,
)
replay = n_heads == 7 and (
(dp_shared_memory and sequence_parallel == (tp_size > 1))
or (
cpu_offload
and not dp_shared_memory
and sequence_parallel == (tp_size > 1)
)
)
if replay:
captures = ("full", "breakable")
for capture in captures:
_check_prefetch_replay(
engram,
head_sizes,
weight,
scale_inv,
dp_shared_memory,
capture,
)
if dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1):
with m.context() as storage_patch:
storage_patch.setattr(layer.weight, "data", layer.weight.data.clone())
with pytest.raises(RuntimeError, match="storage must not be replaced"):
layer(ids)
def _check_dummy_hash_model_forward(
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
):
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
from vllm.models.deepseek_v4_1.common.engram import NgramHashState
from vllm.models.deepseek_v4_1.nvidia import model as model_ops
dp_rank = vllm_config.parallel_config.data_parallel_rank
if dp_shared_memory and dp_rank != 1:
return
tokens = 4
ids = _make_ids(head_sizes, tokens, seed=300 + dp_rank)
state = NgramHashState.__new__(NgramHashState)
torch.nn.Module.__init__(state)
state.multipliers = torch.empty(1, 2, dtype=torch.int64, device="cuda")
state.primes = torch.empty(1, 1, len(head_sizes), dtype=torch.int64, device="cuda")
state.ensure_cache = lambda: True
# Hash arithmetic is covered separately; keep the real model's branch and
# NgramHashState.dummy_hashes, plus real preparation/lookup collectives.
state.forward = lambda *args: ids.unsqueeze(1)
class Decoder(SimpleNamespace):
def __call__(self, hidden, positions, input_ids, *args):
hashes, keep = args[-2:]
if dp_shared_memory and dp_rank == 1:
assert hashes is None and keep is None
output = hidden.new_zeros(
tokens, len(head_sizes), DIM, dtype=torch.bfloat16
)
else:
assert hashes is not None and keep is not None
assert bool(keep.all()) == (dp_rank != 1)
if dp_rank == 1:
assert torch.all(hashes == engram_ops.DEAD_ID)
output = engram.embed(hashes[:, 0])
return output, None, None, None, None
model = SimpleNamespace(
use_mega_moe=False,
use_sequence_parallel=False,
engram_hash=state,
engram_swa_prefix="swa",
engram_dp_shared_memory=dp_shared_memory,
layers=[Decoder(engram=engram)],
start_layer=0,
end_layer=1,
aux_hidden_state_layers=(),
)
metadata = (
None
if dp_rank == 1
else {
"swa": SimpleNamespace(
query_start_loc=None, slot_mapping=None, block_table=None
)
}
)
with pytest.MonkeyPatch.context() as patch:
patch.setattr(
model_ops,
"get_pp_group",
lambda: SimpleNamespace(
is_first_rank=True,
is_last_rank=False,
),
)
patch.setattr(model_ops, "mhc_post_tilelang", lambda hidden, *args: hidden)
with set_forward_context(
metadata,
vllm_config,
num_tokens=tokens,
num_tokens_across_dp=torch.full(
(vllm_config.parallel_config.data_parallel_size,),
tokens,
dtype=torch.int32,
),
):
output = model_ops.DeepseekV4Model.forward(
model,
torch.arange(tokens, device="cuda"),
torch.arange(tokens, device="cuda"),
intermediate_tensors=None,
inputs_embeds=torch.zeros(tokens, DIM, device="cuda"),
lookback_token_ids=torch.full((1, 1), -1, device="cuda"),
)["hidden_states"]
expected_ids = torch.full_like(ids, engram_ops.DEAD_ID) if dp_rank == 1 else ids
expected = _reference(weight.cuda(), scales.cuda(), expected_ids)
torch.testing.assert_close(output, expected, atol=0, rtol=0)
def _load_shared_multithread(layer, weight, scale_inv, load_config):
"""Opposite shard completion orders must leave every shared parameter ready."""
from safetensors.torch import save_file
from vllm.model_executor.model_loader import weight_utils
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
tensors = {
"weight": weight,
"weight_scale_inv": scale_inv.view(torch.float8_e8m0fnu),
}
order = list(tensors)
if get_engram_dp_group().rank_in_group % 2:
order.reverse()
first_loaded = Event()
original_load = weight_utils.load_file
def load_file(path, **kwargs):
if Path(path).stem != order[0]:
assert first_loaded.wait(timeout=30), "First shared weight load stalled"
return original_load(path, **kwargs)
with TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as m:
for name, tensor in tensors.items():
save_file({name: tensor}, str(Path(directory) / f"{name}.safetensors"))
m.setattr(weight_utils, "load_file", load_file)
loader = DefaultModelLoader(load_config)
source = DefaultModelLoader.Source(directory, revision=None)
loaded = []
for name, tensor in loader._get_weights_iterator(source):
param = getattr(layer, name)
param.weight_loader(param, tensor)
loaded.append(name)
first_loaded.set()
assert loaded == order
# Check before any further collective could mask incomplete shared writes.
for name, tensor in tensors.items():
param = getattr(layer, name)
expected = tensor.narrow(0, param.engram_vocab_start, param.shape[0])
torch.testing.assert_close(
param.view(torch.uint8), expected.view(torch.uint8), atol=0, rtol=0
)
def _check_prefetch_replay(
engram, head_sizes, weight, scale_inv, dp_shared_memory, capture
):
"""DP-sharded and shared prefetches must replay with new IDs."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
ids = _make_ids(head_sizes, 8, seed=501)
tp_rank = engram_ops.get_tensor_model_parallel_rank()
tokens = 8 // engram.embed_tokens.tp_size if engram.use_sequence_parallel else 8
output = torch.empty(
tokens, len(head_sizes), DIM, device="cuda", dtype=torch.bfloat16
)
def step():
gathered = gather_engram_hashes(ids.clone(), dp_shared_memory=dp_shared_memory)
engram.prepare_embeddings(gathered)
output.copy_(engram.embed(ids))
with pytest.MonkeyPatch.context() as patch:
patch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 8)
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup):
step()
if capture == "breakable":
graph = BreakableCUDAGraphCapture()
with torch.cuda.stream(warmup), graph:
step()
assert graph.num_eager_breaks == 2
else:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=warmup):
step()
torch.cuda.current_stream().wait_stream(warmup)
for iteration in range(2):
ids.copy_(_make_ids(head_sizes, 8, seed=600 + iteration))
graph.replay()
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
if engram.use_sequence_parallel:
expected = expected[tp_rank * tokens : (tp_rank + 1) * tokens]
torch.testing.assert_close(output, expected, atol=0, rtol=0)
torch.accelerator.synchronize()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("tp_size", [1, 2])
def test_engram_table_shared_across_dp_replicas(tp_size, monkeypatch):
"""Check all storage modes with one process launch per TP/DP topology."""
if torch.accelerator.device_count() < WORLD_SIZE:
pytest.skip(f"Need {WORLD_SIZE} GPUs to run the test.")
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1")
mp.spawn(_worker, args=(tp_size, get_open_port()), nprocs=WORLD_SIZE)
cleanup_dist_env_and_memory()
@pytest.mark.parametrize(
"cpu_offload,dp_size,error",
[
(False, 4, "requires cpu_offload=True"),
(True, 1, "effective Engram DP size is 1"),
],
)
def test_engram_dp_shared_memory_runtime_requirements(
monkeypatch, cpu_offload, dp_size, error
):
"""Report unmet storage/topology requirements before allocating CUDA memory."""
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
with pytest.raises(ValueError, match=error):
ParallelEngramEmbedding(
sum(HEAD_SIZES),
DIM,
HEAD_SIZES,
cpu_offload=cpu_offload,
dp_shared_memory=True,
)
@pytest.mark.parametrize(
"node_ids,dp_size,replica_size,expected",
[
([0] * 4, 1, 4, 1),
([0] * 4, 4, 1, 4),
([0] * 4, 2, 2, 2),
([0] * 8 + [1] * 8, 8, 2, 4),
([0] * 6 + [1] * 6, 4, 1, 2),
([0] * 3 + [1] * 4, 7, 1, 1),
([0] * 8 + [1] * 8 + [2] * 8, 8, 3, 1),
([0, 1] * 4, 4, 2, 1),
([0] * 3 + [1] * 5, 4, 2, 1),
],
)
def test_engram_dp_shard_size_respects_node_boundaries(
node_ids, dp_size, replica_size, expected, monkeypatch
):
"""Sharing must fall back when replicas or node rank ranges do not align."""
monkeypatch.setattr(parallel_state, "get_node_count", lambda: len(set(node_ids)))
monkeypatch.setattr(
parallel_state, "get_world_group", lambda: SimpleNamespace(cpu_group=None)
)
monkeypatch.setattr(
parallel_state,
"in_the_same_node_as",
lambda group, src: [node == node_ids[src] for node in node_ids],
)
assert (
parallel_state._engram_dp_shard_size(len(node_ids), dp_size, replica_size)
== expected
)
@pytest.mark.parametrize("num_tokens", [0, 2, 4, 5])
def test_engram_hash_padding_has_no_valid_rows(num_tokens, monkeypatch):
"""Idle/padded slots must not query row zero; oversized batches are rejected."""
monkeypatch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 4)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(all_gather=lambda ids, dim: torch.cat([ids, ids], dim)),
)
ids = torch.full((num_tokens, 2, 3), 17, dtype=torch.int32)
if num_tokens > 4:
with pytest.raises(ValueError, match="exceeds the DP token slot"):
gather_engram_hashes(ids)
return
gathered = gather_engram_hashes(ids).reshape(2, 4, 2, 3)
for replica in gathered:
torch.testing.assert_close(replica[:num_tokens], ids)
assert torch.all(replica[num_tokens:] == engram_ops.DEAD_ID)
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("uniform", [False, True])
def test_engram_hash_padding_ignores_other_nodes(tp_size, uniform, monkeypatch):
"""Two four-GPU nodes pad only to their own maximum, including TP x DP."""
group_size = 4 // tp_size
counts = (
[4] * (8 // tp_size)
if uniform
else [4096] + [2] * (group_size - 1) + [1, 3] + [0] * (group_size - 2)
)
monkeypatch.setattr(
engram_ops,
"get_forward_context",
lambda: SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor(counts))
),
)
for dp_rank in range(8 // tp_size):
slot = 4 if uniform else (4096 if dp_rank < group_size else 3)
ids = torch.full((counts[dp_rank], 2, 3), 17, dtype=torch.int32)
monkeypatch.setattr(
engram_ops,
"get_dp_group",
lambda rank=dp_rank: SimpleNamespace(rank_in_group=rank),
)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda rank=dp_rank: SimpleNamespace(
rank_in_group=rank % group_size,
world_size=group_size,
all_gather=lambda tensor, dim: torch.cat([tensor] * group_size, dim),
),
)
gathered = gather_engram_hashes(ids)
expected = ids.new_full((slot, 2, 3), engram_ops.DEAD_ID)
expected[: ids.shape[0]] = ids
torch.testing.assert_close(
gathered, expected.repeat(group_size, 1, 1), msg=f"DP rank {dp_rank}"
)
+22 -8
View File
@@ -89,7 +89,10 @@ def test_watermark_config_cli():
@pytest.mark.parametrize(
"options",
[
["--engram-config", '{"cpu_offload": false, "embedding_across_dp": true}'],
[
"--engram-config",
'{"cpu_offload": false, "embedding_across_dp": true}',
],
[
"--engram-config.cpu_offload",
"false",
@@ -98,9 +101,8 @@ def test_watermark_config_cli():
],
],
)
def test_engram_config_cli(options, monkeypatch):
"""CLI settings take precedence over the legacy offload environment."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
def test_engram_config_cli(options):
"""JSON and dotted CLI options independently control Engram settings."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = EngineArgs.from_cli_args(parser.parse_args(options))
assert args.engram_config is not None
@@ -109,14 +111,26 @@ def test_engram_config_cli(options, monkeypatch):
@pytest.mark.parametrize(
"options, provided",
[([], False), (["--engram-config", "{}"], True)],
"options,provided,dp_shared_memory",
[
([], False, False),
(["--engram-config", "{}"], True, False),
(
["--engram-config", '{"dp_shared_memory": true}'],
True,
True,
),
(["--engram-config.dp_shared_memory", "true"], True, True),
],
)
def test_engram_config_cli_optional(options, provided):
"""An explicit empty config must remain distinct from an omitted config."""
def test_engram_config_cli_optional(options, provided, dp_shared_memory):
"""Explicit configs honor defaults and the JSON/dotted DP shared-memory flag."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
args = EngineArgs.from_cli_args(parser.parse_args(options))
assert (args.engram_config is not None) == provided
if provided:
assert args.engram_config.cpu_offload is True
assert args.engram_config.dp_shared_memory is dp_shared_memory
@pytest.mark.parametrize(
+227 -15
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import bisect
import inspect
from types import SimpleNamespace
import pytest
@@ -8,10 +9,13 @@ import torch
from vllm.models.deepseek_v4_1.common import engram as engram_ops
from vllm.models.deepseek_v4_1.common.engram import (
Engram,
NgramHashState,
ParallelEngramEmbedding,
Engram as CommonEngram,
)
from vllm.models.deepseek_v4_1.common.engram import (
NgramHashState,
)
from vllm.models.deepseek_v4_1.nvidia import engram as nvidia_engram_ops
from vllm.models.deepseek_v4_1.nvidia.engram import Engram, ParallelEngramEmbedding
from vllm.platforms import current_platform
@@ -107,7 +111,7 @@ def test_fused_engram_post_wkv_matches_reference(
)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
module = Engram.__new__(Engram)
module = CommonEngram.__new__(CommonEngram)
torch.nn.Module.__init__(module)
module.dim = dim
module.hc_mult = hc_mult
@@ -116,7 +120,7 @@ def test_fused_engram_post_wkv_matches_reference(
module.use_sequence_parallel = use_sequence_parallel
module.embed_tokens = torch.nn.Identity()
# `forward` reads rows staged by `prepare_embeddings`, so inject kv there.
module.embed_tokens.tp_size = 1
module.embed_tokens.tp_size = module.embed_tokens.n_hash_cols = 1
module.staged_rows = kv.unsqueeze(1)
if use_sequence_parallel:
padded = torch.nn.functional.pad(kv, (0, 0, 0, (-num_kv_tokens) % tp_size))
@@ -151,6 +155,25 @@ def _hash_state(use_slot_cache: bool) -> NgramHashState:
return state
@pytest.mark.parametrize("num_tokens", [0, 7])
def test_engram_dummy_hashes_leave_history_untouched(num_tokens):
"""Dummy forwards have no valid table IDs and do not alter cached history."""
state = _hash_state(use_slot_cache=True)
state.multipliers = torch.empty(2, 4, dtype=torch.int64)
state.primes = torch.empty(2, 3, 8, dtype=torch.int64)
state._cache = torch.arange(16, dtype=torch.int32)
history = state._cache.clone()
hashes, keep = state.dummy_hashes(torch.arange(num_tokens))
assert hashes.shape == (num_tokens, 2, 24)
assert hashes.dtype == torch.int32
assert torch.all(hashes == engram_ops.DEAD_ID)
assert keep.shape == (num_tokens,) and keep.dtype == torch.bool
assert not keep.any()
torch.testing.assert_close(state._cache, history)
@pytest.mark.parametrize("num_blocks", [4, 100])
def test_engram_cache_follows_kv_cache_binding(num_blocks):
"""Discard profiling history when the real KV cache replaces its storage."""
@@ -533,9 +556,11 @@ def _make_embedding(cpu_offload, rows=4096, dim=256, block=32):
layer = ParallelEngramEmbedding.__new__(ParallelEngramEmbedding)
torch.nn.Module.__init__(layer)
layer.dim, layer.block_size, layer.tp_size = dim, block, 1
layer.dp_size = 1
layer.n_hash_cols = layer.part_n_hash_cols = 24
layer.head_start = 0
layer.cpu_offload = cpu_offload
layer.dp_shared_memory = False
layer.part_num_embeddings = rows
# A window strictly inside the table, so unowned rows are exercised too.
layer.vocab_start_idx, layer.vocab_end_idx = rows // 4, rows // 4 + rows // 2
@@ -559,12 +584,27 @@ def _make_embedding(cpu_offload, rows=4096, dim=256, block=32):
return layer
@pytest.mark.parametrize(
"tp_size,dp_size,n_heads", [(1, 4, 5), (2, 2, 5), (4, 1, 6), (8, 1, 6)]
)
def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch):
"""Reject empty owners before allocating weights or accessing CUDA."""
monkeypatch.setattr(
engram_ops, "get_tensor_model_parallel_world_size", lambda: tp_size
)
monkeypatch.setattr(nvidia_engram_ops, "get_engram_dp_size", lambda: dp_size)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(nvidia_engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
with pytest.raises(AssertionError, match="ranks without hash heads"):
ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch):
"""Keep complete buckets and reconstruct head order, including TP padding."""
head_sizes = (17, 19, 23, 29, 31, 37)
head_sizes = (17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73)
num_rows, dim = sum(head_sizes), 64
torch.manual_seed(0)
weight = torch.randn(num_rows + 7, dim).to(torch.float8_e4m3fn)
@@ -623,7 +663,7 @@ def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeyp
module.staged_rows = torch.empty_like(shards[0])
module.prepare_embeddings(ids)
torch.testing.assert_close(module.embed(ids), expected, rtol=0, atol=0)
# Slicing before head reordering must preserve padded heads and empty owners.
# Slicing before head reordering must preserve padded heads and tokens.
module.use_sequence_parallel = True
chunk = (len(ids) + tp_size - 1) // tp_size
padded = torch.nn.functional.pad(expected, (0, 0, 0, 0, 0, (-len(ids)) % tp_size))
@@ -636,6 +676,31 @@ def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeyp
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
def test_engram_lookup_reuses_jit_across_token_shapes():
"""Runtime token counts and launch grids must share one JIT variant."""
layer = _make_embedding(cpu_offload=False)
kernel = engram_ops._engram_lookup_kernel
kernel_cache = kernel.device_caches[torch.accelerator.current_device_index()][0]
kernel_cache.clear()
cache_sizes = []
for background in (False, True):
for num_tokens in (1, 7, 256):
ids = torch.zeros(num_tokens, 24, dtype=torch.int32, device="cuda")
out = torch.empty(
num_tokens,
24,
layer.dim,
dtype=torch.bfloat16,
device="cuda",
)
layer.lookup(ids, out, background=background)
cache_sizes.append(len(kernel_cache))
assert cache_sizes == [1] * len(cache_sizes)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize("background", [False, True])
@@ -660,19 +725,147 @@ def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize(
"backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)]
)
def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
"""Default offload uses pinned storage and a side stream; False uses HBM."""
from vllm.config import EngramConfig
config = SimpleNamespace(hidden_size=16, hc_mult=1, rms_norm_eps=1e-6)
layout = SimpleNamespace(
num_embeddings=(72,),
head_dim=32,
max_ngram_size=2,
n_heads=24,
primes=(((3,) * 24,),),
)
vllm_config = SimpleNamespace(
engram_config=EngramConfig()
if cpu_offload is None
else EngramConfig(cpu_offload=cpu_offload),
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
)
offloaded = backend == "nvidia" and vllm_config.engram_config.cpu_offload
if backend == "common":
vllm_config.engram_config = None
monkeypatch.setattr(engram_ops, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(
nvidia_engram_ops, "get_current_vllm_config", lambda: vllm_config
)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
engram_ops, "ReplicatedLinear", lambda *a, **k: torch.nn.Identity()
)
with torch.device("cuda"):
cls = Engram if backend == "nvidia" else CommonEngram
module = cls(config, None, layout, 0, False, "engram")
layer = module.embed_tokens
if offloaded:
assert layer.weight.device.type == "cpu" and layer.weight.is_pinned()
else:
assert layer.weight.device.type == "cuda"
layer.weight.data.copy_(torch.randn(layer.weight.shape).to(torch.float8_e4m3fn))
layer.weight_scale_inv.data.fill_(127)
streams = []
lookup = layer.lookup
def track_lookup(ids, out, background=False):
streams.append(torch.cuda.current_stream())
lookup(ids, out, background=background)
layer.lookup = track_lookup
main = torch.cuda.current_stream()
ids = torch.randint(
layer.vocab_start_idx,
layer.vocab_end_idx,
(8, 24),
device="cuda",
dtype=torch.int32,
)
module.prepare_embeddings(ids)
expected = _reference_lookup(
layer.weight.cuda(),
layer.weight_scale_inv.cuda(),
ids,
layer.vocab_start_idx,
layer.vocab_end_idx,
)
torch.testing.assert_close(module.embed(ids), expected, atol=0, rtol=0)
if offloaded:
assert streams == [module._prefetch_stream]
assert streams[0] != main
else:
assert streams == [main]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize(
"cpu_offload,delay",
[(False, None), (True, None), (True, "producer"), (True, "lookup")],
)
@pytest.mark.parametrize("capture", ["eager", "full", "breakable"])
def test_engram_prepared_rows_survive_graph_breaks(cpu_offload, capture):
"""Consume early lookup results after a break, with fresh IDs each replay."""
_run_engram_prepared_rows(cpu_offload, capture)
def test_engram_prepared_rows_survive_graph_breaks(cpu_offload, capture, delay):
"""Temporary lookup IDs survive allocator reuse and graph replay."""
_run_engram_prepared_rows(cpu_offload, capture, delay=delay)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize(
"missing_dependency", ["record_stream", "producer_wait", "consumer_wait"]
)
def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_dependency):
"""Negative controls must expose incorrect rows when a dependency is removed."""
if (
missing_dependency == "record_stream"
and torch.cuda.memory.get_allocator_backend() != "native"
):
pytest.skip(
"The allocator reuse negative control requires the native allocator"
)
def start(self, hash_ids, rows, stream):
if missing_dependency != "producer_wait":
stream.wait_stream(torch.cuda.current_stream())
if missing_dependency != "record_stream":
hash_ids.record_stream(stream)
with torch.cuda.stream(stream):
self.embed_tokens.lookup(hash_ids, rows, background=True)
def finish(self, stream):
pass
monkeypatch.setattr(Engram, "_start_prefetch", start)
if missing_dependency == "consumer_wait":
monkeypatch.setattr(Engram, "_finish_prefetch", finish)
delay = "producer" if missing_dependency == "producer_wait" else "lookup"
try:
with pytest.raises(
AssertionError, match=r"Tensor-likes are not (?:equal|close)!"
):
_run_engram_prepared_rows(True, "eager", delay=delay)
finally:
torch.accelerator.synchronize()
def _run_engram_prepared_rows(
cpu_offload, capture, tp_size=1, rank=0, use_sequence_parallel=False
cpu_offload, capture, tp_size=1, rank=0, use_sequence_parallel=False, delay=None
):
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
from vllm.compilation.breakable_cudagraph import (
BreakableCUDAGraphCapture,
eager_break_during_capture,
)
layer = _make_embedding(cpu_offload)
if delay == "lookup":
lookup = layer.lookup
def delayed_lookup(indices, out, background=False):
torch.cuda._sleep(2_000_000)
lookup(indices, out, background=background)
layer.lookup = delayed_lookup
cols, num_tokens = (23, 65) if use_sequence_parallel else (24, 64)
layer.n_hash_cols = cols
layer.tp_size = tp_size
@@ -681,6 +874,17 @@ def _run_engram_prepared_rows(
engram = Engram.__new__(Engram)
torch.nn.Module.__init__(engram)
engram.embed_tokens = layer
engram._prefetch_stream = torch.cuda.Stream() if cpu_offload else None
# Exercise the production eager boundaries even when the test process
# imported Engram before breakable graphs were enabled.
if capture == "breakable":
import vllm.envs as envs
with pytest.MonkeyPatch.context() as patch:
patch.setattr(envs, "VLLM_USE_BREAKABLE_CUDAGRAPH", True)
for name in ("_start_prefetch", "_finish_prefetch"):
fn = eager_break_during_capture(inspect.unwrap(getattr(Engram, name)))
setattr(engram, name, fn.__get__(engram, Engram))
engram.use_sequence_parallel = use_sequence_parallel
engram.staged_rows = torch.empty(
num_tokens,
@@ -709,7 +913,12 @@ def _run_engram_prepared_rows(
embed = torch.compile(embed, backend="eager", fullgraph=True, dynamic=True)
def step(cap=None):
engram.prepare_embeddings(src)
if delay == "producer":
torch.cuda._sleep(2_000_000)
# Drop the last reference to a non-contiguous input after launching lookup.
engram.prepare_embeddings(hashes.clone()[:, 1])
# Exercise same-size allocator reuse before consuming the prefetched rows.
torch.empty_like(hashes).fill_(layer.part_num_embeddings - 1)
if cap is not None:
cap.add_eager(lambda: None)
out.copy_(embed(src))
@@ -730,9 +939,12 @@ def _run_engram_prepared_rows(
with torch.cuda.stream(warmup), graph:
step(graph)
torch.cuda.current_stream().wait_stream(warmup)
assert graph.num_graphs == 2
assert graph.num_eager_breaks == (3 if cpu_offload else 1)
for _ in range(3):
# Eager prefill can overwrite staging rows between decode replays.
hashes.random_(0, layer.part_num_embeddings)
step()
hashes.random_(0, layer.part_num_embeddings)
if graph is None:
step()
+110 -48
View File
@@ -1158,31 +1158,68 @@ def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: in
assert config.get_parallel_size(parallel) == expected
def test_engram_rejects_elastic_cross_dp():
@pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"])
def test_engram_rejects_elastic_cross_dp(option):
parallel = ParallelConfig(
tensor_parallel_size=4,
data_parallel_size=2,
distributed_executor_backend="mp",
)
parallel.enable_elastic_ep = True
with pytest.raises(ValueError, match="embedding_across_dp.*elastic EP"):
EngramConfig(embedding_across_dp=True).verify_parallel_config(parallel)
with pytest.raises(ValueError, match=f"{option}.*elastic EP"):
EngramConfig(**{option: True}).verify_parallel_config(parallel)
@pytest.mark.parametrize("legacy", [None, "0", "1"])
def test_engram_cpu_offload_environment_fallback(monkeypatch, legacy):
"""Explicit settings must override the legacy environment fallback."""
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
if legacy is not None:
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
assert EngramConfig().cpu_offload == (legacy == "1")
assert EngramConfig(cpu_offload=False).cpu_offload is False
assert EngramConfig(cpu_offload=True).cpu_offload is True
def test_engram_dp_shared_memory_requires_cpu_offload():
with pytest.raises(ValueError, match="requires cpu_offload"):
EngramConfig(cpu_offload=False, dp_shared_memory=True)
@pytest.mark.parametrize(
"dp_size,load_format,multithread,error",
[
(1, "auto", False, "requires data_parallel_size > 1"),
(2, "dummy", False, "requires load_format"),
(2, "sharded_state", False, "requires load_format"),
(2, "auto", False, None),
(2, "safetensors", True, None),
(2, "pt", True, None),
],
)
def test_engram_dp_shared_memory_config_validation(
monkeypatch, dp_size, load_format, multithread, error
):
"""Reject invalid shared configs before distributed init; allow threaded loads."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
config = cast(
VllmConfig,
SimpleNamespace(
model_config=SimpleNamespace(
architecture="DeepseekV41ForCausalLM",
hf_text_config=SimpleNamespace(engram_layer_ids=[1]),
),
speculative_config=None,
engram_config=EngramConfig(cpu_offload=True, dp_shared_memory=True),
parallel_config=ParallelConfig(data_parallel_size=dp_size),
load_config=LoadConfig(
load_format=load_format,
model_loader_extra_config={"enable_multithread_load": multithread},
),
),
)
if error:
with pytest.raises(ValueError, match=error):
VllmConfig._resolve_and_verify_engram_config(config)
else:
VllmConfig._resolve_and_verify_engram_config(config)
@pytest.mark.parametrize(
"architecture, ple_layers, cuda, supported",
[
("DeepseekV41ForCausalLM", [1], True, True),
("DeepseekV41ForCausalLM", [], True, False),
("DeepseekV41ForCausalLM", [1], False, False),
("Qwen4ExpForCausalLM", [1], True, True),
("Qwen4ExpForConditionalGeneration", [1], True, True),
("Qwen4ExpForCausalLM", [], True, False),
@@ -1201,7 +1238,9 @@ def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, suppo
ModelConfig,
SimpleNamespace(
architecture=architecture,
hf_text_config=SimpleNamespace(ple_layer_ids=ple_layers),
hf_text_config=SimpleNamespace(
ple_layer_ids=ple_layers, engram_layer_ids=ple_layers
),
),
)
if architecture is not None
@@ -1214,55 +1253,73 @@ def test_engram_model_support(monkeypatch, architecture, ple_layers, cuda, suppo
with pytest.raises(ValueError, match="requires a model with supported Engram"):
config.verify_model_config(model)
resolved = cast(
VllmConfig,
SimpleNamespace(
model_config=model,
speculative_config=None,
parallel_config=ParallelConfig(),
load_config=LoadConfig(load_format="dummy"),
engram_config=None,
),
)
VllmConfig._resolve_and_verify_engram_config(resolved)
assert (resolved.engram_config is not None) == supported
if supported:
assert resolved.engram_config.cpu_offload is True
def test_engram_config_defaults_to_none(monkeypatch):
@pytest.mark.parametrize(
("value", "expected"), [(None, True), ("0", False), ("1", True)]
)
def test_engram_cpu_offload_environment_default(monkeypatch, value, expected):
monkeypatch.delenv("VLLM_PLE_CPU_OFFLOAD", raising=False)
if value is not None:
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", value)
assert EngramConfig().cpu_offload is expected
assert EngramConfig(cpu_offload=False).cpu_offload is False
assert EngramConfig(cpu_offload=True).cpu_offload is True
def test_engram_config_defaults_to_none():
config = VllmConfig()
assert config.engram_config is None
assert config.compute_hash()
@pytest.mark.parametrize("legacy", ["0", "1"])
def test_engram_none_resolves_legacy_offload(monkeypatch, legacy):
"""Legacy enablement materializes a config before model validation."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
@pytest.mark.parametrize("explicit", [False, True])
@pytest.mark.parametrize("enable_dbo,ubatch_size", [(True, 0), (False, 2)])
def test_deepseek_engram_rejects_microbatching(explicit, enable_dbo, ubatch_size):
"""Reject overlapping Engram staging even without an explicit EngramConfig."""
config = cast(
VllmConfig,
SimpleNamespace(
model_config=SimpleNamespace(
architecture="Qwen4ExpForCausalLM",
hf_text_config=SimpleNamespace(ple_layer_ids=[1]),
architecture="DeepseekV41ForCausalLM",
hf_text_config=SimpleNamespace(engram_layer_ids=[1]),
),
speculative_config=None,
engram_config=None,
parallel_config=ParallelConfig(),
engram_config=EngramConfig(cpu_offload=False) if explicit else None,
parallel_config=ParallelConfig(
enable_dbo=enable_dbo, ubatch_size=ubatch_size
),
),
)
VllmConfig._resolve_and_verify_engram_config(config)
if legacy == "1":
assert config.engram_config is not None
assert config.engram_config.cpu_offload is True
else:
assert config.engram_config is None
with pytest.raises(
ValueError, match="Engram does not support DBO or microbatching"
):
VllmConfig._resolve_and_verify_engram_config(config)
@pytest.mark.parametrize("legacy", ["0", "1"])
def test_engram_explicit_config_requires_supported_model(monkeypatch, legacy):
def test_engram_explicit_config_requires_supported_model():
"""Explicit all-false settings still opt into model validation."""
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", legacy)
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig(engram_config=EngramConfig(cpu_offload=False))
def test_engram_legacy_offload_requires_supported_model(monkeypatch):
monkeypatch.setenv("VLLM_PLE_CPU_OFFLOAD", "1")
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig()
@pytest.mark.parametrize("target_has_ple", [False, True])
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
@pytest.mark.parametrize("explicit", [False, True])
def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, explicit):
"""MTP may inherit cross-DP sharding without having its own PLE layers."""
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
target = SimpleNamespace(
@@ -1277,24 +1334,29 @@ def test_engram_draft_config_validates_target(monkeypatch, target_has_ple):
speculative_config=SimpleNamespace(
draft_model_config=draft, target_model_config=target
),
engram_config=EngramConfig(embedding_across_dp=True),
engram_config=EngramConfig(embedding_across_dp=True) if explicit else None,
parallel_config=ParallelConfig(),
load_config=LoadConfig(load_format="dummy"),
),
)
if target_has_ple:
VllmConfig._resolve_and_verify_engram_config(config)
else:
if explicit and not target_has_ple:
with pytest.raises(ValueError, match="requires a model with supported Engram"):
VllmConfig._resolve_and_verify_engram_config(config)
else:
VllmConfig._resolve_and_verify_engram_config(config)
assert (config.engram_config is not None) == target_has_ple
if target_has_ple:
assert config.engram_config.cpu_offload is True
def test_engram_hash_tracks_storage_and_sharding():
def test_engram_hash_tracks_execution_options():
configs = [
EngramConfig(cpu_offload=offload, embedding_across_dp=across_dp)
for offload in (False, True)
for across_dp in (False, True)
EngramConfig(cpu_offload=True),
EngramConfig(cpu_offload=False),
EngramConfig(cpu_offload=True, embedding_across_dp=True),
EngramConfig(cpu_offload=True, dp_shared_memory=True),
]
assert len({config.compute_hash() for config in configs}) == 4
assert len({config.compute_hash() for config in configs}) == len(configs)
@pytest.mark.parametrize("port", [1, 29550, 65535])