[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-727
View File
@@ -1,727 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import pytest
import ray
import torch
import torch.distributed as dist
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators import custom_all_reduce as car
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_process_parallel,
)
random.seed(42)
test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
def _bf16_ulps(a: torch.Tensor, b: torch.Tensor) -> int:
"""Largest distance between two BF16 tensors in units in the last place."""
def ordered(t: torch.Tensor) -> torch.Tensor:
bits = t.view(torch.int16).int() & 0xFFFF
return torch.where(bits >= 0x8000, 0x8000 - bits, bits)
return int((ordered(a) - ordered(b)).abs().max())
def _unfused_all_reduce_mhc(peers, residual, post, comb, pre, weight, eps):
"""All-reduce, mHC post, collapse and RMSNorm in the fused kernel's FP32
order, rounding to BF16 where the unfused path returns BF16."""
reduced = peers[0].float()
for peer in peers[1:]:
reduced = reduced + peer.float()
reduced = reduced.bfloat16().float()
output = torch.empty_like(residual)
collapse = torch.zeros_like(reduced)
for target in range(4):
mixed = reduced * post[:, target : target + 1]
for source in range(4):
mixed = torch.addcmul(
mixed, residual[:, source].float(), comb[:, source, target, None]
)
output[:, target] = mixed.bfloat16()
collapse = torch.addcmul(
collapse, output[:, target].float(), pre[:, target : target + 1]
)
prenorm = collapse.bfloat16().float()
inv_rms = torch.rsqrt(prenorm.square().mean(-1, keepdim=True) + eps)
return output, (prenorm * inv_rms * weight.float()).bfloat16()
@ray.remote(num_gpus=1, max_calls=1)
def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
from vllm.models.deepseek_v41.nvidia.ops.cute_dsl import AllReduceMHC
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
op = AllReduceMHC(
hidden_size=5120, hc_mult=4, max_num_tokens=16, top_k=6, device=device
)
def check_eager_and_replayed(fused, check, halved, doubled):
check(fused())
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(5):
fused()
captured = fused()
for _ in range(20):
graph.replay()
check(captured)
# Replays must pick up in-place input changes.
halved.mul_(0.5)
doubled.mul_(2)
graph.replay()
check(captured)
def mhc_inputs(n):
torch.manual_seed(123)
residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16)
post = torch.rand(n, 4, device=device)
comb = torch.randn(n, 4, 4, device=device) * 0.1
pre = torch.rand(n, 4, device=device)
weight = torch.randn(5120, device=device, dtype=torch.bfloat16)
return residual, post, comb, pre, weight
def run(n):
torch.manual_seed(42 + rank)
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
# Packed +0/-0 pairs collide with the Lamport sentinel.
x[:, :16] = 0
x[:, 9:16:2] = -0.0
residual, post, comb, pre, weight = mhc_inputs(n)
def fused():
return op(x, residual, post, comb, pre, weight, 1e-6)
def check(outputs):
peers = get_tp_group().all_gather(x, dim=0).view(tp_size, n, 5120)
output, normalized = _unfused_all_reduce_mhc(
peers, residual, post, comb, pre, weight, 1e-6
)
# The mixed hc streams match bit for bit. The RMSNorm sums the
# squares in another order and uses an approximate rsqrt.
assert torch.equal(outputs[0], output)
assert _bf16_ulps(outputs[1], normalized) <= 1
check_eager_and_replayed(fused, check, x, residual)
def run_finalize(n):
torch.manual_seed(7 + rank)
# A padded permuted GEMM2 buffer, like the MoE's.
rows = n * 6 + 5
gemm2 = torch.randn(rows, 5120, device=device, dtype=torch.bfloat16)
permuted = torch.randperm(rows, device=device)[: n * 6].view(n, 6).int()
# A route to an expert this rank does not hold.
permuted[0, -1] = -1
weights = torch.rand(n, 6, device=device)
shared = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
residual, post, comb, pre, weight = mhc_inputs(n)
def fused():
return op.finalize(
gemm2,
weights,
permuted,
shared,
residual,
post,
comb,
pre,
weight,
1e-6,
)
def check(outputs):
# One FP32 FMA per route in route order, the shared add, one
# rounding, then the plain path: only the finalize differs.
acc = torch.zeros(n, 5120, device=device)
for k in range(6):
valid = (permuted[:, k] >= 0).unsqueeze(-1)
rows_k = gemm2[permuted[:, k].clamp_min(0).long()].float()
torch.addcmul(
acc,
torch.where(valid, rows_k, 0.0),
torch.where(valid, weights[:, k : k + 1], 0.0),
out=acc,
)
x = (acc + shared.float()).bfloat16()
expected = op(x, residual, post, comb, pre, weight, 1e-6)
assert torch.equal(outputs[0], expected[0])
assert torch.equal(outputs[1], expected[1])
check_eager_and_replayed(fused, check, gemm2, shared)
# Changing shapes cover shrinking and growing batches in one mailbox.
for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1):
run(n)
run_finalize(n)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100"
)
def test_all_reduce_mhc_matches_unfused_path(monkeypatch):
if torch.accelerator.device_count() < 4:
pytest.skip("Requires four GPUs with NVLink multicast")
multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc)
@pytest.mark.parametrize(
("dtype", "expected"),
[
(torch.float32, True),
(torch.float16, True),
(torch.bfloat16, True),
(torch.int8, False),
(torch.float8_e4m3fn, False),
],
)
def test_custom_allreduce_filters_dtype(
dtype: torch.dtype,
expected: bool,
) -> None:
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = 2
communicator.max_size = 1024
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
batch_invariant: bool,
) -> None:
"""Batch invariance must not switch backends based on the input size."""
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator.batch_invariant = batch_invariant
communicator._ptr = 0
oversized = torch.empty(1024, dtype=torch.float16)
assert communicator.should_custom_ar(oversized) is batch_invariant
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_reduce_scatter_disabled_under_batch_invariance(
monkeypatch, batch_invariant: bool
) -> None:
"""Reduce-scatter stays on one backend under batch invariance."""
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.fully_connected = True
communicator.mnnvl_only = False
communicator.mnnvl_multicast_ptr = 0
communicator.max_reduce_scatter_size = 1024
communicator.max_mnnvl_reduce_scatter_size = 1024
communicator.batch_invariant = batch_invariant
in_range = torch.empty(64, dtype=torch.float16)
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
(8, True, False),
(9, True, False),
(10, False, False),
(10, True, True),
],
)
def test_cross_node_mnnvl_gate_checks_generation_and_multicast(
monkeypatch,
major,
local_multicast,
expected,
):
def has_device_capability(capability, device_id):
assert capability == 100
assert device_id == 3
return major >= 10
monkeypatch.setattr(
car.current_platform,
"has_device_capability",
has_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
monkeypatch.setattr(car.dist, "all_reduce", lambda *_args, **_kwargs: None)
assert car._group_can_attempt_mnnvl(object(), torch.device("cuda:3")) is expected
def test_cross_node_mnnvl_gate_requires_support_on_every_rank(monkeypatch):
monkeypatch.setattr(
car.current_platform,
"has_device_capability",
lambda *_args: True,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: True,
)
def report_unsupported_peer(support, **_kwargs):
support.zero_()
monkeypatch.setattr(car.dist, "all_reduce", report_unsupported_peer)
assert not car._group_can_attempt_mnnvl(object(), torch.device("cuda:0"))
def test_local_multicast_support_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: False)
assert not car._has_local_multicast_support(torch.device("cuda:0"))
@pytest.mark.parametrize(
("world_size", "device_capability", "local_multicast", "expected"),
[
(2, (10, 0), True, True),
(4, (10, 3), True, True),
(8, (10, 0), True, True),
(8, (10, 3), True, True),
(6, (10, 3), True, False),
(8, (10, 1), True, False),
(8, (9, 0), True, False),
(8, (10, 3), False, False),
],
)
def test_mnnvl_multimem_reduce_scatter_platform_gate(
monkeypatch,
world_size,
device_capability,
local_multicast,
expected,
):
def is_device_capability(capability, device_id):
assert capability in ((10, 0), (10, 3))
assert device_id == 3
return device_capability == capability
monkeypatch.setattr(
car.current_platform,
"is_device_capability",
is_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
supported = car._supports_mnnvl_multimem_reduce_scatter(
torch.device("cuda:3"), world_size
)
assert supported is expected
@pytest.mark.parametrize(
(
"message_bytes",
"multimem_ptr",
"multimem_initialized",
"batch_invariant",
"expected",
),
[
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024 + 128, 1, True, False, None),
(32 * 1024 * 1024, 0, True, False, None),
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
(32 * 1024 * 1024, 1, True, True, None),
],
)
@pytest.mark.parametrize("world_size", [2, 4, 8])
def test_mnnvl_reduce_scatter_backend_gate(
monkeypatch,
world_size,
message_bytes,
multimem_ptr,
multimem_initialized,
batch_invariant,
expected,
):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = world_size
communicator.mnnvl_only = False
communicator.fully_connected = True
communicator.mnnvl_multicast_ptr = 1
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
inp = torch.empty(
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
dtype=torch.bfloat16,
)
assert inp.nbytes == message_bytes
assert communicator._select_reduce_scatter_backend(inp) == expected
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
expected == "mnnvl_multimem"
)
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
monkeypatch,
):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
return None
def report_peer_allocation_failure(group_value, **_kwargs):
events.append("all_reduce")
assert group_value.item() == 1
group_value.zero_()
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
warnings = []
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = object()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory allocation "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
raise RuntimeError("rendezvous failed")
def preserve_local_result(_group_value, **_kwargs):
events.append("all_reduce")
warnings = []
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
events = []
buffers = []
class FakeHandle:
multicast_ptr = 0x3000
class FakeSymmMem:
@staticmethod
def empty(size, **_kwargs):
events.append(("empty", size))
buffer = torch.ones(size, dtype=torch.uint8)
buffers.append(buffer)
return buffer
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append(("rendezvous", None))
return FakeHandle()
def preserve_local_result(*_args, **_kwargs):
events.append(("all_reduce", None))
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(
car.torch.accelerator,
"synchronize",
lambda: events.append(("synchronize", None)),
)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cpu")
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == [
("empty", 257),
("all_reduce", None),
("rendezvous", None),
("synchronize", None),
("all_reduce", None),
]
assert torch.all(buffers[0][:128] == 0)
assert torch.all(buffers[0][128:] == 1)
assert communicator.mnnvl_multimem_rs_buffer_size == 129
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
# A small all_reduce for warmup.
# this is needed because device communicators might be created lazily
# (e.g. NCCL). This will ensure that the communicator is initialized
# before any communication happens, so that this group can be used for
# graph capture immediately.
data = torch.zeros(1)
data = data.to(device=device)
torch.distributed.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
# we use the first group to communicate once
# and the second group to communicate twice
# and so on
# this is used to demonstrate that each group can
# communicate independently
num_communication = rank // tp_size + 1
for sz in test_sizes:
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
with graph_capture(device=device) as graph_capture_context:
# use integers so result matches NCCL exactly
device_idx = torch.accelerator.current_device_index()
inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
inp2 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for i in range(num_communication):
out1 = tensor_model_parallel_all_reduce(inp1)
# the input buffer is immediately modified to test
# synchronization
dist.all_reduce(inp1, group=group)
out2 = tensor_model_parallel_all_reduce(inp2)
dist.all_reduce(inp2, group=group)
graph.replay()
torch.testing.assert_close(out1, inp1)
torch.testing.assert_close(out2, inp2)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
# we use the first group to communicate once
# and the second group to communicate twice
# and so on
# this is used to demonstrate that each group can
# communicate independently
num_communication = rank // tp_size + 1
sz = 1024
fa = get_tp_group().device_communicator.ca_comm
inp = torch.ones(sz, dtype=torch.float32, device=device)
out = inp
for _ in range(num_communication):
out = fa.all_reduce(out, registered=False)
torch.testing.assert_close(out, inp * (tp_size**num_communication))
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
out = inp
for _ in range(num_communication):
out = fa.all_reduce(out, registered=False)
torch.testing.assert_close(out, inp * (tp_size**num_communication))
@ray.remote(num_gpus=1, max_calls=1)
def chunked_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
"""Inputs above max_size are reduced in chunks and match the fixed-order
reference bitwise, both eagerly and inside a CUDA graph."""
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
fa = get_tp_group().device_communicator.ca_comm
# Two full chunks plus a partial one.
dtype = torch.bfloat16
chunk_numel = fa.max_size // dtype.itemsize
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
dist.all_gather(gathered, inp, group=group)
ref = gathered[0].float()
for peer in gathered[1:]:
ref = ref + peer.float()
ref = ref.to(torch.bfloat16)
out = fa.all_reduce(inp, registered=False)
assert torch.equal(out, ref)
# Weak-contiguous but not C-contiguous: a transposed matrix.
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
with fa.capture():
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
out = fa.all_reduce(inp, registered=True)
graph.replay()
assert torch.equal(out, ref)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
if tp_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_custom_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
test_target,
):
world_size = tp_size * pipeline_parallel_size
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
File diff suppressed because it is too large Load Diff
-883
View File
@@ -1,883 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Engram tables shared by co-located DP replicas.
Four GPUs on one node split the hash heads four ways, whether as DEP
(TP1 x DP4, the single-node MoE layout) or TP2 x DP2. A lookup only returns
the right rows if the ids of every replica are gathered, each rank looks up
its own heads, and each replica trades those tokens back for the heads its
peers own.
With shared host storage, DP replicas instead map the same TP slice and
prefetch only their own tokens without DP gathers.
"""
from itertools import product
from pathlib import Path
from tempfile import TemporaryDirectory
from threading import Event
from types import SimpleNamespace
import pytest
import torch
import torch.multiprocessing as mp
from vllm.config import (
EngramConfig,
VllmConfig,
get_current_vllm_config_or_none,
set_current_vllm_config,
)
from vllm.config.load import LoadConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.scheduler import SchedulerConfig
from vllm.distributed import cleanup_dist_env_and_memory, parallel_state
from vllm.distributed.parallel_state import (
get_engram_dp_group,
get_engram_dp_size,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
convert_swizzled_to_linear,
)
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram,
EngramBatch,
EngramLayout,
ParallelEngramEmbedding,
engram_head_shard_rank,
gather_engram_hashes,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
WORLD_SIZE = 4
DIM, BLOCK = 64, 32
# Prime-sized buckets, as the model's layout builds them.
HEAD_SIZES = (97, 101, 103, 107, 109, 113, 127, 131)
def _token_counts(dp_size: int) -> tuple[tuple[int, ...], ...]:
"""Even loads, then uneven ones with an idle replica to pad around."""
return ((8,) * dp_size, (7, 0, 5, 1)[:dp_size], (7,) * dp_size)
def _full_table(rows: int) -> tuple[torch.Tensor, torch.Tensor]:
"""The unsharded table, identical on every rank."""
torch.manual_seed(0)
weight = (torch.randn(rows, DIM) * 4).to(torch.float8_e4m3fn)
scale_inv = torch.randint(120, 134, (rows, DIM // BLOCK), dtype=torch.uint8)
return weight, scale_inv
def _make_ids(head_sizes: tuple[int, ...], num_tokens: int, seed: int) -> torch.Tensor:
"""Ids of each head, drawn from that head's own bucket range."""
torch.manual_seed(seed)
offset = 0
columns = []
for size in head_sizes:
columns.append(torch.randint(offset, offset + size, (num_tokens, 1)))
offset += size
return torch.cat(columns, dim=1).to(torch.int32).cuda()
def _reference(weight, scale_inv, ids) -> torch.Tensor:
"""Dequantized lookup against the whole table."""
valid = ids >= 0
safe_ids = ids.clamp_min(0).long()
values = torch.nn.functional.embedding(safe_ids, weight).float()
scales = torch.nn.functional.embedding(safe_ids, scale_inv)
scales = (scales.to(torch.int32) << 23).view(torch.float32)
values = values.unflatten(-1, (-1, BLOCK)) * scales.unsqueeze(-1)
return values.flatten(-2).to(torch.bfloat16).masked_fill(~valid.unsqueeze(-1), 0)
@pytest.mark.parametrize(
"architecture,options,edp_ranks,etp_ranks",
[
("DeepseekV41ForCausalLM", {}, [0, 2], [0, 1]),
("Qwen4ExpForCausalLM", {}, None, [0, 1]),
(
"Qwen4ExpForCausalLM",
{"embedding_across_dp": True},
None,
[0, 1, 2, 3],
),
("Qwen4ExpForConditionalGeneration", {}, None, [0, 1]),
],
)
def test_engram_group_creation_honors_model_and_options(
monkeypatch, architecture, options, edp_ranks, etp_ranks
):
config = SimpleNamespace(
model_config=SimpleNamespace(architecture=architecture, is_moe=False),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=2,
distributed_executor_backend="mp",
),
engram_config=EngramConfig(**options),
)
for name in (
"_TP",
"_ETP",
"_ENGRAM_DP",
"_DCP",
"_PCP",
"_PP",
"_DP",
"_EP",
"_EPLB",
):
monkeypatch.setattr(parallel_state, name, None)
monkeypatch.setattr(parallel_state, "_WORLD", SimpleNamespace(local_rank=0))
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 4)
monkeypatch.setattr(torch.distributed, "get_rank", lambda: 0)
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
def make_group(group_ranks, *args, **kwargs):
ranks = next(ranks for ranks in group_ranks if 0 in ranks)
return SimpleNamespace(ranks=ranks, rank_in_group=0, world_size=len(ranks))
monkeypatch.setattr(parallel_state, "init_model_parallel_group", make_group)
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=2, backend="nccl")
group = get_engram_dp_group()
assert (group.ranks if group is not None else None) == edp_ranks
assert parallel_state.get_etp_group().ranks == etp_ranks
def _worker(rank: int, tp_size: int, port: int) -> None:
torch.accelerator.set_device_index(rank)
update_environment_variables(
{
"RANK": str(rank),
"LOCAL_RANK": str(rank),
"WORLD_SIZE": str(WORLD_SIZE),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(port),
}
)
dp_size = WORLD_SIZE // tp_size
dp_rank, tp_rank = divmod(rank, tp_size)
vllm_config = VllmConfig(
parallel_config=ParallelConfig(
tensor_parallel_size=tp_size,
data_parallel_size=dp_size,
data_parallel_rank=dp_rank,
),
scheduler_config=SchedulerConfig(
max_model_len=8,
is_encoder_decoder=False,
max_num_batched_tokens=8,
max_num_seqs=8,
),
)
# Supply the resolved Engram settings without constructing a full model.
vllm_config.engram_config = EngramConfig()
vllm_config.model_config = SimpleNamespace(
architecture="DeepseekV41ForCausalLM",
is_moe=True,
sleep_mode_offload_cudagraph=False,
)
try:
init_distributed_environment()
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=tp_size)
assert get_engram_dp_size() == dp_size
assert get_engram_dp_group().ranks == [
tp_rank + replica * tp_size for replica in range(dp_size)
]
assert engram_head_shard_rank() == tp_rank * dp_size + dp_rank
for failure in ("create", "open", "mmap", "register", "pinned", None):
_check_shared_storage_lifetime_and_failures(failure)
# Reuse the processes and communication groups across storage modes.
for (cpu_offload, dp_shared_memory), n_heads, sequence_parallel in product(
[(False, False), (True, False), (True, True)],
[8, 7],
[False, True] if tp_size > 1 else [False],
):
_check_table(
vllm_config,
cpu_offload,
dp_shared_memory,
n_heads,
sequence_parallel,
)
if tp_size == 1:
_check_fp8_batch(vllm_config)
finally:
cleanup_dist_env_and_memory()
def _check_fp8_batch(vllm_config):
"""MXFP8 staging matches the tables across peer/all-to-all slots, idle
replicas and graph replays."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
dp_rank = vllm_config.parallel_config.data_parallel_rank
layers = []
for index in range(2):
weight, scale = _full_table(sum(HEAD_SIZES))
weight = (weight.float() * (1 - 2 * index)).to(torch.float8_e4m3fn)
table = ParallelEngramEmbedding(
sum(HEAD_SIZES), DIM, HEAD_SIZES, cpu_offload=True
)
table.weight.weight_loader(table.weight, weight)
table.weight_scale_inv.weight_loader(table.weight_scale_inv, scale)
staged = torch.empty(0, device="cuda")
reference = (weight.cuda(), scale.cuda())
layers.append(SimpleNamespace(embed_tokens=table, staged_rows=staged))
layers[-1].layer_hash_index, layers[-1].reference = index, reference
batch = EngramBatch(layers, 257)
def check(ids):
tokens = ids.shape[0]
for layer in layers:
x = batch.wkv_input(layer, tokens)
sf = convert_swizzled_to_linear(x.scale, tokens, x.data.shape[1], BLOCK)
actual = dequant_mxfp8_to_bf16(x.data, sf).unflatten(-1, (-1, DIM))
weight, scale = layer.reference
expected = _reference(weight, scale, ids[:, layer.layer_hash_index])
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
for counts in ((7, 0, 5, 1), (127,) * 4, (128,) * 4, (257, 0, 129, 1)):
tokens = counts[dp_rank]
seed = 700 + 10 * dp_rank
ids = torch.stack([_make_ids(HEAD_SIZES, tokens, seed + i) for i in (0, 1)], 1)
with set_forward_context(
None,
vllm_config,
num_tokens=tokens,
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
):
batch.prepare_embeddings(ids)
check(ids)
for capture in ("full", "breakable") if len(set(counts)) == 1 else ():
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
if capture == "full":
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=warmup):
batch.prepare_embeddings(ids)
else:
graph = BreakableCUDAGraphCapture()
with torch.cuda.stream(warmup), graph:
batch.prepare_embeddings(ids)
torch.cuda.current_stream().wait_stream(warmup)
for step in (2, 3):
replay_ids = _make_ids(HEAD_SIZES, tokens, seed + step)
ids.copy_(torch.stack([replay_ids] * 2, 1))
graph.replay()
check(ids)
del graph
def _check_shared_storage_lifetime_and_failures(failure):
"""One peer's failure must roll back every mapping/registration, without hangs."""
import gc
import weakref
group = get_engram_dp_group()
runtime = torch.cuda.cudart()
make_file = engram_ops.tempfile.NamedTemporaryFile
map_file = engram_ops.mmap.mmap
paths, mappings, registrations, unregistrations = [], [], [], []
failed_rank = 0 if failure == "create" else 1
def fail():
raise OSError(f"injected {failure} failure")
def temporary_file(*args, **kwargs):
if failure == "create":
fail()
file = make_file(*args, **kwargs)
paths.append(Path(file.name))
return file
def open_file(*args, **kwargs):
if failure == "open" and group.rank_in_group == failed_rank:
fail()
return open(*args, **kwargs)
def mapping(*args, **kwargs):
if failure == "mmap" and group.rank_in_group == failed_rank:
fail()
result = map_file(*args, **kwargs)
mappings.append(weakref.ref(result))
return result
def register(pointer, size, flags):
if failure == "register" and group.rank_in_group == failed_rank:
return SimpleNamespace(value=1)
result = runtime.cudaHostRegister(pointer, size, flags)
assert result.value == 0
registrations.append(pointer)
return result
def unregister(pointer):
unregistrations.append(pointer)
result = runtime.cudaHostUnregister(pointer)
assert result.value == 0
return result
with pytest.MonkeyPatch.context() as patch:
patch.setattr(
engram_ops,
"tempfile",
SimpleNamespace(NamedTemporaryFile=temporary_file),
)
patch.setattr(engram_ops, "open", open_file, raising=False)
patch.setattr(
engram_ops,
"mmap",
SimpleNamespace(mmap=mapping, MAP_SHARED=engram_ops.mmap.MAP_SHARED),
)
patch.setattr(
torch.cuda,
"cudart",
lambda: SimpleNamespace(
cudaHostRegister=register,
cudaHostUnregister=unregister,
),
)
if failure == "pinned" and group.rank_in_group == failed_rank:
patch.setattr(torch.Tensor, "is_pinned", lambda self: False)
if failure is not None:
stage = "cudaHostRegister" if failure in ("register", "pinned") else failure
with pytest.raises(RuntimeError, match=f"EDP rank {failed_rank}.*{stage}"):
engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
assert unregistrations == registrations
else:
storage = engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
assert all(not path.exists() for path in paths)
storage_ref = weakref.ref(storage)
parameter = torch.nn.Parameter(storage.weight, requires_grad=False)
views = storage.get_views(parameter, storage.weight_scale_inv)
del storage
gc.collect()
assert storage_ref() is None
assert not unregistrations
del parameter
gc.collect()
assert not unregistrations # UVA views still retain the CPU allocation.
del views
gc.collect()
assert unregistrations == registrations
for ref in mappings:
mapped = ref()
assert mapped is None or mapped.closed
assert all(not path.exists() for path in paths)
def _check_table(
vllm_config, cpu_offload, dp_shared_memory, n_heads, sequence_parallel
):
parallel = vllm_config.parallel_config
dp_size, tp_size = parallel.data_parallel_size, parallel.tensor_parallel_size
dp_rank = parallel.data_parallel_rank
tp_rank = engram_ops.get_tensor_model_parallel_rank()
multithread = (
dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1)
)
load_config = (
LoadConfig(
load_format="safetensors",
use_tqdm_on_load=False,
model_loader_extra_config={
"enable_multithread_load": True,
"num_threads": 2,
},
)
if multithread
else vllm_config.load_config
)
with pytest.MonkeyPatch.context() as m:
head_sizes = HEAD_SIZES[:n_heads]
rows = sum(head_sizes)
config = SimpleNamespace(
hidden_size=DIM,
hc_mult=2,
rms_norm_eps=1e-6,
engram_layer_ids=[0],
engram_num_embeddings=[rows],
engram_max_ngram_size=2,
engram_n_heads=n_heads,
engram_head_dim=DIM,
engram_compressed_vocab_size=32,
engram_pad_token_id=0,
engram_vocab_size=HEAD_SIZES[0],
)
with set_current_vllm_config(vllm_config):
weight, scale_inv = _full_table(rows)
layout = EngramLayout(config)
# Stub configuration inputs, not Engram's initialization or execution.
component_config = SimpleNamespace(
engram_config=EngramConfig(
cpu_offload=cpu_offload,
dp_shared_memory=dp_shared_memory,
),
scheduler_config=vllm_config.scheduler_config,
load_config=load_config,
)
with m.context() as init_patch, torch.device("cuda"):
init_patch.setattr(
engram_ops, "get_current_vllm_config", lambda: component_config
)
engram = Engram(
config,
quant_config=None,
layout=layout,
layer_hash_index=0,
use_sequence_parallel=sequence_parallel,
prefix="model.layers.0.engram",
)
layer = engram.embed_tokens
if multithread:
_load_shared_multithread(layer, weight, scale_inv, load_config)
else:
# Only the leader has valid checkpoint payload in shared mode.
loader_weight = weight if not dp_shared_memory or dp_rank == 0 else None
loader_scale = (
scale_inv if not dp_shared_memory or dp_rank == 0 else None
)
layer.weight.weight_loader(layer.weight, loader_weight)
layer.weight_scale_inv.weight_loader(
layer.weight_scale_inv, loader_scale
)
if dp_shared_memory:
assert layer.dp_size == 1
assert layer.head_start == tp_rank * layer.part_n_hash_cols
assert layer.weight.is_pinned() and layer.weight_scale_inv.is_pinned()
def unexpected_collective(*args, **kwargs):
raise AssertionError("shared lookup must not use DP collectives")
m.setattr(get_engram_dp_group(), "all_gather", unexpected_collective)
token_counts = _token_counts(dp_size)
# Production forward runs after the construction-time config has exited.
assert get_current_vllm_config_or_none() is None
for batch_idx, counts in enumerate(token_counts):
num_tokens = counts[dp_rank]
# Ids differ per replica but must agree across its TP ranks.
ids = _make_ids(head_sizes, num_tokens, seed=100 + dp_rank)
if batch_idx == 2 and dp_rank == 1:
ids.fill_(engram_ops.DEAD_ID)
with set_forward_context(
None,
vllm_config,
num_tokens=num_tokens,
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
):
gathered = gather_engram_hashes(
ids, dp_shared_memory=layer.dp_shared_memory
)
expected_tokens = (
num_tokens if dp_shared_memory else max(counts) * dp_size
)
assert gathered.shape[0] == expected_tokens
engram.prepare_embeddings(gathered)
looked_up = engram.embed(ids)
standalone = layer(ids)
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
torch.testing.assert_close(standalone, expected, atol=0, rtol=0)
if sequence_parallel:
# SP hands back only this rank's token shard, zero-padded.
chunk = -(-num_tokens // tp_size)
expected = torch.nn.functional.pad(
expected, (0, 0, 0, 0, 0, chunk * tp_size - num_tokens)
)
expected = expected[tp_rank * chunk : (tp_rank + 1) * chunk]
torch.testing.assert_close(looked_up, expected, atol=0, rtol=0)
if (
cpu_offload
and n_heads == 7
and not sequence_parallel
and (not dp_shared_memory or tp_size == 1)
):
_check_dummy_hash_model_forward(
vllm_config,
engram,
head_sizes,
weight,
scale_inv,
dp_shared_memory,
)
replay = n_heads == 7 and (
(dp_shared_memory and sequence_parallel == (tp_size > 1))
or (
cpu_offload
and not dp_shared_memory
and sequence_parallel == (tp_size > 1)
)
)
if replay:
captures = ("full", "breakable")
for capture in captures:
_check_prefetch_replay(
engram,
head_sizes,
weight,
scale_inv,
dp_shared_memory,
capture,
)
if dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1):
with m.context() as storage_patch:
storage_patch.setattr(layer.weight, "data", layer.weight.data.clone())
with pytest.raises(RuntimeError, match="storage must not be replaced"):
layer(ids)
def _check_dummy_hash_model_forward(
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
):
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
from vllm.models.deepseek_v41.common.engram import NgramHashState
from vllm.models.deepseek_v41.nvidia import model as model_ops
dp_rank = vllm_config.parallel_config.data_parallel_rank
if dp_shared_memory and dp_rank != 1:
return
tokens = 4
ids = _make_ids(head_sizes, tokens, seed=300 + dp_rank)
state = NgramHashState.__new__(NgramHashState)
torch.nn.Module.__init__(state)
state.multipliers = torch.empty(1, 2, dtype=torch.int64, device="cuda")
state.primes = torch.empty(1, 1, len(head_sizes), dtype=torch.int64, device="cuda")
state.ensure_cache = lambda: True
# Hash arithmetic is covered separately; keep the real model's branch and
# NgramHashState.dummy_hashes, plus real preparation/lookup collectives.
state.forward = lambda *args: ids.unsqueeze(1)
class Decoder(SimpleNamespace):
# Engram hashes and the mask stay the last positional arguments;
# **kwargs absorbs the decoder's keyword-only flags.
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
hashes, keep = args[-2:]
if dp_shared_memory and dp_rank == 1:
assert hashes is None and keep is None
output = hidden.new_zeros(
tokens, len(head_sizes), DIM, dtype=torch.bfloat16
)
else:
assert hashes is not None and keep is not None
assert bool(keep.all()) == (dp_rank != 1)
if dp_rank == 1:
assert torch.all(hashes == engram_ops.DEAD_ID)
output = engram.embed(hashes[:, 0])
# Trailing None is previous_aux; this stub captures no aux states.
return output, hidden, hidden, hidden, hidden, None
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.parallel_config = vllm_config.parallel_config
model.use_native_mega_moe = False
model.use_sequence_parallel = False
model.fuse_mhc_all_reduce = False
model.engram_batch = None
model.engram_hash = state
model.engram_swa_prefix = "swa"
model.engram_dp_shared_memory = dp_shared_memory
model.layers = [Decoder(engram=engram)]
model.start_layer = 0
model.end_layer = 1
model.decoder_replay_start = model.end_layer
model.decoder_replay_layers = None
model.aux_hidden_state_layers = ()
metadata = (
None
if dp_rank == 1
else {
"swa": SimpleNamespace(
query_start_loc=None, slot_mapping=None, block_table=None
)
}
)
with pytest.MonkeyPatch.context() as patch:
patch.setattr(
model_ops,
"get_pp_group",
lambda: SimpleNamespace(
is_first_rank=True,
is_last_rank=False,
),
)
patch.setattr(model_ops, "mhc_post_tilelang", lambda hidden, *args: hidden)
with set_forward_context(
metadata,
vllm_config,
num_tokens=tokens,
num_tokens_across_dp=torch.full(
(vllm_config.parallel_config.data_parallel_size,),
tokens,
dtype=torch.int32,
),
):
output = model(
torch.arange(tokens, device="cuda"),
torch.arange(tokens, device="cuda"),
intermediate_tensors=None,
inputs_embeds=torch.zeros(tokens, DIM, device="cuda"),
lookback_token_ids=torch.full((1, 1), -1, device="cuda"),
)["hidden_states"]
expected_ids = torch.full_like(ids, engram_ops.DEAD_ID) if dp_rank == 1 else ids
expected = _reference(weight.cuda(), scales.cuda(), expected_ids)
torch.testing.assert_close(output, expected, atol=0, rtol=0)
def _load_shared_multithread(layer, weight, scale_inv, load_config):
"""Opposite shard completion orders must leave every shared parameter ready."""
from safetensors.torch import save_file
from vllm.model_executor.model_loader import weight_utils
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
tensors = {
"weight": weight,
"weight_scale_inv": scale_inv.view(torch.float8_e8m0fnu),
}
order = list(tensors)
if get_engram_dp_group().rank_in_group % 2:
order.reverse()
first_loaded = Event()
original_load = weight_utils.load_file
def load_file(path, **kwargs):
if Path(path).stem != order[0]:
assert first_loaded.wait(timeout=30), "First shared weight load stalled"
return original_load(path, **kwargs)
with TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as m:
for name, tensor in tensors.items():
save_file({name: tensor}, str(Path(directory) / f"{name}.safetensors"))
m.setattr(weight_utils, "load_file", load_file)
loader = DefaultModelLoader(load_config)
source = DefaultModelLoader.Source(directory, revision=None)
loaded = []
for name, tensor in loader._get_weights_iterator(source):
param = getattr(layer, name)
param.weight_loader(param, tensor)
loaded.append(name)
first_loaded.set()
assert loaded == order
# Check before any further collective could mask incomplete shared writes.
for name, tensor in tensors.items():
param = getattr(layer, name)
expected = tensor.narrow(0, param.engram_vocab_start, param.shape[0])
torch.testing.assert_close(
param.view(torch.uint8), expected.view(torch.uint8), atol=0, rtol=0
)
def _check_prefetch_replay(
engram, head_sizes, weight, scale_inv, dp_shared_memory, capture
):
"""DP-sharded and shared prefetches must replay with new IDs."""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
ids = _make_ids(head_sizes, 8, seed=501)
tp_rank = engram_ops.get_tensor_model_parallel_rank()
tokens = 8 // engram.embed_tokens.tp_size if engram.use_sequence_parallel else 8
output = torch.empty(
tokens, len(head_sizes), DIM, device="cuda", dtype=torch.bfloat16
)
def step():
gathered = gather_engram_hashes(ids.clone(), dp_shared_memory=dp_shared_memory)
engram.prepare_embeddings(gathered)
output.copy_(engram.embed(ids))
with pytest.MonkeyPatch.context() as patch:
patch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 8)
warmup = torch.cuda.Stream()
warmup.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup):
step()
if capture == "breakable":
graph = BreakableCUDAGraphCapture()
with torch.cuda.stream(warmup), graph:
step()
assert graph.num_eager_breaks == 2
else:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=warmup):
step()
torch.cuda.current_stream().wait_stream(warmup)
for iteration in range(2):
ids.copy_(_make_ids(head_sizes, 8, seed=600 + iteration))
graph.replay()
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
if engram.use_sequence_parallel:
expected = expected[tp_rank * tokens : (tp_rank + 1) * tokens]
torch.testing.assert_close(output, expected, atol=0, rtol=0)
torch.accelerator.synchronize()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("tp_size", [1, 2])
def test_engram_table_shared_across_dp_replicas(tp_size, monkeypatch):
"""Check all storage modes with one process launch per TP/DP topology."""
if torch.accelerator.device_count() < WORLD_SIZE:
pytest.skip(f"Need {WORLD_SIZE} GPUs to run the test.")
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1")
mp.spawn(_worker, args=(tp_size, get_open_port()), nprocs=WORLD_SIZE)
cleanup_dist_env_and_memory()
@pytest.mark.parametrize(
"cpu_offload,dp_size,error",
[
(False, 4, "requires cpu_offload=True"),
(True, 1, "effective Engram DP size is 1"),
],
)
def test_engram_dp_shared_memory_runtime_requirements(
monkeypatch, cpu_offload, dp_size, error
):
"""Report unmet storage/topology requirements before allocating CUDA memory."""
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
with pytest.raises(ValueError, match=error):
ParallelEngramEmbedding(
sum(HEAD_SIZES),
DIM,
HEAD_SIZES,
cpu_offload=cpu_offload,
dp_shared_memory=True,
)
@pytest.mark.parametrize("query_fails", [False, True])
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
"""A /dev/shm that is too small or cannot be queried must fall back, and the
leader must still broadcast that decision so its peers do not hang."""
broadcasts = []
def broadcast(obj):
broadcasts.append(obj)
return obj
def query(*args, **kwargs):
raise OSError("statvfs failed")
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
)
if query_fails:
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
assert not engram_ops.can_share_engram_tables(layout)
assert len(broadcasts) == 1 and broadcasts[0] is not None
@pytest.mark.parametrize(
"node_ids,dp_size,replica_size,expected",
[
([0] * 4, 1, 4, 1),
([0] * 4, 4, 1, 4),
([0] * 4, 2, 2, 2),
([0] * 8 + [1] * 8, 8, 2, 4),
([0] * 6 + [1] * 6, 4, 1, 2),
([0] * 3 + [1] * 4, 7, 1, 1),
([0] * 8 + [1] * 8 + [2] * 8, 8, 3, 1),
([0, 1] * 4, 4, 2, 1),
([0] * 3 + [1] * 5, 4, 2, 1),
],
)
def test_engram_dp_shard_size_respects_node_boundaries(
node_ids, dp_size, replica_size, expected, monkeypatch
):
"""Sharing must fall back when replicas or node rank ranges do not align."""
monkeypatch.setattr(parallel_state, "get_node_count", lambda: len(set(node_ids)))
monkeypatch.setattr(
parallel_state, "get_world_group", lambda: SimpleNamespace(cpu_group=None)
)
monkeypatch.setattr(
parallel_state,
"in_the_same_node_as",
lambda group, src: [node == node_ids[src] for node in node_ids],
)
assert (
parallel_state._engram_dp_shard_size(len(node_ids), dp_size, replica_size)
== expected
)
@pytest.mark.parametrize("num_tokens", [0, 2, 4, 5])
def test_engram_hash_padding_has_no_valid_rows(num_tokens, monkeypatch):
"""Idle/padded slots must not query row zero; oversized batches are rejected."""
monkeypatch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 4)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(all_gather=lambda ids, dim: torch.cat([ids, ids], dim)),
)
ids = torch.full((num_tokens, 2, 3), 17, dtype=torch.int32)
if num_tokens > 4:
with pytest.raises(ValueError, match="exceeds the DP token slot"):
gather_engram_hashes(ids)
return
gathered = gather_engram_hashes(ids).reshape(2, 4, 2, 3)
for replica in gathered:
torch.testing.assert_close(replica[:num_tokens], ids)
assert torch.all(replica[num_tokens:] == engram_ops.DEAD_ID)
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("uniform", [False, True])
def test_engram_hash_padding_ignores_other_nodes(tp_size, uniform, monkeypatch):
"""Two four-GPU nodes pad only to their own maximum, including TP x DP."""
group_size = 4 // tp_size
counts = (
[4] * (8 // tp_size)
if uniform
else [4096] + [2] * (group_size - 1) + [1, 3] + [0] * (group_size - 2)
)
monkeypatch.setattr(
engram_ops,
"get_forward_context",
lambda: SimpleNamespace(
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor(counts))
),
)
for dp_rank in range(8 // tp_size):
slot = 4 if uniform else (4096 if dp_rank < group_size else 3)
ids = torch.full((counts[dp_rank], 2, 3), 17, dtype=torch.int32)
monkeypatch.setattr(
engram_ops,
"get_dp_group",
lambda rank=dp_rank: SimpleNamespace(rank_in_group=rank),
)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda rank=dp_rank: SimpleNamespace(
rank_in_group=rank % group_size,
world_size=group_size,
all_gather=lambda tensor, dim: torch.cat([tensor] * group_size, dim),
),
)
gathered = gather_engram_hashes(ids)
expected = ids.new_full((slot, 2, 3), engram_ops.DEAD_ID)
expected[: ids.shape[0]] = ids
torch.testing.assert_close(
gathered, expected.repeat(group_size, 1, 1), msg=f"DP rank {dp_rank}"
)
-355
View File
@@ -1,355 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import lm_eval
import pytest
import torch
from transformers import DeepseekV4Config
from tests.utils import large_gpu_mark
from vllm.distributed.eplb.eplb_state import EplbState
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.eplb_utils import (
EPLBController,
draft_model_supports_eplb,
)
def get_model_args(
model_name: str,
spec_model_name: str | None,
spec_method: str,
tp_size: int,
model_max_len: int,
num_speculative_tokens: int = 1,
use_async: bool = True,
) -> dict:
speculative_config = {
"method": spec_method,
"model": spec_model_name,
"num_speculative_tokens": num_speculative_tokens,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
speculative_config["model"] = model_name
speculative_config["draft_sample_method"] = "probabilistic"
eplb_config = {
"num_redundant_experts": tp_size,
"window_size": 128,
"step_interval": 1024,
"log_balancedness": False,
"use_async": use_async,
}
model_args = {
"pretrained": model_name,
"dtype": "auto",
"add_bos_token": True,
"tensor_parallel_size": tp_size,
"gpu_memory_utilization": 0.7,
"speculative_config": speculative_config,
"enable_expert_parallel": True,
"eplb_config": eplb_config,
"enable_eplb": True,
"max_model_len": model_max_len,
}
if spec_method == "dspark":
model_args["trust_remote_code"] = True
return model_args
pytestmark = pytest.mark.skipif(
current_platform.is_rocm(),
reason="EPLB with Spec Decode is a work in progress on ROCm.",
)
class FakeEplbState:
instances: list[FakeEplbState] = []
def __init__(self, parallel_config, device: torch.device):
self.add_model_calls: list[tuple[object, object, str | None]] = []
FakeEplbState.instances.append(self)
def add_model(
self,
model: object,
model_config: object,
*,
model_name: str | None = None,
) -> None:
self.add_model_calls.append((model, model_config, model_name))
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
return DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"],
hidden_size=128,
num_hidden_layers=2,
n_routed_experts=8,
num_experts_per_tok=2,
num_hash_layers=0,
n_shared_experts=1,
moe_intermediate_size=128,
hc_mult=1,
hc_eps=1e-5,
rms_norm_eps=1e-5,
dspark_target_layer_ids=[0],
dspark_markov_rank=8,
index_topk=4,
head_dim=64,
num_attention_heads=4,
vocab_size=256,
n_mtp_layers=2,
enable_confidence_head=False,
layer_types=["sliding_attention"] * 2,
)
@pytest.fixture
def dspark_vllm_config():
hf_config = _make_dsv4_dspark_hf_config()
model_config = SimpleNamespace(
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
)
return SimpleNamespace(
model_config=model_config,
quant_config=None,
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
parallel_config=SimpleNamespace(
pipeline_parallel_size=1,
tensor_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=True,
enable_eplb=True,
enable_elastic_ep=False,
eplb_config=SimpleNamespace(num_redundant_experts=4),
),
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
compilation_config=SimpleNamespace(static_forward_context={}),
speculative_config=SimpleNamespace(
method="dspark",
draft_model_config=model_config,
),
)
def _make_moe_topology(
*,
num_routed_experts: int = 8,
num_redundant_experts: int = 4,
num_expert_groups: int = 1,
) -> SimpleNamespace:
num_physical_experts = num_routed_experts + num_redundant_experts
return SimpleNamespace(
num_routed_experts=num_routed_experts,
num_redundant_experts=num_redundant_experts,
num_physical_experts=num_physical_experts,
num_logical_experts=num_routed_experts,
num_expert_groups=num_expert_groups,
)
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
draft = _make_moe_topology()
target = _make_moe_topology()
EplbState.assert_confs_equal(draft, target)
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
draft = _make_moe_topology(num_redundant_experts=0)
target = _make_moe_topology(num_redundant_experts=4)
with pytest.raises(RuntimeError, match="mismatch"):
EplbState.assert_confs_equal(draft, target)
@pytest.mark.parametrize(
("model_type", "expected"),
[
("deepseek_v4", True),
("deepseek_v41", False),
],
)
def test_dspark_draft_supports_eplb(
dspark_vllm_config, model_type: str, expected: bool
):
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
assert dspark_draft_supports_eplb(draft_model_config) is expected
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
assert not draft_model_supports_eplb(
dspark_vllm_config.speculative_config,
None,
)
@pytest.mark.parametrize(
"model_type,load_dummy_weights,expected_registered",
[
("deepseek_v4", False, True),
("deepseek_v4", True, False),
("deepseek_v41", False, False),
],
)
def test_eplb_dspark_draft_registration(
dspark_vllm_config,
monkeypatch: pytest.MonkeyPatch,
model_type,
load_dummy_weights: bool,
expected_registered: bool,
):
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
FakeEplbState.instances.clear()
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.EplbState",
FakeEplbState,
)
monkeypatch.setattr(
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
lambda model: model,
)
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
draft_model_config.hf_config.model_type = model_type
draft = SimpleNamespace()
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
controller.prepare_load()
speculator = SimpleNamespace(model=draft, eplb_state=None)
def set_eplb_state(state) -> None:
speculator.eplb_state = state
speculator.set_eplb_state = set_eplb_state
registered = controller.maybe_register_speculator(
speculator,
dspark_vllm_config.speculative_config,
load_dummy_weights=load_dummy_weights,
)
assert registered is expected_registered
assert controller.state is not None
if expected_registered:
assert controller.state.add_model_calls == [
(draft, draft_model_config, "dspark (draft)")
]
assert speculator.eplb_state is controller.state
else:
assert controller.state.add_model_calls == []
assert speculator.eplb_state is None
@pytest.mark.parametrize(
"model_setup",
[
pytest.param(
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
marks=large_gpu_mark(min_gb=80),
),
pytest.param(
(
"eagle",
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
4,
0.92,
1,
),
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
),
pytest.param(
(
"dspark",
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
None,
4,
0.95,
7,
),
marks=large_gpu_mark(min_gb=80),
),
],
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
)
def test_eplb_spec_decode(
monkeypatch: pytest.MonkeyPatch,
model_setup: tuple[str, str, str | None, int, float, int],
):
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
Applicable to MoE models with mtp, eagle, or dspark spec decode.
"""
(
method,
model_name,
spec_model_name,
tp_size,
expected_gsm8k_value,
num_speculative_tokens,
) = model_setup
if method == "dspark" and not current_platform.is_device_capability_family(100):
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.03
model_args = get_model_args(
model_name=model_name,
spec_model_name=spec_model_name,
spec_method=method,
tp_size=tp_size,
model_max_len=4096,
num_speculative_tokens=num_speculative_tokens,
)
results = lm_eval.simple_evaluate(
model="vllm",
model_args=model_args,
tasks=TASK,
batch_size=64,
num_fewshot=8,
)
measured_value = results["results"][TASK][FILTER]
assert (
measured_value - RTOL < expected_gsm8k_value
and measured_value + RTOL > expected_gsm8k_value
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
@large_gpu_mark(min_gb=80)
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
RTOL = 0.03
expected_gsm8k_value = 0.86
model_args = get_model_args(
model_name="Qwen/Qwen3-Next-80B-A3B-Instruct",
spec_model_name=None,
spec_method="mtp",
tp_size=4,
model_max_len=4096,
use_async=True,
)
results = lm_eval.simple_evaluate(
model="vllm",
model_args=model_args,
tasks=TASK,
batch_size=64,
num_fewshot=8,
)
measured_value = results["results"][TASK][FILTER]
assert (
measured_value - RTOL < expected_gsm8k_value
and measured_value + RTOL > expected_gsm8k_value
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
File diff suppressed because it is too large Load Diff