[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,727 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
|
||||
from vllm.distributed.device_communicators import custom_all_reduce as car
|
||||
from vllm.distributed.parallel_state import get_tp_group, graph_capture
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
random.seed(42)
|
||||
test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
|
||||
for i, v in enumerate(test_sizes):
|
||||
test_sizes[i] -= v % 8
|
||||
|
||||
|
||||
def _bf16_ulps(a: torch.Tensor, b: torch.Tensor) -> int:
|
||||
"""Largest distance between two BF16 tensors in units in the last place."""
|
||||
|
||||
def ordered(t: torch.Tensor) -> torch.Tensor:
|
||||
bits = t.view(torch.int16).int() & 0xFFFF
|
||||
return torch.where(bits >= 0x8000, 0x8000 - bits, bits)
|
||||
|
||||
return int((ordered(a) - ordered(b)).abs().max())
|
||||
|
||||
|
||||
def _unfused_all_reduce_mhc(peers, residual, post, comb, pre, weight, eps):
|
||||
"""All-reduce, mHC post, collapse and RMSNorm in the fused kernel's FP32
|
||||
order, rounding to BF16 where the unfused path returns BF16."""
|
||||
reduced = peers[0].float()
|
||||
for peer in peers[1:]:
|
||||
reduced = reduced + peer.float()
|
||||
reduced = reduced.bfloat16().float()
|
||||
output = torch.empty_like(residual)
|
||||
collapse = torch.zeros_like(reduced)
|
||||
for target in range(4):
|
||||
mixed = reduced * post[:, target : target + 1]
|
||||
for source in range(4):
|
||||
mixed = torch.addcmul(
|
||||
mixed, residual[:, source].float(), comb[:, source, target, None]
|
||||
)
|
||||
output[:, target] = mixed.bfloat16()
|
||||
collapse = torch.addcmul(
|
||||
collapse, output[:, target].float(), pre[:, target : target + 1]
|
||||
)
|
||||
prenorm = collapse.bfloat16().float()
|
||||
inv_rms = torch.rsqrt(prenorm.square().mean(-1, keepdim=True) + eps)
|
||||
return output, (prenorm * inv_rms * weight.float()).bfloat16()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
|
||||
from vllm.models.deepseek_v41.nvidia.ops.cute_dsl import AllReduceMHC
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
op = AllReduceMHC(
|
||||
hidden_size=5120, hc_mult=4, max_num_tokens=16, top_k=6, device=device
|
||||
)
|
||||
|
||||
def check_eager_and_replayed(fused, check, halved, doubled):
|
||||
check(fused())
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for _ in range(5):
|
||||
fused()
|
||||
captured = fused()
|
||||
for _ in range(20):
|
||||
graph.replay()
|
||||
check(captured)
|
||||
# Replays must pick up in-place input changes.
|
||||
halved.mul_(0.5)
|
||||
doubled.mul_(2)
|
||||
graph.replay()
|
||||
check(captured)
|
||||
|
||||
def mhc_inputs(n):
|
||||
torch.manual_seed(123)
|
||||
residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16)
|
||||
post = torch.rand(n, 4, device=device)
|
||||
comb = torch.randn(n, 4, 4, device=device) * 0.1
|
||||
pre = torch.rand(n, 4, device=device)
|
||||
weight = torch.randn(5120, device=device, dtype=torch.bfloat16)
|
||||
return residual, post, comb, pre, weight
|
||||
|
||||
def run(n):
|
||||
torch.manual_seed(42 + rank)
|
||||
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
|
||||
# Packed +0/-0 pairs collide with the Lamport sentinel.
|
||||
x[:, :16] = 0
|
||||
x[:, 9:16:2] = -0.0
|
||||
residual, post, comb, pre, weight = mhc_inputs(n)
|
||||
|
||||
def fused():
|
||||
return op(x, residual, post, comb, pre, weight, 1e-6)
|
||||
|
||||
def check(outputs):
|
||||
peers = get_tp_group().all_gather(x, dim=0).view(tp_size, n, 5120)
|
||||
output, normalized = _unfused_all_reduce_mhc(
|
||||
peers, residual, post, comb, pre, weight, 1e-6
|
||||
)
|
||||
# The mixed hc streams match bit for bit. The RMSNorm sums the
|
||||
# squares in another order and uses an approximate rsqrt.
|
||||
assert torch.equal(outputs[0], output)
|
||||
assert _bf16_ulps(outputs[1], normalized) <= 1
|
||||
|
||||
check_eager_and_replayed(fused, check, x, residual)
|
||||
|
||||
def run_finalize(n):
|
||||
torch.manual_seed(7 + rank)
|
||||
# A padded permuted GEMM2 buffer, like the MoE's.
|
||||
rows = n * 6 + 5
|
||||
gemm2 = torch.randn(rows, 5120, device=device, dtype=torch.bfloat16)
|
||||
permuted = torch.randperm(rows, device=device)[: n * 6].view(n, 6).int()
|
||||
# A route to an expert this rank does not hold.
|
||||
permuted[0, -1] = -1
|
||||
weights = torch.rand(n, 6, device=device)
|
||||
shared = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
|
||||
residual, post, comb, pre, weight = mhc_inputs(n)
|
||||
|
||||
def fused():
|
||||
return op.finalize(
|
||||
gemm2,
|
||||
weights,
|
||||
permuted,
|
||||
shared,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre,
|
||||
weight,
|
||||
1e-6,
|
||||
)
|
||||
|
||||
def check(outputs):
|
||||
# One FP32 FMA per route in route order, the shared add, one
|
||||
# rounding, then the plain path: only the finalize differs.
|
||||
acc = torch.zeros(n, 5120, device=device)
|
||||
for k in range(6):
|
||||
valid = (permuted[:, k] >= 0).unsqueeze(-1)
|
||||
rows_k = gemm2[permuted[:, k].clamp_min(0).long()].float()
|
||||
torch.addcmul(
|
||||
acc,
|
||||
torch.where(valid, rows_k, 0.0),
|
||||
torch.where(valid, weights[:, k : k + 1], 0.0),
|
||||
out=acc,
|
||||
)
|
||||
x = (acc + shared.float()).bfloat16()
|
||||
expected = op(x, residual, post, comb, pre, weight, 1e-6)
|
||||
assert torch.equal(outputs[0], expected[0])
|
||||
assert torch.equal(outputs[1], expected[1])
|
||||
|
||||
check_eager_and_replayed(fused, check, gemm2, shared)
|
||||
|
||||
# Changing shapes cover shrinking and growing batches in one mailbox.
|
||||
for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1):
|
||||
run(n)
|
||||
run_finalize(n)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100), reason="Requires SM100"
|
||||
)
|
||||
def test_all_reduce_mhc_matches_unfused_path(monkeypatch):
|
||||
if torch.accelerator.device_count() < 4:
|
||||
pytest.skip("Requires four GPUs with NVLink multicast")
|
||||
multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "expected"),
|
||||
[
|
||||
(torch.float32, True),
|
||||
(torch.float16, True),
|
||||
(torch.bfloat16, True),
|
||||
(torch.int8, False),
|
||||
(torch.float8_e4m3fn, False),
|
||||
],
|
||||
)
|
||||
def test_custom_allreduce_filters_dtype(
|
||||
dtype: torch.dtype,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator._ptr = 0
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
|
||||
batch_invariant: bool,
|
||||
) -> None:
|
||||
"""Batch invariance must not switch backends based on the input size."""
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
communicator._ptr = 0
|
||||
|
||||
oversized = torch.empty(1024, dtype=torch.float16)
|
||||
assert communicator.should_custom_ar(oversized) is batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_reduce_scatter_disabled_under_batch_invariance(
|
||||
monkeypatch, batch_invariant: bool
|
||||
) -> None:
|
||||
"""Reduce-scatter stays on one backend under batch invariance."""
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_only = False
|
||||
communicator.mnnvl_multicast_ptr = 0
|
||||
communicator.max_reduce_scatter_size = 1024
|
||||
communicator.max_mnnvl_reduce_scatter_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
|
||||
in_range = torch.empty(64, dtype=torch.float16)
|
||||
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
(8, True, False),
|
||||
(9, True, False),
|
||||
(10, False, False),
|
||||
(10, True, True),
|
||||
],
|
||||
)
|
||||
def test_cross_node_mnnvl_gate_checks_generation_and_multicast(
|
||||
monkeypatch,
|
||||
major,
|
||||
local_multicast,
|
||||
expected,
|
||||
):
|
||||
def has_device_capability(capability, device_id):
|
||||
assert capability == 100
|
||||
assert device_id == 3
|
||||
return major >= 10
|
||||
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"has_device_capability",
|
||||
has_device_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: local_multicast,
|
||||
)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", lambda *_args, **_kwargs: None)
|
||||
|
||||
assert car._group_can_attempt_mnnvl(object(), torch.device("cuda:3")) is expected
|
||||
|
||||
|
||||
def test_cross_node_mnnvl_gate_requires_support_on_every_rank(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"has_device_capability",
|
||||
lambda *_args: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: True,
|
||||
)
|
||||
|
||||
def report_unsupported_peer(support, **_kwargs):
|
||||
support.zero_()
|
||||
|
||||
monkeypatch.setattr(car.dist, "all_reduce", report_unsupported_peer)
|
||||
|
||||
assert not car._group_can_attempt_mnnvl(object(), torch.device("cuda:0"))
|
||||
|
||||
|
||||
def test_local_multicast_support_rejects_non_cuda(monkeypatch):
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: False)
|
||||
|
||||
assert not car._has_local_multicast_support(torch.device("cuda:0"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("world_size", "device_capability", "local_multicast", "expected"),
|
||||
[
|
||||
(2, (10, 0), True, True),
|
||||
(4, (10, 3), True, True),
|
||||
(8, (10, 0), True, True),
|
||||
(8, (10, 3), True, True),
|
||||
(6, (10, 3), True, False),
|
||||
(8, (10, 1), True, False),
|
||||
(8, (9, 0), True, False),
|
||||
(8, (10, 3), False, False),
|
||||
],
|
||||
)
|
||||
def test_mnnvl_multimem_reduce_scatter_platform_gate(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
device_capability,
|
||||
local_multicast,
|
||||
expected,
|
||||
):
|
||||
def is_device_capability(capability, device_id):
|
||||
assert capability in ((10, 0), (10, 3))
|
||||
assert device_id == 3
|
||||
return device_capability == capability
|
||||
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"is_device_capability",
|
||||
is_device_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: local_multicast,
|
||||
)
|
||||
|
||||
supported = car._supports_mnnvl_multimem_reduce_scatter(
|
||||
torch.device("cuda:3"), world_size
|
||||
)
|
||||
assert supported is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"message_bytes",
|
||||
"multimem_ptr",
|
||||
"multimem_initialized",
|
||||
"batch_invariant",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
|
||||
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
|
||||
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
|
||||
(64 * 1024 * 1024 + 128, 1, True, False, None),
|
||||
(32 * 1024 * 1024, 0, True, False, None),
|
||||
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
|
||||
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
|
||||
(32 * 1024 * 1024, 1, True, True, None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
def test_mnnvl_reduce_scatter_backend_gate(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
message_bytes,
|
||||
multimem_ptr,
|
||||
multimem_initialized,
|
||||
batch_invariant,
|
||||
expected,
|
||||
):
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator._ptr = 0
|
||||
communicator.world_size = world_size
|
||||
communicator.mnnvl_only = False
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_multicast_ptr = 1
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
|
||||
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
|
||||
inp = torch.empty(
|
||||
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert inp.nbytes == message_bytes
|
||||
assert communicator._select_reduce_scatter_backend(inp) == expected
|
||||
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
|
||||
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
|
||||
expected == "mnnvl_multimem"
|
||||
)
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
|
||||
monkeypatch,
|
||||
):
|
||||
events = []
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(*_args, **_kwargs):
|
||||
events.append("empty")
|
||||
return torch.empty(1, dtype=torch.uint8)
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append("rendezvous")
|
||||
return None
|
||||
|
||||
def report_peer_allocation_failure(group_value, **_kwargs):
|
||||
events.append("all_reduce")
|
||||
assert group_value.item() == 1
|
||||
group_value.zero_()
|
||||
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
|
||||
warnings = []
|
||||
monkeypatch.setattr(
|
||||
car.logger,
|
||||
"warning_once",
|
||||
lambda message, *_args, **_kwargs: warnings.append(message),
|
||||
)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = object()
|
||||
communicator.device = torch.device("cuda:0")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == ["empty", "all_reduce"]
|
||||
assert communicator.mnnvl_multimem_rs_initialized
|
||||
assert communicator.mnnvl_multimem_rs_buffer is None
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
|
||||
assert warnings == [
|
||||
"MNNVL multimem reduce-scatter symmetric-memory allocation "
|
||||
"failed on at least one rank; falling back to NCCL."
|
||||
]
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
|
||||
events = []
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(*_args, **_kwargs):
|
||||
events.append("empty")
|
||||
return torch.empty(1, dtype=torch.uint8)
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append("rendezvous")
|
||||
raise RuntimeError("rendezvous failed")
|
||||
|
||||
def preserve_local_result(_group_value, **_kwargs):
|
||||
events.append("all_reduce")
|
||||
|
||||
warnings = []
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
|
||||
monkeypatch.setattr(
|
||||
car.logger,
|
||||
"warning_once",
|
||||
lambda message, *_args, **_kwargs: warnings.append(message),
|
||||
)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = type("Group", (), {"group_name": "test"})()
|
||||
communicator.device = torch.device("cuda:0")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
|
||||
assert communicator.mnnvl_multimem_rs_initialized
|
||||
assert communicator.mnnvl_multimem_rs_buffer is None
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
|
||||
assert warnings == [
|
||||
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
|
||||
"failed on at least one rank; falling back to NCCL."
|
||||
]
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
|
||||
events = []
|
||||
buffers = []
|
||||
|
||||
class FakeHandle:
|
||||
multicast_ptr = 0x3000
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(size, **_kwargs):
|
||||
events.append(("empty", size))
|
||||
buffer = torch.ones(size, dtype=torch.uint8)
|
||||
buffers.append(buffer)
|
||||
return buffer
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append(("rendezvous", None))
|
||||
return FakeHandle()
|
||||
|
||||
def preserve_local_result(*_args, **_kwargs):
|
||||
events.append(("all_reduce", None))
|
||||
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(
|
||||
car.torch.accelerator,
|
||||
"synchronize",
|
||||
lambda: events.append(("synchronize", None)),
|
||||
)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = type("Group", (), {"group_name": "test"})()
|
||||
communicator.device = torch.device("cpu")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == [
|
||||
("empty", 257),
|
||||
("all_reduce", None),
|
||||
("rendezvous", None),
|
||||
("synchronize", None),
|
||||
("all_reduce", None),
|
||||
]
|
||||
assert torch.all(buffers[0][:128] == 0)
|
||||
assert torch.all(buffers[0][128:] == 1)
|
||||
assert communicator.mnnvl_multimem_rs_buffer_size == 129
|
||||
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
# A small all_reduce for warmup.
|
||||
# this is needed because device communicators might be created lazily
|
||||
# (e.g. NCCL). This will ensure that the communicator is initialized
|
||||
# before any communication happens, so that this group can be used for
|
||||
# graph capture immediately.
|
||||
data = torch.zeros(1)
|
||||
data = data.to(device=device)
|
||||
torch.distributed.all_reduce(data, group=group)
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
# we use the first group to communicate once
|
||||
# and the second group to communicate twice
|
||||
# and so on
|
||||
# this is used to demonstrate that each group can
|
||||
# communicate independently
|
||||
num_communication = rank // tp_size + 1
|
||||
|
||||
for sz in test_sizes:
|
||||
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
# use integers so result matches NCCL exactly
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
|
||||
inp2 = torch.randint(1, 16, (sz,), dtype=dtype, device=device_idx)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for i in range(num_communication):
|
||||
out1 = tensor_model_parallel_all_reduce(inp1)
|
||||
# the input buffer is immediately modified to test
|
||||
# synchronization
|
||||
dist.all_reduce(inp1, group=group)
|
||||
out2 = tensor_model_parallel_all_reduce(inp2)
|
||||
dist.all_reduce(inp2, group=group)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(out1, inp1)
|
||||
torch.testing.assert_close(out2, inp2)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
# we use the first group to communicate once
|
||||
# and the second group to communicate twice
|
||||
# and so on
|
||||
# this is used to demonstrate that each group can
|
||||
# communicate independently
|
||||
num_communication = rank // tp_size + 1
|
||||
sz = 1024
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
inp = torch.ones(sz, dtype=torch.float32, device=device)
|
||||
out = inp
|
||||
for _ in range(num_communication):
|
||||
out = fa.all_reduce(out, registered=False)
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
inp = torch.ones(sz * 4, dtype=torch.bfloat16, device=device)
|
||||
out = inp
|
||||
for _ in range(num_communication):
|
||||
out = fa.all_reduce(out, registered=False)
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def chunked_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
"""Inputs above max_size are reduced in chunks and match the fixed-order
|
||||
reference bitwise, both eagerly and inside a CUDA graph."""
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
|
||||
# Two full chunks plus a partial one.
|
||||
dtype = torch.bfloat16
|
||||
chunk_numel = fa.max_size // dtype.itemsize
|
||||
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
|
||||
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
|
||||
dist.all_gather(gathered, inp, group=group)
|
||||
ref = gathered[0].float()
|
||||
for peer in gathered[1:]:
|
||||
ref = ref + peer.float()
|
||||
ref = ref.to(torch.bfloat16)
|
||||
|
||||
out = fa.all_reduce(inp, registered=False)
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
# Weak-contiguous but not C-contiguous: a transposed matrix.
|
||||
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
|
||||
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
|
||||
|
||||
with fa.capture():
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
out = fa.all_reduce(inp, registered=True)
|
||||
graph.replay()
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
|
||||
if tp_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
def test_custom_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
test_target,
|
||||
):
|
||||
world_size = tp_size * pipeline_parallel_size
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,883 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Engram tables shared by co-located DP replicas.
|
||||
|
||||
Four GPUs on one node split the hash heads four ways, whether as DEP
|
||||
(TP1 x DP4, the single-node MoE layout) or TP2 x DP2. A lookup only returns
|
||||
the right rows if the ids of every replica are gathered, each rank looks up
|
||||
its own heads, and each replica trades those tokens back for the heads its
|
||||
peers own.
|
||||
|
||||
With shared host storage, DP replicas instead map the same TP slice and
|
||||
prefetch only their own tokens without DP gathers.
|
||||
"""
|
||||
|
||||
from itertools import product
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from threading import Event
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm.config import (
|
||||
EngramConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config_or_none,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory, parallel_state
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_engram_dp_group,
|
||||
get_engram_dp_size,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
convert_swizzled_to_linear,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
EngramBatch,
|
||||
EngramLayout,
|
||||
ParallelEngramEmbedding,
|
||||
engram_head_shard_rank,
|
||||
gather_engram_hashes,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
WORLD_SIZE = 4
|
||||
DIM, BLOCK = 64, 32
|
||||
# Prime-sized buckets, as the model's layout builds them.
|
||||
HEAD_SIZES = (97, 101, 103, 107, 109, 113, 127, 131)
|
||||
|
||||
|
||||
def _token_counts(dp_size: int) -> tuple[tuple[int, ...], ...]:
|
||||
"""Even loads, then uneven ones with an idle replica to pad around."""
|
||||
return ((8,) * dp_size, (7, 0, 5, 1)[:dp_size], (7,) * dp_size)
|
||||
|
||||
|
||||
def _full_table(rows: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The unsharded table, identical on every rank."""
|
||||
torch.manual_seed(0)
|
||||
weight = (torch.randn(rows, DIM) * 4).to(torch.float8_e4m3fn)
|
||||
scale_inv = torch.randint(120, 134, (rows, DIM // BLOCK), dtype=torch.uint8)
|
||||
return weight, scale_inv
|
||||
|
||||
|
||||
def _make_ids(head_sizes: tuple[int, ...], num_tokens: int, seed: int) -> torch.Tensor:
|
||||
"""Ids of each head, drawn from that head's own bucket range."""
|
||||
torch.manual_seed(seed)
|
||||
offset = 0
|
||||
columns = []
|
||||
for size in head_sizes:
|
||||
columns.append(torch.randint(offset, offset + size, (num_tokens, 1)))
|
||||
offset += size
|
||||
return torch.cat(columns, dim=1).to(torch.int32).cuda()
|
||||
|
||||
|
||||
def _reference(weight, scale_inv, ids) -> torch.Tensor:
|
||||
"""Dequantized lookup against the whole table."""
|
||||
valid = ids >= 0
|
||||
safe_ids = ids.clamp_min(0).long()
|
||||
values = torch.nn.functional.embedding(safe_ids, weight).float()
|
||||
scales = torch.nn.functional.embedding(safe_ids, scale_inv)
|
||||
scales = (scales.to(torch.int32) << 23).view(torch.float32)
|
||||
values = values.unflatten(-1, (-1, BLOCK)) * scales.unsqueeze(-1)
|
||||
return values.flatten(-2).to(torch.bfloat16).masked_fill(~valid.unsqueeze(-1), 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"architecture,options,edp_ranks,etp_ranks",
|
||||
[
|
||||
("DeepseekV41ForCausalLM", {}, [0, 2], [0, 1]),
|
||||
("Qwen4ExpForCausalLM", {}, None, [0, 1]),
|
||||
(
|
||||
"Qwen4ExpForCausalLM",
|
||||
{"embedding_across_dp": True},
|
||||
None,
|
||||
[0, 1, 2, 3],
|
||||
),
|
||||
("Qwen4ExpForConditionalGeneration", {}, None, [0, 1]),
|
||||
],
|
||||
)
|
||||
def test_engram_group_creation_honors_model_and_options(
|
||||
monkeypatch, architecture, options, edp_ranks, etp_ranks
|
||||
):
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(architecture=architecture, is_moe=False),
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=2,
|
||||
distributed_executor_backend="mp",
|
||||
),
|
||||
engram_config=EngramConfig(**options),
|
||||
)
|
||||
for name in (
|
||||
"_TP",
|
||||
"_ETP",
|
||||
"_ENGRAM_DP",
|
||||
"_DCP",
|
||||
"_PCP",
|
||||
"_PP",
|
||||
"_DP",
|
||||
"_EP",
|
||||
"_EPLB",
|
||||
):
|
||||
monkeypatch.setattr(parallel_state, name, None)
|
||||
monkeypatch.setattr(parallel_state, "_WORLD", SimpleNamespace(local_rank=0))
|
||||
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
|
||||
monkeypatch.setattr(torch.distributed, "get_world_size", lambda: 4)
|
||||
monkeypatch.setattr(torch.distributed, "get_rank", lambda: 0)
|
||||
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
|
||||
|
||||
def make_group(group_ranks, *args, **kwargs):
|
||||
ranks = next(ranks for ranks in group_ranks if 0 in ranks)
|
||||
return SimpleNamespace(ranks=ranks, rank_in_group=0, world_size=len(ranks))
|
||||
|
||||
monkeypatch.setattr(parallel_state, "init_model_parallel_group", make_group)
|
||||
with set_current_vllm_config(config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=2, backend="nccl")
|
||||
group = get_engram_dp_group()
|
||||
assert (group.ranks if group is not None else None) == edp_ranks
|
||||
assert parallel_state.get_etp_group().ranks == etp_ranks
|
||||
|
||||
|
||||
def _worker(rank: int, tp_size: int, port: int) -> None:
|
||||
torch.accelerator.set_device_index(rank)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(rank),
|
||||
"LOCAL_RANK": str(rank),
|
||||
"WORLD_SIZE": str(WORLD_SIZE),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(port),
|
||||
}
|
||||
)
|
||||
dp_size = WORLD_SIZE // tp_size
|
||||
dp_rank, tp_rank = divmod(rank, tp_size)
|
||||
vllm_config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=tp_size,
|
||||
data_parallel_size=dp_size,
|
||||
data_parallel_rank=dp_rank,
|
||||
),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=8,
|
||||
is_encoder_decoder=False,
|
||||
max_num_batched_tokens=8,
|
||||
max_num_seqs=8,
|
||||
),
|
||||
)
|
||||
# Supply the resolved Engram settings without constructing a full model.
|
||||
vllm_config.engram_config = EngramConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
try:
|
||||
init_distributed_environment()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=tp_size)
|
||||
assert get_engram_dp_size() == dp_size
|
||||
assert get_engram_dp_group().ranks == [
|
||||
tp_rank + replica * tp_size for replica in range(dp_size)
|
||||
]
|
||||
assert engram_head_shard_rank() == tp_rank * dp_size + dp_rank
|
||||
|
||||
for failure in ("create", "open", "mmap", "register", "pinned", None):
|
||||
_check_shared_storage_lifetime_and_failures(failure)
|
||||
|
||||
# Reuse the processes and communication groups across storage modes.
|
||||
for (cpu_offload, dp_shared_memory), n_heads, sequence_parallel in product(
|
||||
[(False, False), (True, False), (True, True)],
|
||||
[8, 7],
|
||||
[False, True] if tp_size > 1 else [False],
|
||||
):
|
||||
_check_table(
|
||||
vllm_config,
|
||||
cpu_offload,
|
||||
dp_shared_memory,
|
||||
n_heads,
|
||||
sequence_parallel,
|
||||
)
|
||||
if tp_size == 1:
|
||||
_check_fp8_batch(vllm_config)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _check_fp8_batch(vllm_config):
|
||||
"""MXFP8 staging matches the tables across peer/all-to-all slots, idle
|
||||
replicas and graph replays."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
layers = []
|
||||
for index in range(2):
|
||||
weight, scale = _full_table(sum(HEAD_SIZES))
|
||||
weight = (weight.float() * (1 - 2 * index)).to(torch.float8_e4m3fn)
|
||||
table = ParallelEngramEmbedding(
|
||||
sum(HEAD_SIZES), DIM, HEAD_SIZES, cpu_offload=True
|
||||
)
|
||||
table.weight.weight_loader(table.weight, weight)
|
||||
table.weight_scale_inv.weight_loader(table.weight_scale_inv, scale)
|
||||
staged = torch.empty(0, device="cuda")
|
||||
reference = (weight.cuda(), scale.cuda())
|
||||
layers.append(SimpleNamespace(embed_tokens=table, staged_rows=staged))
|
||||
layers[-1].layer_hash_index, layers[-1].reference = index, reference
|
||||
batch = EngramBatch(layers, 257)
|
||||
|
||||
def check(ids):
|
||||
tokens = ids.shape[0]
|
||||
for layer in layers:
|
||||
x = batch.wkv_input(layer, tokens)
|
||||
sf = convert_swizzled_to_linear(x.scale, tokens, x.data.shape[1], BLOCK)
|
||||
actual = dequant_mxfp8_to_bf16(x.data, sf).unflatten(-1, (-1, DIM))
|
||||
weight, scale = layer.reference
|
||||
expected = _reference(weight, scale, ids[:, layer.layer_hash_index])
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
for counts in ((7, 0, 5, 1), (127,) * 4, (128,) * 4, (257, 0, 129, 1)):
|
||||
tokens = counts[dp_rank]
|
||||
seed = 700 + 10 * dp_rank
|
||||
ids = torch.stack([_make_ids(HEAD_SIZES, tokens, seed + i) for i in (0, 1)], 1)
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=tokens,
|
||||
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
|
||||
):
|
||||
batch.prepare_embeddings(ids)
|
||||
check(ids)
|
||||
for capture in ("full", "breakable") if len(set(counts)) == 1 else ():
|
||||
warmup = torch.cuda.Stream()
|
||||
warmup.wait_stream(torch.cuda.current_stream())
|
||||
if capture == "full":
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=warmup):
|
||||
batch.prepare_embeddings(ids)
|
||||
else:
|
||||
graph = BreakableCUDAGraphCapture()
|
||||
with torch.cuda.stream(warmup), graph:
|
||||
batch.prepare_embeddings(ids)
|
||||
torch.cuda.current_stream().wait_stream(warmup)
|
||||
for step in (2, 3):
|
||||
replay_ids = _make_ids(HEAD_SIZES, tokens, seed + step)
|
||||
ids.copy_(torch.stack([replay_ids] * 2, 1))
|
||||
graph.replay()
|
||||
check(ids)
|
||||
del graph
|
||||
|
||||
|
||||
def _check_shared_storage_lifetime_and_failures(failure):
|
||||
"""One peer's failure must roll back every mapping/registration, without hangs."""
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
group = get_engram_dp_group()
|
||||
runtime = torch.cuda.cudart()
|
||||
make_file = engram_ops.tempfile.NamedTemporaryFile
|
||||
map_file = engram_ops.mmap.mmap
|
||||
paths, mappings, registrations, unregistrations = [], [], [], []
|
||||
failed_rank = 0 if failure == "create" else 1
|
||||
|
||||
def fail():
|
||||
raise OSError(f"injected {failure} failure")
|
||||
|
||||
def temporary_file(*args, **kwargs):
|
||||
if failure == "create":
|
||||
fail()
|
||||
file = make_file(*args, **kwargs)
|
||||
paths.append(Path(file.name))
|
||||
return file
|
||||
|
||||
def open_file(*args, **kwargs):
|
||||
if failure == "open" and group.rank_in_group == failed_rank:
|
||||
fail()
|
||||
return open(*args, **kwargs)
|
||||
|
||||
def mapping(*args, **kwargs):
|
||||
if failure == "mmap" and group.rank_in_group == failed_rank:
|
||||
fail()
|
||||
result = map_file(*args, **kwargs)
|
||||
mappings.append(weakref.ref(result))
|
||||
return result
|
||||
|
||||
def register(pointer, size, flags):
|
||||
if failure == "register" and group.rank_in_group == failed_rank:
|
||||
return SimpleNamespace(value=1)
|
||||
result = runtime.cudaHostRegister(pointer, size, flags)
|
||||
assert result.value == 0
|
||||
registrations.append(pointer)
|
||||
return result
|
||||
|
||||
def unregister(pointer):
|
||||
unregistrations.append(pointer)
|
||||
result = runtime.cudaHostUnregister(pointer)
|
||||
assert result.value == 0
|
||||
return result
|
||||
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(
|
||||
engram_ops,
|
||||
"tempfile",
|
||||
SimpleNamespace(NamedTemporaryFile=temporary_file),
|
||||
)
|
||||
patch.setattr(engram_ops, "open", open_file, raising=False)
|
||||
patch.setattr(
|
||||
engram_ops,
|
||||
"mmap",
|
||||
SimpleNamespace(mmap=mapping, MAP_SHARED=engram_ops.mmap.MAP_SHARED),
|
||||
)
|
||||
patch.setattr(
|
||||
torch.cuda,
|
||||
"cudart",
|
||||
lambda: SimpleNamespace(
|
||||
cudaHostRegister=register,
|
||||
cudaHostUnregister=unregister,
|
||||
),
|
||||
)
|
||||
if failure == "pinned" and group.rank_in_group == failed_rank:
|
||||
patch.setattr(torch.Tensor, "is_pinned", lambda self: False)
|
||||
if failure is not None:
|
||||
stage = "cudaHostRegister" if failure in ("register", "pinned") else failure
|
||||
with pytest.raises(RuntimeError, match=f"EDP rank {failed_rank}.*{stage}"):
|
||||
engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
|
||||
assert unregistrations == registrations
|
||||
else:
|
||||
storage = engram_ops.DPSharedEngramStorage(128, DIM, BLOCK, group)
|
||||
assert all(not path.exists() for path in paths)
|
||||
storage_ref = weakref.ref(storage)
|
||||
parameter = torch.nn.Parameter(storage.weight, requires_grad=False)
|
||||
views = storage.get_views(parameter, storage.weight_scale_inv)
|
||||
del storage
|
||||
gc.collect()
|
||||
assert storage_ref() is None
|
||||
assert not unregistrations
|
||||
del parameter
|
||||
gc.collect()
|
||||
assert not unregistrations # UVA views still retain the CPU allocation.
|
||||
del views
|
||||
gc.collect()
|
||||
assert unregistrations == registrations
|
||||
for ref in mappings:
|
||||
mapped = ref()
|
||||
assert mapped is None or mapped.closed
|
||||
assert all(not path.exists() for path in paths)
|
||||
|
||||
|
||||
def _check_table(
|
||||
vllm_config, cpu_offload, dp_shared_memory, n_heads, sequence_parallel
|
||||
):
|
||||
parallel = vllm_config.parallel_config
|
||||
dp_size, tp_size = parallel.data_parallel_size, parallel.tensor_parallel_size
|
||||
dp_rank = parallel.data_parallel_rank
|
||||
tp_rank = engram_ops.get_tensor_model_parallel_rank()
|
||||
multithread = (
|
||||
dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1)
|
||||
)
|
||||
load_config = (
|
||||
LoadConfig(
|
||||
load_format="safetensors",
|
||||
use_tqdm_on_load=False,
|
||||
model_loader_extra_config={
|
||||
"enable_multithread_load": True,
|
||||
"num_threads": 2,
|
||||
},
|
||||
)
|
||||
if multithread
|
||||
else vllm_config.load_config
|
||||
)
|
||||
with pytest.MonkeyPatch.context() as m:
|
||||
head_sizes = HEAD_SIZES[:n_heads]
|
||||
rows = sum(head_sizes)
|
||||
config = SimpleNamespace(
|
||||
hidden_size=DIM,
|
||||
hc_mult=2,
|
||||
rms_norm_eps=1e-6,
|
||||
engram_layer_ids=[0],
|
||||
engram_num_embeddings=[rows],
|
||||
engram_max_ngram_size=2,
|
||||
engram_n_heads=n_heads,
|
||||
engram_head_dim=DIM,
|
||||
engram_compressed_vocab_size=32,
|
||||
engram_pad_token_id=0,
|
||||
engram_vocab_size=HEAD_SIZES[0],
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
weight, scale_inv = _full_table(rows)
|
||||
layout = EngramLayout(config)
|
||||
# Stub configuration inputs, not Engram's initialization or execution.
|
||||
component_config = SimpleNamespace(
|
||||
engram_config=EngramConfig(
|
||||
cpu_offload=cpu_offload,
|
||||
dp_shared_memory=dp_shared_memory,
|
||||
),
|
||||
scheduler_config=vllm_config.scheduler_config,
|
||||
load_config=load_config,
|
||||
)
|
||||
with m.context() as init_patch, torch.device("cuda"):
|
||||
init_patch.setattr(
|
||||
engram_ops, "get_current_vllm_config", lambda: component_config
|
||||
)
|
||||
engram = Engram(
|
||||
config,
|
||||
quant_config=None,
|
||||
layout=layout,
|
||||
layer_hash_index=0,
|
||||
use_sequence_parallel=sequence_parallel,
|
||||
prefix="model.layers.0.engram",
|
||||
)
|
||||
layer = engram.embed_tokens
|
||||
if multithread:
|
||||
_load_shared_multithread(layer, weight, scale_inv, load_config)
|
||||
else:
|
||||
# Only the leader has valid checkpoint payload in shared mode.
|
||||
loader_weight = weight if not dp_shared_memory or dp_rank == 0 else None
|
||||
loader_scale = (
|
||||
scale_inv if not dp_shared_memory or dp_rank == 0 else None
|
||||
)
|
||||
layer.weight.weight_loader(layer.weight, loader_weight)
|
||||
layer.weight_scale_inv.weight_loader(
|
||||
layer.weight_scale_inv, loader_scale
|
||||
)
|
||||
if dp_shared_memory:
|
||||
assert layer.dp_size == 1
|
||||
assert layer.head_start == tp_rank * layer.part_n_hash_cols
|
||||
assert layer.weight.is_pinned() and layer.weight_scale_inv.is_pinned()
|
||||
|
||||
def unexpected_collective(*args, **kwargs):
|
||||
raise AssertionError("shared lookup must not use DP collectives")
|
||||
|
||||
m.setattr(get_engram_dp_group(), "all_gather", unexpected_collective)
|
||||
|
||||
token_counts = _token_counts(dp_size)
|
||||
|
||||
# Production forward runs after the construction-time config has exited.
|
||||
assert get_current_vllm_config_or_none() is None
|
||||
for batch_idx, counts in enumerate(token_counts):
|
||||
num_tokens = counts[dp_rank]
|
||||
# Ids differ per replica but must agree across its TP ranks.
|
||||
ids = _make_ids(head_sizes, num_tokens, seed=100 + dp_rank)
|
||||
if batch_idx == 2 and dp_rank == 1:
|
||||
ids.fill_(engram_ops.DEAD_ID)
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=torch.tensor(counts, dtype=torch.int32),
|
||||
):
|
||||
gathered = gather_engram_hashes(
|
||||
ids, dp_shared_memory=layer.dp_shared_memory
|
||||
)
|
||||
expected_tokens = (
|
||||
num_tokens if dp_shared_memory else max(counts) * dp_size
|
||||
)
|
||||
assert gathered.shape[0] == expected_tokens
|
||||
engram.prepare_embeddings(gathered)
|
||||
looked_up = engram.embed(ids)
|
||||
standalone = layer(ids)
|
||||
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
|
||||
torch.testing.assert_close(standalone, expected, atol=0, rtol=0)
|
||||
if sequence_parallel:
|
||||
# SP hands back only this rank's token shard, zero-padded.
|
||||
chunk = -(-num_tokens // tp_size)
|
||||
expected = torch.nn.functional.pad(
|
||||
expected, (0, 0, 0, 0, 0, chunk * tp_size - num_tokens)
|
||||
)
|
||||
expected = expected[tp_rank * chunk : (tp_rank + 1) * chunk]
|
||||
torch.testing.assert_close(looked_up, expected, atol=0, rtol=0)
|
||||
|
||||
if (
|
||||
cpu_offload
|
||||
and n_heads == 7
|
||||
and not sequence_parallel
|
||||
and (not dp_shared_memory or tp_size == 1)
|
||||
):
|
||||
_check_dummy_hash_model_forward(
|
||||
vllm_config,
|
||||
engram,
|
||||
head_sizes,
|
||||
weight,
|
||||
scale_inv,
|
||||
dp_shared_memory,
|
||||
)
|
||||
|
||||
replay = n_heads == 7 and (
|
||||
(dp_shared_memory and sequence_parallel == (tp_size > 1))
|
||||
or (
|
||||
cpu_offload
|
||||
and not dp_shared_memory
|
||||
and sequence_parallel == (tp_size > 1)
|
||||
)
|
||||
)
|
||||
if replay:
|
||||
captures = ("full", "breakable")
|
||||
for capture in captures:
|
||||
_check_prefetch_replay(
|
||||
engram,
|
||||
head_sizes,
|
||||
weight,
|
||||
scale_inv,
|
||||
dp_shared_memory,
|
||||
capture,
|
||||
)
|
||||
if dp_shared_memory and n_heads == 7 and sequence_parallel == (tp_size > 1):
|
||||
with m.context() as storage_patch:
|
||||
storage_patch.setattr(layer.weight, "data", layer.weight.data.clone())
|
||||
with pytest.raises(RuntimeError, match="storage must not be replaced"):
|
||||
layer(ids)
|
||||
|
||||
|
||||
def _check_dummy_hash_model_forward(
|
||||
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
|
||||
):
|
||||
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
|
||||
from vllm.models.deepseek_v41.common.engram import NgramHashState
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_ops
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
if dp_shared_memory and dp_rank != 1:
|
||||
return
|
||||
tokens = 4
|
||||
ids = _make_ids(head_sizes, tokens, seed=300 + dp_rank)
|
||||
state = NgramHashState.__new__(NgramHashState)
|
||||
torch.nn.Module.__init__(state)
|
||||
state.multipliers = torch.empty(1, 2, dtype=torch.int64, device="cuda")
|
||||
state.primes = torch.empty(1, 1, len(head_sizes), dtype=torch.int64, device="cuda")
|
||||
state.ensure_cache = lambda: True
|
||||
# Hash arithmetic is covered separately; keep the real model's branch and
|
||||
# NgramHashState.dummy_hashes, plus real preparation/lookup collectives.
|
||||
state.forward = lambda *args: ids.unsqueeze(1)
|
||||
|
||||
class Decoder(SimpleNamespace):
|
||||
# Engram hashes and the mask stay the last positional arguments;
|
||||
# **kwargs absorbs the decoder's keyword-only flags.
|
||||
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
|
||||
hashes, keep = args[-2:]
|
||||
if dp_shared_memory and dp_rank == 1:
|
||||
assert hashes is None and keep is None
|
||||
output = hidden.new_zeros(
|
||||
tokens, len(head_sizes), DIM, dtype=torch.bfloat16
|
||||
)
|
||||
else:
|
||||
assert hashes is not None and keep is not None
|
||||
assert bool(keep.all()) == (dp_rank != 1)
|
||||
if dp_rank == 1:
|
||||
assert torch.all(hashes == engram_ops.DEAD_ID)
|
||||
output = engram.embed(hashes[:, 0])
|
||||
# Trailing None is previous_aux; this stub captures no aux states.
|
||||
return output, hidden, hidden, hidden, hidden, None
|
||||
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.parallel_config = vllm_config.parallel_config
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_batch = None
|
||||
model.engram_hash = state
|
||||
model.engram_swa_prefix = "swa"
|
||||
model.engram_dp_shared_memory = dp_shared_memory
|
||||
model.layers = [Decoder(engram=engram)]
|
||||
model.start_layer = 0
|
||||
model.end_layer = 1
|
||||
model.decoder_replay_start = model.end_layer
|
||||
model.decoder_replay_layers = None
|
||||
model.aux_hidden_state_layers = ()
|
||||
metadata = (
|
||||
None
|
||||
if dp_rank == 1
|
||||
else {
|
||||
"swa": SimpleNamespace(
|
||||
query_start_loc=None, slot_mapping=None, block_table=None
|
||||
)
|
||||
}
|
||||
)
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(
|
||||
model_ops,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(
|
||||
is_first_rank=True,
|
||||
is_last_rank=False,
|
||||
),
|
||||
)
|
||||
patch.setattr(model_ops, "mhc_post_tilelang", lambda hidden, *args: hidden)
|
||||
with set_forward_context(
|
||||
metadata,
|
||||
vllm_config,
|
||||
num_tokens=tokens,
|
||||
num_tokens_across_dp=torch.full(
|
||||
(vllm_config.parallel_config.data_parallel_size,),
|
||||
tokens,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
):
|
||||
output = model(
|
||||
torch.arange(tokens, device="cuda"),
|
||||
torch.arange(tokens, device="cuda"),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=torch.zeros(tokens, DIM, device="cuda"),
|
||||
lookback_token_ids=torch.full((1, 1), -1, device="cuda"),
|
||||
)["hidden_states"]
|
||||
expected_ids = torch.full_like(ids, engram_ops.DEAD_ID) if dp_rank == 1 else ids
|
||||
expected = _reference(weight.cuda(), scales.cuda(), expected_ids)
|
||||
torch.testing.assert_close(output, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _load_shared_multithread(layer, weight, scale_inv, load_config):
|
||||
"""Opposite shard completion orders must leave every shared parameter ready."""
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
|
||||
tensors = {
|
||||
"weight": weight,
|
||||
"weight_scale_inv": scale_inv.view(torch.float8_e8m0fnu),
|
||||
}
|
||||
order = list(tensors)
|
||||
if get_engram_dp_group().rank_in_group % 2:
|
||||
order.reverse()
|
||||
first_loaded = Event()
|
||||
original_load = weight_utils.load_file
|
||||
|
||||
def load_file(path, **kwargs):
|
||||
if Path(path).stem != order[0]:
|
||||
assert first_loaded.wait(timeout=30), "First shared weight load stalled"
|
||||
return original_load(path, **kwargs)
|
||||
|
||||
with TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as m:
|
||||
for name, tensor in tensors.items():
|
||||
save_file({name: tensor}, str(Path(directory) / f"{name}.safetensors"))
|
||||
m.setattr(weight_utils, "load_file", load_file)
|
||||
loader = DefaultModelLoader(load_config)
|
||||
source = DefaultModelLoader.Source(directory, revision=None)
|
||||
loaded = []
|
||||
for name, tensor in loader._get_weights_iterator(source):
|
||||
param = getattr(layer, name)
|
||||
param.weight_loader(param, tensor)
|
||||
loaded.append(name)
|
||||
first_loaded.set()
|
||||
assert loaded == order
|
||||
|
||||
# Check before any further collective could mask incomplete shared writes.
|
||||
for name, tensor in tensors.items():
|
||||
param = getattr(layer, name)
|
||||
expected = tensor.narrow(0, param.engram_vocab_start, param.shape[0])
|
||||
torch.testing.assert_close(
|
||||
param.view(torch.uint8), expected.view(torch.uint8), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
def _check_prefetch_replay(
|
||||
engram, head_sizes, weight, scale_inv, dp_shared_memory, capture
|
||||
):
|
||||
"""DP-sharded and shared prefetches must replay with new IDs."""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
ids = _make_ids(head_sizes, 8, seed=501)
|
||||
tp_rank = engram_ops.get_tensor_model_parallel_rank()
|
||||
tokens = 8 // engram.embed_tokens.tp_size if engram.use_sequence_parallel else 8
|
||||
output = torch.empty(
|
||||
tokens, len(head_sizes), DIM, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
def step():
|
||||
gathered = gather_engram_hashes(ids.clone(), dp_shared_memory=dp_shared_memory)
|
||||
engram.prepare_embeddings(gathered)
|
||||
output.copy_(engram.embed(ids))
|
||||
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 8)
|
||||
warmup = torch.cuda.Stream()
|
||||
warmup.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(warmup):
|
||||
step()
|
||||
if capture == "breakable":
|
||||
graph = BreakableCUDAGraphCapture()
|
||||
with torch.cuda.stream(warmup), graph:
|
||||
step()
|
||||
assert graph.num_eager_breaks == 2
|
||||
else:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=warmup):
|
||||
step()
|
||||
torch.cuda.current_stream().wait_stream(warmup)
|
||||
for iteration in range(2):
|
||||
ids.copy_(_make_ids(head_sizes, 8, seed=600 + iteration))
|
||||
graph.replay()
|
||||
expected = _reference(weight.cuda(), scale_inv.cuda(), ids)
|
||||
if engram.use_sequence_parallel:
|
||||
expected = expected[tp_rank * tokens : (tp_rank + 1) * tokens]
|
||||
torch.testing.assert_close(output, expected, atol=0, rtol=0)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
def test_engram_table_shared_across_dp_replicas(tp_size, monkeypatch):
|
||||
"""Check all storage modes with one process launch per TP/DP topology."""
|
||||
if torch.accelerator.device_count() < WORLD_SIZE:
|
||||
pytest.skip(f"Need {WORLD_SIZE} GPUs to run the test.")
|
||||
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "1")
|
||||
mp.spawn(_worker, args=(tp_size, get_open_port()), nprocs=WORLD_SIZE)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cpu_offload,dp_size,error",
|
||||
[
|
||||
(False, 4, "requires cpu_offload=True"),
|
||||
(True, 1, "effective Engram DP size is 1"),
|
||||
],
|
||||
)
|
||||
def test_engram_dp_shared_memory_runtime_requirements(
|
||||
monkeypatch, cpu_offload, dp_size, error
|
||||
):
|
||||
"""Report unmet storage/topology requirements before allocating CUDA memory."""
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
ParallelEngramEmbedding(
|
||||
sum(HEAD_SIZES),
|
||||
DIM,
|
||||
HEAD_SIZES,
|
||||
cpu_offload=cpu_offload,
|
||||
dp_shared_memory=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_fails", [False, True])
|
||||
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
|
||||
"""A /dev/shm that is too small or cannot be queried must fall back, and the
|
||||
leader must still broadcast that decision so its peers do not hang."""
|
||||
broadcasts = []
|
||||
|
||||
def broadcast(obj):
|
||||
broadcasts.append(obj)
|
||||
return obj
|
||||
|
||||
def query(*args, **kwargs):
|
||||
raise OSError("statvfs failed")
|
||||
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
|
||||
)
|
||||
if query_fails:
|
||||
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
|
||||
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
|
||||
assert not engram_ops.can_share_engram_tables(layout)
|
||||
assert len(broadcasts) == 1 and broadcasts[0] is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"node_ids,dp_size,replica_size,expected",
|
||||
[
|
||||
([0] * 4, 1, 4, 1),
|
||||
([0] * 4, 4, 1, 4),
|
||||
([0] * 4, 2, 2, 2),
|
||||
([0] * 8 + [1] * 8, 8, 2, 4),
|
||||
([0] * 6 + [1] * 6, 4, 1, 2),
|
||||
([0] * 3 + [1] * 4, 7, 1, 1),
|
||||
([0] * 8 + [1] * 8 + [2] * 8, 8, 3, 1),
|
||||
([0, 1] * 4, 4, 2, 1),
|
||||
([0] * 3 + [1] * 5, 4, 2, 1),
|
||||
],
|
||||
)
|
||||
def test_engram_dp_shard_size_respects_node_boundaries(
|
||||
node_ids, dp_size, replica_size, expected, monkeypatch
|
||||
):
|
||||
"""Sharing must fall back when replicas or node rank ranges do not align."""
|
||||
monkeypatch.setattr(parallel_state, "get_node_count", lambda: len(set(node_ids)))
|
||||
monkeypatch.setattr(
|
||||
parallel_state, "get_world_group", lambda: SimpleNamespace(cpu_group=None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parallel_state,
|
||||
"in_the_same_node_as",
|
||||
lambda group, src: [node == node_ids[src] for node in node_ids],
|
||||
)
|
||||
assert (
|
||||
parallel_state._engram_dp_shard_size(len(node_ids), dp_size, replica_size)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 2, 4, 5])
|
||||
def test_engram_hash_padding_has_no_valid_rows(num_tokens, monkeypatch):
|
||||
"""Idle/padded slots must not query row zero; oversized batches are rejected."""
|
||||
monkeypatch.setattr(engram_ops, "engram_gathered_num_tokens", lambda: 4)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda: SimpleNamespace(all_gather=lambda ids, dim: torch.cat([ids, ids], dim)),
|
||||
)
|
||||
ids = torch.full((num_tokens, 2, 3), 17, dtype=torch.int32)
|
||||
if num_tokens > 4:
|
||||
with pytest.raises(ValueError, match="exceeds the DP token slot"):
|
||||
gather_engram_hashes(ids)
|
||||
return
|
||||
gathered = gather_engram_hashes(ids).reshape(2, 4, 2, 3)
|
||||
for replica in gathered:
|
||||
torch.testing.assert_close(replica[:num_tokens], ids)
|
||||
assert torch.all(replica[num_tokens:] == engram_ops.DEAD_ID)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@pytest.mark.parametrize("uniform", [False, True])
|
||||
def test_engram_hash_padding_ignores_other_nodes(tp_size, uniform, monkeypatch):
|
||||
"""Two four-GPU nodes pad only to their own maximum, including TP x DP."""
|
||||
group_size = 4 // tp_size
|
||||
counts = (
|
||||
[4] * (8 // tp_size)
|
||||
if uniform
|
||||
else [4096] + [2] * (group_size - 1) + [1, 3] + [0] * (group_size - 2)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=torch.tensor(counts))
|
||||
),
|
||||
)
|
||||
for dp_rank in range(8 // tp_size):
|
||||
slot = 4 if uniform else (4096 if dp_rank < group_size else 3)
|
||||
ids = torch.full((counts[dp_rank], 2, 3), 17, dtype=torch.int32)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_dp_group",
|
||||
lambda rank=dp_rank: SimpleNamespace(rank_in_group=rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda rank=dp_rank: SimpleNamespace(
|
||||
rank_in_group=rank % group_size,
|
||||
world_size=group_size,
|
||||
all_gather=lambda tensor, dim: torch.cat([tensor] * group_size, dim),
|
||||
),
|
||||
)
|
||||
gathered = gather_engram_hashes(ids)
|
||||
expected = ids.new_full((slot, 2, 3), engram_ops.DEAD_ID)
|
||||
expected[: ids.shape[0]] = ids
|
||||
torch.testing.assert_close(
|
||||
gathered, expected.repeat(group_size, 1, 1), msg=f"DP rank {dp_rank}"
|
||||
)
|
||||
@@ -1,355 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.models.deepseek_v4.common.eplb_util import dspark_draft_supports_eplb
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.eplb_utils import (
|
||||
EPLBController,
|
||||
draft_model_supports_eplb,
|
||||
)
|
||||
|
||||
|
||||
def get_model_args(
|
||||
model_name: str,
|
||||
spec_model_name: str | None,
|
||||
spec_method: str,
|
||||
tp_size: int,
|
||||
model_max_len: int,
|
||||
num_speculative_tokens: int = 1,
|
||||
use_async: bool = True,
|
||||
) -> dict:
|
||||
speculative_config = {
|
||||
"method": spec_method,
|
||||
"model": spec_model_name,
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
speculative_config["model"] = model_name
|
||||
speculative_config["draft_sample_method"] = "probabilistic"
|
||||
eplb_config = {
|
||||
"num_redundant_experts": tp_size,
|
||||
"window_size": 128,
|
||||
"step_interval": 1024,
|
||||
"log_balancedness": False,
|
||||
"use_async": use_async,
|
||||
}
|
||||
model_args = {
|
||||
"pretrained": model_name,
|
||||
"dtype": "auto",
|
||||
"add_bos_token": True,
|
||||
"tensor_parallel_size": tp_size,
|
||||
"gpu_memory_utilization": 0.7,
|
||||
"speculative_config": speculative_config,
|
||||
"enable_expert_parallel": True,
|
||||
"eplb_config": eplb_config,
|
||||
"enable_eplb": True,
|
||||
"max_model_len": model_max_len,
|
||||
}
|
||||
if spec_method == "dspark":
|
||||
model_args["trust_remote_code"] = True
|
||||
return model_args
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="EPLB with Spec Decode is a work in progress on ROCm.",
|
||||
)
|
||||
|
||||
|
||||
class FakeEplbState:
|
||||
instances: list[FakeEplbState] = []
|
||||
|
||||
def __init__(self, parallel_config, device: torch.device):
|
||||
self.add_model_calls: list[tuple[object, object, str | None]] = []
|
||||
FakeEplbState.instances.append(self)
|
||||
|
||||
def add_model(
|
||||
self,
|
||||
model: object,
|
||||
model_config: object,
|
||||
*,
|
||||
model_name: str | None = None,
|
||||
) -> None:
|
||||
self.add_model_calls.append((model, model_config, model_name))
|
||||
|
||||
|
||||
def _make_dsv4_dspark_hf_config() -> DeepseekV4Config:
|
||||
return DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
layer_types=["sliding_attention"] * 2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dspark_vllm_config():
|
||||
hf_config = _make_dsv4_dspark_hf_config()
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16, hf_config=hf_config, model="dspark"
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(moe_backend="deep_gemm_mega_moe"),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
enable_elastic_ep=False,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(
|
||||
method="dspark",
|
||||
draft_model_config=model_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_moe_topology(
|
||||
*,
|
||||
num_routed_experts: int = 8,
|
||||
num_redundant_experts: int = 4,
|
||||
num_expert_groups: int = 1,
|
||||
) -> SimpleNamespace:
|
||||
num_physical_experts = num_routed_experts + num_redundant_experts
|
||||
return SimpleNamespace(
|
||||
num_routed_experts=num_routed_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_logical_experts=num_routed_experts,
|
||||
num_expert_groups=num_expert_groups,
|
||||
)
|
||||
|
||||
|
||||
def test_eplb_state_accepts_matching_dsv4_draft_and_target():
|
||||
draft = _make_moe_topology()
|
||||
target = _make_moe_topology()
|
||||
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
def test_eplb_state_rejects_mismatched_dsv4_draft_redundant_experts():
|
||||
draft = _make_moe_topology(num_redundant_experts=0)
|
||||
target = _make_moe_topology(num_redundant_experts=4)
|
||||
|
||||
with pytest.raises(RuntimeError, match="mismatch"):
|
||||
EplbState.assert_confs_equal(draft, target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "expected"),
|
||||
[
|
||||
("deepseek_v4", True),
|
||||
("deepseek_v41", False),
|
||||
],
|
||||
)
|
||||
def test_dspark_draft_supports_eplb(
|
||||
dspark_vllm_config, model_type: str, expected: bool
|
||||
):
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
assert dspark_draft_supports_eplb(draft_model_config) is expected
|
||||
|
||||
|
||||
def test_draft_model_supports_eplb_requires_draft_moe(dspark_vllm_config):
|
||||
assert not draft_model_supports_eplb(
|
||||
dspark_vllm_config.speculative_config,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_type,load_dummy_weights,expected_registered",
|
||||
[
|
||||
("deepseek_v4", False, True),
|
||||
("deepseek_v4", True, False),
|
||||
("deepseek_v41", False, False),
|
||||
],
|
||||
)
|
||||
def test_eplb_dspark_draft_registration(
|
||||
dspark_vllm_config,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type,
|
||||
load_dummy_weights: bool,
|
||||
expected_registered: bool,
|
||||
):
|
||||
"""Register V4 drafts; skip incompatible V4.1 drafts and dummy weights."""
|
||||
FakeEplbState.instances.clear()
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.EplbState",
|
||||
FakeEplbState,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.eplb_utils.get_mixture_of_experts_model",
|
||||
lambda model: model,
|
||||
)
|
||||
|
||||
draft_model_config = dspark_vllm_config.speculative_config.draft_model_config
|
||||
draft_model_config.hf_config.model_type = model_type
|
||||
draft = SimpleNamespace()
|
||||
controller = EPLBController(dspark_vllm_config.parallel_config, torch.device("cpu"))
|
||||
controller.prepare_load()
|
||||
speculator = SimpleNamespace(model=draft, eplb_state=None)
|
||||
|
||||
def set_eplb_state(state) -> None:
|
||||
speculator.eplb_state = state
|
||||
|
||||
speculator.set_eplb_state = set_eplb_state
|
||||
|
||||
registered = controller.maybe_register_speculator(
|
||||
speculator,
|
||||
dspark_vllm_config.speculative_config,
|
||||
load_dummy_weights=load_dummy_weights,
|
||||
)
|
||||
|
||||
assert registered is expected_registered
|
||||
assert controller.state is not None
|
||||
if expected_registered:
|
||||
assert controller.state.add_model_calls == [
|
||||
(draft, draft_model_config, "dspark (draft)")
|
||||
]
|
||||
assert speculator.eplb_state is controller.state
|
||||
else:
|
||||
assert controller.state.add_model_calls == []
|
||||
assert speculator.eplb_state is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_setup",
|
||||
[
|
||||
pytest.param(
|
||||
("mtp", "Qwen/Qwen3-Next-80B-A3B-Instruct", None, 4, 0.86, 1),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
"eagle",
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
|
||||
4,
|
||||
0.92,
|
||||
1,
|
||||
),
|
||||
marks=pytest.mark.skip(reason="Skipping due to CI OOM issues"),
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
"dspark",
|
||||
"deepseek-ai/DeepSeek-V4-Flash-DSpark",
|
||||
None,
|
||||
4,
|
||||
0.95,
|
||||
7,
|
||||
),
|
||||
marks=large_gpu_mark(min_gb=80),
|
||||
),
|
||||
],
|
||||
ids=["qwen3_next_mtp", "llama4_eagle", "dsv4_dspark"],
|
||||
)
|
||||
def test_eplb_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str | None, int, float, int],
|
||||
):
|
||||
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
|
||||
Applicable to MoE models with mtp, eagle, or dspark spec decode.
|
||||
"""
|
||||
(
|
||||
method,
|
||||
model_name,
|
||||
spec_model_name,
|
||||
tp_size,
|
||||
expected_gsm8k_value,
|
||||
num_speculative_tokens,
|
||||
) = model_setup
|
||||
if method == "dspark" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DSV4 DSpark EPLB requires SM100 for MegaMoE")
|
||||
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
|
||||
model_args = get_model_args(
|
||||
model_name=model_name,
|
||||
spec_model_name=spec_model_name,
|
||||
spec_method=method,
|
||||
tp_size=tp_size,
|
||||
model_max_len=4096,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="vllm",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
batch_size=64,
|
||||
num_fewshot=8,
|
||||
)
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
assert (
|
||||
measured_value - RTOL < expected_gsm8k_value
|
||||
and measured_value + RTOL > expected_gsm8k_value
|
||||
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=80)
|
||||
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
|
||||
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
expected_gsm8k_value = 0.86
|
||||
|
||||
model_args = get_model_args(
|
||||
model_name="Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
spec_model_name=None,
|
||||
spec_method="mtp",
|
||||
tp_size=4,
|
||||
model_max_len=4096,
|
||||
use_async=True,
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="vllm",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
batch_size=64,
|
||||
num_fewshot=8,
|
||||
)
|
||||
measured_value = results["results"][TASK][FILTER]
|
||||
assert (
|
||||
measured_value - RTOL < expected_gsm8k_value
|
||||
and measured_value + RTOL > expected_gsm8k_value
|
||||
), f"Expected: {expected_gsm8k_value} | Measured: {measured_value}"
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user