[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
-775
View File
@@ -1,775 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for packed tensor broadcasting functionality.
Unit tests for packed_nccl_broadcast_producer and packed_nccl_broadcast_consumer.
These utilities enable efficient batched tensor transfer over NCCL.
"""
import pytest
import torch
from vllm.distributed.weight_transfer.packed_tensor import (
pack_tensors,
packed_ipc_consumer,
packed_ipc_producer,
packed_nccl_broadcast_consumer,
packed_nccl_broadcast_producer,
unpack_tensor,
)
class MockCommunicationGroup:
"""Mock communication group for testing producer broadcast operations."""
def __init__(self):
self.broadcasted_tensors: list[torch.Tensor] = []
self.broadcast_count = 0
self.device = torch.device("cuda:0")
def broadcast(self, tensor, src):
"""Mock broadcast that stores the tensor for later verification."""
self.broadcasted_tensors.append(tensor.clone())
self.broadcast_count += 1
class MockConsumerCommunicationGroup:
"""Mock communication group for consumer that returns pre-stored tensors."""
def __init__(self, tensors_to_return: list[torch.Tensor]):
self.tensors_to_return = tensors_to_return
self.current_index = 0
self.device = torch.device("cuda:0")
def broadcast(self, tensor, src):
"""Mock broadcast that fills the tensor with pre-stored data."""
if self.current_index < len(self.tensors_to_return):
tensor.copy_(self.tensors_to_return[self.current_index])
self.current_index += 1
def create_mock_model_params(
num_layers: int = 3,
dtype: torch.dtype = torch.float32,
) -> list[tuple[str, torch.Tensor]]:
"""Create mock model parameters for testing."""
params = []
for i in range(num_layers):
params.append((f"layer{i}.weight", torch.randn(10, 20, dtype=dtype)))
params.append((f"layer{i}.bias", torch.randn(10, dtype=dtype)))
return params
def create_state_dict_info(
params: list[tuple[str, torch.Tensor]],
) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
"""Create state dict info (name -> (shape, dtype)) from params."""
return {name: (tuple(tensor.shape), tensor.dtype) for name, tensor in params}
# --- Unit Tests: packed_nccl_broadcast_producer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedBroadcastProducer:
"""Test packed_nccl_broadcast_producer function."""
def test_producer_empty_iterator(self):
"""Test producer handles empty iterator gracefully."""
mock_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter([]),
group=mock_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
# No broadcasts for empty iterator
assert mock_group.broadcast_count == 0
# --- Integration Tests: Producer-Consumer Roundtrip ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedBroadcastRoundtrip:
"""Test producer-consumer roundtrip behavior."""
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_roundtrip_different_dtypes(self, dtype):
"""Test roundtrip with different data types."""
params = create_mock_model_params(num_layers=2, dtype=dtype)
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
buffer_size = 1000
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params_cuda)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify roundtrip preserves data
for name, original_tensor in params_cuda:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.dtype == dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
def test_roundtrip_mixed_dtypes(self):
"""Test roundtrip with mixed data types."""
# Create params with mixed dtypes
params = [
("layer1.weight", torch.randn(10, 20, dtype=torch.float32).cuda()),
("layer1.bias", torch.randn(10, dtype=torch.float16).cuda()),
("layer2.weight", torch.randn(20, 30, dtype=torch.bfloat16).cuda()),
]
buffer_size = 500
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify all params roundtrip correctly with correct dtypes
for name, original_tensor in params:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.shape == original_tensor.shape
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
@pytest.mark.parametrize("target_size", [100, 100000])
def test_roundtrip_different_batch_sizes(self, target_size):
"""Test roundtrip with different target batch sizes."""
params = create_mock_model_params(num_layers=5)
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=target_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params_cuda)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=target_size,
)
# Verify all params roundtrip correctly
assert len(unpacked_tensors) == len(params)
for name, original_tensor in params_cuda:
assert name in unpacked_tensors
assert torch.allclose(
unpacked_tensors[name], original_tensor, rtol=1e-5, atol=1e-7
)
def test_roundtrip_non_contiguous_tensors(self):
"""Test roundtrip with non-contiguous tensors from the trainer."""
# Create non-contiguous tensors (simulating trainer outputs)
# Transposed tensors are non-contiguous
weight1 = torch.randn(20, 10, dtype=torch.float32).cuda().T
# Sliced tensors with step are non-contiguous
weight2 = torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]
# Permuted tensors are non-contiguous
weight3 = torch.randn(5, 10, 15, dtype=torch.bfloat16).cuda().permute(2, 0, 1)
params = [
("layer1.weight", weight1),
("layer2.weight", weight2),
("layer3.weight", weight3),
]
# Verify tensors are indeed non-contiguous
for name, tensor in params:
assert not tensor.is_contiguous(), f"{name} should be non-contiguous"
buffer_size = 500
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify all non-contiguous params roundtrip correctly
for name, original_tensor in params:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.shape == original_tensor.shape
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
# --- Unit Tests: unpack_tensor ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestUnpackTensor:
"""Test the shared unpack_tensor function."""
def test_unpack_produces_independent_copies(self):
"""Verify unpacked tensors don't share memory with packed buffer."""
original = torch.randn(10, dtype=torch.float32).cuda()
packed = original.contiguous().view(torch.uint8).view(-1)
result = unpack_tensor(
packed,
names=["w"],
shapes=[[10]],
dtypes=[torch.float32],
tensor_sizes=[packed.numel()],
)
# Mutate the packed buffer
packed.zero_()
# Unpacked tensor should be unaffected
assert torch.allclose(result[0][1], original)
# --- Unit Tests: pack_tensors ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackTensors:
"""Test the shared pack_tensors function."""
def test_pack_basic(self):
"""Test packing a few tensors into one buffer."""
params = [
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
("w2", torch.randn(5, dtype=torch.float16).cuda()),
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
assert chunk is not None
assert len(chunk.names) == 2
assert chunk.names == ["w1", "w2"]
assert chunk.shapes == [[10, 20], [5]]
assert chunk.dtypes == [torch.float32, torch.float16]
assert chunk.packed_tensor.dtype == torch.uint8
def test_pack_respects_buffer_limit(self):
"""Test that packing stops when buffer_size_bytes is exceeded."""
params = [
(f"w{i}", torch.randn(100, 100, dtype=torch.float32).cuda())
for i in range(10)
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=50_000,
)
assert chunk is not None
assert len(chunk.names) < 10
def test_pack_empty_iterator(self):
"""Test that an empty iterator returns None."""
chunk = pack_tensors(
iterator=iter([]),
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
assert chunk is None
def test_pack_single_tensor_larger_than_buffer_warns(self):
"""Test that a tensor exceeding buffer_size_bytes emits a warning."""
big = torch.randn(1000, 1000, dtype=torch.float32).cuda()
params = [("big", big)]
import warnings
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=100,
)
assert chunk is not None
assert len(chunk.names) == 1
assert any("exceeds buffer_size_bytes" in str(wi.message) for wi in w)
def test_pack_unpack_roundtrip(self):
"""Test pack then unpack produces identical tensors."""
params = [
("a", torch.randn(8, 16, dtype=torch.float32).cuda()),
("b", torch.randn(4, dtype=torch.float16).cuda()),
("c", torch.randn(3, 5, 7, dtype=torch.bfloat16).cuda()),
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
assert chunk is not None
result = unpack_tensor(
chunk.packed_tensor,
chunk.names,
chunk.shapes,
chunk.dtypes,
chunk.tensor_sizes,
)
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor, rtol=1e-4, atol=1e-6)
def test_pack_multiple_chunks(self):
"""Test consuming an iterator across multiple pack_tensors calls."""
params = [
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
]
it = iter(params)
all_names = []
chunks = []
while True:
chunk = pack_tensors(it, lambda x: x[1], buffer_size_bytes=12_000)
if chunk is None:
break
chunks.append(chunk)
all_names.extend(chunk.names)
assert len(chunks) > 1
assert all_names == [f"w{i}" for i in range(6)]
# --- Unit Tests: packed_ipc_producer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedIpcProducer:
"""Test the packed_ipc_producer generator."""
def test_producer_yields_chunks(self):
"""Test that the producer yields PackedIpcChunk objects."""
params = [
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="test-uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=12_000,
)
)
assert len(chunks) > 1
def test_producer_ipc_handle_has_uuid(self):
"""Test that each chunk's ipc_handle is keyed by the given UUID."""
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="my-gpu-uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
assert "my-gpu-uuid" in chunks[0].ipc_handle
def test_producer_dtype_names_are_strings(self):
"""Test that dtype_names are string representations."""
params = [
("a", torch.randn(10, dtype=torch.float32).cuda()),
("b", torch.randn(10, dtype=torch.float16).cuda()),
]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
assert chunks[0].dtype_names == ["float32", "float16"]
def test_producer_empty_iterator(self):
"""Test producer with empty iterator yields nothing."""
chunks = list(
packed_ipc_producer(
iterator=iter([]),
gpu_uuid="uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
)
assert len(chunks) == 0
# --- Integration Tests: IPC Producer-Consumer Roundtrip ---
def _ipc_consumer_worker(cmd_q, ack_q, result_q, done_event, device_index):
"""Worker that consumes chunks streamed one at a time from the parent.
CUDA IPC requires the consumer to be in a separate process from the
producer. The producer reuses a single IPC buffer between chunks, so
the parent must wait for our ack (sent after we copy the chunk to
CPU) before advancing the producer.
"""
try:
torch.accelerator.set_device_index(device_index)
all_results = []
while True:
cd = cmd_q.get()
if cd is None:
break
result = packed_ipc_consumer(
ipc_handle=cd["ipc_handle"],
names=cd["names"],
shapes=cd["shapes"],
dtype_names=cd["dtype_names"],
tensor_sizes=cd["tensor_sizes"],
device_index=device_index,
)
# .cpu() forces a GPU→CPU copy off the shared IPC buffer, so
# the producer is free to overwrite it once we ack.
all_results.extend([(name, tensor.cpu()) for name, tensor in result])
del result
ack_q.put("ack")
result_q.put(("ok", all_results))
except Exception as e:
result_q.put(("error", str(e)))
# Keep the process alive until the parent has finished reading from
# the result queue — torch serializes CPU tensors via fd sharing,
# which requires this process's resource-sharer server to be running.
done_event.wait(timeout=60)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedIpcRoundtrip:
"""Test IPC producer-consumer roundtrip using real CUDA IPC.
These tests spawn a child process for the consumer because
rebuild_cuda_tensor requires a separate process from the one that
called reduce_tensor.
"""
def _get_gpu_uuid(self) -> str:
device_index = torch.cuda.current_device()
props = torch.cuda.get_device_properties(device_index)
return str(props.uuid)
def _run_roundtrip(self, chunk_iter, device_index, timeout=30):
"""Stream chunks through a child consumer one at a time.
``packed_ipc_producer`` reuses a single IPC buffer for every
chunk, so the producer must not be advanced until the consumer
has finished reading the current chunk. We enforce that with an
ack queue: the consumer puts ``"ack"`` after it has copied the
chunk to CPU, and only then do we pull the next chunk from the
generator.
Returns ``(num_chunks, results)``.
"""
import multiprocessing as mp
ctx = mp.get_context("spawn")
cmd_q = ctx.Queue()
ack_q = ctx.Queue()
result_q = ctx.Queue()
done_event = ctx.Event()
proc = ctx.Process(
target=_ipc_consumer_worker,
args=(cmd_q, ack_q, result_q, done_event, device_index),
)
proc.start()
num_chunks = 0
try:
for chunk in chunk_iter:
cmd_q.put(
{
"ipc_handle": chunk.ipc_handle,
"names": chunk.names,
"shapes": chunk.shapes,
"dtype_names": chunk.dtype_names,
"tensor_sizes": chunk.tensor_sizes,
}
)
if ack_q.get(timeout=timeout) != "ack":
raise RuntimeError("Consumer did not ack chunk")
num_chunks += 1
cmd_q.put(None)
status, payload = result_q.get(timeout=timeout)
finally:
done_event.set()
proc.join(timeout=10)
if proc.is_alive():
proc.kill()
if status == "error":
raise RuntimeError(f"Consumer process failed: {payload}")
# Reclaim IPC-shared memory now that the child has released it
torch.cuda.ipc_collect()
return num_chunks, payload
def test_roundtrip_basic(self):
"""Test basic IPC producer -> consumer roundtrip."""
params = [
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
("scale", torch.tensor(1.5, dtype=torch.float32).cuda()),
("w2", torch.randn(5, dtype=torch.float16).cuda()),
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
num_chunks, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
assert num_chunks == 1
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_roundtrip_dtypes(self, dtype):
"""Test IPC roundtrip with different dtypes."""
params = create_mock_model_params(num_layers=2, dtype=dtype)
params_cuda = [(n, t.cuda()) for n, t in params]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
_, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params_cuda),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
assert len(result) == len(params_cuda)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(
params_cuda, result
):
assert orig_name == res_name
assert res_tensor.dtype == dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
def test_roundtrip_multiple_chunks(self):
"""Test IPC roundtrip across multiple chunks."""
params = [
(f"layer{i}.weight", torch.randn(100, 100, dtype=torch.float32).cuda())
for i in range(8)
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
num_chunks, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=50_000,
),
device_index,
)
assert num_chunks > 1
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-5, atol=1e-7)
def test_roundtrip_non_contiguous(self):
"""Test IPC roundtrip with non-contiguous tensors."""
params = [
("transposed", torch.randn(20, 10, dtype=torch.float32).cuda().T),
("sliced", torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]),
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
for _, t in params:
assert not t.is_contiguous()
_, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
def test_consumer_wrong_uuid_raises(self):
"""Test that consumer raises ValueError for unknown GPU UUID."""
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
gpu_uuid = self._get_gpu_uuid()
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
c = chunks[0]
fake_handle = {"fake-uuid-12345": c.ipc_handle[gpu_uuid]}
with pytest.raises(ValueError, match="IPC handle not found"):
packed_ipc_consumer(
ipc_handle=fake_handle,
names=c.names,
shapes=c.shapes,
dtype_names=c.dtype_names,
tensor_sizes=c.tensor_sizes,
device_index=torch.cuda.current_device(),
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_pack_unpack_scalar(dtype):
"""Preserve scalar shape, dtype and value alongside a matrix on CPU."""
params = [
("weight", torch.randn(2, 3, dtype=dtype)),
("scale", torch.tensor(1.5, dtype=dtype)),
]
chunk = pack_tensors(iter(params), lambda item: item[1], buffer_size_bytes=1024)
assert chunk is not None
result = unpack_tensor(
chunk.packed_tensor,
chunk.names,
chunk.shapes,
chunk.dtypes,
chunk.tensor_sizes,
)
for (expected_name, expected), (name, actual) in zip(params, result, strict=True):
assert name == expected_name
assert actual.shape == expected.shape
assert actual.dtype == expected.dtype
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@@ -1,628 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the sharded-RDT trainer-side engine.
The trainer engine's own surface: init info and factory registration, the
send_weights round trip against an in-process fake producer server, the
ownership table it resolves and ships to the workers, and the producer's packed
destination-view cache.
The plan/replay half (including `RdtRouter`, which is consumer-side) lives in
test_sharded_rdt_plan.py; the producer server's protocol in
test_sharded_rdt_producer.py.
"""
import itertools
from dataclasses import asdict
import pytest
import torch
from tests.distributed.test_weight_transfer import RecordingClient
from vllm.distributed.weight_transfer import (
ModuleSource,
ParamMeta,
WeightSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.base import layerwise_groups
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
ShardedRDTTrainerInitInfo,
ShardedRDTTrainerWeightTransferEngine,
)
class _ListSource(WeightSource):
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
sharded-RDT group/order logic can be tested without a real model."""
def __init__(self, pairs):
self._pairs = list(pairs)
def metadata(self):
return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs]
def __iter__(self):
return iter(self._pairs)
class _FakeProducerServer:
"""In-process stand-in for the _RDTProducerServer Ray actor. Records the
engine->server call sequence and, by default, frees each group as soon as
it is published (simulating the consumers' free_group barrier) so the
gather loop's credit gate never parks. Mirrors the real server's
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
to the ``begin_sync`` live total, publish_group returns nothing, and freed
groups flow back only through wait_freed / end_sync (see
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
def __init__(self, auto_free=True):
self.order: list[str] = []
self.published: list[int] = []
self.live_count = 1
self._inflight_groups: list[int] = []
self.auto_free = auto_free
self.free_counts: dict[int, int] = {}
self._pending_freed: list[int] = []
# Alias for tests that inspect in-flight groups.
@property
def inflight(self):
return self._inflight_groups
def begin_sync(self, live_count, live_consumer_ids=None):
self.order.append("begin")
self.live_count = max(1, int(live_count))
self.live_ids = live_consumer_ids
def publish_group(self, group_idx, entries):
self.order.append("publish")
self.published.append(group_idx)
self._inflight_groups.append(group_idx)
if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count:
self._inflight_groups.remove(group_idx)
self._pending_freed.append(group_idx)
def free_group(self, group_idx):
"""Consumer back-edge; may arrive before the group's publish."""
self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1
if (
group_idx in self._inflight_groups
and self.free_counts[group_idx] >= self.live_count
):
self._inflight_groups.remove(group_idx)
self._pending_freed.append(group_idx)
def wait_freed(self):
"""The engine's credit gate. The real server blocks here; the fake
must already have a banked credit when the gate asks (auto_free, or a
test's own free_group) — anything else is the deadlock the real
watchdog would kill, so fail loudly. Not appended to ``order``: it is
pacing, not a lifecycle milestone."""
assert self._pending_freed, (
"wait_freed with nothing freed: the gather loop would deadlock"
)
freed = self._pending_freed
self._pending_freed = []
return freed
def end_sync(self):
self.order.append("end")
freed = self._pending_freed
self._pending_freed = []
return freed
def set_gather_error(self, message):
self.order.append("error")
def _rdt_engine_with_fake_server(
source, *, is_sender, client, server, monkeypatch, fleet_owned=None
):
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
group-major metadata, and route _rpc to the fake."""
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",)))
init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1)
engine = ShardedRDTTrainerWeightTransferEngine(
client=client, source=source, is_sender=is_sender, init_info=init_info
)
engine._meta = list(source.metadata())
names = [m.name for m in engine._meta]
engine._groups = layerwise_groups(names)
engine._server = server
engine._rpc = lambda method, *args: getattr(server, method)(*args)
# What trainer_init resolves from the source's held names + the fleet's
# all-gather. ``fleet_owned`` stands in for that all-gather so a partial-
# ownership rank can be tested without a real process group; the fleet must
# cover every name or the transpose rejects it (nothing would serve the rest).
if fleet_owned is None:
engine._resolve_ownership(1, 0)
else:
# Stands in for the (metadata digest, held-name bitmask) all-gather:
# each rank's group list becomes the mask its held names would set.
names = [m.name for m in engine._meta]
groups = engine._groups
def _fake_gather(_world, mine):
out = []
for owned in fleet_owned:
held = {n for gi in owned for n in groups[gi]}
mask = bytearray((len(names) + 7) // 8)
for i, n in enumerate(names):
if n in held:
mask[i >> 3] |= 1 << (i & 7)
out.append((mine[0], bytes(mask)))
return out
monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather)
engine._resolve_ownership(len(fleet_owned), 0)
return engine
def _rdt_source_two_layers():
return _ListSource(
[
("embed.weight", torch.zeros(2)),
("model.layers.0.w", torch.zeros(2)),
("model.layers.1.w", torch.zeros(2)),
("norm.weight", torch.zeros(2)),
]
)
class TestShardedRDTTrainerInitInfo:
def test_declares_backend(self):
assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt"
def test_rank_is_keyword_only_and_drives_is_sender(self):
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False
with pytest.raises(TypeError):
# rank is keyword-only.
ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc]
def test_registered_in_trainer_factory(self):
cls = WeightTransferTrainerFactory._registry["sharded_rdt"]()
assert cls is ShardedRDTTrainerWeightTransferEngine
def test_sharded_rdt_trainer_init_requires_source():
with pytest.raises(ValueError, match="requires a WeightSource"):
ShardedRDTTrainerWeightTransferEngine.trainer_init(
ShardedRDTTrainerInitInfo(num_consumers=1, rank=0),
client=RecordingClient(),
source=None,
)
def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch):
source = _rdt_source_two_layers()
engine = _rdt_engine_with_fake_server(
source,
is_sender=True,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
)
worker_init = engine._build_worker_init_info(["srv_rk0"])
# 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1.
assert worker_init.names == [
"embed.weight",
"model.layers.0.w",
"model.layers.1.w",
"norm.weight",
]
assert worker_init.group_lens == [1, 1, 1, 1]
assert worker_init.trainer_actor_names == ["srv_rk0"]
assert worker_init.produce_method_name == "rdt_produce_weights_batched"
assert sum(worker_init.group_lens) == len(worker_init.names)
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch):
server = _FakeProducerServer(auto_free=True)
client = RecordingClient()
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=True,
client=client,
server=server,
monkeypatch=monkeypatch,
)
engine.send_weights()
assert client.order == ["start", "update", "finish"]
# begin, one publish per group (4), end.
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
assert len(server.published) == 4
# every group freed -> no engine-held refs remain.
assert engine._inflight == {}
def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch):
# Source whose iteration order disagrees with its metadata order.
class _BadSource(_ListSource):
def __iter__(self):
reordered = list(self._pairs)
reordered[0], reordered[1] = reordered[1], reordered[0]
return iter(reordered)
server = _FakeProducerServer()
engine = _rdt_engine_with_fake_server(
_BadSource(_rdt_source_two_layers()._pairs),
is_sender=True,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
)
with pytest.raises(RuntimeError, match="iteration order must match"):
engine.send_weights()
assert "error" in server.order # gather error propagated to the server
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_non_sender_skips_client(monkeypatch):
class _RaisingClient(RecordingClient):
def start_weight_update(self):
raise AssertionError("non-sender must not touch the client")
def update_weights(self, update_info):
raise AssertionError("non-sender must not touch the client")
def finish_weight_update(self, weight_version: str | None = None):
raise AssertionError("non-sender must not touch the client")
server = _FakeProducerServer(auto_free=True)
client = _RaisingClient()
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=False,
client=client,
server=server,
monkeypatch=monkeypatch,
)
engine.send_weights() # gathers only; must not raise
assert client.order == []
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
class _FailingUpdateClient(RecordingClient):
def update_weights(self, update_info):
self.order.append("update")
raise RuntimeError("inference side rejected update")
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=True,
client=_FailingUpdateClient(),
server=server,
monkeypatch=monkeypatch,
)
with pytest.raises(RuntimeError, match="inference side rejected update"):
engine.send_weights()
class _OwnedSource(_ListSource):
"""A source holding only some groups' names, like a pipeline-parallel rank.
Takes group indices for convenience and declares the names inside them."""
def __init__(self, pairs, owned_group_idx):
super().__init__(pairs)
self._owned = list(owned_group_idx)
groups = layerwise_groups([n for n, _ in pairs])
self._owned_names = [n for gi in self._owned for n in groups[gi]]
def held_names(self):
return list(self._owned_names)
def __iter__(self):
by_name = dict(self._pairs)
return iter([(n, by_name[n]) for n in self._owned_names])
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch):
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest
)
engine.send_weights()
assert server.published == [1, 2]
assert server.order == ["begin", "publish", "publish", "end"]
assert engine._inflight == {}
# embed + norm on rank 1, the two layers here: two distinct owner sets,
# numbered by first appearance in metadata order.
assert engine._owner_sets == [[1], [0]]
assert engine._name_owner_class == [0, 1, 1, 0]
def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
class _MisorderedOwned(_OwnedSource):
def __iter__(self):
return iter(list(super().__iter__())[::-1])
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
fleet_owned=[[1, 2], [0, 3]],
)
with pytest.raises(RuntimeError, match="iteration order must match"):
engine.send_weights()
assert "error" in server.order
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
"""The free barrier's target is one integer per sync: the live consumer
count, defaulting to the whole provisioned fleet."""
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=False,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
)
engine.send_weights()
assert server.live_count == 1 # num_consumers=1 in the harness
engine.send_weights(live_consumer_ids=[0])
assert server.live_count == 1
class TestLiveCountPlumbing:
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
provisioned geometry is frozen; a degraded sync only lowers the target and
narrows each slot-sharing group's rendezvous."""
@staticmethod
def _engine(num_consumers, world=2, rank=0):
engine = ShardedRDTTrainerWeightTransferEngine.__new__(
ShardedRDTTrainerWeightTransferEngine
)
engine._init_info = ShardedRDTTrainerInitInfo(
rank=rank, num_consumers=num_consumers
)
engine.source = object() # send_weights asserts a source is present
received: list = []
engine._send_weights_inner = lambda count, ids: received.append((count, ids))
return engine, received
def test_none_counts_the_whole_provisioned_fleet(self):
engine, got = self._engine(num_consumers=8)
engine.send_weights(None)
assert got == [(8, list(range(8)))]
def test_a_live_set_counts_its_distinct_members(self):
engine, got = self._engine(num_consumers=8)
engine.send_weights([0, 1, 4, 5, 5])
assert got == [(4, [0, 1, 4, 5])]
def test_a_full_live_set_matches_the_provisioned_count(self):
engine, got = self._engine(num_consumers=8)
engine.send_weights(list(range(8)))
engine.send_weights(None)
assert got == [(8, list(range(8)))] * 2
def test_which_consumers_died_does_not_matter_only_how_many(self):
"""The whole point of the barrier: no routed per-producer targets, so
the identity of the dead consumer is irrelevant to the producers."""
counts = []
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
engine, got = self._engine(num_consumers=8)
engine.send_weights(live)
counts += [c for c, _ids in got]
assert counts == [4, 4, 4]
def test_the_live_ids_travel_with_the_count(self):
"""The count sizes the free barrier, the ids size the slot-sharing
rendezvous, so a producer that shares slots can tell WHICH consumers it
is still waiting for. They must describe the same set."""
engine, got = self._engine(num_consumers=8)
engine.send_weights([6, 0, 2, 2])
((count, ids),) = got
assert ids == [0, 2, 6] and count == len(ids)
def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch):
import json
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
is_sender=True,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
fleet_owned=[[0, 1], [2, 3]],
)
worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"])
assert worker_init.owner_sets == [[0], [1]]
assert worker_init.name_owner_class == [0, 0, 1, 1]
assert len(worker_init.name_owner_class) == len(worker_init.names)
# The payload crosses the control plane as JSON: nested lists must survive.
round_tripped = json.loads(json.dumps(asdict(worker_init)))
assert round_tripped["owner_sets"] == [[0], [1]]
assert round_tripped["name_owner_class"] == [0, 0, 1, 1]
def test_weight_source_holds_everything_by_default():
"""The contract's default: a source that says nothing holds the whole model."""
src = _rdt_source_two_layers()
assert src.held_names() is None
assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None
def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch):
class _BadHeld(_OwnedSource):
def held_names(self):
return ["embed.weight", "not.a.real.name"]
with pytest.raises(ValueError, match="not"):
_rdt_engine_with_fake_server(
_BadHeld(_rdt_source_two_layers()._pairs, [0]),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
)
def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
class _HoldsNothing(_ListSource):
def held_names(self):
return []
with pytest.raises(ValueError, match="empty"):
_rdt_engine_with_fake_server(
_HoldsNothing(_rdt_source_two_layers()._pairs),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
)
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
"""Every name must be held somewhere or it can never be served — caught
when the holdings are transposed, naming the orphan."""
with pytest.raises(ValueError, match="no trainer rank holds"):
_rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
fleet_owned=[[0, 1], [2]], # nobody holds group 3
)
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
"""Only the sender's metadata reaches the consumers, so a rank describing
just its own share must fail loudly rather than silently drop the rest."""
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(auto_free=True),
monkeypatch=monkeypatch,
fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds
)
# Now rank 1 reports a DIFFERENT metadata digest for the same model.
monkeypatch.setattr(
engine,
"_all_gather_owned",
lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])],
)
with pytest.raises(ValueError, match="disagrees across trainer ranks"):
engine._resolve_ownership(2, 0)
def _serve_ring_server(src_name, src):
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
pull needs no Ray and no NIXL registration. Returns (server, serve) where
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
what puts the destination-view cache, and only it, under test."""
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
_RDTProducerServer,
)
srv = _RDTProducerServer(
num_rdt_buffers=2,
buffer_presize_gb=0.0,
gather_lookahead=2,
)
srv._cache[src_name] = src
srv._serve_rings[0] = [
torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2)
]
# The slot is ``seq % nring``, so stepping seq by nring keeps every pull in
# slot 0 while still making each one its own generation -- reusing a seq
# would rendezvous with the finished pack and return it unchanged.
seqs = itertools.count(0, srv._nring)
def serve(chain):
return srv.rdt_produce_weights_batched(
[(src_name, chain)], consumer_id=0, seq=next(seqs)
)[0]
return srv, serve
@pytest.mark.skipif(
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
)
def test_serve_does_not_reuse_packed_views_of_another_shape():
"""Two requests can share a name yet pack different slices of it.
The producer caches the destination views it carves into a serve ring slot.
Keyed by name alone, the second request is packed through the first's views.
Reachable when one name's copies split across owner-class chunks.
"""
name = "model.layers.0.w"
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
_srv, serve = _serve_ring_server(name, src)
serve((("narrow", (0, 0, 2), ()),)) # 2 rows
wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot
blob = serve((("narrow", (0, 0, 6), ()),))
got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8)
assert torch.equal(got, wide)
@pytest.mark.skipif(
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
)
def test_serve_does_not_reuse_packed_views_of_another_dtype():
"""The SILENT case of the same cache: two requests whose slices have the same
name and the same shape but different dtypes pack at identical offsets, so
reusing the stale views raises nothing — ``copy_`` just casts, and the blob
carries the wrong bytes with no check downstream.
``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under
``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte
length identical, which is exactly what makes the stale view fit.
"""
name = "model.layers.0.w"
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
_srv, serve = _serve_ring_server(name, src)
serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16
blob = serve(()) # same name and shape, bf16
got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8)
assert torch.equal(got, src), "packed through a cached view of the wrong dtype"
File diff suppressed because it is too large Load Diff
@@ -1,465 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous.
The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX
trainer) initialize NCCL weight transfer from a unique id minted out of band.
* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL
init infos.
* Two-GPU integration tests driving the real worker engine against a peer: one
via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer
(cupy buffer + raw ``ncclBroadcast``).
The shared Ray helpers come from the sibling module (unchanged); the worker task
is local so the sibling's TCP test and its worker stay untouched.
"""
from unittest.mock import MagicMock
import pybase64 as base64
import pytest
import ray
import torch
from tests.distributed.test_weight_transfer import (
_init_ray_for_weight_transfer,
_set_ray_assigned_device,
create_mock_vllm_config,
)
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerInitInfo,
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
)
# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId.
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
class TestNCCLRendezvousValidation:
"""Test the xor-of-rendezvous-modes validation on the NCCL init infos."""
def test_worker_init_info_is_keyword_only(self):
# kw_only so a stale positional call fails loudly rather than silently
# swapping fields once the rendezvous fields became optional.
with pytest.raises(TypeError):
NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg]
def test_trainer_init_info_accepts_positional_fields(self):
# NCCLTrainerInitInfo keeps positional subclass fields; only `rank`
# (from TrainerInitInfo) is keyword-only.
trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0)
trainer_keyword = NCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=12345,
world_size=3,
rank=0,
)
assert trainer_positional == trainer_keyword
def test_tcp_mode_valid(self):
info = NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
master_port=12345,
)
assert info.master_address == "127.0.0.1"
assert info.nccl_unique_id_b64 is None
def test_uid_mode_valid(self):
info = NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64=VALID_UID_B64,
)
assert info.nccl_unique_id_b64 == VALID_UID_B64
assert info.master_address is None
assert info.master_port is None
assert VALID_UID_B64 not in repr(info)
def test_both_modes_raises(self):
with pytest.raises(ValueError, match="not both"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
master_port=12345,
nccl_unique_id_b64=VALID_UID_B64,
)
def test_neither_mode_raises(self):
with pytest.raises(ValueError, match="need nccl_unique_id_b64"):
NCCLWeightTransferInitInfo(rank_offset=1, world_size=3)
def test_half_specified_tcp_raises(self):
with pytest.raises(ValueError, match="master_port"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
)
def test_bad_base64_uid_raises(self):
with pytest.raises(ValueError, match="not valid standard base64"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64="!!!!",
)
def test_wrong_length_uid_raises(self):
short_uid = base64.b64encode(b"\x00" * 64).decode()
with pytest.raises(ValueError, match="128"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64=short_uid,
)
def test_parse_init_info_uid_valid(self):
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda"),
MagicMock(spec=torch.nn.Module),
)
init_info = engine.parse_init_info(
{
"nccl_unique_id_b64": VALID_UID_B64,
"rank_offset": 1,
"world_size": 3,
}
)
assert isinstance(init_info, NCCLWeightTransferInitInfo)
assert init_info.nccl_unique_id_b64 == VALID_UID_B64
assert init_info.master_address is None
def test_parse_init_info_both_modes_raises(self):
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda"),
MagicMock(spec=torch.nn.Module),
)
with pytest.raises(ValueError, match="not both"):
engine.parse_init_info(
{
"master_address": "127.0.0.1",
"master_port": 12345,
"nccl_unique_id_b64": VALID_UID_B64,
"rank_offset": 1,
"world_size": 3,
}
)
@pytest.mark.parametrize(
("unique_id_bytes", "rank", "world_size", "match"),
[
(b"\x00" * 64, 0, 2, "128-byte"),
(b"\x00" * 128, -1, 2, "out of range"),
(b"\x00" * 128, 2, 2, "out of range"),
],
)
def test_from_unique_id_bytes_validation(
self, unique_id_bytes, rank, world_size, match
):
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
with pytest.raises(ValueError, match=match):
PyNcclCommunicator.from_unique_id_bytes(
unique_id_bytes,
rank=rank,
world_size=world_size,
device=0,
)
@ray.remote(num_gpus=1)
def inference_receive_tensor(
init_info_dict: dict,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Worker that joins via the given init dict and records the tensor it gets.
Local to this module (rather than the sibling's TCP-only worker) so it can
take a UID init payload -- the same dict boundary an external trainer POSTs.
"""
import contextlib
import torch
_set_ray_assigned_device()
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
recorder = Recorder()
engine = NCCLWeightTransferEngine(
config, vllm_config, torch.device("cuda"), recorder
)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
try:
engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict))
engine.receive_weights(
NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=[tensor_dtype],
shapes=[tensor_shape],
)
)
torch.accelerator.synchronize()
received_shape = None
received_sum = None
if len(recorder.received) == 1:
_name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
return {
"success": received_shape == tensor_shape
and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01,
"received_shape": received_shape,
"received_sum": received_sum,
}
finally:
engine.shutdown()
@ray.remote(num_gpus=1)
def trainer_broadcast_tensor_uid(
uid_b64: str,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Drive rank 0 via vLLM's own communicator from a pre-shared unique id.
A real torch-free trainer would mint the id and drive rank 0 itself; here we
reuse the real send protocol by injecting the communicator into a sender
engine, exercising the worker-side UID path end to end.
"""
import pybase64 as base64
import torch
_set_ray_assigned_device()
from vllm.distributed.weight_transfer import ModuleSource
from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerWeightTransferEngine,
)
class NoopClient:
def init_weight_transfer_engine(self, init_info):
pass
def start_weight_update(self):
pass
def update_weights(self, update_info):
pass
def finish_weight_update(self, weight_version=None):
pass
dtype = getattr(torch, tensor_dtype)
model = torch.nn.Module()
test_module = torch.nn.Module()
test_module.register_parameter(
"weight",
torch.nn.Parameter(
torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False
),
)
model.add_module("test", test_module)
engine = NCCLTrainerWeightTransferEngine(
client=NoopClient(),
source=ModuleSource(model),
is_sender=True,
packed=False,
)
engine.model_update_group = uid_init_process_group(
base64.b64decode(uid_b64),
rank=0,
world_size=world_size,
device=torch.accelerator.current_device_index(),
)
try:
engine.send_weights()
torch.accelerator.synchronize()
return {"success": True}
finally:
engine.shutdown()
@ray.remote(num_gpus=1)
def trainer_broadcast_tensor_torch_free(
uid_b64: str,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in).
Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it
exercises interop with a foreign NCCL peer rather than vLLM against itself.
``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right
after ``ncclCommInitRank``; that is a collective, so this peer must issue the
matching all_reduce before the broadcast or both ranks deadlock.
"""
import cupy as cp
import numpy as np
import pybase64 as base64
device = _set_ray_assigned_device()
cp.cuda.Device(device.index).use()
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCLLibrary,
buffer_type,
cudaStream_t,
ncclDataTypeEnum,
ncclRedOpTypeEnum,
)
assert tensor_dtype == "float32", "harness only wires float32"
# Fill on the host and copy up (a plain H2D memcpy) so we neither use torch
# nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets
# the worker verify by sum == numel.
sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32))
warmup = cp.asarray(np.zeros(1, dtype=np.float32))
numel = int(sendbuff.size)
# The copies above run on the default stream; drain them before the NCCL
# collectives read the buffers on our own stream.
cp.cuda.Device(device.index).synchronize()
stream = cp.cuda.Stream()
nccl = NCCLLibrary()
comm = nccl.ncclCommInitRank(
world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0
)
try:
# Mirror the worker's init-time warm-up all_reduce (rank 0 must join it).
nccl.ncclAllReduce(
buffer_type(warmup.data.ptr),
buffer_type(warmup.data.ptr),
1,
ncclDataTypeEnum.ncclFloat32,
ncclRedOpTypeEnum.ncclSum,
comm,
cudaStream_t(stream.ptr),
)
nccl.ncclBroadcast(
buffer_type(sendbuff.data.ptr),
buffer_type(sendbuff.data.ptr),
numel,
ncclDataTypeEnum.ncclFloat32,
0, # root
comm,
cudaStream_t(stream.ptr),
)
stream.synchronize()
return {"success": True}
finally:
# Abort (not the collective destroy) for an uncoordinated teardown.
nccl.ncclCommAbort(comm)
def _run_uid_transfer(trainer_task) -> None:
"""Mint a unique id, run the worker and the given trainer peer concurrently,
and assert the tensor round-trips. Both ranks must enter init together --
there is no store barrier on the UID path."""
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
_init_ray_for_weight_transfer()
nccl = NCCLLibrary()
uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode()
world_size = 2 # 1 trainer + 1 inference worker
tensor_shape = [100, 100]
tensor_dtype = "float32"
inference_future = inference_receive_tensor.remote(
{
"rank_offset": 1,
"world_size": world_size,
"nccl_unique_id_b64": uid_b64,
"packed": False,
},
tensor_shape,
tensor_dtype,
)
trainer_future = trainer_task.remote(
uid_b64, world_size, tensor_shape, tensor_dtype
)
try:
trainer_result, result = ray.get(
[trainer_future, inference_future], timeout=120
)
finally:
ray.shutdown()
assert trainer_result["success"], "Trainer should complete successfully"
assert result["success"], (
f"Weight transfer failed. "
f"Received shape: {result['received_shape']}, "
f"Received sum: {result['received_sum']}"
)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_between_processes_uid():
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
ranks on vLLM's ``PyNcclCommunicator``."""
_run_uid_transfer(trainer_broadcast_tensor_uid)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_torch_free_trainer():
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
pytest.importorskip("cupy")
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
import requests
import torch
from prometheus_client.parser import text_string_to_metric_families
from transformers import AutoModelForCausalLM
from tests.entrypoints.rl.conftest import MODEL_NAME
from tests.utils import RemoteOpenAIServer
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
SYNCS = 8
def scrape(url: str) -> dict[tuple[str, str], float]:
text = requests.get(f"{url}/metrics", timeout=30).text
return {
(sample.name, sample.labels["operation"]): sample.value
for family in text_string_to_metric_families(text)
for sample in family.samples
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
}
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
args = [
"--load-format",
"dummy",
"--enforce-eager",
"--max-model-len",
"512",
"--gpu-memory-utilization",
"0.5",
"--api-server-count",
"2",
"--weight-transfer-config",
'{"backend": "ipc"}',
]
env = {
"VLLM_SERVER_DEV_MODE": "1",
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
url = server.url_root
trainer = AutoModelForCausalLM.from_pretrained(
MODEL_NAME, torch_dtype=torch.bfloat16
).cuda()
engine = WeightTransferTrainerFactory.trainer_init(
IPCTrainerInitInfo(rank=0),
client=HTTPVLLMWeightSyncClient(url),
source=ModuleSource(trainer),
)
for _ in range(SYNCS):
engine.send_weights()
metrics = scrape(url)
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
for operation, count in calls.items():
assert metrics[(f"{DURATION}_count", operation)] == count, operation
assert metrics[(IN_FLIGHT, operation)] == 0, operation
@@ -1,234 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.testing import assert_close
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
def test_pack_seq_basic_fp8():
"""Test basic functionality of pack_seq_triton with fp8 and 3D tensors."""
device = "cuda"
dtype = torch.float8_e4m3fn
# Test cases with 3D tensors (N, H, D)
test_cases = [
(6, 8, 4, 2, [3, 3]), # (6, 8, 4) -> (2, 3, 8, 4)
(10, 4, 8, 3, [2, 4, 4]), # (10, 4, 8) -> (3, 4, 4, 8)
(20, 16, 32, 4, [5, 5, 5, 5]), # (20, 16, 32) -> (4, 5, 16, 32)
]
for N, H, D, B, lengths_list in test_cases:
# Create input tensor with small values for fp8
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor(lengths_list, device=device)
# Pack the data
packed = pack_seq_triton(x, lengths)
# Check output shape and properties
expected_shape = (B, max(lengths_list), H, D)
assert packed.shape == expected_shape
assert packed.dtype == dtype
assert packed.device == x.device
# Check that valid data is preserved (within fp8 precision)
for b in range(B):
start_idx = sum(lengths_list[:b])
seq_len = lengths_list[b]
expected_data = x[start_idx : start_idx + seq_len].to(torch.float32)
actual_data = packed[b, :seq_len].to(torch.float32)
assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2)
def test_pack_seq_custom_padding_fp8():
"""Test pack_seq_triton with custom padding values for fp8."""
device = "cuda"
dtype = torch.float8_e4m3fn
N, H, D, B = 20, 8, 16, 2
lengths = torch.tensor([10, 10], device=device)
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
# Test with different padding values
for pad_value in [-100.0, -10.0, 0.0, 10.0, 100.0]:
result = pack_seq_triton(x, lengths, pad_value=pad_value)
# Check valid data
for b in range(B):
start_idx = b * 10
expected_data = x[start_idx : start_idx + 10].to(torch.float32)
actual_data = result[b, :10].to(torch.float32)
assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2)
# Check padding (fp8 has limited range, so check for large values)
padded_data = result[:, 10:].to(torch.float32)
if pad_value < 0:
assert torch.all(padded_data < -50) # Large negative values
elif pad_value > 0:
assert torch.all(padded_data > 50) # Large positive values
else:
assert torch.allclose(padded_data, torch.zeros_like(padded_data), atol=1e-2)
def test_pack_seq_default_negative_inf_padding_fp8():
"""Test that pack_seq_triton uses -inf padding by default for fp8."""
device = "cuda"
dtype = torch.float8_e4m3fn
# B = 2
N, H, D = 20, 8, 16
lengths = torch.tensor([10, 10], device=device)
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
result = pack_seq_triton(x, lengths)
# Check that padding is large negative values (fp8 representation of -inf)
padded_data = result[:, 10:].to(torch.float32)
assert torch.all(
padded_data < -100
) # fp8 -inf is represented as large negative number
def test_pack_seq_edge_cases_fp8():
"""Test pack_seq_triton with edge cases for fp8."""
device = "cuda"
dtype = torch.float8_e4m3fn
# Test with single batch element
x = torch.randn(10, 8, 16, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor([10], device=device)
result = pack_seq_triton(x, lengths)
assert result.shape == (1, 10, 8, 16)
# Test with very short sequences
x = torch.randn(20, 4, 8, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor([1, 1, 1], device=device)
result = pack_seq_triton(x, lengths)
assert result.shape == (3, 1, 4, 8)
# Test with different sequence lengths
x = torch.randn(15, 8, 16, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor([5, 7, 3], device=device)
result = pack_seq_triton(x, lengths)
assert result.shape == (3, 7, 8, 16)
def test_pack_seq_different_block_sizes_fp8():
"""Test pack_seq_triton with different block sizes for fp8."""
device = "cuda"
dtype = torch.float8_e4m3fn
N, H, D, B = 100, 16, 32, 4
lengths = torch.tensor([25, 25, 25, 25], device=device)
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
# Test different block sizes
for block_t, block_d in [(32, 32), (64, 64), (128, 128)]:
result = pack_seq_triton(x, lengths, block_t=block_t, block_d=block_d)
assert result.shape == (B, 25, H, D)
# Check that valid data is preserved (within fp8 precision)
for b in range(B):
start_idx = b * 25
expected_data = x[start_idx : start_idx + 25].to(torch.float32)
actual_data = result[b, :25].to(torch.float32)
assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2)
def test_pack_seq_shape_consistency():
"""Test that pack_seq_triton maintains shape consistency."""
device = "cuda"
dtype = torch.float8_e4m3fn
N, H, D, B = 20, 8, 16, 2
lengths = torch.tensor([10, 10], device=device)
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
result = pack_seq_triton(x, lengths)
# Check shape consistency
assert result.shape[0] == B # Batch dimension
assert result.shape[1] == lengths.max().item() # Max sequence length
assert result.shape[2:] == x.shape[1:] # Feature dimensions preserved
def test_pack_unpack_roundtrip_fp8():
"""Test that pack -> unpack gives us back the original data for fp8."""
device = "cuda"
dtype = torch.float8_e4m3fn
# Test cases with 3D tensors
test_cases = [
(6, 8, 4, 2, [3, 3]),
(10, 4, 8, 3, [2, 4, 4]),
(20, 16, 32, 4, [5, 5, 5, 5]),
(15, 8, 16, 3, [7, 5, 3]),
]
for N, H, D, B, lengths_list in test_cases:
# Create input tensor with small values for fp8
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor(lengths_list, device=device)
# Pack the data
packed = pack_seq_triton(x, lengths)
# Unpack the data
unpacked = unpack_seq_triton(packed, lengths)
# Check that we get back the original data (within fp8 precision)
assert unpacked.shape == x.shape
x_f32 = x.to(torch.float32)
unpacked_f32 = unpacked.to(torch.float32)
assert_close(x_f32, unpacked_f32, rtol=1e-3, atol=1e-3)
# Unpack without explicit start locations (computed in kernel)
unpacked_with_loc = unpack_seq_triton(packed, lengths)
assert_close(x_f32, unpacked_with_loc.to(torch.float32), rtol=1e-3, atol=1e-2)
def test_unpack_seq_triton_edge_cases_fp8():
"""Test unpack function with edge cases for fp8."""
device = "cuda"
dtype = torch.float8_e4m3fn
# Test with single batch element
x = torch.randn(10, 8, 16, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor([10], device=device)
packed = pack_seq_triton(x, lengths)
unpacked = unpack_seq_triton(packed, lengths)
assert unpacked.shape == x.shape
assert_close(x.to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2)
# Test with very short sequences
x = torch.randn(20, 4, 8, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor([1, 1, 1], device=device)
packed = pack_seq_triton(x, lengths)
unpacked = unpack_seq_triton(packed, lengths)
# Only compare the first 3 elements that were actually packed
assert_close(
x[:3].to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2
)
x = torch.randn(15, 8, 16, dtype=torch.float32, device=device) * 0.1
x = x.to(dtype=dtype)
lengths = torch.tensor([5, 7, 3], device=device)
packed = pack_seq_triton(x, lengths)
unpacked = unpack_seq_triton(packed, lengths)
assert unpacked.shape == x.shape
assert_close(x.to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2)
@@ -250,8 +250,8 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
return BatchFeature(data=inputs, tensor_type="pt")
hf_model.processor = processor
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language.model.embed_tokens
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language.model.embed_tokens
)
return hf_model
@@ -353,8 +353,8 @@ def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
)
hf_model.processor = processor
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.transformer.output_layer
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.transformer.output_layer
)
return hf_model
@@ -442,8 +442,8 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = H2OVLProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
@@ -678,8 +678,8 @@ def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = SkyworkR1VProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
@@ -787,8 +787,8 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = InternVLProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
@@ -941,8 +941,8 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.llm.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.llm.get_output_embeddings()
)
def processor(*args, text="", images=None, **kwargs):
@@ -977,8 +977,8 @@ def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.llm.get_output_embeddings()
hf_model.model.get_output_embeddings = lambda: (
hf_model.model.llm.get_output_embeddings()
)
def processor(*args, text="", images=None, videos=None, **kwargs):
@@ -1209,221 +1209,3 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
hf_model.model.generate = patched_generate # type: ignore[method-assign]
return hf_model
def moondream3_processor(model: str):
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
return Moondream3Processor.from_pretrained(model, trust_remote_code=True)
def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patch HfRunner for Moondream3."""
moondream_processor = hf_model.processor
def processor(*args, text="", images=None, **kwargs):
if images is None:
return moondream_processor(text=text, **kwargs)
images_list = [images] if isinstance(images, Image) else images
return moondream_processor(images=images_list, text=text, **kwargs)
hf_model.processor = processor
# Expose the LM head for logprob extraction.
hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head
native_model = hf_model.model.model # MoondreamModel instance
from torch.nn import functional as F
from vllm.model_executor.models.moondream3 import reconstruct_from_crops
# Resolve the placeholder tokens from the tokenizer instead of hard-coding.
image_placeholder_ids = moondream_processor.tokenizer.encode(
"<image>", add_special_tokens=False
)
def _normalize_tiling(tilings):
"""Extract (h, w) tuple from various tiling container formats."""
tiling = tilings
if isinstance(tiling, torch.Tensor):
tiling = tuple(tiling.squeeze().tolist())
elif isinstance(tiling, (list, tuple)):
t0 = tiling[0]
if isinstance(t0, torch.Tensor):
tiling = tuple(t0.tolist())
elif isinstance(t0, (list, tuple)):
tiling = tuple(t0)
return tiling
def _encode_vision(pixel_values, tilings):
"""Run preprocessed crops through vision encoder + projection."""
device = native_model.device
dtype = native_model.vision.pos_emb.dtype
config = native_model.config
pv = pixel_values
while pv.dim() > 4:
pv = pv.squeeze(0)
pv = pv.to(device=device, dtype=dtype)
features = native_model._vis_enc(pv)
grid_size = config.vision.crop_size // config.vision.enc_patch_size
global_feat = features[0]
if features.shape[0] > 1 and tilings is not None:
tiling = _normalize_tiling(tilings)
local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim)
reconstructed = reconstruct_from_crops(
local,
tiling,
config.vision.overlap_margin,
patch_size=1,
)
else:
reconstructed = global_feat.view(
grid_size, grid_size, config.vision.enc_dim
)
return native_model._vis_proj(global_feat, reconstructed)
def _find_subsequence(seq, subseq):
"""Find start index of subseq in seq, or None."""
n = len(subseq)
for i in range(len(seq) - n + 1):
if seq[i : i + n] == subseq:
return i
return None
def _generate(
self,
input_ids=None,
pixel_values=None,
tilings=None,
attention_mask=None,
**kwargs,
):
max_new_tokens = kwargs.get("max_new_tokens", 128)
return_dict = kwargs.get("return_dict_in_generate", False)
output_hs = kwargs.get("output_hidden_states", False)
if pixel_values is None:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
# Processor may return lists; extract the single element.
if isinstance(pixel_values, (list, tuple)):
pixel_values = pixel_values[0]
if (
isinstance(tilings, (list, tuple))
and tilings
and not isinstance(tilings[0], int)
):
tilings = tilings[0]
hf_model.model._setup_caches()
native_model.use_flex_decoding = False
device = native_model.device
config = native_model.config
with torch.inference_mode():
for block in native_model.text.blocks:
block.kv_cache.k_cache.zero_()
block.kv_cache.v_cache.zero_()
img_emb = _encode_vision(pixel_values, tilings)
bos_emb = F.embedding(
torch.tensor([[config.tokenizer.bos_id]], device=device),
native_model.text.wte,
)
img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1)
prefix_len = img_input.size(1)
mask = native_model.attn_mask[:, :, :prefix_len, :]
pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device)
native_model._prefill(img_input, mask, pos_ids, None)
ids = input_ids.squeeze(0).tolist()
img_start = _find_subsequence(ids, image_placeholder_ids)
if img_start is None:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
prompt_tokens = ids[img_start + len(image_placeholder_ids) :]
if not prompt_tokens:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
prompt_tensor = torch.tensor([prompt_tokens], device=device)
prompt_emb = F.embedding(prompt_tensor, native_model.text.wte)
prompt_len = prompt_emb.size(1)
mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :]
pos_ids = torch.arange(
prefix_len,
prefix_len + prompt_len,
dtype=torch.long,
device=device,
)
hidden = native_model._prefill(prompt_emb, mask, pos_ids, None)
pos = prefix_len + prompt_len
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
logits = native_model.text.lm_head(hidden_last.squeeze(1))
generated = []
all_hidden_states = []
# Record the hidden state that predicted each generated token.
prev_hs = hidden_last
for _ in range(max_new_tokens):
next_token = logits.argmax(dim=-1).item()
if next_token == 0:
break
generated.append(next_token)
if output_hs:
all_hidden_states.append((prev_hs,))
next_emb = F.embedding(
torch.tensor([[next_token]], device=device),
native_model.text.wte,
)
mask = native_model.attn_mask[:, :, pos : pos + 1, :]
pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device)
hidden = native_model._prefill(next_emb, mask, pos_ids_step, None)
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
prev_hs = hidden_last
logits = native_model.text.lm_head(hidden_last.squeeze(1))
pos += 1
result_ids = ids + generated
sequences = torch.tensor([result_ids], device=device)
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=tuple(all_hidden_states) if output_hs else None,
)
return sequences
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
return hf_model
-436
View File
@@ -15,7 +15,6 @@ from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
from torch.nn import functional as F
from transformers import Qwen4ExpTextConfig
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
@@ -31,10 +30,6 @@ from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer
from vllm.models.qwen4_exp.amd.ple_layer import (
Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD,
)
from vllm.models.qwen4_exp.common.ple import (
PLEShardOverlap,
compute_ple_shard_overlap,
@@ -51,7 +46,6 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
from vllm.platforms import current_platform
from vllm.utils.torch_utils import weak_ref_tensor
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionMetadata,
@@ -732,10 +726,6 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
)
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
@@ -2289,162 +2279,6 @@ def _make_conv_metadata(
return metadata, token_offset
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused conv needs CUDA")
@pytest.mark.parametrize("state_layout", ["SD", "DS"])
@pytest.mark.parametrize(
"case",
[
pytest.param(
_ConvBatchCase(num_decodes=1, channels=512),
id="decode-single",
),
pytest.param(
_ConvBatchCase(
num_decodes=33,
channels=2048,
kernel_size=5,
dilation=1,
),
id="decode-padded",
),
pytest.param(
_ConvBatchCase(prefill_query_lens=(37, 0, 5, 128)),
id="prefill",
),
pytest.param(
_ConvBatchCase(
prefill_query_lens=(16, 676),
channels=10240,
state_index_stride=4,
include_null_state=False,
),
id="prefill-strided-state-indices",
),
pytest.param(
_ConvBatchCase(
spec_query_lens=(1, 4, 4, 0),
num_accepted=(1, 2, 4, 1),
spec_query_len=4,
graph_padding=4,
),
id="spec-graph-padded",
),
pytest.param(
_ConvBatchCase(
spec_query_lens=(3, 4),
num_accepted=(2, 4),
num_decodes=1,
prefill_query_lens=(5,),
spec_query_len=4,
),
id="mixed",
),
pytest.param(
_ConvBatchCase(
spec_query_lens=(1, 2),
num_accepted=(0, 1),
num_decodes=2,
prefill_query_lens=(0, 8, 3),
channels=640,
kernel_size=3,
dilation=2,
spec_query_len=3,
),
id="mixed-varied",
),
],
)
@pytest.mark.parametrize(
("layer_cls", "fuse_residual"),
[
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
],
)
def test_fused_conv_correctness(
case: _ConvBatchCase,
state_layout: str,
layer_cls: type[nn.Module],
fuse_residual: bool,
) -> None:
device = torch.device("cuda")
metadata, num_real_tokens = _make_conv_metadata(case, device)
if case.state_index_stride > 1:
assert metadata.state_indices_tensor.stride(0) == case.state_index_stride
# layer_cls is type[nn.Module], so layer_cls.__new__ is type.__new__.
module = object.__new__(layer_cls)
nn.Module.__init__(module)
module.conv_state_len = (case.kernel_size - 1) * case.dilation
module.short_conv_dilation = case.dilation
rng, state_reference, conv_state, weights = _make_conv_case(
device,
seed=num_real_tokens + case.channels,
channels=case.channels,
kernel_size=case.kernel_size,
dilation=case.dilation,
state_layout=state_layout,
spec_query_len=case.spec_query_len,
)
inputs = torch.randn(
metadata.num_actual_tokens,
case.channels,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
residual = torch.randn(
inputs.shape,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
outer_residual = torch.randn(
inputs.shape,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
null_state = conv_state[NULL_BLOCK_ID].clone()
residual_kernel = residual.clone()
# The unfused kernel must overwrite the buffer without reading its contents.
residual_reference = (
residual.clone() if fuse_residual else torch.zeros_like(residual)
)
module._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual_kernel,
outer_residual=outer_residual if fuse_residual else None,
metadata=metadata,
conv_state=conv_state,
conv_weights=weights,
)
_short_conv_dilated_dispatch_pytorch(
inputs=inputs,
residual=residual_reference,
metadata=metadata,
conv_state=state_reference,
conv_weights=weights,
conv_state_len=module.conv_state_len,
dilation=module.short_conv_dilation,
)
if fuse_residual:
residual_reference = outer_residual + residual_reference
assert torch.equal(
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
)
assert torch.equal(conv_state, state_reference)
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
if case.graph_padding:
assert torch.equal(
residual_kernel[num_real_tokens:],
residual_reference[num_real_tokens:],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused gate needs CUDA")
@pytest.mark.parametrize(("num_tokens", "strided_kv"), [(1, False), (64, True)])
def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
@@ -2509,273 +2343,3 @@ def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
expected_normed = grouped_norm(expected_gated, norm_conv)
assert torch.equal(gated, expected_gated)
torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2)
def _build_amd_ngram_embedding(
monkeypatch: pytest.MonkeyPatch,
*,
device: str,
cpu_offload: bool,
fp8_checkpoint: bool,
) -> tuple[amd_ple_layer.Qwen4ExpNGramEmbedding, torch.Tensor]:
"""Construct and load a small AMD embedding through its checkpoint loader."""
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(
amd_ple_layer,
"get_current_vllm_config",
lambda: SimpleNamespace(engram_config=SimpleNamespace(cpu_offload=cpu_offload)),
)
config = Qwen4ExpTextConfig(
ngram_size=3,
heads_per_ngram=1,
ngram_vocab_size_base=5,
make_ngram_vocab_size_divisible_by=8,
split_ngram_parts=2,
ple_embed_dim=6,
eos_token_id=0,
vocab_size=64,
)
with torch.device(device):
module = amd_ple_layer.Qwen4ExpNGramEmbedding(
config,
embedding_dim=config.ple_embed_dim,
ple_dense_layer_id=0,
max_total_tokens=4,
prefix="test.ple_embedding",
layer_name="test.ple",
quant_config=(
Fp8Config(is_checkpoint_fp8_serialized=True) if fp8_checkpoint else None
),
params_dtype=torch.bfloat16,
)
# Only the outer owner is a stub; the embedding constructor and loader run.
layer = Qwen4ExpPLELayerAMD.__new__(Qwen4ExpPLELayerAMD)
nn.Module.__init__(layer)
layer.ple_embedding = module
monkeypatch.setattr(
amd_ple_layer,
"get_forward_context",
lambda: SimpleNamespace(no_compile_layers={module.layer_name: layer}),
)
embedding = module.ngram_embedding
loaded_weight = (
torch.arange(embedding.org_vocab_size * embedding.embedding_dim)
.reshape(embedding.org_vocab_size, embedding.embedding_dim)
.to(embedding.weight.dtype)
)
shard_size = (embedding.org_vocab_size + 1) // 2
weights = [
(f"ngram_embedding.shard_{i}.weight", shard)
for i, shard in enumerate(loaded_weight.split(shard_size))
]
expected_loaded = {"ngram_embedding.weight"}
if fp8_checkpoint:
weights.append(("ngram_embedding.weight_scale", torch.tensor([0.25])))
expected_loaded.add("ngram_embedding.weight_scale")
assert module.load_weights(weights) == expected_loaded
embedding.quant_method.process_weights_after_loading(embedding)
return module, loaded_weight
@pytest.mark.parametrize(
"device,cpu_offload",
[
pytest.param("cpu", False, id="cpu-resident"),
pytest.param(
"cuda:0",
False,
id="gpu-resident",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
),
),
pytest.param(
"cuda:0",
True,
id="gpu-offloaded",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
),
),
],
)
def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
monkeypatch: pytest.MonkeyPatch,
device: str,
cpu_offload: bool,
) -> None:
"""AMD checkpoint loading must accept and apply the global FP8 scale."""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device=device, cpu_offload=cpu_offload, fp8_checkpoint=True
)
embedding = module.ngram_embedding
assert embedding.weight.dtype == torch.float8_e4m3fn
assert embedding.weight.device == torch.device("cpu" if cpu_offload else device)
if cpu_offload:
assert embedding.weight.is_pinned()
assert embedding.weight_scale.device == torch.device(device)
torch.testing.assert_close(embedding.weight.float().cpu(), loaded_weight.float())
dequantized = embedding.dequantize(embedding.weight.to(device), torch.bfloat16)
torch.testing.assert_close(
dequantized.cpu(), loaded_weight.to(torch.bfloat16) * 0.25, rtol=0, atol=0
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
first ``start_prefetch`` — the eager profile run, always before cudagraph
capture — allocates them lazily.
"""
module, _ = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
)
embedding = module.ngram_embedding
assert embedding._prefetch_buffer is None
assert embedding._prefetch_stream is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
def test_amd_pinned_embedding_output_written_under_compile(
monkeypatch: pytest.MonkeyPatch,
fp8_checkpoint: bool,
) -> None:
"""The AMD pinned lookup op must survive aot_autograd/Inductor.
The op writes via the declared-mutated ``output``. Without
``mutates_args=["output"]`` the call is dead-code-eliminated, the UVA lookup
never runs, and the returned ``output`` stays untouched.
"""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = torch.zeros(
ngram_ids.shape[0],
module.embedding_dim,
dtype=module.ngram_embedding.weight.dtype,
device="cuda:0",
)
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
ids, out, module.layer_name
)
return out
torch.compile(lookup, fullgraph=True)(ngram_ids, output)
torch.cuda.current_stream().synchronize()
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
def test_amd_pinned_embedding_output_written_under_cudagraph_capture(
monkeypatch: pytest.MonkeyPatch,
fp8_checkpoint: bool,
) -> None:
"""The AMD pinned lookup op must be capture-safe.
It launches a single kernel on the current stream, so a cudagraph capture
reproduces the lookup into ``output`` without cross-stream work.
"""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
output = torch.zeros(
ngram_ids.shape[0],
module.embedding_dim,
dtype=module.ngram_embedding.weight.dtype,
device="cuda:0",
)
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
ids, out, module.layer_name
)
return out
graph = torch.cuda.CUDAGraph()
warmup_stream = torch.cuda.Stream(device="cuda:0")
warmup_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup_stream):
for _ in range(3):
lookup(ngram_ids, output)
torch.cuda.current_stream().wait_stream(warmup_stream)
with torch.cuda.graph(graph):
lookup(ngram_ids, output)
for _ in range(2):
ngram_ids.copy_(ngram_ids.flip(-1) + 3)
output.fill_(float("nan"))
graph.replay()
torch.cuda.current_stream().synchronize()
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="only the ROCm backend runs the n-gram embedding under torch.compile",
)
@pytest.mark.parametrize(
("query_lens", "contexts"),
[([3, 1], [[5, 6], [0, 7]]), ([1, 0, 2], [[1, 2], [3, 4], [0, 0]])],
ids=["two-requests", "empty-request"],
)
def test_amd_ngram_embedding_matches_reference_under_compile(
monkeypatch: pytest.MonkeyPatch,
query_lens: list[int],
contexts: list[list[int]],
) -> None:
"""The AMD n-gram ID op must run under Dynamo with a symbolic request count."""
module, loaded_weight = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=False, fp8_checkpoint=False
)
device = torch.device("cuda:0")
query_start_loc = torch.tensor(
[0, *accumulate(query_lens)], dtype=torch.int32, device=device
)
input_ids = torch.tensor(
[9, 0, 11, 12][: sum(query_lens)], dtype=torch.int32, device=device
)
ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device)
compiled = torch.compile(module, fullgraph=True, dynamic=True)
output = compiled(input_ids, query_start_loc, ngram_context)
ngram_ids = _reference_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
module.layer_multipliers,
module.ngram_heads_vocab_sizes,
module.ngram_heads_offsets,
module.eos_token_id,
module.heads_per_ngram,
)
expected = loaded_weight[ngram_ids.cpu()].flatten(-2).to(device)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
File diff suppressed because it is too large Load Diff
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's MLA fuser.
The fuser must discover every MLA submodule structurally (never by assuming the
Transformers attribute names) and, when the query is low-rank, merge the checkpoint's
separate `q_a_proj`/`kv_a_proj_with_mqa` weights into the single fused down-projection.
"""
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.fx_utils import trace
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
def _attention(q_lora_rank: int | None):
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
DeepseekV2Config,
)
from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
DeepseekV2Attention,
)
cfg = DeepseekV2Config(
hidden_size=256,
num_attention_heads=16,
kv_lora_rank=128,
qk_rope_head_dim=32,
qk_nope_head_dim=32,
v_head_dim=32,
q_lora_rank=q_lora_rank,
num_hidden_layers=1,
)
with torch.device("meta"):
return DeepseekV2Attention(cfg, layer_idx=0)
def _match(q_lora_rank: int | None) -> MLAFuser | None:
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
attn = _attention(q_lora_rank)
return MLAFuser.match(trace(attn), attn)
def test_discovers_modules_without_q_lora():
fuser = _match(q_lora_rank=None)
assert isinstance(fuser, MLAFuser)
assert fuser.q_proj_name == "q_proj"
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
assert fuser.kv_b_proj_name == "kv_b_proj"
assert fuser.o_proj_name == "o_proj"
assert fuser.q_a_proj_name is None
# Nothing is stacked without a query LoRA.
assert fuser.packed_modules_mapping == {}
assert fuser.orig_to_new_stacked("model.layers.0.self_attn") == {}
def test_discovers_modules_with_q_lora():
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
assert fuser.q_a_proj_name == "q_a_proj"
assert fuser.q_a_layernorm_name == "q_a_layernorm"
assert fuser.q_b_proj_name == "q_b_proj"
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
assert fuser.kv_b_proj_name == "kv_b_proj"
assert fuser.o_proj_name == "o_proj"
assert fuser.q_proj_name is None
def test_q_lora_stacks_qkv_a_proj():
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
down-projection, so both checkpoint weights must remap into it."""
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
prefix = "model.layers.0.self_attn"
merged = f"{prefix}.{_FUSED_QKV_A_PROJ}"
assert fuser.packed_modules_mapping == {
_FUSED_QKV_A_PROJ: ["q_a_proj", "kv_a_proj_with_mqa"]
}
assert fuser.orig_to_new_stacked(prefix) == {
f"{prefix}.q_a_proj": (merged, 0),
f"{prefix}.kv_a_proj_with_mqa": (merged, 1),
}
@pytest.mark.parametrize("q_lora_rank", [64, None])
def test_update_forward_rewrites_real_attention(q_lora_rank):
"""The source rewrite must survive the input-stability check.
The q-LoRA path hoists the merged down-projection to a top-level statement
while its calls sit inside the `q_lora_rank` branches, so the region it
spans is wide; a check that over-approximates aliasing would silently refuse
here and cost the fusion rather than fail loudly."""
fuser = _match(q_lora_rank)
assert isinstance(fuser, MLAFuser)
fuser.update_forward(_attention(q_lora_rank))
names = set(fuser.fused_forward.__code__.co_names)
if fuser.q_a_proj_name is not None:
assert _FUSED_QKV_A_PROJ in names
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
class _Norm(nn.Module):
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""
def __init__(self, n: int):
super().__init__()
self.weight = nn.Parameter(torch.ones(n))
def forward(self, x):
variance = x.pow(2).mean(-1, keepdim=True)
return self.weight * (x * torch.rsqrt(variance + 1e-6))
class RenamedMLA(nn.Module):
"""An MLA-shaped attention whose children have non-standard names, proving
discovery is by structure and not attribute name."""
def __init__(self):
super().__init__()
heads, kv_lora, rope, nope, v, hidden = 4, 32, 8, 8, 16, 64
self.alpha = nn.Linear(hidden, heads * (nope + rope)) # q_proj
self.beta = nn.Linear(hidden, kv_lora + rope) # kv_a_proj_with_mqa
self.gamma = _Norm(kv_lora) # kv_a_layernorm
self.delta = nn.Linear(kv_lora, heads * (nope + v)) # kv_b_proj
self.omega = nn.Linear(heads * v, hidden) # o_proj (uncalled)
self.kv_lora, self.rope = kv_lora, rope
def forward(self, hidden_states):
q = self.alpha(hidden_states)
kv_lora, k_pe = torch.split(
self.beta(hidden_states), [self.kv_lora, self.rope], dim=-1
)
expanded = self.delta(self.gamma(kv_lora))
# Stand-in for the attention interface; `match` finds `o_proj` (omega) from
# the source as the Linear producing the returned value.
attn_output = expanded.sum() + q.sum() + k_pe.sum()
attn_output = self.omega(attn_output)
return attn_output
def test_discovers_modules_under_arbitrary_names():
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
names, and `match` still locates each projection by dataflow."""
with torch.device("meta"):
module = RenamedMLA()
fuser = MLAFuser.match(trace(module), module)
assert isinstance(fuser, MLAFuser)
assert fuser.q_a_proj_name is None
assert fuser.q_proj_name == "alpha"
assert fuser.kv_a_proj_name == "beta"
assert fuser.kv_a_layernorm_name == "gamma"
assert fuser.kv_b_proj_name == "delta"
assert fuser.o_proj_name == "omega"
class GLU(nn.Module):
"""A gated MLP: matches the GLU fuser, never MLA."""
def __init__(self):
super().__init__()
self.gate = nn.Linear(16, 16)
self.up = nn.Linear(16, 16)
def forward(self, x):
return self.up(F.silu(self.gate(x)))
def test_non_mla_is_not_matched():
with torch.device("meta"):
assert get_fuser(GLU(), MLAFuser) is None
@@ -1,611 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's MoE fuser."""
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fusers import MoEBlockFuser
from .test_linear import GLUMLP
class TopKRouter(nn.Module):
"""HF v5 top-k router: `linear -> softmax -> topk (-> renorm)`."""
def __init__(self, num_experts=8, hidden=16, top_k=2, sigmoid=False):
super().__init__()
self.top_k = top_k
self.sigmoid = sigmoid
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits) if self.sigmoid else F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class SelectionScoredRouter(TopKRouter):
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight).float()
value, index = torch.topk(logits, self.top_k, dim=-1)
value = (
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
).type_as(hidden_states)
return logits, value, index
class EpsilonRenormRouter(TopKRouter):
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits)
value, index = torch.topk(scores, self.top_k, dim=-1)
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
value /= denominator
return logits, value, index
class UntraceableRouter(TopKRouter):
"""Scores and selects, then loses the trace before the renormalization."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
if value.sum() > 0: # data-dependent branch: tracing stops here
value = value / value.sum(dim=-1, keepdim=True)
return logits, value, index
class ScaledRouter(TopKRouter):
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class Fp32Router(TopKRouter):
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
def forward(self, hidden_states):
logits = F.linear(
hidden_states.type(torch.float32), self.weight.type(torch.float32)
)
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value.to(hidden_states.dtype), index
class GroupedRouter(TopKRouter):
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = F.softmax(logits, dim=-1)
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
group_mask = torch.zeros_like(group_scores)
group_mask.scatter_(1, group_idx, 1)
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
scores = scores.masked_fill(~score_mask.bool(), 0.0)
value, index = torch.topk(scores, self.top_k, dim=-1)
value = value * 16.0
return logits, value, index
class CorrectionRouter(nn.Module):
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
def __init__(self, num_experts=8, hidden=16):
super().__init__()
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
self.register_buffer("e_score_correction_bias", torch.zeros(num_experts))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
scores = torch.sigmoid(logits) + self.e_score_correction_bias
_, index = torch.topk(scores, 2, dim=-1)
return logits, scores, index
class BiasedRouter(TopKRouter):
"""A valid top-k router but not `weight`-only (extra `bias` param) -> declined."""
def __init__(self):
super().__init__()
self.bias = nn.Parameter(torch.zeros(8))
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight) + self.bias
scores = F.softmax(logits, dim=-1)
value, index = torch.topk(scores, self.top_k, dim=-1)
return logits, value, index
class DisconnectedRouter(TopKRouter):
"""linear+softmax+top-k present but top-k ignores the logits -> not a router."""
def forward(self, hidden_states):
logits = F.linear(hidden_states, self.weight)
_ = F.softmax(logits, dim=-1) # scored, but not consumed by top-k
value, index = torch.topk(hidden_states, self.top_k, dim=-1)
return logits, value, index
class ExtraInputRouter(TopKRouter):
"""Router taking a block-owned correction bias as a second input (HYV3)."""
def forward(self, hidden_states, bias=0.0):
logits = F.linear(hidden_states, self.weight)
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
value = logits.sigmoid().gather(1, index)
return logits, value, index
class MoEExperts(nn.Module):
"""Packed experts (3D weights); only its name (`experts`) matters here."""
def __init__(self, num_experts=8, hidden=16, inter=32):
super().__init__()
self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * inter, hidden))
self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden, inter))
def forward(self, hidden_states, index, weights):
return hidden_states
class MoEBlock(nn.Module):
"""Single-tensor MoE block (Qwen3-style); subclasses override `_shared`."""
def __init__(self, router_cls=TopKRouter):
super().__init__()
self.experts = MoEExperts()
self.gate = router_cls()
def _shared(self, x, logits):
"""The term added to the experts' output (none for a plain block)."""
return None
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
logits, weights, index = self.gate(x)
out = self.experts(x, index, weights)
if (shared := self._shared(x, logits)) is not None:
out = out + shared
return out.reshape(hidden_states.shape)
class MoEBlockNoShared(MoEBlock):
"""No shared-expert child but a gate-derived add -> declined, not misread."""
def _shared(self, x, logits):
return logits.sum()
class MoEBlockShared(MoEBlock):
"""A block with a shared expert and its sigmoid gate (Qwen2-style)."""
def __init__(self):
super().__init__()
self.shared_expert = GLUMLP()
self.shared_expert_gate = nn.Linear(16, 1, bias=False)
def _shared(self, x, logits):
return torch.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x)
class MoEBlockSharedNoGate(MoEBlock):
"""A block with an ungated shared expert -> native, shared passed through."""
def __init__(self):
super().__init__()
self.shared_expert = GLUMLP()
def _shared(self, x, logits):
return self.shared_expert(x)
class MoEBlockTuple(MoEBlock):
"""A tuple-returning block (gpt-oss-style) -> must decline."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights), index
class MoEBlockTupleVar(MoEBlock):
"""Returns a name bound to a tuple, not a literal tuple -> must still decline."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
result = self.experts(x, index, weights), index
return result
class MoEBlockNestedTupleReturn(MoEBlock):
"""Tuple `return` in a nested helper; block returns one tensor -> still fuses."""
def forward(self, hidden_states):
def keep(a, b):
return a, b
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out, _ = keep(self.experts(x, index, weights), index)
return out.reshape(hidden_states.shape)
class PlainMLP(nn.Module):
"""A non-GLU FFN: `down(act(up(x)))`, no gating multiply."""
def __init__(self, hidden: int = 16, inter: int = 32):
super().__init__()
self.up_proj = nn.Linear(hidden, inter, bias=False)
self.down_proj = nn.Linear(inter, hidden, bias=False)
self.act_fn = nn.SiLU()
def forward(self, x):
return self.down_proj(self.act_fn(self.up_proj(x)))
class MoEBlockSharedNonGLU(MoEBlock):
"""A non-GLU shared expert -> detected by dataflow (no gate/up merge)."""
def __init__(self):
super().__init__()
self.shared_expert = PlainMLP()
def _shared(self, x, logits):
return self.shared_expert(x)
class MoEBlockUnaccounted(MoEBlock):
"""A weight-bearing child outside the fused dataflow (pre-router) -> declined."""
def __init__(self):
super().__init__()
self.extra = nn.Linear(16, 16, bias=False)
def forward(self, hidden_states):
x = self.extra(hidden_states.reshape(-1, hidden_states.shape[-1]))
_, weights, index = self.gate(x)
return self.experts(x, index, weights).reshape(hidden_states.shape)
class BufferScale(nn.Module):
"""A stateful child carrying only a buffer (no parameters)."""
def __init__(self, hidden: int = 16):
super().__init__()
self.register_buffer("scale", torch.ones(hidden))
def forward(self, x):
return x * self.scale
class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
"""Like `MoEBlockUnaccounted`, but the extra child holds only a buffer."""
def __init__(self):
super().__init__()
self.extra = BufferScale()
class MoEBlockExpertsKeywords(MoEBlock):
"""Passes the experts' routing by keyword -> still fuses."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, weights=weights, index=index)
class ScaledExperts(MoEExperts):
def forward(self, hidden_states, index, weights, scale=1.0):
return hidden_states * scale
class MoEBlockExpertsExtraArg(MoEBlock):
"""Passes the experts an extra argument the fused forward would drop."""
def __init__(self):
super().__init__()
self.experts = ScaledExperts()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights, scale=2.0)
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
def forward(self, hidden_states):
_, weights, index = self.gate(hidden_states)
x = hidden_states.view(-1, hidden_states.shape[-1])
out = self.experts(x, index, weights).view(*hidden_states.shape)
return out + self.shared_expert(hidden_states)
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
def forward(self, hidden_states):
return super().forward(hidden_states) / 2
class MoEBlockScaled(MoEBlockSharedNoGate):
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
"""An unrecognized router with a scaled experts' output."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
"""An unrecognized router with a shared expert."""
def __init__(self):
super().__init__()
self.gate = BiasedRouter()
class MoEBlockGateExtraInput(MoEBlock):
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
def __init__(self):
super().__init__(ExtraInputRouter)
self.register_buffer("e_score_correction_bias", torch.zeros(8))
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x, self.e_score_correction_bias)
return self.experts(x, index, weights)
class MoEBlockCastCombine(MoEBlockSharedNoGate):
"""Combines the outputs in fp32 (HYV3)."""
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
return out.to(x.dtype)
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
def __init__(self):
super().__init__()
self.fc1_latent_proj = nn.Identity()
self.fc2_latent_proj = nn.Identity()
def forward(self, hidden_states):
x = hidden_states.reshape(-1, hidden_states.shape[-1])
_, weights, index = self.gate(x)
out = self.fc2_latent_proj(
self.experts(self.fc1_latent_proj(x), index, weights)
)
return out + self.shared_expert(x)
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_detects_router(sigmoid):
with torch.device("meta"):
block = MoEBlock(lambda: TopKRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.shared_name is None and fuser.shared_gate_name is None
@pytest.mark.parametrize("sigmoid", [False, True])
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
"""Scoring the selection rather than all logits still allows the fuser to match."""
with torch.device("meta"):
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.gate_name == "gate"
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
assert fuser.renormalize is (not sigmoid)
assert fuser.router_dtype == torch.float32
def test_moe_fuser_matches_scaled_router():
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
with torch.device("meta"):
block = MoEBlock(ScaledRouter)
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
@pytest.mark.parametrize(
"router_cls,expected",
[
(TopKRouter, True), # divides by the sum of the selection
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
(ScaledRouter, False), # scales the weights instead of renormalizing
(UntraceableRouter, None), # trace stops before it could say
],
)
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
"""The router's graph, not the config, decides whether vLLM renormalizes: it
is the code being replaced, and it already bakes in `norm_topk_prob` because
the flag is a Python bool that fx resolves at trace time. Absence of the
division only means "no" on a complete trace."""
with torch.device("meta"):
block = MoEBlock(router_cls)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.renormalize is expected
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
with torch.device("meta"):
block = MoEBlock(GroupedRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "softmax"
def test_moe_fuser_matches_correction_router():
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
with torch.device("meta"):
block = MoEBlock(CorrectionRouter)
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.scoring_func == "sigmoid"
def test_moe_fuser_reads_router_dtype_from_the_gate():
"""A router that computes its logits in fp32 must keep routing in fp32 when
rebuilt, even though no config field names the dtype. The cast back to the
activation dtype after the top-k must not be mistaken for the routing dtype."""
with torch.device("meta"):
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
assert fp32.router_dtype == torch.float32
assert default.router_dtype is None
def test_moe_fuser_detects_shared_experts():
with torch.device("meta"):
block = MoEBlockShared()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name == "shared_expert_gate"
def test_moe_fuser_shared_without_gate():
with torch.device("meta"):
block = MoEBlockSharedNoGate()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name is None
def test_moe_fuser_detects_non_glu_shared_expert():
with torch.device("meta"):
block = MoEBlockSharedNonGLU()
fuser = MoEBlockFuser.match(block, "experts")
assert isinstance(fuser, MoEBlockFuser)
# Recognised by dataflow (added to the experts' output), though not a GLU.
assert fuser.shared_name == "shared_expert"
assert fuser.shared_gate_name is None
@pytest.mark.parametrize(
"block_cls",
[
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
MoEBlockNoShared, # gate-derived term added to the experts' output
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
MoEBlockGateExtraInput, # gate input the fused forward drops
],
)
def test_moe_fuser_declines_unsupported(block_cls):
with torch.device("meta"):
block = block_cls()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_ignores_nested_returns():
"""A tuple `return` inside a nested helper must not decline a block whose own
forward returns a single tensor."""
with torch.device("meta"):
block = MoEBlockNestedTupleReturn()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_router_requires_connected_dataflow():
"""A gate with linear + softmax + top-k present but not wired as a router
(top-k selects over the input, not the scored logits) is not detected."""
with torch.device("meta"):
block = MoEBlock(DisconnectedRouter)
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_unrecognized_router_moves_shared_expert():
"""Without vLLM routing, the shared expert still moves to vLLM."""
with torch.device("meta"):
block = MoEBlockSharedBiasedRouter()
fuser = MoEBlockFuser.match(block, "experts")
assert fuser.scoring_func is None
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize(
("block_cls", "scoring_func"),
[
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
(MoEBlock, "softmax"), # no shared output to divide
],
)
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
assert fuser.scoring_func == scoring_func
def test_moe_fuser_scaled_output_needs_vllm_routing():
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
with torch.device("meta"):
block = MoEBlockScaledBiasedRouter()
assert MoEBlockFuser.match(block, "experts") is None
def test_moe_fuser_matches_experts_called_by_keyword():
with torch.device("meta"):
block = MoEBlockExpertsKeywords()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
def test_moe_fuser_scaled_output_with_vllm_routing():
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
with torch.device("meta"):
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
assert fuser.scoring_func == "softmax"
assert fuser.shared_name == "shared_expert"
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
with torch.device("meta"):
fuser = MoEBlockFuser.match(block_cls(), "experts")
assert fuser.shared_name == "shared_expert"
@@ -1,409 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers modeling backend's RMSNorm fuser."""
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers import RMSNormFuser
class RMSNorm(nn.Module):
"""The canonical HF RMSNorm: `weight * x * rsqrt(mean(x**2) + eps)`."""
def __init__(self, hidden: int = 16, eps: float = 1e-5, weight: bool = True):
super().__init__()
if weight:
self.weight = nn.Parameter(torch.ones(hidden))
self.variance_epsilon = eps
def _rms(self, x):
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon)
def forward(self, x):
return self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
class GemmaRMSNorm(RMSNorm):
"""Zero-centered weight: `(1 + weight) * normalized`."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
self.weight = nn.Parameter(torch.zeros(hidden))
def forward(self, x):
return (1.0 + self.weight) * self._rms(x.to(torch.float32)).to(x.dtype)
class WeightlessRMSNorm(RMSNorm):
"""No scale parameter (e.g. Gemma3n `with_scale=False`)."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps, weight=False)
def forward(self, x):
return self._rms(x.to(torch.float32)).to(x.dtype)
class Gemma4RMSNorm(RMSNorm):
"""`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`).
Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep
Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not*
zero-centered, so this fuses to a plain `RMSNorm`.
"""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(mean_squared, -0.5)
def forward(self, x):
return (self._rms(x.float()) * self.weight.float()).type_as(x)
class PowOperatorRMSNorm(Gemma4RMSNorm):
"""The `v ** -0.5` operator spelling of the same reciprocal square root."""
def _rms(self, x):
return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5
class KwargPowRMSNorm(Gemma4RMSNorm):
"""The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt."""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(mean_squared, exponent=-0.5)
class KwargBasePowRMSNorm(Gemma4RMSNorm):
"""The fully functional spelling `torch.pow(input=v, exponent=-0.5)`."""
def _rms(self, x):
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
return x * torch.pow(input=mean_squared, exponent=-0.5)
class WeightlessGemma4RMSNorm(Gemma4RMSNorm):
"""Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps, weight=False)
def forward(self, x):
return self._rms(x.float()).type_as(x)
class LayerNorm(RMSNorm):
"""An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast)."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
def forward(self, x):
return self.weight * self._rms(x)
class NamedEpsRMSNorm(nn.Module):
"""Holds eps under `attr`, with no `variance_epsilon` to find instead."""
attr = "eps"
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
setattr(self, self.attr, eps)
def forward(self, x):
eps = getattr(self, self.attr)
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps))
class ToleranceRMSNorm(NamedEpsRMSNorm):
"""Names eps something other than `eps`, so only its value identifies it."""
attr = "tolerance"
class LiteralEpsRMSNorm(RMSNorm):
"""eps is a literal in the source, and an unrelated attribute happens to
hold the same value: matching on the value alone would bind to it."""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__(hidden, eps)
self.epsilon_lookalike = 1e-4
def forward(self, x):
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4))
class AmbiguousEpsRMSNorm(nn.Module):
"""Two attributes hold the eps value, and the forward reads the second, so
their order cannot pick it."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
self.decoy_eps = eps
self.variance_epsilon = eps
def forward(self, x):
var = x.pow(2).mean(-1, True)
return self.weight * (x * torch.rsqrt(var + self.variance_epsilon))
class NotAnRMSNorm(RMSNorm):
"""Mean-subtracting LayerNorm-like math -> not an RMSNorm."""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__(hidden, eps)
def forward(self, x):
x = x - x.mean(-1, keepdim=True)
variance = x.var(-1, keepdim=True)
return self.weight * x / torch.sqrt(variance + self.variance_epsilon)
class GatedRMSNorm(RMSNorm):
"""Second input and tail compute -> not an RMSNorm."""
def forward(self, x, gate=None):
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
return normed * F.silu(gate)
class GatedFusedRMSNorm(nn.Module):
"""Same as GatedRMSNorm, but built on the fused `rms_norm` op -> not an RMSNorm."""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden))
self.eps = eps
def forward(self, x, gate=None):
return F.rms_norm(x, (x.shape[-1],), self.weight, self.eps) * F.silu(gate)
class UntraceableGatedRMSNorm(RMSNorm):
"""Tracer can't see tail compute in forward, but still has a second input (gate)."""
def forward(self, x, gate=None):
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
if gate.sum() > 0: # untraceable -> partial graph, no visible tail
normed = normed * F.silu(gate)
return normed
@pytest.mark.parametrize(
"cls,eps,zero_centered",
[
(RMSNorm, 1e-5, False),
(GemmaRMSNorm, 1e-6, True),
(WeightlessRMSNorm, 1e-6, False),
(Gemma4RMSNorm, 1e-6, False),
(PowOperatorRMSNorm, 1e-6, False),
(KwargPowRMSNorm, 1e-6, False),
(KwargBasePowRMSNorm, 1e-6, False),
(WeightlessGemma4RMSNorm, 1e-6, False),
(LayerNorm, 1e-6, False),
(torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op
],
)
def test_detects_rms_norm_variants(cls, eps, zero_centered):
with torch.device("meta"):
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert fuser.zero_centered == zero_centered
@pytest.mark.parametrize("cls", [NotAnRMSNorm, nn.LayerNorm, nn.SiLU])
def test_non_rms_norms_are_not_matched(cls):
with torch.device("meta"):
module = cls(16) if cls is nn.LayerNorm else cls()
assert get_fuser(module, RMSNormFuser) is None
@pytest.mark.parametrize(
"cls", [GatedRMSNorm, GatedFusedRMSNorm, UntraceableGatedRMSNorm]
)
def test_gated_rms_norm_is_not_fused(cls):
with torch.device("meta"):
assert get_fuser(cls(), RMSNormFuser) is None
@pytest.mark.parametrize(
"cls,expected,zero_centered",
[
(RMSNorm, "RMSNorm", False),
(GemmaRMSNorm, "GemmaRMSNorm", True),
(WeightlessRMSNorm, "RMSNorm", False),
(Gemma4RMSNorm, "RMSNorm", False),
(KwargPowRMSNorm, "RMSNorm", False),
(KwargBasePowRMSNorm, "RMSNorm", False),
(WeightlessGemma4RMSNorm, "RMSNorm", False),
],
)
def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config):
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
module = cls()
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
)
types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm}
assert isinstance(built, types_by_name[expected])
assert isinstance(built, TPAwareNormMixin)
assert built.variance_epsilon == module.variance_epsilon
# The weight states the hidden size (and is what the TP check reads); a norm
# without one has none to state.
weight = getattr(module, "weight", None)
assert isinstance(built.weight, nn.Parameter) == (weight is not None)
assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0)
@pytest.mark.parametrize(
"cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm]
)
def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config):
"""Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square
root: the fused norm must reproduce the unfused forward, not just replace it.
`pow` and `rsqrt` take different paths to the same value, so compare at float32
tolerance rather than bit-for-bit."""
torch.manual_seed(0)
module = cls(16)
with torch.no_grad():
module.weight.copy_(torch.randn(16))
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
with torch.no_grad():
built.weight.copy_(module.weight)
x = torch.randn(4, 16)
torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6)
def test_weightless_norm_has_no_hidden_size(default_vllm_config):
"""A weightless norm states no hidden size, and the model's (LM) hidden size
would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes
head_dim, so sizing it at hidden_size made the TP gather reject its input.
It fuses with none, and so normalizes the width it is given at any TP size,
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
x = torch.randn(4, 72)
torch.testing.assert_close(built(x), module(x))
def test_fused_rms_norm_op_default_eps(default_vllm_config):
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
fuser = get_fuser(module, RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
built = fuser.fuse(module, "norm", vllm_config)
assert isinstance(built, VLLMRMSNorm)
assert built.variance_epsilon == torch.finfo(torch.float32).eps
def test_eps_is_derived_per_instance(default_vllm_config):
"""Two instances of the same norm class with different eps must fuse to their
own eps: the type-cached fuser holds only structure, not this value."""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
"""The eps attribute is identified by holding the traced value, so a norm
stays per-instance correct whatever it names it."""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == cls.attr
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
"""A literal eps is recognised as coming from no attribute, even when one
holds the same value, and is taken from the traced source instead."""
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert fuser.eps_attr is None
assert built.variance_epsilon == 1e-4
assert "does not hold its eps" in caplog.text
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
"""When several attributes hold the eps value, the one the forward actually
reads is identified, and they are all left as they were found."""
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == "variance_epsilon"
assert vars(module) == before
other = AmbiguousEpsRMSNorm(16, eps=1e-5)
other.decoy_eps = 1e-3
built = fuser.fuse(other, "norm", default_vllm_config)
assert built.variance_epsilon == 1e-5
def test_fused_norm_is_gather_capable(default_vllm_config):
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
plumbing. A full-width input skips the gather and equals a plain norm."""
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers import rms_norm
torch.manual_seed(0)
x = torch.randn(4, 16)
for gathered_cls, plain_cls in [
(rms_norm.TPAwareRMSNorm, RMSNorm),
(rms_norm.TPAwareGemmaRMSNorm, GemmaRMSNorm),
]:
gathered = gathered_cls(hidden_size=16, eps=1e-6)
assert isinstance(gathered, rms_norm.TPAwareNormMixin)
plain = plain_cls(hidden_size=16, eps=1e-6)
with torch.no_grad():
weight = torch.randn(16)
gathered.weight.copy_(weight)
plain.weight.copy_(weight)
torch.testing.assert_close(gathered(x), plain(x))
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
"""A sharded input (narrower than the full-width weight) that does not tile
the weight evenly across ranks is rejected before any collective."""
from vllm.model_executor.models.transformers.fusers import rms_norm
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
norm.tp_size = 2 # emulate TP=2 without a real process group
with pytest.raises(ValueError, match="does not tile it evenly"):
norm(torch.randn(2, 3)) # 3 * 2 != 8
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for skipping dynamic rope's device sync where it can never rescale."""
from types import SimpleNamespace
import pytest
import torch
from transformers import HunYuanDenseV1Config, LlamaConfig
from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import (
HunYuanDenseV1RotaryEmbedding,
)
from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding
from vllm.model_executor.models.transformers.utils import (
can_enable_torch_compile,
disable_noop_dynamic_rope,
)
MAX_POSITION_EMBEDDINGS = 128
CONFIG_KWARGS = dict(
hidden_size=64,
num_attention_heads=4,
num_hidden_layers=1,
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
)
DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4}
def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace:
model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len)
return SimpleNamespace(model_config=model_config)
@pytest.mark.parametrize(
"rope_parameters,max_model_len,compilable",
[
(DYNAMIC, MAX_POSITION_EMBEDDINGS, True),
(DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False),
# The longrope layer type still syncs
(
{"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}},
MAX_POSITION_EMBEDDINGS,
False,
),
],
)
def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable):
"""Dynamic rope only blocks compilation when it can still rescale."""
config = SimpleNamespace(
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
rope_parameters=rope_parameters,
)
config.get_text_config = lambda: config
vllm_config = make_vllm_config(config, max_model_len)
assert can_enable_torch_compile(vllm_config) is compilable
@pytest.mark.parametrize(
"config,rotary_cls",
[
(LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding),
# Hunyuan's dynamic init also applies `alpha`, which must be kept
(
HunYuanDenseV1Config(
**CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3}
),
HunYuanDenseV1RotaryEmbedding,
),
],
)
def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls):
"""The update no longer syncs and the frequencies are unchanged."""
inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :]
rotary_emb = rotary_cls(config)
expected = rotary_emb(*inputs)
vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS)
disable_noop_dynamic_rope(rotary_emb, vllm_config)
def no_sync(*args, **kwargs):
raise AssertionError("dynamic rope update still runs")
monkeypatch.setattr(torch, "max", no_sync)
actual = rotary_emb(*inputs)
assert all(torch.equal(e, a) for e, a in zip(expected, actual))
@@ -1,80 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Transformers backend's hw-agnostic layer resolution.
`layers._resolve` imports a layer symbol from
`vllm.model_executor.hw_agnostic.layers.<module>` when `VLLM_USE_HW_AGNOSTIC`
is set and the symbol exists, and otherwise falls back to
`vllm.model_executor.layers.<module>`. These tests pin that contract and the
logging that reports which source was used.
CPU-only: no engine is built. The end-to-end counterpart that serves a model
through `vllm_runner` lives in `test_layer_registry_e2e.py` (GPU job).
"""
import logging
import sys
import types
import pytest
from vllm.model_executor.models.transformers import layers
HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm"
@pytest.fixture
def fake_hw_layernorm(monkeypatch):
"""Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`.
A `SimpleNamespace` stands in for the module: `importlib.import_module`
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
attributes are set at construction (no `ModuleType` attribute-set that mypy
rejects, no constant `setattr` that ruff rejects)."""
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
monkeypatch.setitem(sys.modules, HW_MODULE, module)
return module
def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm):
"""Disabled: the vLLM class is used even if a hw-agnostic one exists."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm
def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog_vllm):
"""Enabled and available: the hw-agnostic class is used and logged."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
with caplog_vllm.at_level(logging.INFO):
resolved = layers._resolve("layernorm", "RMSNorm")
assert resolved is fake_hw_layernorm.RMSNorm
assert "Using hw-agnostic layer: RMSNorm" in caplog_vllm.text
def test_falls_back_when_symbol_missing(monkeypatch, caplog_vllm):
"""Enabled but the symbol is not ported: fall back to vLLM and warn."""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
# A hw-agnostic module without the requested attribute triggers fallback.
empty = types.ModuleType(HW_MODULE)
monkeypatch.setitem(sys.modules, HW_MODULE, empty)
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
with caplog_vllm.at_level(logging.WARNING):
resolved = layers._resolve("layernorm", "RMSNorm")
assert resolved is VllmRMSNorm
assert "falling back to default" in caplog_vllm.text
def test_act_and_mul_falls_back_for_unknown_activation(
monkeypatch, default_vllm_config
):
"""An activation with no hw-agnostic equivalent falls back to vLLM's.
`default_vllm_config` supplies the config context the CustomOp needs.
"""
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
from vllm.model_executor.layers.activation import GeluAndMul
assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul)
-14
View File
@@ -418,20 +418,6 @@ def test_allowed_image_formats_exclude_external_renderer_plugins():
assert denied.isdisjoint(ALLOWED_IMAGE_FORMATS)
def test_kimi_fused_vision_rejects_postscript_payload():
"""Kimi fused vision must use the same image format allowlist."""
from vllm.transformers_utils.processors.kimi_k25_vision_fused import _to_pil
eps = b"""%!PS-Adobe-3.0 EPSF-3.0
%%BoundingBox: 0 0 10 10
%%EndComments
0 0 moveto 10 10 lineto stroke
showpage
"""
with pytest.raises(Image.UnidentifiedImageError):
_to_pil(eps)
def test_open_image_does_not_accept_format_override():
"""The allowlist helper must not expose a formats override."""
assert "formats" not in inspect.signature(open_image).parameters
-172
View File
@@ -1,172 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Citation source resolution and the streamed citation wire shape."""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
from vllm.parser.cohere_command import (
_melody_citations_to_vllm,
_melody_sources_to_vllm,
)
from .utils import make_parser, stream_parser
class TestMelodySourceResolution:
"""Pin the parser-side source resolution (see
``_melody_sources_to_vllm``). The parser receives melody's numeric
``(tool_call_index, tool_result_indices)`` addressing and resolves
it against a ``position_to_source`` map handed in by
``CohereServingChatV2._apply_cohere_template_kwargs`` -- the same
map that would otherwise live in the serving layer's resolver.
"""
@staticmethod
def _fake_melody_source(bucket: int, indices: list[int]) -> Any:
return SimpleNamespace(tool_call_index=bucket, tool_result_indices=indices)
def test_multi_index_source_fans_out(self):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
position_map: dict[tuple[int, int], CitationSource] = {
(0, 0): CitationSource(type="document", id="d0", document={"id": "d0"}),
(0, 1): CitationSource(type="document", id="d1", document={"id": "d1"}),
}
raw = [self._fake_melody_source(0, [0, 1])]
out = _melody_sources_to_vllm(raw, position_map)
assert [s.id for s in out] == ["d0", "d1"]
# Verify type / payload were plumbed through, not just the id.
assert out[0].type == "document"
assert out[0].document == {"id": "d0"}
def test_unresolvable_position_skipped(self):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
position_map: dict[tuple[int, int], CitationSource] = {
(0, 0): CitationSource(type="document", id="d0"),
}
raw = [self._fake_melody_source(9, [0])]
assert _melody_sources_to_vllm(raw, position_map) == []
def test_missing_position_map_drops_all_sources(self):
# A parser instance without a position map (parser wired
# outside of ``CohereServingChatV2``) can't attribute anything,
# so every source is dropped. Callers downstream will see the
# citation with empty ``sources`` and drop it entirely.
raw = [self._fake_melody_source(0, [0])]
assert _melody_sources_to_vllm(raw, None) == []
def test_citations_pass_through_is_thinking_tag(self):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
position_map: dict[tuple[int, int], CitationSource] = {
(0, 0): CitationSource(type="document", id="d0"),
}
raw = [
SimpleNamespace(
start_index=0,
end_index=5,
text="hello",
is_thinking=True,
sources=[self._fake_melody_source(0, [0])],
)
]
out = _melody_citations_to_vllm(raw, position_map)
assert out is not None
assert out[0].type == "THINKING_CONTENT"
assert out[0].sources[0].id == "d0"
class TestParserStreamingEndToEnd:
"""End-to-end streaming shape: raw model output text is tokenized
with the mock byte-level tokenizer and fed one token at a time
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
We collect the ordered sequence of ``DeltaMessage``s the parser
hands back -- i.e. what the client would receive on the wire --
and pin the exact shape.
Combines the three flavors that reach ``delta``: a thinking block
(populates ``reasoning``), a text block (populates ``content``),
and a resolved citation (populates ``citations``) referencing
``(start, end)`` offsets into the already-streamed content.
"""
_MODEL_OUTPUT = (
"<|START_THINKING|>Let me check.<|END_THINKING|>"
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
)
def test_full_wire_stream_shape(self, tokenizer, request_obj):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
# installs on a request whose tool_call_index=0 tool result at
# index 1 is the "France" document.
position_to_source = {
(0, 1): CitationSource(
type="document",
id="doc-paris",
document={"id": "doc-paris", "title": "France"},
),
}
parser = make_parser(
tokenizer,
"cohere_command3",
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
)
deltas = stream_parser(parser, request_obj, tokenizer, self._MODEL_OUTPUT)
# -- 1. Reasoning and content stream separately and reach
# the wire only as their real payload. Framing tokens
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
# state machine and never produce a delta.
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
content_stream = [d.content for d in deltas if d.content is not None]
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
assert "".join(reasoning_stream) == "Let me check."
assert "".join(content_stream) == "The capital is Paris."
# -- 2. Exactly one citation reaches the wire, resolved to the
# source from ``position_to_source`` (not the raw
# ``(bucket, idx)`` coordinates).
assert len(citation_deltas) == 1
(cite_delta,) = citation_deltas
assert cite_delta.citations is not None
assert len(cite_delta.citations) == 1
cite = cite_delta.citations[0]
assert cite.text == "Paris"
# Char offsets are into the accumulated ``content``: "The
# capital is " is 15 chars, "Paris" is 5.
assert cite.start == 15
assert cite.end == 20
assert len(cite.sources) == 1
assert cite.sources[0].id == "doc-paris"
assert cite.sources[0].type == "document"
# -- 3. The citation's ``(start, end)`` offsets point into
# bytes that were already emitted as content-deltas earlier
# in the stream -- i.e. the citation is anchored to real
# already-shipped content.
cite_delta_idx = deltas.index(cite_delta)
earlier_content = "".join(
d.content for d in deltas[:cite_delta_idx] if d.content is not None
)
assert earlier_content[cite.start : cite.end] == "Paris"
# Everything after the citation delta is the trailing period.
later_content = "".join(
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
)
assert later_content == "."
# -- 4. Reasoning and content are never mixed on the same
# delta: the parser flips modes on the ``<|END_THINKING|>``
# boundary and never emits a delta with both fields set.
for d in deltas:
assert not (d.reasoning is not None and d.content is not None)
-476
View File
@@ -1,476 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``adjust_request``: folding tools and JSON schemas into Cohere structural tags."""
from __future__ import annotations
import json
from collections import UserDict
from types import SimpleNamespace
import pytest
from pydantic import ValidationError
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.generate.base.protocol import (
JsonSchemaResponseFormat,
ResponseFormat,
StructuralTagResponseFormat,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.parser.cohere_command import (
CohereCommandParser,
CohereNormalizedTool,
_has_effective_tools,
_response_format_type,
_schema_dict_from_structured_outputs,
collect_tool_schema,
convert_schema_to_structural_tags,
)
from vllm.sampling_params import StructuredOutputsParams
from .utils import MockCohereTokenizer, make_parser
SCHEMA_A = {"type": "object", "properties": {"a": {"type": "string"}}}
SCHEMA_B = {"type": "object", "properties": {"b": {"type": "number"}}}
GET_WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
NESTED_DEFS_PARAMETERS = {
"$defs": {
"Location": {
"type": "object",
"properties": {
"city": {"type": "string"},
"country": {"type": "string"},
},
"required": ["city"],
}
},
"type": "object",
"properties": {"location": {"$ref": "#/$defs/Location"}},
"required": ["location"],
}
NESTED_DEFS_TOOL = {
"type": "function",
"function": {"name": "get_weather", "parameters": NESTED_DEFS_PARAMETERS},
}
RECURSIVE_DEFS_PARAMETERS = {
"$defs": {
"Finding": {
"type": "object",
"properties": {
"note": {"type": "string"},
"related": {"type": "array", "items": {"$ref": "#/$defs/Finding"}},
},
}
},
"type": "object",
"properties": {"finding": {"$ref": "#/$defs/Finding"}},
}
DRAFT07_DEFINITIONS_PARAMETERS = {
"definitions": {
"Location": {
"type": "object",
"properties": {"city": {"type": "string"}},
}
},
"type": "object",
"properties": {"location": {"$ref": "#/definitions/Location"}},
}
VALID_STRUCTURAL_TAG = {
"type": "structural_tag",
"format": {
"type": "triggered_tags",
"tags": [
{
"begin": "<tool>",
"content": {"type": "any_text"},
"end": "</tool>",
}
],
"triggers": ["<tool>"],
},
}
def _model_config(arch: str) -> SimpleNamespace:
return SimpleNamespace(
architecture=arch,
architectures=[arch],
hf_text_config=SimpleNamespace(architectures=[arch]),
)
def _make_chat_request(**kwargs) -> ChatCompletionRequest:
data = {"messages": [{"role": "user", "content": "hi"}], "model": "m"}
data.update(kwargs)
return ChatCompletionRequest.model_validate(data)
def _first_json_schema(tag_json: str) -> dict | None:
outer = json.loads(tag_json)
for t in (outer.get("format") or {}).get("tags") or []:
c = t.get("content") or {}
if c.get("type") == "json_schema":
js = c.get("json_schema")
return js if isinstance(js, dict) else None
return None
def _content_types(tag_json: str) -> set[str]:
outer = json.loads(tag_json)
out: set[str] = set()
for t in (outer.get("format") or {}).get("tags") or []:
ty = (t.get("content") or {}).get("type")
if isinstance(ty, str):
out.add(ty)
return out
@pytest.fixture(scope="module")
def parser(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
"""Parser configured with a supported Cohere architecture."""
return make_parser(
tokenizer, "cohere_command4", model_config=_model_config("Cohere2ForCausalLM")
)
@pytest.fixture(scope="module")
def parser_no_model_config(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
"""Parser with no ``model_config`` (cannot resolve architecture)."""
return make_parser(tokenizer, "cohere_command4")
@pytest.fixture(scope="module")
def parser_unsupported_arch(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
"""Parser configured with an architecture that has no structural tag style."""
return make_parser(
tokenizer, "cohere_command4", model_config=_model_config("LlamaForCausalLM")
)
class TestAdjustRequestPassthrough:
def test_structured_outputs_structural_tag_not_modified(self, parser) -> None:
tag = json.dumps(VALID_STRUCTURAL_TAG)
r = _make_chat_request(structured_outputs={"structural_tag": tag})
o = parser.adjust_request(r)
assert o.structured_outputs.structural_tag == tag
def test_response_format_structural_tag_short_circuit(self, parser) -> None:
# ``ChatCompletionRequest`` validates ``response_format`` as a union;
# bare ``{"type": "structural_tag"}`` is invalid (use pydantic model).
rf = StructuralTagResponseFormat(
type="structural_tag",
format=VALID_STRUCTURAL_TAG["format"],
)
r = _make_chat_request(response_format=rf)
o = parser.adjust_request(r)
assert _response_format_type(o.response_format) == "structural_tag"
assert o.structured_outputs is None
class TestAdjustRequestNoOp:
def test_no_schema_no_tools(self, parser) -> None:
o = parser.adjust_request(_make_chat_request())
assert o.structured_outputs is None
assert o.response_format is None
def test_no_model_config(self, parser_no_model_config) -> None:
inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
)
o = parser_no_model_config.adjust_request(r)
assert o.response_format is not None
assert o.structured_outputs is None
class TestAdjustRequestUnsupportedArchitecture:
def test_json_schema_raises(self, parser_unsupported_arch) -> None:
inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
)
with pytest.raises(ValueError, match="does not support"):
parser_unsupported_arch.adjust_request(r)
class TestAdjustRequestFoldFromResponseFormat:
@pytest.mark.parametrize(
"response_format, expected_schema",
[
pytest.param(
ResponseFormat(
type="json_schema",
json_schema=JsonSchemaResponseFormat(
name="n", json_schema=SCHEMA_A
),
),
SCHEMA_A,
id="json_schema_pydantic",
),
pytest.param(
{
"type": "json_schema",
"json_schema": {"name": "n", "schema": SCHEMA_A},
},
SCHEMA_A,
id="json_schema_dict",
),
pytest.param(
{"type": "json_object"},
{"type": "object"},
id="json_object",
),
],
)
def test_response_format_cleared(
self, parser, response_format, expected_schema
) -> None:
r = _make_chat_request(response_format=response_format)
o = parser.adjust_request(r)
assert o.response_format is None
assert (
_first_json_schema(o.structured_outputs.structural_tag) == expected_schema
)
class TestHasEffectiveTools:
@pytest.mark.parametrize(
"tools, expected",
[
pytest.param(None, False, id="none"),
pytest.param([], False, id="empty_list"),
pytest.param(" ", False, id="blank_str"),
pytest.param(
[{"type": "function", "function": {"name": "f"}}],
True,
id="non_empty_list",
),
pytest.param('{"x": 1}', True, id="non_empty_str"),
],
)
def test_has_effective_tools(self, tools, expected) -> None:
assert _has_effective_tools(tools) is expected
def test_convert_schema_json_only_with_empty_tools_list(self) -> None:
tag = convert_schema_to_structural_tags(
schema=SCHEMA_B,
tools=[],
model_architecture="Cohere2ForCausalLM",
)
assert tag is not None
assert _first_json_schema(tag) == SCHEMA_B
class TestAdjustRequestFoldFromStructuredOutputs:
@pytest.mark.parametrize(
"structured_outputs, expected_schema",
[
pytest.param({"json": SCHEMA_B}, SCHEMA_B, id="json_dict"),
pytest.param({"json": json.dumps(SCHEMA_B)}, SCHEMA_B, id="json_string"),
pytest.param(
{"json_object": True}, {"type": "object"}, id="json_object_flag"
),
pytest.param(
StructuredOutputsParams(json=SCHEMA_B),
SCHEMA_B,
id="structured_outputs_dataclass",
),
pytest.param(
{"json": {"name": "n", "schema": SCHEMA_A}},
SCHEMA_A,
id="openai_wrapper_dict_unwrapped",
),
],
)
def test_structured_outputs_folded(
self, parser, structured_outputs, expected_schema
) -> None:
o = parser.adjust_request(
_make_chat_request(structured_outputs=structured_outputs),
)
assert (
_first_json_schema(o.structured_outputs.structural_tag) == expected_schema
)
def test_responses_request_default_empty_tools(self, parser) -> None:
"""``ResponsesRequest.tools`` defaults to ``[]``, not ``None``."""
r = ResponsesRequest.model_validate(
{
"input": "hi",
"model": "m",
"structured_outputs": {"json": SCHEMA_B},
}
)
assert r.tools == []
o = parser.adjust_request(r)
assert _first_json_schema(o.structured_outputs.structural_tag) == SCHEMA_B
def test_json_userdict_mapping_unwrapped(self) -> None:
inner = {"type": "object", "properties": {"u": {"type": "number"}}}
so = StructuredOutputsParams(json=UserDict(inner))
assert _schema_dict_from_structured_outputs(so) == inner
@pytest.mark.parametrize(
"json_value, match",
[
pytest.param("{not json}", "valid JSON", id="invalid_json_string"),
pytest.param(
json.dumps(["a", "b"]), "JSON object", id="non_object_json_string"
),
pytest.param(" ", "empty", id="empty_json_string"),
],
)
def test_structured_outputs_json_string_raises(
self, parser, json_value, match
) -> None:
with pytest.raises(ValueError, match=match):
parser.adjust_request(
_make_chat_request(structured_outputs={"json": json_value}),
)
def test_deeply_nested_json_string_rejected(self, parser) -> None:
"""json.loads raises RecursionError on a string nested this deeply."""
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
with pytest.raises(VLLMValidationError, match="nested too deeply"):
parser.adjust_request(
_make_chat_request(structured_outputs={"json": schema}),
)
@pytest.mark.parametrize(
"construct",
[
pytest.param(
lambda: _make_chat_request(structured_outputs={"json": [1, 2, 3]}),
id="chat_completion_request",
),
pytest.param(
lambda: StructuredOutputsParams(json=[1, 2, 3]),
id="structured_outputs_params",
),
],
)
def test_json_wrong_type_raises(self, construct) -> None:
"""Non-str / non-dict ``json`` fails at Pydantic validation."""
with pytest.raises(ValidationError):
construct()
class TestAdjustRequestPrecedence:
def test_response_format_over_structured_outputs_json(self, parser) -> None:
s_rf = {"type": "object", "properties": {"rf": {"type": "string"}}}
s_so = {"type": "object", "properties": {"so": {"type": "number"}}}
inner = JsonSchemaResponseFormat(name="n", json_schema=s_rf)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
structured_outputs={"json": s_so},
)
o = parser.adjust_request(r)
assert _first_json_schema(o.structured_outputs.structural_tag) == s_rf
class TestAdjustRequestTextPlusStructuredOutputs:
def test_text_response_format_preserved(self, parser) -> None:
sch = {"type": "object", "properties": {"k": {"type": "string"}}}
r = _make_chat_request(
response_format=ResponseFormat(type="text"),
structured_outputs={"json": sch},
)
o = parser.adjust_request(r)
assert o.response_format is not None
assert o.response_format.type == "text"
assert _first_json_schema(o.structured_outputs.structural_tag) == sch
class TestAdjustRequestTools:
def test_tools_only_command_a_grammar(self, parser) -> None:
o = parser.adjust_request(
_make_chat_request(tools=[GET_WEATHER_TOOL], tool_choice="auto"),
)
assert "grammar" in _content_types(o.structured_outputs.structural_tag)
def test_tools_plus_json_schema_both_kinds(self, parser) -> None:
inner = JsonSchemaResponseFormat(
name="n",
json_schema={"type": "object", "properties": {"r": {"type": "string"}}},
)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
tools=[GET_WEATHER_TOOL],
tool_choice="auto",
)
o = parser.adjust_request(r)
types = _content_types(o.structured_outputs.structural_tag)
assert "grammar" in types
assert "json_schema" in types
class TestCollectToolSchemaDefs:
def test_nested_defs_tool_adjust_request(self, parser) -> None:
o = parser.adjust_request(
_make_chat_request(tools=[NESTED_DEFS_TOOL], tool_choice="auto"),
)
assert "grammar" in _content_types(o.structured_outputs.structural_tag)
@staticmethod
def _tool_calls_json(name: str, parameters: dict) -> str:
return json.dumps(
[{"tool_call_id": "0", "tool_name": name, "parameters": parameters}]
)
def test_recursive_defs(self) -> None:
grammar = collect_tool_schema(
[CohereNormalizedTool(name="report", parameters=RECURSIVE_DEFS_PARAMETERS)]
)
nested = {"note": "a", "related": [{"note": "b", "related": []}]}
assert _is_grammar_accept_string(
grammar, self._tool_calls_json("report", {"finding": nested})
)
bad = {"note": 1, "related": []}
assert not _is_grammar_accept_string(
grammar, self._tool_calls_json("report", {"finding": bad})
)
def test_draft07_definitions(self) -> None:
grammar = collect_tool_schema(
[
CohereNormalizedTool(
name="get_weather", parameters=DRAFT07_DEFINITIONS_PARAMETERS
)
]
)
assert _is_grammar_accept_string(
grammar,
self._tool_calls_json("get_weather", {"location": {"city": "Prague"}}),
)
assert not _is_grammar_accept_string(
grammar,
self._tool_calls_json("get_weather", {"location": {"city": 1}}),
)
def test_deeply_nested_parameters_rejected(self) -> None:
"""Tool schemas are compiled here, before request validation, and end up
inside an EBNF grammar that the nesting check cannot see later."""
parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200)
with pytest.raises(VLLMValidationError, match="nested too deeply"):
collect_tool_schema(
[CohereNormalizedTool(name="deep", parameters=parameters)]
)
def test_input_parameters_not_mutated(self) -> None:
params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS))
collect_tool_schema(
[CohereNormalizedTool(name="get_weather", parameters=params)]
)
assert params == NESTED_DEFS_PARAMETERS
@@ -1,211 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Replay tests for DelegatingParser with engine adapters.
Exercises DelegatingParser in engine-adapter mode to verify that delegated
routing produces correct output across chunk sizes.
See test_replay.py for tests that target engine parsers directly.
Parser discovery is automatic: any engine parser in ``registered_adapters``
that has both tool and reasoning adapters and a builder in
``trace_builder._BUILDERS`` is picked up with zero manual wiring.
"""
from __future__ import annotations
from typing import NamedTuple
import pytest
from pydantic import TypeAdapter
from tests.parser.engine.replay_harness import (
CHUNK_SIZES,
DUMMY_TOOLS,
MockTokenizer,
_test_request,
assert_no_terminal_leakage,
assert_parse_output,
collect_output,
make_mock_tokenizer,
parse_non_streaming,
replay_streaming,
)
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import DelegatingParser, Parser
from vllm.parser.engine import registered_adapters as _adapters_mod
from vllm.parser.engine.adapters import (
ParserEngineReasoningAdapter,
ParserEngineToolAdapter,
)
from vllm.parser.mistral import MistralParser
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
# ── Pairing discovery ────────────────────────────────────────────────
class _PairingInfo(NamedTuple):
parser_cls: type[Parser]
name: str
samples: tuple
def _discover_pairings() -> list[_PairingInfo]:
"""Discover valid delegating pairings from registered engine adapters.
Groups tool and reasoning adapters by their engine class, then builds
a DelegatingParser subclass for each engine that has both adapters
and a test builder.
"""
bare_tok = MockTokenizer(vocab={}, tokens=[])
engines: dict[type, dict[str, type]] = {}
for obj in vars(_adapters_mod).values():
if not isinstance(obj, type):
continue
if (
issubclass(obj, ParserEngineToolAdapter)
and obj is not ParserEngineToolAdapter
):
tool_adapter: type[ParserEngineToolAdapter] = obj
engines.setdefault(tool_adapter._parser_engine_cls, {})["tool"] = obj
elif (
issubclass(obj, ParserEngineReasoningAdapter)
and obj is not ParserEngineReasoningAdapter
):
reasoning_adapter: type[ParserEngineReasoningAdapter] = obj
engines.setdefault(reasoning_adapter._parser_engine_cls, {})[
"reasoning"
] = obj
found: list[_PairingInfo] = []
missing_builders: list[str] = []
for engine_cls, adapters in engines.items():
if "tool" not in adapters or "reasoning" not in adapters:
continue
if engine_cls is MistralParser:
# Mistral uses brace-balanced JSON tool args with no TOOL_END
# token, so it does not fit this TOOL_END-based replay harness.
# It is covered by tests/parser/mistral/ instead.
continue
cfg = engine_cls(bare_tok, None).parser_engine_config
if cfg.name not in _BUILDERS:
missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
continue
if cfg.name == "inkling":
# Inkling uses typed structural blocks and opts out of token-id
# terminal matching; combined-parser replay coverage lives in
# test_inkling.py.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal;
# its replay coverage lives in test_granite.py.
continue
parser_cls = type(
f"_Delegating{engine_cls.__name__}",
(DelegatingParser,),
{
"reasoning_parser_cls": adapters["reasoning"],
"tool_parser_cls": adapters["tool"],
},
)
found.append(
_PairingInfo(
parser_cls=parser_cls,
name=cfg.name,
samples=build_samples(cfg.name),
)
)
if missing_builders:
raise RuntimeError(
f"Engine adapters in registered_adapters have no test builder "
f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
f"Add a builder to _BUILDERS for each new parser."
)
found.sort(key=lambda p: p.name)
return found
_PAIRINGS = _discover_pairings()
_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples]
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
@pytest.mark.parametrize(
"parser_cls,sample",
_ALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else "",
)
def test_delegating_replay(parser_cls, sample, chunk_size):
tokenizer = make_mock_tokenizer(sample)
validated_tools = (
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
)
parser = parser_cls(
tokenizer,
validated_tools,
chat_template_kwargs=sample.chat_template_kwargs,
)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=chunk_size,
finished_on_last=True,
tools=sample.tools,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
_TOOL_CALL_SAMPLES = [
(p.parser_cls, p.name, s)
for p in _PAIRINGS
for s in p.samples
if s.expected_tool_calls
]
@pytest.mark.parametrize(
"parser_cls,parser_name,sample",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else "",
)
def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
"""Non-streaming parse() with tool_choice='none' via DelegatingParser
must not leak special tokens into content."""
tokenizer = make_mock_tokenizer(sample)
validated_tools = (
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
)
parser = parser_cls(
tokenizer,
validated_tools,
chat_template_kwargs=sample.chat_template_kwargs,
)
request = _test_request(tools=DUMMY_TOOLS)
request.tool_choice = "none"
output = parse_non_streaming(parser, sample, request)
assert output.tool_calls == [], (
f"Expected no tool calls but got {output.tool_calls}"
)
cfg = parser._tool_parser._parser_engine.parser_engine_config
terminals = sorted(
v
for v in cfg.terminal_literals | set(cfg.token_id_terminals.values())
if len(v) > 1
)
assert_no_terminal_leakage(
output,
terminals,
context=f"parser={parser_name}",
)
-632
View File
@@ -1,632 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
Replays dynamically built token sequences at different chunk sizes and
holdback depths to verify chunk-size invariance and terminal-token hygiene.
Parser discovery is automatic: any ``ParserEngine`` subclass registered in
``registered_adapters`` that also has a builder in ``trace_builder._BUILDERS``
is picked up with zero manual wiring.
"""
from __future__ import annotations
import dataclasses
from typing import NamedTuple
import pytest
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
MockTokenizer,
Sample,
_test_request,
assert_no_terminal_leakage,
assert_parse_output,
collect_output,
make_mock_tokenizer,
parse_non_streaming,
replay_streaming,
replay_with_text_holdback,
)
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
from vllm.parser.engine import registered_adapters as _adapters_mod
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import ParserState
from vllm.parser.mistral import MistralParser
# ── Parser discovery ─────────────────────────────────────────────────
class _ParserInfo(NamedTuple):
parser_cls: type[ParserEngine]
name: str
samples: tuple
terminals: list[str]
tool_end: str
think_end: str
tool_start: str
def _discover_parsers() -> list[_ParserInfo]:
"""Discover engine parsers from registered_adapters that have test builders.
Returns one ``_ParserInfo`` per parser, sorted by config name.
Raises ``RuntimeError`` if any registered parser lacks a builder.
"""
bare_tok = MockTokenizer(vocab={}, tokens=[])
found: list[_ParserInfo] = []
missing_builders: list[str] = []
for obj in vars(_adapters_mod).values():
if not (
isinstance(obj, type)
and issubclass(obj, ParserEngine)
and obj is not ParserEngine
):
continue
if obj is MistralParser:
# Mistral uses brace-balanced JSON tool args with no TOOL_END
# token, so it does not fit this TOOL_END-based replay harness.
# It is covered by tests/parser/mistral/ instead.
continue
cfg = obj(bare_tok, None).parser_engine_config
if cfg.name not in _BUILDERS:
missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
continue
if cfg.name in ("inkling", "plamo3", "minimax_m3"):
# Inkling uses typed structural blocks. PLaMo and MiniMax M3
# markers can span several token IDs. None fits this TOOL_END-token
# harness; their direct and delegating coverage lives in dedicated
# tests.
continue
if cfg.name == "granite":
# Granite has a JSON-array tool body with no TOOL_END terminal, so
# it does not fit this token-terminal harness; its replay coverage
# lives in test_granite.py.
continue
tool_end = cfg.token_id_terminals.get("TOOL_END")
if not tool_end:
raise RuntimeError(
f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals"
)
all_vals = cfg.terminal_literals | set(cfg.token_id_terminals.values())
found.append(
_ParserInfo(
parser_cls=obj,
name=cfg.name,
samples=build_samples(cfg.name),
terminals=sorted(v for v in all_vals if len(v) > 1),
tool_end=tool_end,
think_end=cfg.terminal_literal("THINK_END") or "",
tool_start=(
cfg.terminal_literal("TOOL_SECTION_START")
if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions
else cfg.terminal_literal("TOOL_START")
)
or "",
)
)
if missing_builders:
raise RuntimeError(
f"Engine parsers in registered_adapters have no test builder "
f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
f"Add a builder to _BUILDERS for each new parser."
)
found.sort(key=lambda p: p.name)
return found
_PARSERS = _discover_parsers()
def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra):
kwargs = dict(extra)
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
return parser_cls(tokenizer, sample.tools, **kwargs)
_ENGINE_PARSERS: dict[str, type[ParserEngine]] = {
f"{p.name}_engine": p.parser_cls for p in _PARSERS
}
# ── Parametrize sample lists ─────────────────────────────────────────
HOLDBACK_CONFIGS = [6, 12, 24]
_REPLAY_SAMPLES = [(p.parser_cls, s, p.terminals) for p in _PARSERS for s in p.samples]
@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
@pytest.mark.parametrize(
"parser_cls,sample,terminals",
_REPLAY_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else "",
)
class TestReplayWithHoldback:
"""Replay all parsers with simulated detokenizer holdback."""
def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback):
tokenizer = make_mock_tokenizer(sample)
parser = _make_parser(parser_cls, tokenizer, sample)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=chunk_size,
holdback_chars=holdback,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
assert_no_terminal_leakage(
output,
terminals,
context=f"chunk_size={chunk_size}, holdback={holdback}",
)
TEXT_HOLDBACK_DELAYS = [1, 2, 3]
@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
@pytest.mark.parametrize(
"parser_cls,sample,terminals",
_REPLAY_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else "",
)
class TestTextHoldback:
"""Replay with production-like text/token-ID misalignment.
In production the detokenizer sends token IDs immediately but holds
back text by N tokens. This exercises the TokenIDScanner deferred
terminal path that aligned-holdback tests do not cover.
"""
def test_replay(self, parser_cls, sample, terminals, delay):
tokenizer = make_mock_tokenizer(sample)
parser = _make_parser(parser_cls, tokenizer, sample)
deltas = replay_with_text_holdback(
parser,
sample.tokens,
text_delay=delay,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
assert_no_terminal_leakage(
output,
terminals,
context=f"text_delay={delay}",
)
@pytest.mark.parametrize(
"chunk_size", [1, 2, 3, 5, 10, 19, 20, None], ids=lambda c: f"chunk{c}"
)
@pytest.mark.parametrize(
"parser_cls,sample,terminals",
_REPLAY_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else "",
)
class TestReplay:
"""Replay all parsers at varied chunk sizes without holdback."""
def test_replay(self, parser_cls, sample, terminals, chunk_size):
tokenizer = make_mock_tokenizer(sample)
parser = _make_parser(parser_cls, tokenizer, sample)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=chunk_size,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
assert_no_terminal_leakage(output, terminals)
_DEFERRAL_SAMPLES = [
(p.parser_cls, s, p.tool_end)
for p in _PARSERS
for s in p.samples
if s.expected_tool_calls
]
@pytest.mark.parametrize(
"parser_cls,sample,tool_end_text",
_DEFERRAL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestDeferralFinish:
"""Test that parse_delta(finished=True) resolves deferred scanner state.
Simulates a production failure where delta_text is missing the
tool-call-end text but delta_token_ids has the token, causing the
scanner to defer it. Without finish(), the deferred state is lost
and tool call arguments are empty.
"""
def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text):
tokenizer = make_mock_tokenizer(sample)
parser = _make_parser(parser_cls, tokenizer, sample)
request = _test_request()
all_ids = [tid for tid, _ in sample.tokens]
all_texts = [text for _, text in sample.tokens]
tool_end_id = sample.vocab.get(tool_end_text)
split_idx = None
for i in range(len(all_ids) - 1, -1, -1):
if all_ids[i] == tool_end_id:
split_idx = i
break
if split_idx is None:
pytest.skip(f"no {tool_end_text} token found")
first_ids = all_ids[:split_idx]
first_text = "".join(all_texts[:split_idx])
last_ids = all_ids[split_idx:]
last_text_missing = "".join(all_texts[split_idx:]).replace(tool_end_text, "")
result1 = parser.parse_delta(
first_text,
first_ids,
request,
prompt_token_ids=[],
finished=False,
)
result2 = parser.parse_delta(
last_text_missing, last_ids, request, finished=True
)
output = collect_output([result1, result2])
tool_calls_only = dataclasses.replace(
sample, expected_reasoning=None, expected_content=None
)
assert_parse_output(output, tool_calls_only)
@pytest.mark.parametrize(
"parser_cls,sample",
[(p.parser_cls, p.samples[0]) for p in _PARSERS],
ids=[p.name for p in _PARSERS],
)
class TestParserEngineAdjustRequest:
"""Verify ParserEngine and its adapters set skip_special_tokens=False."""
def test_adjust_request_disables_skip_special_tokens(self, parser_cls, sample):
tokenizer = make_mock_tokenizer(sample)
parser = parser_cls(tokenizer, sample.tools)
request = _test_request()
assert request.skip_special_tokens is True
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is False
_TOOL_CALL_SAMPLES = [
(p.parser_cls, s, p.think_end, p.tool_start)
for p in _PARSERS
for s in p.samples
if s.expected_tool_calls
and s.expected_reasoning
# Inkling's typed-block format (structural role/kind tokens, shared
# block-end token) doesn't fit the generic reasoning/tool split
# below; its filtering modes are covered in test_inkling.py instead.
and p.name != "inkling"
]
def _tool_suppression_expectations(
sample, think_end: str, tool_start: str, *, include_tool_block: bool
) -> tuple[str, str]:
reasoning, content = _raw_tool_suppression_expectations(
sample, think_end, tool_start, include_tool_block=include_tool_block
)
if sample.content_lstrip:
content = content.lstrip(sample.content_lstrip)
return reasoning, content
def _raw_tool_suppression_expectations(
sample, think_end: str, tool_start: str, *, include_tool_block: bool
) -> tuple[str, str]:
"""Expected (reasoning, content) from the raw token text.
With ``include_tool_block=True`` (skip_tool_parsing / reasoning
adapter first pass), tool terminal text is preserved as content so
a second-pass parser can see it.
With ``include_tool_block=False`` (_suppress_tool_calls /
tool_choice='none'), the state machine consumes tool blocks and
only non-tool content survives.
"""
full_text = "".join(text for _, text in sample.tokens)
reasoning = sample.expected_reasoning
idx = full_text.find(reasoning)
if idx < 0:
return (full_text, "")
after_reasoning = full_text[idx + len(reasoning) :]
if think_end:
pos = after_reasoning.find(think_end)
if pos >= 0:
if include_tool_block:
return (reasoning, after_reasoning[pos + len(think_end) :])
after_reasoning = after_reasoning[pos + len(think_end) :]
if tool_start:
pos = after_reasoning.find(tool_start)
if pos >= 0:
if include_tool_block:
return (reasoning, after_reasoning[pos:])
return (reasoning, after_reasoning[:pos])
if include_tool_block:
return (full_text, "")
return (reasoning, after_reasoning)
@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
@pytest.mark.parametrize(
"mode",
["skip_tool_parsing", "suppress_tool_calls"],
ids=["skip_tool_parsing", "suppress_tool_calls"],
)
@pytest.mark.parametrize(
"parser_cls,sample,think_end,tool_start",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestToolCallFilteringReplay:
"""Replay with tool calls not extracted, in both filtering modes.
``skip_tool_parsing`` (reasoning adapter first pass): tool terminal
text is preserved as content for a second-pass tool parser.
``suppress_tool_calls`` (tool_choice='none'): tool call blocks are
consumed by the state machine and do not leak into content.
"""
def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
parser = parser_cls(tokenizer, **kwargs)
request = _test_request()
request.tools = DUMMY_TOOLS
if mode == "skip_tool_parsing":
parser.skip_tool_parsing = True
else:
request.tool_choice = "none"
all_ids = [tid for tid, _ in sample.tokens]
all_texts = [text for _, text in sample.tokens]
if chunk_size is None:
chunk_size = len(all_ids)
results = []
chunks = list(range(0, len(all_ids), chunk_size))
for i, start in enumerate(chunks):
end = min(start + chunk_size, len(all_ids))
is_last = i == len(chunks) - 1
result = parser.parse_delta(
"".join(all_texts[start:end]),
all_ids[start:end],
request,
prompt_token_ids=(sample.prompt_token_ids or [])
if start == 0
else None,
finished=is_last,
)
results.append(result)
output = collect_output(results)
include_block = mode == "skip_tool_parsing"
expected_reasoning, expected_content = _tool_suppression_expectations(
sample, think_end, tool_start, include_tool_block=include_block
)
assert output.reasoning == expected_reasoning, (
f"Reasoning mismatch (mode={mode}):\n"
f" expected: {expected_reasoning!r}\n"
f" actual: {output.reasoning!r}"
)
assert output.tool_calls == [], (
f"Expected no tool calls (mode={mode}) but got {output.tool_calls}"
)
assert output.content == expected_content, (
f"Content mismatch (mode={mode}):\n"
f" expected: {expected_content!r}\n"
f" actual: {output.content!r}"
)
@pytest.mark.parametrize(
"parser_cls,sample,think_end,tool_start",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestToolCallFilteringNonStreaming:
"""Non-streaming parse() with tool_choice='none' must suppress tool
calls and not leak special tokens into content."""
def test_parse(self, parser_cls, sample, think_end, tool_start):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
parser = parser_cls(tokenizer, **kwargs)
request = _test_request()
request.tools = DUMMY_TOOLS
request.tool_choice = "none"
output = parse_non_streaming(parser, sample, request)
expected_reasoning, expected_content = _tool_suppression_expectations(
sample, think_end, tool_start, include_tool_block=False
)
assert output.reasoning == expected_reasoning, (
f"Reasoning mismatch:\n"
f" expected: {expected_reasoning!r}\n"
f" actual: {output.reasoning!r}"
)
assert output.tool_calls == [], (
f"Expected no tool calls but got {output.tool_calls}"
)
assert output.content == expected_content, (
f"Content mismatch:\n"
f" expected: {expected_content!r}\n"
f" actual: {output.content!r}"
)
_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id]
@pytest.mark.parametrize(
"parser_cls,sample",
_WS_TOOL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestToolChoiceNoneStreamingParity:
"""Streaming and non-streaming must return the same content
when tool_choice='none' suppresses tool calls."""
def test_content_matches(self, parser_cls, sample):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
request = _test_request()
request.tools = DUMMY_TOOLS
request.tool_choice = "none"
ns_output = parse_non_streaming(
parser_cls(tokenizer, **kwargs),
sample,
request,
)
s_parser = parser_cls(tokenizer, **kwargs)
results = []
for i, (tid, text) in enumerate(sample.tokens):
is_last = i == len(sample.tokens) - 1
results.append(
s_parser.parse_delta(
text,
[tid],
request,
prompt_token_ids=(sample.prompt_token_ids or [])
if i == 0
else None,
finished=is_last,
)
)
s_output = collect_output(results)
assert ns_output.content == s_output.content, (
f"Streaming/non-streaming content mismatch:\n"
f" streaming: {s_output.content!r}\n"
f" non-streaming: {ns_output.content!r}"
)
_DROP_TOKENS = {"<bos>": 99990, "<eos>": 99991}
def _inject_drop_tokens(sample):
"""Insert <bos> at stream start and <eos> between the first two tokens."""
new_vocab = {**sample.vocab, **_DROP_TOKENS}
tokens = list(sample.tokens)
tokens.insert(0, (99990, "<bos>"))
if len(tokens) >= 3:
tokens.insert(2, (99991, "<eos>"))
else:
tokens.append((99991, "<eos>"))
return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens)
class TestDropTokenReplay:
"""Verify unconfigured special tokens are silently dropped across
all parsers and chunk sizes."""
@pytest.mark.parametrize(
"parser_info",
_PARSERS,
ids=[p.name for p in _PARSERS],
)
@pytest.mark.parametrize("chunk_size", [1, 3, None])
def test_drop_tokens_removed_from_output(self, parser_info, chunk_size):
for sample in parser_info.samples:
injected = _inject_drop_tokens(sample)
tokenizer = make_mock_tokenizer(injected)
parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
results = replay_streaming(
parser,
injected.tokens,
chunk_size=chunk_size,
tools=sample.tools,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(results)
assert_no_terminal_leakage(
output,
list(_DROP_TOKENS.keys()),
context=f"parser={parser_info.name}, chunk={chunk_size}",
)
assert_parse_output(output, sample)
class TestDropTokenNonStreaming:
"""Non-streaming parse() must also strip unconfigured special tokens."""
@pytest.mark.parametrize(
"parser_info",
_PARSERS,
ids=[p.name for p in _PARSERS],
)
def test_drop_tokens_removed_from_output(self, parser_info):
for sample in parser_info.samples:
injected = _inject_drop_tokens(sample)
tokenizer = make_mock_tokenizer(injected)
parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
request = _test_request(tools=sample.tools)
output = parse_non_streaming(parser, injected, request)
assert_no_terminal_leakage(
output,
list(_DROP_TOKENS.keys()),
context=f"parser={parser_info.name}",
)
class TestAdapterReferences:
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
parser classes so the serving layer finds them and calls adjust_request."""
@pytest.mark.parametrize(
"parser_name",
list(_ENGINE_PARSERS.keys()),
)
def test_adapter_cls_refs_set(self, parser_name):
parser_cls = _ENGINE_PARSERS[parser_name]
assert parser_cls.reasoning_parser_cls is not None, (
f"{parser_name}: reasoning_parser_cls is None"
)
assert parser_cls.tool_parser_cls is not None, (
f"{parser_name}: tool_parser_cls is None"
)
@@ -23,8 +23,6 @@ from tests.parser.engine.replay_harness import (
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.engine.registered_adapters import (
Glm47MoeParserReasoningAdapter,
Glm47MoeParserToolAdapter,
Qwen3ParserReasoningAdapter,
Qwen3ParserToolAdapter,
)
@@ -62,11 +60,6 @@ class ByteFallbackMockTokenizer(MockTokenizer):
# ── Model-specific DelegatingParser subclasses ───────────────────────
class _Glm47Delegating(DelegatingParser):
reasoning_parser_cls = Glm47MoeParserReasoningAdapter
tool_parser_cls = Glm47MoeParserToolAdapter
class _Qwen3Delegating(DelegatingParser):
reasoning_parser_cls = Qwen3ParserReasoningAdapter
tool_parser_cls = Qwen3ParserToolAdapter
@@ -92,20 +85,6 @@ EXPECTED_REASONING = "Let me think about Samsung."
EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다."
_MODEL_CONFIGS = [
pytest.param(
{
"<think>": 50,
"</think>": 51,
"<tool_call>": 60,
"</tool_call>": 61,
"<arg_key>": 62,
"</arg_key>": 63,
"<arg_value>": 64,
"</arg_value>": 65,
},
_Glm47Delegating,
id="glm47",
),
pytest.param(
{
"<think>": 50,
@@ -165,7 +144,7 @@ class TestUfffdReasoningTransition:
vocab = dict(_MODEL_CONFIGS[0].values[0])
tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids)
parser = _Glm47Delegating(tokenizer)
parser = _Qwen3Delegating(tokenizer)
deltas = replay_streaming(
parser,
tokens,
File diff suppressed because it is too large Load Diff
-224
View File
@@ -1,224 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from types import SimpleNamespace
import pytest
import torch
from vllm.config import CacheConfig, KVTransferConfig, VllmConfig
from vllm.platforms.cpu import CpuPlatform
class _FakeModel:
@classmethod
def get_mamba_state_dtype_from_config(
cls, vllm_config: VllmConfig
) -> tuple[torch.dtype, torch.dtype]:
state_dtype = {
"float16": torch.float16,
"bfloat16": torch.bfloat16,
}.get(vllm_config.cache_config.mamba_ssm_cache_dtype, torch.float32)
return torch.float32, state_dtype
class _FakeRegistry:
@staticmethod
def resolve_model_cls(
architecture: str,
model_config: SimpleNamespace,
) -> tuple[type[_FakeModel], str]:
return _FakeModel, architecture
def _cpu_config(
cache_config: CacheConfig,
*,
model_type: str,
resolved_dtype: str,
architecture: str | None = None,
layer_types: tuple[str, ...] = ("linear_attention",),
) -> SimpleNamespace:
model_config = SimpleNamespace(
architecture=architecture or model_type,
has_inner_state=True,
use_mla=False,
hf_config=SimpleNamespace(model_type=model_type),
hf_text_config=SimpleNamespace(
model_type=model_type,
layer_types=layer_types,
),
registry=_FakeRegistry,
)
config = VllmConfig(cache_config=cache_config)
config.model_config = model_config
cache_config.mamba_ssm_cache_dtype = resolved_dtype
return config
@pytest.mark.parametrize(
(
"model_type",
"requested_dtype",
"resolved_dtype",
"layer_types",
"expected_dtype",
),
[
pytest.param(
"qwen3_5",
"bfloat16",
"bfloat16",
("linear_attention",),
"bfloat16",
id="gdn-explicit-bf16",
),
pytest.param(
"qwen3_5",
"float16",
"float16",
("linear_attention",),
"float16",
id="gdn-explicit-fp16",
),
pytest.param(
"qwen3_5",
"auto",
"bfloat16",
("linear_attention",),
"bfloat16",
id="gdn-model-bf16",
),
pytest.param(
"qwen3_5",
"auto",
"float16",
("linear_attention",),
"float16",
id="gdn-model-fp16",
),
pytest.param(
"nemotron_h",
"auto",
"bfloat16",
("mamba",),
"float32",
id="unsupported-model-bf16",
),
pytest.param(
"nemotron_h",
"auto",
"float16",
("mamba",),
"float32",
id="unsupported-model-fp16",
),
pytest.param(
"nemotron_h",
"bfloat16",
"bfloat16",
("mamba",),
"float32",
id="unsupported-explicit-bf16",
),
pytest.param(
"nemotron_h",
"float16",
"float16",
("mamba",),
"float32",
id="unsupported-explicit-fp16",
),
],
)
def test_cpu_accelerated_gdn_dtype_policy(
monkeypatch: pytest.MonkeyPatch,
model_type: str,
requested_dtype: str,
resolved_dtype: str,
layer_types: tuple[str, ...],
expected_dtype: str,
) -> None:
monkeypatch.setattr("torch.cpu._is_avx512_bf16_supported", lambda: True)
cache_config = CacheConfig(mamba_ssm_cache_dtype=requested_dtype)
config = _cpu_config(
cache_config,
model_type=model_type,
resolved_dtype=resolved_dtype,
layer_types=layer_types,
)
CpuPlatform.check_and_update_config(config)
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
@pytest.mark.parametrize(
(
"connector",
"extra_config",
"explicit_layout",
"avx512_bf16_supported",
"expected_layout",
),
[
pytest.param("NixlConnector", None, None, True, "DS", id="nixl"),
pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"),
pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"),
pytest.param(
"MultiConnector",
{
"connectors": [
{
"kv_connector": "ExampleConnector",
"kv_connector_extra_config": {},
},
{
"kv_connector": "NixlConnector",
"kv_connector_extra_config": {},
},
]
},
None,
True,
"DS",
id="multi-nixl",
),
pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"),
pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"),
pytest.param(
"OffloadingConnector", None, None, False, None, id="non-nixl-no-avx"
),
],
)
def test_cpu_conv_state_layout_selection(
monkeypatch: pytest.MonkeyPatch,
connector: str,
extra_config: dict | None,
explicit_layout: str | None,
avx512_bf16_supported: bool,
expected_layout: str | None,
) -> None:
layout_env = "VLLM_SSM_CONV_STATE_LAYOUT"
monkeypatch.setattr(
"torch.cpu._is_avx512_bf16_supported",
lambda: avx512_bf16_supported,
)
kv_transfer_config = KVTransferConfig(
kv_connector=connector,
kv_connector_extra_config=extra_config or {},
kv_role="kv_both",
)
config = _cpu_config(
CacheConfig(mamba_ssm_cache_dtype="float32"),
model_type="qwen3_5",
resolved_dtype="float32",
)
config.kv_transfer_config = kv_transfer_config
monkeypatch.delenv(layout_env, raising=False)
if explicit_layout is not None:
monkeypatch.setenv(layout_env, explicit_layout)
CpuPlatform.check_and_update_config(config)
assert os.environ.get(layout_env) == expected_layout
@@ -1,436 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from tests.reasoning.utils import run_reasoning_extraction
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
# Create a concrete test implementation of BaseThinkingReasoningParser
class TestThinkingReasoningParser(BaseThinkingReasoningParser):
"""Test implementation of BaseThinkingReasoningParser."""
@property
def start_token(self) -> str:
return "<test:think>"
@property
def end_token(self) -> str:
return "</test:think>"
class TestThinkingReasoningParserAlt(BaseThinkingReasoningParser):
"""Alternative test implementation with different tokens."""
@property
def start_token(self) -> str:
return "<alt:start>"
@property
def end_token(self) -> str:
return "<alt:end>"
# Use a test model
REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
@pytest.fixture(scope="module")
def test_tokenizer() -> PreTrainedTokenizerBase:
tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(
REASONING_MODEL_NAME
)
# Add custom test tokens
test_tokens = ["<test:think>", "</test:think>", "<alt:start>", "<alt:end>"]
existing_tokens = set(tokenizer.get_vocab().keys())
new_tokens = [token for token in test_tokens if token not in existing_tokens]
if new_tokens:
tokenizer.add_tokens(new_tokens)
return tokenizer
class TestBaseThinkingReasoningParserInit:
"""Test initialization and basic properties of
BaseThinkingReasoningParser.
"""
def test_successful_initialization(self, test_tokenizer):
"""Test successful initialization with valid tokens."""
parser = TestThinkingReasoningParser(test_tokenizer)
assert parser.start_token == "<test:think>"
assert parser.end_token == "</test:think>"
assert parser.start_token_id is not None
assert parser.end_token_id is not None
def test_initialization_with_missing_tokenizer(self):
"""Test that initialization fails without tokenizer."""
with pytest.raises(ValueError, match="model tokenizer must be passed"):
TestThinkingReasoningParser(None)
def test_initialization_with_missing_tokens(self, test_tokenizer):
"""Test that initialization fails when tokens are not in vocabulary."""
# Create a parser with tokens not in vocabulary
class MissingTokenParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return "<missing:start>"
@property
def end_token(self) -> str:
return "<missing:end>"
with pytest.raises(
RuntimeError, match="could not locate think start/end tokens"
):
MissingTokenParser(test_tokenizer)
def test_initialization_with_empty_tokens(self, test_tokenizer):
"""Test that initialization fails with empty token strings."""
class EmptyTokenParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return ""
@property
def end_token(self) -> str:
return ""
with pytest.raises(
ValueError, match="start_token and end_token must be defined"
):
EmptyTokenParser(test_tokenizer)
class TestBaseThinkingReasoningParserMethods:
"""Test the methods of BaseThinkingReasoningParser."""
def test_is_reasoning_end(self, test_tokenizer):
"""Test the is_reasoning_end method."""
parser = TestThinkingReasoningParser(test_tokenizer)
end_token_id = parser.end_token_id
start_token_id = parser.start_token_id
# Test with end token present
assert parser.is_reasoning_end([1, 2, end_token_id, 4]) is True
# Test without end token
assert parser.is_reasoning_end([1, 2, 3, 4]) is False
# Test with empty list
assert parser.is_reasoning_end([]) is False
# Test with interleaved thinking
assert parser.is_reasoning_end([1, start_token_id, 2, end_token_id]) is True
assert parser.is_reasoning_end([1, start_token_id, 2, 3]) is False
assert (
parser.is_reasoning_end(
[1, start_token_id, 2, end_token_id, 2, 2, start_token_id]
)
is False
)
def test_is_reasoning_end_streaming(self, test_tokenizer):
"""Test the is_reasoning_end_streaming method."""
parser = TestThinkingReasoningParser(test_tokenizer)
end_token_id = parser.end_token_id
start_token_id = parser.start_token_id
assert (
parser.is_reasoning_end_streaming([1, 2, end_token_id], [end_token_id])
is True
)
assert parser.is_reasoning_end_streaming([1, 2, 3, 4], [4]) is False
assert parser.is_reasoning_end_streaming([], []) is False
assert (
parser.is_reasoning_end_streaming(
[1, start_token_id, 2, end_token_id], [end_token_id]
)
is True
)
assert (
parser.is_reasoning_end_streaming([1, start_token_id, 2, 3], [3]) is False
)
assert (
parser.is_reasoning_end_streaming(
[1, start_token_id, 2, end_token_id, 2, start_token_id, 2],
[2],
)
is False
)
assert (
parser.is_reasoning_end_streaming(
[1, start_token_id, 2, end_token_id, 2, 2], [2]
)
is False
)
def test_count_reasoning_tokens(self, test_tokenizer):
"""Count tokens between start/end markers."""
parser = TestThinkingReasoningParser(test_tokenizer)
start = parser.start_token_id
end = parser.end_token_id
token_ids = [0, start, 11, 12, end, 99]
assert parser.count_reasoning_tokens(token_ids) == 2
def test_count_reasoning_tokens_nested(self, test_tokenizer):
"""Ensure nested thinking spans count all inner tokens safely."""
parser = TestThinkingReasoningParser(test_tokenizer)
s = parser.start_token_id
e = parser.end_token_id
token_ids = [s, 1, s, 2, e, 3, e]
# Tokens 1,2,3 are inside reasoning (depth>0) => 3 tokens
assert parser.count_reasoning_tokens(token_ids) == 3
def test_extract_content_ids(self, test_tokenizer):
"""Test the extract_content_ids method."""
parser = TestThinkingReasoningParser(test_tokenizer)
end_token_id = parser.end_token_id
# Test with end token in the middle
input_ids = [1, 2, end_token_id, 4, 5]
content_ids = parser.extract_content_ids(input_ids)
assert content_ids == [4, 5]
# Test with end token at the end
input_ids = [1, 2, 3, end_token_id]
content_ids = parser.extract_content_ids(input_ids)
assert content_ids == []
# Test without end token
input_ids = [1, 2, 3, 4]
content_ids = parser.extract_content_ids(input_ids)
assert content_ids == []
# Test with end token as last element (should not extract)
input_ids = [1, 2, 3, end_token_id]
content_ids = parser.extract_content_ids(input_ids)
assert content_ids == []
class TestBaseThinkingReasoningParserExtraction:
"""Test reasoning content extraction methods."""
def test_extract_reasoning_with_both_tokens(self, test_tokenizer):
"""Test extraction when both start and end tokens are present."""
parser = TestThinkingReasoningParser(test_tokenizer)
request = ChatCompletionRequest(messages=[], model="test-model")
model_output = "<test:think>This is reasoning</test:think>This is content"
reasoning, content = parser.extract_reasoning(model_output, request)
assert reasoning == "This is reasoning"
assert content == "This is content"
def test_extract_reasoning_only_end_token(self, test_tokenizer):
"""Test extraction when only end token is present."""
parser = TestThinkingReasoningParser(test_tokenizer)
request = ChatCompletionRequest(messages=[], model="test-model")
model_output = "This is reasoning</test:think>This is content"
reasoning, content = parser.extract_reasoning(model_output, request)
assert reasoning == "This is reasoning"
assert content == "This is content"
def test_extract_reasoning_no_end_token(self, test_tokenizer):
"""Test extraction when no end token is present."""
parser = TestThinkingReasoningParser(test_tokenizer)
request = ChatCompletionRequest(messages=[], model="test-model")
model_output = "This is just content"
reasoning, content = parser.extract_reasoning(model_output, request)
assert reasoning == "This is just content"
assert content is None
def test_extract_reasoning_empty_output(self, test_tokenizer):
"""Test extraction with empty output."""
parser = TestThinkingReasoningParser(test_tokenizer)
request = ChatCompletionRequest(messages=[], model="test-model")
model_output = ""
reasoning, content = parser.extract_reasoning(model_output, request)
assert reasoning == ""
assert content is None
def test_extract_reasoning_only_tokens(self, test_tokenizer):
"""Test extraction with only tokens and no content."""
parser = TestThinkingReasoningParser(test_tokenizer)
request = ChatCompletionRequest(messages=[], model="test-model")
model_output = "<test:think></test:think>"
reasoning, content = parser.extract_reasoning(model_output, request)
assert reasoning == ""
assert content is None
class TestBaseThinkingReasoningParserStreaming:
"""Test streaming functionality of BaseThinkingReasoningParser."""
@pytest.mark.parametrize("streaming", [True, False])
def test_simple_reasoning_extraction(self, test_tokenizer, streaming):
"""Test basic reasoning extraction in both
streaming and non-streaming modes.
"""
parser = TestThinkingReasoningParser(test_tokenizer)
model_output = [
"<test:think>",
"Some ",
"reasoning ",
"content",
"</test:think>",
"Final ",
"answer",
]
reasoning, content = run_reasoning_extraction(
parser, model_output, streaming=streaming
)
assert reasoning == "Some reasoning content"
assert content == "Final answer"
def test_streaming_with_incremental_deltas(self, test_tokenizer):
"""Test streaming processing with small incremental deltas."""
parser = TestThinkingReasoningParser(test_tokenizer)
deltas = [
"<test:think>",
"Some ",
"reasoning ",
"content",
"</test:think>",
"Final ",
"answer",
]
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
assert reasoning == "Some reasoning content"
assert content == "Final answer"
def test_streaming_with_start_token(self, test_tokenizer):
"""Test streaming with start token included."""
parser = TestThinkingReasoningParser(test_tokenizer)
deltas = [
"<test:think>",
"Some ",
"reasoning",
"</test:think>",
"Answer",
]
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
assert reasoning == "Some reasoning"
assert content == "Answer"
def test_streaming_no_end_token(self, test_tokenizer):
"""Test streaming when no end token is encountered."""
parser = TestThinkingReasoningParser(test_tokenizer)
deltas = [
"<test:think>",
"Some ",
"reasoning ",
"without ",
"end",
]
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
assert reasoning == "Some reasoning without end"
assert content is None
def test_streaming_only_end_token(self, test_tokenizer):
"""Test streaming when only end token appears."""
parser = TestThinkingReasoningParser(test_tokenizer)
deltas = [
"<test:think>",
"Reasoning ",
"content",
"</test:think>",
"Final",
]
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
assert reasoning == "Reasoning content"
assert content == "Final"
class TestBaseThinkingReasoningParserMultipleImplementations:
"""Test that multiple implementations of
BaseThinkingReasoningParser work correctly.
"""
def test_different_token_implementations(self, test_tokenizer):
"""Test that different implementations
with different tokens work independently.
"""
parser1 = TestThinkingReasoningParser(test_tokenizer)
parser2 = TestThinkingReasoningParserAlt(test_tokenizer)
# Test parser1
model_output1 = "Reasoning1</test:think>Content1"
reasoning1, content1 = run_reasoning_extraction(parser1, [model_output1])
assert reasoning1 == "Reasoning1"
assert content1 == "Content1"
# Test parser2
model_output2 = "Reasoning2<alt:end>Content2"
reasoning2, content2 = run_reasoning_extraction(parser2, [model_output2])
assert reasoning2 == "Reasoning2"
assert content2 == "Content2"
# Verify tokens are different
assert parser1.start_token != parser2.start_token
assert parser1.end_token != parser2.end_token
assert parser1.start_token_id != parser2.start_token_id
assert parser1.end_token_id != parser2.end_token_id
class TestBaseThinkingReasoningParserEdgeCases:
"""Test edge cases and error conditions."""
def test_multiple_end_tokens(self, test_tokenizer):
"""Test behavior with multiple end tokens."""
parser = TestThinkingReasoningParser(test_tokenizer)
model_output = "First</test:think>Middle</test:think>Last"
reasoning, content = run_reasoning_extraction(parser, [model_output])
# Should stop at first end token
assert reasoning == "First"
assert content == "Middle</test:think>Last"
def test_nested_tokens(self, test_tokenizer):
"""Test behavior with nested-like token patterns."""
parser = TestThinkingReasoningParser(test_tokenizer)
model_output = "<test:think>Outer<test:think>Inner</test:think>Content"
reasoning, content = run_reasoning_extraction(parser, [model_output])
# Should process normally, start from first start token
assert reasoning == "Outer<test:think>Inner"
assert content == "Content"
def test_malformed_tokens(self, test_tokenizer):
"""Test behavior with malformed token-like strings."""
parser = TestThinkingReasoningParser(test_tokenizer)
model_output = "<test:thinking>Not a real token</test:thinking>Content"
reasoning, content = run_reasoning_extraction(parser, [model_output])
# Should treat as regular content since tokens don't match exactly
assert reasoning == ("<test:thinking>Not a real token</test:thinking>Content")
assert content is None
-161
View File
@@ -1,161 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.config import ParallelConfig
from vllm.distributed.device_communicators.all2all import AgRsAll2AllManager
from vllm.forward_context import DPMetadata
from vllm.v1.attention.ops.pcp import maybe_gather_mla_latent_cache_inputs
@pytest.mark.parametrize(
"pcp_size,sp_size,enable_ep,expected",
[
(1, 1, True, [5, 7]),
(1, 2, True, [3, 3, 4, 4]),
(2, 1, False, [10, 14]),
(2, 1, True, [5, 5, 7, 7]),
(2, 2, True, [3, 3, 3, 3, 4, 4, 4, 4]),
],
)
def test_dispatch_sizes_expand_pcp_before_tp(pcp_size, sp_size, enable_ep, expected):
config = ParallelConfig(
distributed_executor_backend="mp",
data_parallel_size=2,
prefill_context_parallel_size=pcp_size,
tensor_parallel_size=sp_size,
enable_expert_parallel=enable_ep,
)
metadata = DPMetadata.make(config, 5, torch.tensor([5, 7]))
with metadata.sp_local_sizes(sp_size, pcp_size=pcp_size, use_ep=enable_ep) as sizes:
assert sizes == expected
assert metadata.local_sizes is None
assert metadata.num_tokens_across_dp_cpu.tolist() == [5, 7]
@pytest.mark.parametrize(
"dp_size,pcp_size,tp_size,use_ep,is_sp,expected",
[
(2, 2, 1, True, False, "ep"),
(2, 2, 2, True, True, "ep"),
(2, 2, 2, False, False, "dp"),
(2, 1, 2, True, False, "dp"),
(1, 2, 2, True, False, "pcp"),
(2, 2, 2, True, False, None),
],
)
def test_dispatch_reuses_existing_groups(
dp_size, pcp_size, tp_size, use_ep, is_sp, expected, monkeypatch
):
groups = {
"dp": SimpleNamespace(world_size=dp_size),
"pcp": SimpleNamespace(world_size=pcp_size),
"ep": object(),
}
for name, group in groups.items():
monkeypatch.setattr(
f"vllm.distributed.device_communicators.all2all.get_{name}_group",
lambda group=group: group,
)
manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager)
manager.dp_world_size = dp_size
manager.tp_group = SimpleNamespace(world_size=tp_size)
manager.use_ep = use_ep
if expected is None:
with pytest.raises(AssertionError, match="requires sequence-parallel MoE"):
manager._get_comm_group(is_sp)
else:
assert manager._get_comm_group(is_sp) is groups[expected]
@pytest.mark.parametrize("enable_ep", [False, True])
def test_ag_rs_dispatch_and_combine_use_dp_pcp_sizes(monkeypatch, enable_ep):
calls = []
local_tokens = [20, 21] if enable_ep else [20, 21, 22, 23]
class FakeGroup:
world_size = 4 if enable_ep else 2
rank_in_group = 2 if enable_ep else 1
def all_gatherv(self, tensors, dim, sizes):
calls.append(("gather", sizes))
return [torch.tensor([10, 11, 20, 21, 22, 23]) for _ in tensors]
def reduce_scatterv(self, tensor, dim, sizes):
calls.append(("scatter", sizes))
return tensor[2 : 2 + len(local_tokens)]
config = ParallelConfig(
distributed_executor_backend="mp",
data_parallel_size=2,
data_parallel_rank=1,
prefill_context_parallel_size=2,
enable_expert_parallel=enable_ep,
)
metadata = DPMetadata.make(config, 2, torch.tensor([1, 2]))
manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager)
manager.dp_world_size = 2
manager.use_ep = enable_ep
manager.tp_group = SimpleNamespace(world_size=1)
for name in ("dp", "ep"):
monkeypatch.setattr(
f"vllm.distributed.device_communicators.all2all.get_{name}_group",
FakeGroup,
)
monkeypatch.setattr(
"vllm.distributed.device_communicators.all2all.get_pcp_group",
lambda: SimpleNamespace(world_size=2),
)
monkeypatch.setattr(
"vllm.distributed.device_communicators.all2all.get_forward_context",
lambda: SimpleNamespace(dp_metadata=metadata),
)
with metadata.sp_local_sizes(1, pcp_size=2, use_ep=enable_ep):
hidden_states, _, _ = manager.dispatch(
torch.tensor(local_tokens),
torch.ones(len(local_tokens)),
torch.zeros(len(local_tokens)),
)
combined = manager.combine(hidden_states)
assert combined.tolist() == local_tokens
sizes = [1, 1, 2, 2] if enable_ep else [2, 4]
assert calls == [("gather", sizes), ("scatter", sizes)]
@pytest.mark.parametrize("slots", [[3, 4], [3, 4, -1, -1, -1, -1]])
def test_decode_cache_write_ignores_dp_padding(slots):
kv = torch.arange(12).reshape(6, 2)
pe = torch.arange(6).reshape(6, 1, 1)
slots = torch.tensor(slots)
cache_kv, cache_pe, cache_slots = maybe_gather_mla_latent_cache_inputs(
kv, pe, slots, num_decode_tokens=2, use_pcp=True
)
torch.testing.assert_close(cache_kv, kv[:2])
torch.testing.assert_close(cache_pe, pe[:2])
torch.testing.assert_close(cache_slots, slots[:2])
def test_expanded_slot_mapping_keeps_pcp_prefill_padding(monkeypatch):
calls = []
def all_gather(tensor, dim):
calls.append(tensor.shape[0])
return torch.cat((tensor, tensor), dim=dim)
monkeypatch.setattr(
"vllm.v1.attention.ops.pcp.get_pcp_group",
lambda: SimpleNamespace(world_size=2, all_gather=all_gather),
)
kv = torch.zeros(3, 2) # Two decodes followed by one PCP padding row.
pe = torch.zeros(3, 1, 1)
slots = torch.tensor([3, 4, 8, 3, 4, -1])
cache_kv, _, cache_slots = maybe_gather_mla_latent_cache_inputs(
kv, pe, slots, num_decode_tokens=2, use_pcp=True
)
assert calls == [1, 1]
assert cache_kv.shape == (4, 2)
assert cache_slots.tolist() == [3, 4, 8, -1]
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import transformers.image_utils
from PIL import Image
from vllm.transformers_utils.processors.pixtral import MistralCommonImageProcessor
@pytest.fixture(scope="module")
def image_processor() -> MistralCommonImageProcessor:
return MistralCommonImageProcessor(mm_encoder=None)
def test_fetch_images_passes_through_decoded_image(
image_processor: MistralCommonImageProcessor,
):
image = Image.new("RGB", (4, 4))
result = image_processor.fetch_images(image)
assert result is image
def test_fetch_images_recurses_over_list(
image_processor: MistralCommonImageProcessor,
):
a = Image.new("RGB", (4, 4))
b = Image.new("RGB", (8, 8))
result = image_processor.fetch_images([a, b])
assert isinstance(result, list)
assert len(result) == 2
assert result[0] is a
assert result[1] is b
def test_fetch_images_recurses_over_nested_list(
image_processor: MistralCommonImageProcessor,
):
a = Image.new("RGB", (4, 4))
b = Image.new("RGB", (8, 8))
result = image_processor.fetch_images([[a], [b]])
assert result == [[a], [b]]
def test_fetch_images_str_delegates_to_load_image(
monkeypatch, image_processor: MistralCommonImageProcessor
):
sentinel = Image.new("RGB", (2, 2))
received: dict[str, object] = {}
def fake_load_image(path):
received["path"] = path
return sentinel
monkeypatch.setattr(transformers.image_utils, "load_image", fake_load_image)
result = image_processor.fetch_images("/tmp/fake.png")
assert result is sentinel
assert received["path"] == "/tmp/fake.png"
def test_fetch_images_rejects_unsupported_type(
image_processor: MistralCommonImageProcessor,
):
with pytest.raises(TypeError, match="only a single or a list"):
image_processor.fetch_images(42)
@@ -1,127 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ``MistralCommonFeatureExtractor.fetch_audio``.
``transformers>=5.10`` adds a ``ProcessorMixin.prepare_inputs_layout`` helper
that calls ``self.feature_extractor.fetch_audio(...)`` unconditionally. The
duck-typed :class:`MistralCommonFeatureExtractor` previously did not implement
that method, so loading any voxtral model under transformers 5.10.x raised
``AttributeError: 'MistralCommonFeatureExtractor' object has no attribute
'fetch_audio'``. These tests pin the new ``fetch_audio`` method to the same
contract as ``transformers.SequenceFeatureExtractor.fetch_audio``.
"""
import numpy as np
import pytest
import torch
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.transformers_utils.processors.voxtral import (
MistralCommonFeatureExtractor,
)
@pytest.fixture(scope="module")
def feature_extractor() -> MistralCommonFeatureExtractor:
tokenizer = MistralTokenizer.from_pretrained("mistralai/Voxtral-Mini-3B-2507")
return MistralCommonFeatureExtractor(tokenizer.instruct.audio_encoder)
@pytest.mark.parametrize(
"audio",
[
np.zeros(1024, dtype=np.float32),
torch.zeros(1024),
[0.0, 1.0, 2.0],
],
ids=["numpy_array", "torch_tensor", "list_of_floats"],
)
def test_fetch_audio_passes_through(
feature_extractor: MistralCommonFeatureExtractor, audio
):
result = feature_extractor.fetch_audio(audio)
assert result is audio
def test_fetch_audio_recurses_over_list_of_arrays(
feature_extractor: MistralCommonFeatureExtractor,
):
a = np.zeros(8, dtype=np.float32)
b = np.ones(8, dtype=np.float32)
result = feature_extractor.fetch_audio([a, b])
assert isinstance(result, list)
assert len(result) == 2
assert result[0] is a
assert result[1] is b
def test_fetch_audio_uses_self_sampling_rate_when_none(
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
):
"""If ``sampling_rate`` is None, ``self.sampling_rate`` must be used.
Verified indirectly via the recursion path: when we pass a list of arrays
without sampling_rate, recursive calls receive the resolved rate.
"""
captured: list[int | None] = []
original = feature_extractor.fetch_audio
def spy(audio, sampling_rate=None):
captured.append(sampling_rate)
return original(audio, sampling_rate=sampling_rate)
monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)])
# Top-level call has sampling_rate=None; inner recursive call sees the
# resolved rate from self.sampling_rate.
assert captured[0] is None
assert captured[1] == 16000
def test_fetch_audio_explicit_sampling_rate_propagates(
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
):
captured: list[int | None] = []
original = feature_extractor.fetch_audio
def spy(audio, sampling_rate=None):
captured.append(sampling_rate)
return original(audio, sampling_rate=sampling_rate)
monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)], sampling_rate=8000)
assert captured[0] == 8000
assert captured[1] == 8000
def test_fetch_audio_rejects_unsupported_type(
feature_extractor: MistralCommonFeatureExtractor,
):
with pytest.raises(TypeError, match="only a numpy array"):
feature_extractor.fetch_audio(42)
def test_fetch_audio_str_delegates_to_load_audio(
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
):
"""A str input must round-trip through ``transformers.audio_utils.load_audio``.
We monkey-patch ``load_audio`` so the test stays offline (no real URL/path
fetched) and still asserts the delegation contract.
"""
sentinel = np.array([0.5, -0.5], dtype=np.float32)
received: dict[str, object] = {}
def fake_load_audio(path, sampling_rate=None):
received["path"] = path
received["sampling_rate"] = sampling_rate
return sentinel
import transformers.audio_utils
monkeypatch.setattr(transformers.audio_utils, "load_audio", fake_load_audio)
result = feature_extractor.fetch_audio("/tmp/fake.wav")
assert result is sentinel
assert received["path"] == "/tmp/fake.wav"
assert received["sampling_rate"] == 16000
@@ -1,159 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import json
from pathlib import Path
from typing import Any
import pytest
from transformers import AutoConfig
from vllm.transformers_utils.config import get_config, uses_mrope
from vllm.transformers_utils.configs.bailing_moe_v3_vl import (
BailingMoeV3TextConfig,
BailingMoeV3VisionConfig,
BailingMoeV3VLConfig,
)
def _bailing_v3_vl_config() -> dict[str, Any]:
return {
"architectures": ["BailingMoeV3VLForConditionalGeneration"],
"auto_map": {
"AutoConfig": ("configuration_bailing_moe_v3_vl.BailingMoeV3VLConfig")
},
"model_type": "bailing_moe_v3_vl",
"mrope_section": [8, 12, 12],
"text_config": {
"num_hidden_layers": 42,
"layer_group_size": 6,
"rope_theta": 6_000_000,
},
"vision_config": {"disable_merger_proj": True},
}
def _write_config(path: Path, config: dict[str, Any]) -> None:
path.mkdir()
(path / "config.json").write_text(json.dumps(config), encoding="utf-8")
def test_bailing_v3_vl_config_loads_without_remote_code(tmp_path: Path):
model_path = tmp_path / "model"
_write_config(model_path, _bailing_v3_vl_config())
config = get_config(model_path, trust_remote_code=False)
assert isinstance(config, BailingMoeV3VLConfig)
assert isinstance(config.text_config, BailingMoeV3TextConfig)
assert isinstance(config.vision_config, BailingMoeV3VisionConfig)
assert isinstance(
AutoConfig.from_pretrained(model_path, trust_remote_code=False),
BailingMoeV3VLConfig,
)
assert config.mrope_section == [8, 12, 12]
assert uses_mrope(config)
text_config = config.text_config
assert text_config.model_type == "bailing_hybrid"
assert text_config.architectures == ["BailingMoeV3ForCausalLM"]
assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
assert text_config.rope_parameters["rope_theta"] == 6_000_000
assert text_config.layer_types.count("linear_attention") == 35
assert [
layer_idx
for layer_idx, layer_type in enumerate(text_config.layer_types)
if layer_type == "full_attention"
] == [5, 11, 17, 23, 29, 35, 41]
assert config.vision_config.disable_merger_proj is True
normalized = config.to_dict()
restored = BailingMoeV3VLConfig.from_dict(normalized)
assert isinstance(restored.text_config, BailingMoeV3TextConfig)
assert isinstance(restored.vision_config, BailingMoeV3VisionConfig)
assert restored.to_dict() == normalized
def test_bailing_v3_vl_config_preserves_explicit_nested_values():
config_dict = _bailing_v3_vl_config()
config_dict["architectures"] = ["CustomBailingVLForConditionalGeneration"]
text_config = config_dict["text_config"]
text_config["architectures"] = ["CustomBailingForCausalLM"]
text_config["layer_types"] = (["linear_attention"] * 5 + ["full_attention"]) * 7
text_config["rope_theta"] = 1_000_000
text_config["rope_parameters"] = {
"rope_type": "default",
"rope_theta": 1_000_000,
"mrope_section": [8, 12, 12],
}
original = copy.deepcopy(config_dict)
config = BailingMoeV3VLConfig(**config_dict)
assert config_dict == original
assert config.architectures == ["CustomBailingVLForConditionalGeneration"]
assert config.text_config.architectures == ["CustomBailingForCausalLM"]
assert config.text_config.layer_types == text_config["layer_types"]
assert config.text_config.rope_parameters["rope_theta"] == 1_000_000
assert config.text_config.rope_parameters["mrope_section"] == [8, 12, 12]
def test_bailing_v3_vl_config_normalizes_config_instance():
text_config = BailingMoeV3TextConfig()
config = BailingMoeV3VLConfig(
text_config=text_config,
mrope_section=[8, 12, 12],
)
assert config.text_config is text_config
assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
def test_bailing_v3_text_config_trailing_and_legacy_layers():
expected = ["linear_attention", "linear_attention", "full_attention"] * 2 + [
"full_attention"
] * 2
text_config = BailingMoeV3TextConfig(
num_hidden_layers=8,
layer_group_size=3,
)
assert text_config.layer_types == expected
text_config = BailingMoeV3TextConfig(
num_hidden_layers=8,
layer_group_size=3,
layers_block_type=["mamba", "mamba", "attention"] * 2 + ["attention"] * 2,
)
assert text_config.layer_types == expected
def test_bailing_v3_text_config_accepts_legacy_rope_type():
text_config = BailingMoeV3TextConfig(rope_scaling={"type": "linear", "factor": 2.0})
assert text_config.rope_parameters["type"] == "linear"
assert text_config.rope_parameters["rope_type"] == "linear"
def test_bailing_v3_vl_config_rejects_invalid_values():
with pytest.raises(ValueError, match="layer_group_size must be positive"):
BailingMoeV3TextConfig(layer_group_size=0)
with pytest.raises(ValueError, match="must match the layer_group_size schedule"):
BailingMoeV3TextConfig(
num_hidden_layers=8,
layer_group_size=3,
layer_types=["full_attention"] * 8,
)
with pytest.raises(ValueError, match="conflicting M-RoPE sections"):
BailingMoeV3VLConfig(
text_config={
"rope_parameters": {
"rope_type": "default",
"mrope_section": [4, 14, 14],
}
},
mrope_section=[8, 12, 12],
)
@@ -1,153 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
from vllm.transformers_utils.config import get_config
from vllm.transformers_utils.configs.k3_dspark import K3DSparkConfig
def _write_dspark_config(path, **overrides):
path.mkdir()
config = {
"architectures": ["K3DSparkModel"],
"model_type": "k3_dspark",
"hidden_size": 7168,
"intermediate_size": 14336,
"num_hidden_layers": 5,
"num_attention_heads": 96,
"num_key_value_heads": 96,
"q_lora_rank": 1536,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"vocab_size": 163840,
"rms_norm_eps": 1e-5,
"max_position_embeddings": 32768,
"rope_theta": 50000.0,
"num_target_layers": 5,
"target_hidden_size": 7168,
"target_num_hidden_layers": 93,
"target_layer_ids": [2, 23, 47, 71, 89],
"markov_rank": 256,
"draft_vocab_size": 163840,
"torch_dtype": "bfloat16",
}
config.update(overrides)
(path / "config.json").write_text(json.dumps(config))
def _write_target_config(path):
path.mkdir()
config = {
"architectures": ["LlamaForCausalLM"],
"model_type": "llama",
"hidden_size": 7168,
"intermediate_size": 14336,
"num_hidden_layers": 93,
"num_attention_heads": 56,
"num_key_value_heads": 8,
"vocab_size": 163840,
"max_position_embeddings": 32768,
"torch_dtype": "bfloat16",
}
(path / "config.json").write_text(json.dumps(config))
def test_dspark_mla_config_loads_from_local_json(tmp_path):
draft_path = tmp_path / "draft"
_write_dspark_config(draft_path)
config = get_config(draft_path, trust_remote_code=False)
assert isinstance(config, K3DSparkConfig)
assert config.model_type == "k3_dspark"
assert config.architectures == ["K3DSparkModel"]
assert config.hidden_act == "silu"
assert config.rope_parameters == {
"rope_type": "default",
"rope_theta": 50000.0,
}
assert config.n_routed_experts == 0
assert config.draft_vocab_size == config.vocab_size
@pytest.mark.parametrize(
"overrides",
[
{"mla_use_nope": True},
{"mla_use_output_gate": True},
{"mla_use_qk_norm": True},
{"dspark_bonus_anchor": True},
{"q_lora_rank": None},
{"draft_vocab_size": 8192},
{"target_layer_ids": []},
{"num_target_layers": 4},
],
)
def test_dspark_mla_rejects_unsupported_checkpoint_options(tmp_path, overrides):
draft_path = tmp_path / "draft"
_write_dspark_config(draft_path, **overrides)
with pytest.raises(ValueError, match="MLA DSpark"):
get_config(draft_path, trust_remote_code=False)
@pytest.mark.parametrize(
("num_attention_heads", "expected_local_heads"),
[(64, 8), (96, 12)],
ids=["released-64-head", "block5-96-head"],
)
def test_dspark_mla_uses_latent_kv_geometry(
tmp_path, num_attention_heads, expected_local_heads
):
draft_path = tmp_path / "draft"
_write_dspark_config(
draft_path,
num_attention_heads=num_attention_heads,
num_key_value_heads=num_attention_heads,
)
model_config = ModelConfig(
model=str(draft_path),
tokenizer_mode="skip",
runner="draft",
max_model_len=32768,
)
assert model_config.is_deepseek_mla
assert model_config.use_mla
assert model_config.get_head_size() == 576
# external_launcher skips ParallelConfig's local-GPU-count check so the
# config logic can be exercised at TP8 on a single-GPU test node.
parallel_config = ParallelConfig(
tensor_parallel_size=8, distributed_executor_backend="external_launcher"
)
assert model_config.get_num_kv_heads(parallel_config) == 1
assert model_config.get_num_attention_heads(parallel_config) == expected_local_heads
assert model_config.get_num_experts() == 0
def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
target_path = tmp_path / "target"
draft_path = tmp_path / "draft"
_write_target_config(target_path)
_write_dspark_config(draft_path)
target_config = ModelConfig(
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
)
speculative_config = SpeculativeConfig(
model=str(draft_path),
method="dspark",
num_speculative_tokens=8,
target_model_config=target_config,
target_parallel_config=ParallelConfig(),
)
assert speculative_config.parallel_drafting
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
assert speculative_config.draft_model_config.use_mla
File diff suppressed because it is too large Load Diff
-69
View File
@@ -24,12 +24,7 @@ from vllm.distributed.aux_output_connector.connector import (
AuxRequestOutput,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
@@ -49,7 +44,6 @@ from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.engine import FinishReason
from vllm.v1.engine.core import EngineCore
from vllm.v1.executor.uniproc_executor import UniProcExecutor
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -219,30 +213,6 @@ def test_max_num_active_seqs_caps_admission():
assert scheduler.max_num_running_reqs == 16
def _bind_hisparse_connector(scheduler):
connector = object.__new__(HiSparseConnector)
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
connector.connector_scheduler.bind_coordinator(coordinator)
composite = object.__new__(MultiConnector)
composite._connectors = (connector,)
scheduler.connector = composite
return coordinator
def test_pending_hisparse_spill_keeps_scheduler_alive():
"""A final host spill must complete after the last request finishes."""
scheduler = create_scheduler()
coordinator = _bind_hisparse_connector(scheduler)
pending = coordinator.pending_spills
pending[0] = Mock()
assert scheduler.has_requests()
pending.clear()
assert not scheduler.has_requests()
@pytest.mark.parametrize(
"enable_prefix_caching, prompt_logprobs",
[
@@ -1776,45 +1746,6 @@ def test_preempt_during_execution():
assert requests[1].output_token_ids[0] == 42
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
"""Pending reclamation must stall rather than preempt its owning request."""
scheduler = create_scheduler(enable_prefix_caching=False)
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[0]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
monkeypatch.setattr(
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
)
coordinator = _bind_hisparse_connector(scheduler)
monkeypatch.setattr(
scheduler.connector, "build_connector_meta", Mock(return_value=None)
)
monkeypatch.setattr(
coordinator,
"has_pending_reclamation",
Mock(return_value=True),
)
deferred_output = scheduler.schedule()
assert not deferred_output.num_scheduled_tokens
assert scheduler.running == [request]
assert request.status == RequestStatus.RUNNING
assert request.num_preemptions == 0
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry."""
File diff suppressed because it is too large Load Diff
@@ -1,283 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for resource management in hf3fs_client.py: constructor failure cleanup
and idempotent close(). Tests use mock to replace real I/O operations
(hf3fs_fuse.io, SharedMemory, os, CUDA).
Requires hf3fs_fuse.io to be installed; skipped otherwise.
"""
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
HF3FS_AVAILABLE = True
try:
from hf3fs_fuse.io import ( # noqa: F401
deregister_fd,
extract_mount_point,
make_ioring,
make_iovec,
register_fd,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client import (
Hf3fsClient,
)
except Exception:
HF3FS_AVAILABLE = False
requires_hf3fs = pytest.mark.skipif(
not HF3FS_AVAILABLE,
reason="hf3fs_fuse.io is not available on this machine",
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class _FakeShm:
"""Shared-memory stub matching the multiprocessing.shared_memory.SharedMemory
interface used by Hf3fsClient:
Attributes accessed by the constructor:
.buf – memoryview / buffer-protocol object consumed by torch.frombuffer
Methods called during normal lifetime:
.unlink() – called right after the iovec is set up
.close() – called in _release_resources()
"""
def __init__(self, size: int = 1024):
self._data = bytearray(size)
self.buf = memoryview(self._data)
self.closed = False
self.close_call_count = 0
self.unlink_call_count = 0
def close(self):
self.closed = True
self.close_call_count += 1
def unlink(self):
self.unlink_call_count += 1
# ===========================================================================
# TestHf3fsClientResourceManagement
# ===========================================================================
@requires_hf3fs
class TestHf3fsClientResourceManagement:
"""Tests for constructor failure cleanup and idempotent close()."""
_MOD = "vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client"
# ------------------------------------------------------------------
# Helper: build a minimal Hf3fsClient bypassing all real I/O so that
# we can fully control its internal state.
# ------------------------------------------------------------------
def _make_client(self, tmp_path):
"""Return a fully-mocked Hf3fsClient with controllable internals."""
fake_shm_r = _FakeShm()
fake_shm_w = _FakeShm()
patcher_list: list[Any] = [
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
patch(f"{self._MOD}.register_fd"),
patch(f"{self._MOD}.deregister_fd"),
patch(f"{self._MOD}.extract_mount_point", return_value="/mnt/hf3fs"),
patch(f"{self._MOD}.make_ioring", return_value=MagicMock()),
patch(f"{self._MOD}.make_iovec", return_value=MagicMock()),
patch(
"multiprocessing.shared_memory.SharedMemory",
side_effect=[fake_shm_r, fake_shm_w],
),
patch("os.open", return_value=99),
patch("os.ftruncate"),
patch("os.close"),
patch("os.fsync"),
patch("torch.cuda.Stream", return_value=MagicMock()),
patch("torch.frombuffer", return_value=MagicMock()),
patch("torch.empty", return_value=MagicMock()),
]
for p in patcher_list:
p.start()
try:
client = Hf3fsClient(
path=str(tmp_path / "test.bin"),
size=1024,
bytes_per_page=256,
entries=4,
)
finally:
for p in patcher_list:
p.stop()
# Manually point internal handles to our controllable fakes so that
# assertions after close() can inspect them directly.
client.shm_r = fake_shm_r
client.shm_w = fake_shm_w
client.file = 99
return client, fake_shm_r, fake_shm_w
# ------------------------------------------------------------------
# close() idempotency
# ------------------------------------------------------------------
def test_close_idempotent_and_handles_cleared(self, tmp_path):
"""Multiple close() calls must not raise; deregister_fd called exactly
once, all handles set to None, shm.close() invoked."""
client, shm_r, shm_w = self._make_client(tmp_path)
with (
patch(f"{self._MOD}.deregister_fd") as mock_dereg,
patch("os.close"),
):
client.close() # first close
client.close() # second close — must be no-op
client.close() # third close — must be no-op
assert client._closed is True
assert mock_dereg.call_count == 1, (
f"deregister_fd called {mock_dereg.call_count} times; expected 1"
)
for attr in ("iov_r", "iov_w", "ior_r", "ior_w", "shm_r", "shm_w", "file"):
assert getattr(client, attr) is None, f"{attr} should be None after close()"
assert shm_r.closed is True
assert shm_w.closed is True
def test_flush_after_close_is_noop(self, tmp_path):
"""flush() after close() must silently do nothing (no fsync call)."""
client, _, _ = self._make_client(tmp_path)
with (
patch(f"{self._MOD}.deregister_fd"),
patch("os.close"),
patch("os.fsync") as mock_fsync,
):
client.close()
client.flush()
mock_fsync.assert_not_called()
# ------------------------------------------------------------------
# Constructor failure leaves no leaked resources
# ------------------------------------------------------------------
def test_constructor_failure_after_file_open_cleans_file(self, tmp_path):
"""If the constructor raises after os.open(), the fd must be closed."""
with (
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
patch(f"{self._MOD}.register_fd"),
patch(f"{self._MOD}.deregister_fd"),
patch(
f"{self._MOD}.extract_mount_point",
side_effect=RuntimeError("mount point not found"),
),
patch("os.open", return_value=55),
patch("os.ftruncate"),
patch("os.close") as mock_os_close,
patch("torch.cuda.Stream", return_value=MagicMock()),
pytest.raises(RuntimeError, match="mount point not found"),
):
Hf3fsClient(
path=str(tmp_path / "fail.bin"),
size=1024,
bytes_per_page=256,
entries=4,
)
mock_os_close.assert_called_once_with(55)
def test_constructor_failure_after_shm_alloc_closes_shm(self, tmp_path):
"""Constructor raises after SharedMemory creation → both shm objects closed."""
fake_shm_r = _FakeShm()
fake_shm_w = _FakeShm()
with (
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
patch(f"{self._MOD}.register_fd"),
patch(f"{self._MOD}.deregister_fd"),
patch(f"{self._MOD}.extract_mount_point", return_value="/mnt/hf3fs"),
patch(
"multiprocessing.shared_memory.SharedMemory",
side_effect=[fake_shm_r, fake_shm_w],
),
patch("os.open", return_value=66),
patch("os.ftruncate"),
patch("os.close"),
patch("torch.frombuffer", return_value=MagicMock()),
patch("torch.empty", return_value=MagicMock()),
patch(
f"{self._MOD}.make_ioring",
side_effect=RuntimeError("ioring init failed"),
),
patch(f"{self._MOD}.make_iovec", return_value=MagicMock()),
patch("torch.cuda.Stream", return_value=MagicMock()),
pytest.raises(RuntimeError, match="ioring init failed"),
):
Hf3fsClient(
path=str(tmp_path / "fail2.bin"),
size=1024,
bytes_per_page=256,
entries=4,
)
assert fake_shm_r.closed is True, (
"shm_r was not closed after constructor failure"
)
assert fake_shm_w.closed is True, (
"shm_w was not closed after constructor failure"
)
def test_constructor_failure_does_not_close_unallocated_shm(self, tmp_path):
"""Failure before SharedMemory is created must not raise AttributeError
or TypeError from cleanup."""
with (
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
patch(f"{self._MOD}.register_fd"),
patch(f"{self._MOD}.deregister_fd"),
patch(
f"{self._MOD}.extract_mount_point",
side_effect=RuntimeError("early failure"),
),
patch("os.open", return_value=77),
patch("os.ftruncate"),
patch("os.close"),
patch("torch.cuda.Stream", return_value=MagicMock()),
pytest.raises(RuntimeError, match="early failure"),
):
Hf3fsClient(
path=str(tmp_path / "early_fail.bin"),
size=1024,
bytes_per_page=256,
entries=4,
)
# ------------------------------------------------------------------
# _release_resources on already-cleared state must be a no-op
# ------------------------------------------------------------------
def test_release_resources_on_empty_state_is_safe(self, tmp_path):
"""_release_resources() on a fully-cleared client must not raise."""
client, _, _ = self._make_client(tmp_path)
with (
patch(f"{self._MOD}.deregister_fd"),
patch("os.close"),
):
client.close() # clears all handles
with (
patch(f"{self._MOD}.deregister_fd") as mock_dereg2,
patch("os.close") as mock_os_close2,
):
client._release_resources() # must not raise
mock_dereg2.assert_not_called()
mock_os_close2.assert_not_called()
@@ -1,233 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for HF3FS KV Connector high-level components:
- TestHf3fsMockClient : file-backed mock client I/O correctness
- TestHF3FSKVConnectorStats: metric collection, aggregation, serialisation
"""
import os
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector import (
HF3FSKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils.hf3fs_mock_client import (
Hf3fsClient as MockHf3fsClient,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@pytest.fixture
def hf3fs_stats():
"""Fresh HF3FSKVConnectorStats instance."""
return HF3FSKVConnectorStats()
def _make_cuda_event():
"""Return a real CUDA event when available, otherwise a MagicMock."""
if torch.cuda.is_available():
return torch.cuda.Event()
return MagicMock()
# ===========================================================================
# TestHf3fsMockClient
# ===========================================================================
class TestHf3fsMockClient:
"""Tests for hf3fs_mock_client.Hf3fsClient (file-backend mock)."""
def test_init_creates_file(self, tmp_path):
"""Initializing the client should create the backing file."""
path = str(tmp_path / "test_file")
client = MockHf3fsClient(path=path, size=4096, bytes_per_page=512, entries=4)
assert os.path.exists(path), "Backing file should be created on init"
assert os.path.getsize(path) == 4096
client.close()
@pytest.mark.parametrize(
"dtype, bytes_per_page",
[
(torch.float32, 512),
(torch.float16, 256),
(torch.bfloat16, 256),
],
ids=["float32", "float16", "bfloat16"],
)
def test_batch_write_and_read_dtype(self, tmp_path, dtype, bytes_per_page):
"""Write a tensor of the given dtype and verify round-trip correctness."""
path = str(tmp_path / f"rw_{dtype}")
client = MockHf3fsClient(
path=path, size=bytes_per_page * 8, bytes_per_page=bytes_per_page, entries=4
)
elem_size = torch.tensor([], dtype=dtype).element_size()
numel = bytes_per_page // elem_size
tensor_write = torch.arange(numel, dtype=dtype)
event = _make_cuda_event()
results = client.batch_write([0], [tensor_write], event)
assert results == [bytes_per_page], f"Write should succeed, got {results}"
tensor_read = torch.zeros(numel, dtype=dtype)
results = client.batch_read([0], [tensor_read])
assert results == [bytes_per_page], f"Read should succeed, got {results}"
assert torch.equal(tensor_write, tensor_read), (
"Read tensor should match written tensor"
)
client.close()
def test_batch_read_empty_file_returns_error(self, tmp_path):
"""Reading out-of-bounds offset should return -1."""
bytes_per_page = 128
size = bytes_per_page * 4
path = str(tmp_path / "empty_read")
client = MockHf3fsClient(
path=path, size=size, bytes_per_page=bytes_per_page, entries=4
)
numel = bytes_per_page // 4
tensor_read = torch.zeros(numel, dtype=torch.float32)
results = client.batch_read([size], [tensor_read]) # offset == size => OOB
assert results[0] == -1, "Out-of-bounds read should return -1"
client.close()
def test_batch_write_out_of_bounds_returns_error(self, tmp_path):
"""Writing at an offset beyond file size should return -1."""
bytes_per_page = 128
size = bytes_per_page * 4
path = str(tmp_path / "oob_write")
client = MockHf3fsClient(
path=path, size=size, bytes_per_page=bytes_per_page, entries=4
)
numel = bytes_per_page // 4
tensor = torch.ones(numel, dtype=torch.float32)
event = _make_cuda_event()
results = client.batch_write([size], [tensor], event) # OOB offset
assert results[0] == -1, "Out-of-bounds write should return -1"
client.close()
def test_multiple_tensors_rw(self, tmp_path):
"""Write multiple tensors at different offsets, then read all back."""
bytes_per_page = 128
n = 4
path = str(tmp_path / "multi_rw")
client = MockHf3fsClient(
path=path,
size=bytes_per_page * n * 2,
bytes_per_page=bytes_per_page,
entries=8,
)
tensors_write = [
torch.full((bytes_per_page // 4,), float(i), dtype=torch.float32)
for i in range(n)
]
offsets = [i * bytes_per_page for i in range(n)]
event = _make_cuda_event()
results = client.batch_write(offsets, tensors_write, event)
assert all(r == bytes_per_page for r in results)
tensors_read = [
torch.zeros(bytes_per_page // 4, dtype=torch.float32) for _ in range(n)
]
results = client.batch_read(offsets, tensors_read)
assert all(r == bytes_per_page for r in results)
for i, (tw, tr) in enumerate(zip(tensors_write, tensors_read)):
assert torch.allclose(tw, tr), f"Tensor {i} mismatch after round-trip"
client.close()
def test_flush_and_close_no_error(self, tmp_path):
"""flush() and close() should not raise exceptions."""
path = str(tmp_path / "flush_close")
client = MockHf3fsClient(path=path, size=1024, bytes_per_page=128, entries=4)
client.flush()
client.close()
# ===========================================================================
# TestHF3FSKVConnectorStats
# ===========================================================================
class TestHF3FSKVConnectorStats:
"""Tests for HF3FSKVConnectorStats metric collection and aggregation."""
def test_initial_is_empty(self, hf3fs_stats):
"""Fresh stats object should report is_empty() == True."""
assert hf3fs_stats.is_empty() is True
@pytest.mark.parametrize(
"task_type, duration_key",
[
("Saved", "save_duration"),
("Loaded", "load_duration"),
],
ids=["save", "load"],
)
def test_record_success_duration(self, hf3fs_stats, task_type, duration_key):
"""Recording a successful task should update duration list and total count."""
hf3fs_stats.record_success_task_duration(task_type, 0.5)
assert not hf3fs_stats.is_empty()
assert len(hf3fs_stats.data[duration_key]) == 1
assert hf3fs_stats.data[duration_key][0] == pytest.approx(0.5)
assert hf3fs_stats.data["num_transfer_task"] == 1
@pytest.mark.parametrize(
"task_type, failed_key",
[
("Saved", "num_failed_save"),
("Loaded", "num_failed_load"),
],
ids=["save", "load"],
)
def test_record_failed_task(self, hf3fs_stats, task_type, failed_key):
"""Recording a failed task should increment the corresponding counter."""
hf3fs_stats.record_failed_task_count(task_type)
assert hf3fs_stats.data[failed_key] == 1
assert hf3fs_stats.data["num_transfer_task"] == 1
def test_aggregate_two_stats(self):
"""aggregate() should merge save/load duration lists and sum counters."""
stats1 = HF3FSKVConnectorStats()
stats1.record_success_task_duration("Saved", 0.1)
stats1.record_success_task_duration("Loaded", 0.2)
stats2 = HF3FSKVConnectorStats()
stats2.record_success_task_duration("Saved", 0.3)
stats2.record_failed_task_count("Loaded")
stats1.aggregate(stats2)
assert stats1.data["save_duration"] == pytest.approx([0.1, 0.3])
assert stats1.data["load_duration"] == pytest.approx([0.2])
assert stats1.data["num_failed_load"] == 1
assert stats1.data["num_transfer_task"] == 4
def test_reduce_with_data(self):
"""reduce() computes correct averages when data is present."""
stats = HF3FSKVConnectorStats()
stats.record_success_task_duration("Saved", 1.0)
stats.record_success_task_duration("Saved", 3.0)
result = stats.reduce()
assert result["Num save task success"] == pytest.approx(2.0, rel=0.01)
assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01)
assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01)
# Reduced values must be plain Python scalars so CLI logging renders
# them without numpy reprs (eg np.float64(...)).
assert all(not isinstance(v, np.generic) for v in result.values())
def test_clone_and_reset(self, hf3fs_stats):
"""clone_and_reset() returns a copy with data and resets the original."""
hf3fs_stats.record_success_task_duration("Saved", 0.7)
hf3fs_stats.record_success_task_duration("Loaded", 0.4)
clone = hf3fs_stats.clone_and_reset()
assert clone.data["num_transfer_task"] == 2
assert hf3fs_stats.is_empty()
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for HF3FS metadata server data structures and allocation logic:
- RankFileMetadata : page allocation / release primitives
- KeyMetadata : per-key rank-page tracking and completion detection
- GlobalMetadataState : coordinated allocation with cache-hit semantics
"""
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_metadata_server import (
GlobalMetadataState,
KeyMetadata,
RankFileMetadata,
)
# ===========================================================================
# TestRankFileMetadata
# ===========================================================================
class TestRankFileMetadata:
"""Unit tests for RankFileMetadata page allocation primitives."""
@pytest.mark.parametrize(
"alloc_count, expected_pages",
[(3, 3), (5, 0)],
ids=["alloc_partial", "alloc_exceeds"],
)
def test_allocate_pages(self, alloc_count, expected_pages):
"""allocate_pages returns correct pages or empty list when insufficient."""
rank_meta = RankFileMetadata(rank_id=0, num_pages=3, free_pages=list(range(3)))
pages = rank_meta.allocate_pages(alloc_count)
assert len(pages) == expected_pages
if expected_pages > 0:
rank_meta.release_pages(pages)
assert rank_meta.get_free_page_count() == 3
def test_release_pages_restores_count(self):
"""Releasing allocated pages returns them to the free pool."""
rank_meta = RankFileMetadata(rank_id=0, num_pages=4, free_pages=list(range(4)))
pages = rank_meta.allocate_pages(2)
assert rank_meta.get_free_page_count() == 2
rank_meta.release_pages(pages)
assert rank_meta.get_free_page_count() == 4
def test_release_pages_no_duplicates(self):
"""Releasing the same page twice must not create duplicates."""
rank_meta = RankFileMetadata(rank_id=0, num_pages=3, free_pages=list(range(3)))
rank_meta.allocate_pages(1) # takes page 0
rank_meta.release_pages([0])
rank_meta.release_pages([0]) # second release of the same page
assert rank_meta.get_free_page_count() == 3
# ===========================================================================
# TestKeyMetadata
# ===========================================================================
class TestKeyMetadata:
"""Unit tests for KeyMetadata completion tracking."""
def test_is_complete_false_until_all_ranks(self):
"""is_complete() returns True only when all ranks confirmed."""
key_meta = KeyMetadata(key="k", rank_to_page={}, tp_world_size=2)
assert key_meta.is_complete() is False
key_meta.add_rank_page(0, 5)
assert key_meta.is_complete() is False
key_meta.add_rank_page(1, 10)
assert key_meta.is_complete() is True
def test_get_rank_page_returns_none_for_missing_rank(self):
"""get_rank_page() returns None when the rank has no entry."""
key_meta = KeyMetadata(key="k", rank_to_page={0: 3}, tp_world_size=2)
assert key_meta.get_rank_page(0) == 3
assert key_meta.get_rank_page(1) is None
def test_get_all_pages(self):
"""get_all_pages() returns all (rank, page) pairs."""
key_meta = KeyMetadata(key="k", rank_to_page={0: 1, 1: 2}, tp_world_size=2)
pairs = key_meta.get_all_pages()
assert set(pairs) == {(0, 1), (1, 2)}
# ===========================================================================
# TestGlobalMetadataStateAllocation
# ===========================================================================
class TestGlobalMetadataStateAllocation:
"""Tests for GlobalMetadataState allocation and cache-hit semantics."""
def test_uninitialized_rank_raises_on_allocate(self):
"""allocate_pages_for_keys raises ValueError for unknown rank."""
state = GlobalMetadataState()
with pytest.raises((ValueError, Exception)):
state.allocate_pages_for_keys(99, [("key", "")])
def test_uninitialized_rank_raises_on_get_locations(self):
"""get_key_locations raises ValueError for unknown rank."""
state = GlobalMetadataState()
with pytest.raises((ValueError, Exception)):
state.get_key_locations(99, ["any_key"])
def test_basic_allocation_and_confirm(self):
"""Allocating a page and confirming it marks the key as complete."""
state = GlobalMetadataState()
state.initialize_rank(0, 4)
results = state.allocate_pages_for_keys(0, [("K", "")])
assert results["K"] >= 0
state.confirm_write_for_keys(0, [("K", results["K"])])
assert state.batch_key_exists(["K"]) == [True]
locations = state.get_key_locations(0, ["K"])
assert locations == [results["K"]]
def test_allocate_pages_cache_hit_does_not_leak_pages(self):
"""Cache-hit key must not consume a page from the free pool;
the pre-allocated slot must be returned before reusing the existing page.
"""
state = GlobalMetadataState()
state.initialize_rank(0, 5) # 5 free pages: [0,1,2,3,4]
# Simulate a key that has already been fully written and confirmed.
state.key_metadata["K_cached"] = KeyMetadata(
key="K_cached", rank_to_page={0: 2}, tp_world_size=1
)
free_before = state.rank_metadata[0].get_free_page_count() # 5
results = state.allocate_pages_for_keys(0, [("K_cached", ""), ("K_new", "")])
free_after = state.rank_metadata[0].get_free_page_count()
# Cache-hit key must reuse its existing page.
assert results["K_cached"] == 2, (
f"Cache-hit key should reuse page 2, got {results['K_cached']}"
)
# New key must receive a valid page.
assert results["K_new"] >= 0, (
f"New key should get a valid page, got {results['K_new']}"
)
# Exactly one page consumed from the free pool.
assert free_before - free_after == 1, (
f"Expected 1 page consumed, got delta={free_before - free_after}"
)
def test_allocate_pages_all_cache_hits_frees_all_slots(self):
"""When every key in the batch is a cache hit, no pages are consumed."""
state = GlobalMetadataState()
state.initialize_rank(0, 5)
for key, page in (("K1", 0), ("K2", 1)):
state.key_metadata[key] = KeyMetadata(
key=key, rank_to_page={0: page}, tp_world_size=1
)
free_before = state.rank_metadata[0].get_free_page_count()
results = state.allocate_pages_for_keys(0, [("K1", ""), ("K2", "")])
free_after = state.rank_metadata[0].get_free_page_count()
assert results["K1"] == 0
assert results["K2"] == 1
assert free_after == free_before, (
f"All-cache-hit batch must not consume free pages; "
f"before={free_before}, after={free_after}"
)
def test_allocate_returns_minus_one_when_pool_exhausted(self):
"""If the free pool is exhausted, all new keys receive -1."""
state = GlobalMetadataState()
state.initialize_rank(0, 1) # only 1 free page
results = state.allocate_pages_for_keys(0, [("K1", ""), ("K2", "")])
# allocate_pages uses all-or-nothing: 2 needed but only 1 available → []
assert all(v == -1 for v in results.values()), f"Expected all -1, got {results}"
def test_confirm_write_releases_pages(self):
"""confirm_write_for_keys with pages_to_release returns them to pool."""
state = GlobalMetadataState()
state.initialize_rank(0, 3)
results = state.allocate_pages_for_keys(0, [("K", "")])
page = results["K"]
free_after_alloc = state.rank_metadata[0].get_free_page_count()
state.confirm_write_for_keys(0, [("K", page)], pages_to_release=[page])
free_after_release = state.rank_metadata[0].get_free_page_count()
assert free_after_release == free_after_alloc + 1
@@ -1,158 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparseKVConnectorStats,
)
def test_is_empty_on_fresh_stats():
stats = HiSparseKVConnectorStats()
assert stats.is_empty()
def test_record_snapshot_and_reduce():
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
assert not stats.is_empty()
reduced = stats.reduce()
assert reduced["HiSparse hot-buffer hits"] == 12
assert reduced["HiSparse hot-buffer misses"] == 4
assert reduced["HiSparse host-to-device bytes"] == 64
def test_record_host_usage_reports_latest_level():
stats = HiSparseKVConnectorStats()
stats.record_host_usage(usage=0.4, pending_page_transfers=2)
stats.record_host_usage(usage=0.8, pending_page_transfers=0)
reduced = stats.reduce()
assert reduced["HiSparse host KV cache usage %"] == 80.0
def test_aggregate_keeps_latest_host_usage_level():
first = HiSparseKVConnectorStats()
first.record_host_usage(usage=0.4, pending_page_transfers=2)
second = HiSparseKVConnectorStats()
second.record_host_usage(usage=0.8, pending_page_transfers=0)
second.record_host_usage(usage=0.6, pending_page_transfers=1)
first.aggregate(second)
assert first.data["host_cache_usage_perc"] == [0.4, 0.6]
assert first.data["pending_page_transfers"] == [2, 1]
def test_aggregate_extends_snapshot_deltas():
first = HiSparseKVConnectorStats()
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
second = HiSparseKVConnectorStats()
second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
first.aggregate(second)
assert first.data["cache_hits"] == [7, 5]
assert first.data["cache_misses"] == [3, 1]
assert first.data["host_to_device_bytes"] == [48, 16]
def test_aggregate_skips_empty_stats():
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16)
stats.aggregate(HiSparseKVConnectorStats())
assert stats.data["cache_hits"] == [2]
def test_build_kv_connector_stats_round_trip():
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64)
payload = stats.to_dict()
rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload)
assert rebuilt is not None
assert isinstance(rebuilt, HiSparseKVConnectorStats)
assert rebuilt.reduce() == stats.reduce()
@pytest.mark.parametrize("kv_cache_metrics", [True, False])
def test_prom_metrics_observe_host_metrics(kv_cache_metrics: bool):
from types import SimpleNamespace
from typing import Any
from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparsePromMetrics,
)
from vllm.v1.metrics.stats import KVCacheEvictionEvent
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.observed: list[float] = []
def labels(self, *labelvalues: object) -> "_FakeMetric":
return self
def inc(self, value: int | float) -> None:
self.increments.append(value)
def set(self, value: int | float) -> None:
self.set_values.append(value)
def observe(self, value: float) -> None:
self.observed.append(value)
created: dict[str, _FakeMetric] = {}
class _NamedFake(_FakeMetric):
def __init__(self, **kwargs: Any):
super().__init__(**kwargs)
created[kwargs["name"]] = self
prom = HiSparsePromMetrics(
vllm_config=SimpleNamespace(
kv_transfer_config=None,
observability_config=SimpleNamespace(
kv_cache_metrics=kv_cache_metrics, custom_histogram_buckets=None
),
),
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
labelnames=["model_name"],
per_engine_labelvalues={0: ["model"]},
)
stats = HiSparseKVConnectorStats()
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
stats.record_host_evictions([KVCacheEvictionEvent(5.0, 2.0, (1.0, 2.0))])
later = HiSparseKVConnectorStats()
later.record_host_evictions([KVCacheEvictionEvent(3.0, 3.0, ())])
prom.observe(stats.aggregate(later).to_dict())
assert created["vllm:hisparse_cache_hits"].increments == [3]
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
if not kv_cache_metrics:
assert not any("host_block" in name for name in created)
return
assert created["vllm:hisparse_host_block_lifetime_seconds"].observed == [5.0, 3.0]
assert created["vllm:hisparse_host_block_idle_before_evict_seconds"].observed == [
2.0,
3.0,
]
assert created["vllm:hisparse_host_block_reuse_gap_seconds"].observed == [1.0, 2.0]
File diff suppressed because it is too large Load Diff
@@ -1,335 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Hardware-fair request routing in the MoRIIO toy P/D proxy.
Exercises the REAL ``flat_interleaved_dp_route`` from the toy proxy (loaded from
its ``examples/`` path with heavy deps stubbed) — no routing logic is copied
here. This is the request-distribution half of end-to-end fairness: the proxy
must hand every prefill/decode (instance, dp_rank) slot an equal share so no GPU
is starved. The connector-side read routing is covered in
``test_moriio_routing_fairness.py``.
Regression target: the previous scheme derived ``instance = req % n_instances``
and ``dp_rank = req % dp_size`` from the same counter, so when
``n_instances | dp_size`` each instance was locked to a stride-``n`` subset of
its ranks — e.g. 2 prefill instances x DP8 stranded 4 of every node's 8 GPUs.
``flat_interleaved_dp_route`` walks ONE counter over the full
``(instance, dp_rank)`` slot space, so the two selections can never alias.
Role shapes below are exactly those in the RFC (#46107) deployments:
(1, 1) 1P/1D TP8 (2, 1) 2P/2D TP8 (4, 1) 4D TP8
(1, 8) 1D DP8EP (2, 8) 2P DP8EP (3, 8) 3D DP8EP
"""
import contextlib
import ctypes
import importlib.util
import os
import subprocess
import sys
import time
import types
from collections import Counter
from pathlib import Path
from typing import cast
import msgpack
import pytest
import zmq
_MISSING = object()
PROXY_REL = "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
def _module(name, **attrs):
module = types.ModuleType(name)
for attr, value in attrs.items():
setattr(module, attr, value)
return module
def _package(name):
module = _module(name)
module.__path__ = []
return module
class _QuartStub:
def __init__(self, *a, **k):
pass
def route(self, *a, **k):
return lambda fn: fn
def post(self, *a, **k):
return self.route()
async def _make_response_stub(value):
return value
@contextlib.contextmanager
def _proxy_import_stubs():
"""Stub the proxy's external deps so the module imports for a pure unit test.
Only third-party/vllm imports are stubbed; the routing function under test
is executed as-is from the real module.
"""
common = "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
class _MoRIIOConstants:
TRANSFER_PREFIX = "moriio-transfer"
stubs = {
"aiohttp": _module("aiohttp"),
"msgpack": _module("msgpack"),
"zmq": _module("zmq"),
"quart": _module(
"quart",
Quart=_QuartStub,
Request=object,
make_response=_make_response_stub,
request=object(),
),
"vllm": _package("vllm"),
"vllm.distributed": _package("vllm.distributed"),
"vllm.distributed.kv_transfer": _package("vllm.distributed.kv_transfer"),
"vllm.distributed.kv_transfer.kv_connector": _package(
"vllm.distributed.kv_transfer.kv_connector"
),
"vllm.distributed.kv_transfer.kv_connector.v1": _package(
"vllm.distributed.kv_transfer.kv_connector.v1"
),
"vllm.distributed.kv_transfer.kv_connector.v1.moriio": _package(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio"
),
common: _module(common, MoRIIOConstants=_MoRIIOConstants),
}
saved = {}
for name, module in stubs.items():
if name not in sys.modules:
saved[name] = _MISSING
sys.modules[name] = module
try:
yield
finally:
for name, previous in saved.items():
if previous is _MISSING:
sys.modules.pop(name, None)
else:
sys.modules[name] = cast(types.ModuleType, previous)
def _load_proxy_module():
path = Path(__file__).parents[4] / PROXY_REL
spec = importlib.util.spec_from_file_location("moriio_proxy_under_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
with _proxy_import_stubs():
spec.loader.exec_module(module)
return module
@pytest.fixture(scope="module")
def route():
return _load_proxy_module().flat_interleaved_dp_route
def _instances(n: int, dp_size: int):
# Only dp_size is read by the router; tp_size carried for realism.
return [{"dp_size": dp_size, "tp_size": 8 // dp_size} for _ in range(n)]
# (n_instances, dp_size) for every distinct P/D role shape in the RFC configs.
ROLE_SHAPES = [(1, 1), (2, 1), (4, 1), (1, 8), (2, 8), (3, 8)]
SHAPE_IDS = [f"n{n}_dp{dp}" for n, dp in ROLE_SHAPES]
def _route_n(route, instances, count):
# Proxy uses 1-indexed request numbers (slot = (request_number - 1) % ...).
return [route(rn, instances) for rn in range(1, count + 1)]
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
def test_full_slot_space_is_covered_uniformly(route, n, dp):
instances = _instances(n, dp)
period = n * dp
# Three full cycles -> every (instance, dp_rank) slot must be hit the same
# number of times (exactly uniform, no starved slot, no aliasing).
hits = Counter(_route_n(route, instances, period * 3))
expected: set[tuple[int, int | None]]
if dp == 1:
expected = {(inst, None) for inst in range(n)}
else:
expected = {(inst, r) for inst in range(n) for r in range(dp)}
assert set(hits) == expected, f"missing slots: {expected - set(hits)}"
assert max(hits.values()) == min(hits.values())
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
def test_instance_and_dp_rank_marginals_are_balanced(route, n, dp):
instances = _instances(n, dp)
routed = _route_n(route, instances, n * dp * 5)
inst_counts = Counter(inst for inst, _ in routed)
assert set(inst_counts) == set(range(n))
assert max(inst_counts.values()) == min(inst_counts.values())
dp_counts = Counter(r for _, r in routed)
if dp == 1:
assert set(dp_counts) == {None}
else:
assert set(dp_counts) == set(range(dp))
assert max(dp_counts.values()) == min(dp_counts.values())
def test_tp_instance_forwards_no_dp_rank(route):
# dp_size == 1 (a TP instance) must yield dp_rank None so the proxy never
# forwards an out-of-range data-parallel rank.
assert all(r is None for _, r in _route_n(route, _instances(2, 1), 8))
def test_two_instance_dp8_gives_every_node_all_ranks(route):
# Direct regression for the stranded-GPU bug: with 2 prefill instances x DP8
# each instance must receive ALL 8 dp ranks (16 distinct slots), not a
# stride-2 subset of 4.
routed = _route_n(route, _instances(2, 8), 2 * 8)
per_instance: dict[int, set] = {0: set(), 1: set()}
for inst, dp_rank in routed:
per_instance[inst].add(dp_rank)
assert per_instance[0] == set(range(8))
assert per_instance[1] == set(range(8))
def test_consecutive_requests_alternate_instances(route):
# Interleaved order spreads consecutive requests across instances rather
# than filling one instance's ranks before moving on.
insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)]
assert insts == [0, 1, 2]
@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep")
def test_discovery_heartbeat_progresses_while_worker_holds_gil():
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
payload = {
"type": "P",
"http_address": "127.0.0.1:8000",
"zmq_address": "host:127.0.0.1,handshake:6301,notify:61005",
"dp_size": 1,
"tp_size": 8,
"transfer_mode": "READ",
}
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
port = socket.bind_to_random_port("tcp://127.0.0.1")
heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02)
try:
assert socket.poll(5000), "Discovery did not register"
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
# PyDLL retains the GIL throughout the native call.
ctypes.PyDLL(None).usleep(300_000)
received = 0
while socket.poll(0):
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
received += 1
assert received >= 3, "Discovery stalled while the worker held the GIL"
finally:
heartbeat.shutdown()
heartbeat.shutdown()
while socket.poll(0):
socket.recv_multipart()
assert not socket.poll(100), "Heartbeats continued after shutdown"
@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs")
def test_discovery_heartbeat_exits_when_worker_dies():
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat
owner_code = """
import runpy
import sys
Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"]
heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02)
print(heartbeat._process.pid, flush=True)
sys.stdin.read()
"""
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
port = socket.bind_to_random_port("tcp://127.0.0.1")
owner = subprocess.Popen(
[
sys.executable,
"-c",
owner_code,
moriio_heartbeat.__file__,
f"tcp://127.0.0.1:{port}",
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
text=True,
)
child_pid = None
child_exited = False
try:
assert socket.poll(5000), "Discovery did not register"
assert owner.stdout is not None
child_pid = int(owner.stdout.readline())
owner.kill()
owner.wait(timeout=5)
stat = Path(f"/proc/{child_pid}/stat")
deadline = time.monotonic() + 5
while stat.exists():
# A terminated child may await reaping by the container's init.
try:
if stat.read_text().split(") ")[1].startswith("Z "):
break
except FileNotFoundError:
break
assert time.monotonic() < deadline, "Orphan heartbeat survived worker"
time.sleep(0.01)
child_exited = True
finally:
if owner.poll() is None:
owner.kill()
owner.communicate(timeout=5)
if child_pid is not None and not child_exited:
with contextlib.suppress(ProcessLookupError):
os.kill(child_pid, 9)
def test_discovery_heartbeat_recovers_after_router_outage(capfd):
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
payload = {"type": "P", "http_address": "127.0.0.1:8000"}
with zmq.Context() as context:
with context.socket(zmq.ROUTER) as socket:
port = socket.bind_to_random_port("tcp://127.0.0.1")
endpoint = f"tcp://127.0.0.1:{port}"
heartbeat = MoRIIOHeartbeat(endpoint, payload, 0.001, max_retries=1)
try:
assert socket.poll(5000), "Discovery did not register"
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
except BaseException:
heartbeat.shutdown()
raise
try:
# Fill the real DEALER queue while discovery is unavailable.
deadline = time.monotonic() + 10
errors = ""
while "heartbeat send failed" not in errors:
errors += capfd.readouterr().err
assert time.monotonic() < deadline, "No send timeout during outage"
time.sleep(0.01)
with context.socket(zmq.ROUTER) as socket:
socket.bind(endpoint)
assert socket.poll(5000), "Discovery did not recover after outage"
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
assert heartbeat._process.poll() is None
finally:
heartbeat.shutdown()
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MoRIIO router-authoritative DP-rank routing.
These lock in the redesign that addresses the PR #45043 review feedback
(njhill): the connector must NOT self-derive a DP rank, and the prefill leg
must *return* the rank it actually ran on via ``request_finished`` so the
decode->prefill notify is routed by pure propagation (READ / serial WRITE),
not by an independent hash on each side.
Following test_moriio_unmap.py, we bind the unbound method to a lightweight
stand-in rather than constructing a full scheduler.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
)
from vllm.v1.request import RequestStatus
_request_finished = MoRIIOConnectorScheduler.request_finished
_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
s = SimpleNamespace(
is_producer=True,
_is_hma_required=False,
_has_mamba=False,
blocks_per_sw=[0],
_global_dp_rank=global_dp_rank,
engine_id="engine-abc",
host_ip="10.0.0.1",
handshake_port=6300,
side_notify_port=7300,
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_size=dp_size,
data_parallel_size_local=dp_size_local,
tensor_parallel_size=1,
)
),
)
s.get_exchange_clipped_blocks = _clip_blocks.__get__(s)
return s
def _req(
request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED
):
return SimpleNamespace(
request_id=request_id,
kv_transfer_params=params,
status=status,
)
def test_request_finished_returns_prefill_rank_for_propagation():
# 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8.
s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8)
req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"})
delay_free, params = _request_finished(s, req, block_ids=[])
# The prefill leg returns the rank IT actually ran on, marked authoritative,
# so the router forwards it to decode and the notify propagates (no hashing).
assert params is not None
assert params["remote_dp_rank"] == 11
assert params["remote_dp_rank_override"] is True
assert params["remote_dp_size"] == 16
assert params["remote_dp_size_local"] == 8
assert params["transfer_id"] == "t1"
# Empty block_ids => nothing deferred, blocks freed immediately.
assert delay_free is False
def test_request_finished_rank_tracks_the_actual_rank():
# A different rank must be echoed verbatim -- the value is the producer's
# own rank, never a recomputation of the request id.
s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8)
req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"})
_, params = _request_finished(s, req, block_ids=[])
assert params["remote_dp_rank"] == 4
assert params["remote_dp_size_local"] == 8
def test_request_finished_no_kv_params_is_noop():
s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8)
req = _req("r3", None)
delay_free, params = _request_finished(s, req, block_ids=[])
assert delay_free is False
assert params is None
@@ -1,210 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Hardware-fair KV-read routing across MoRIIO heterogeneous P/D configs.
Dependency-light (no GPU / ROCm / mori): builds bare ``MoRIIOConnectorWorker``
instances via ``object.__new__`` and drives the REAL read-source routing
decision (``_resolve_read_source`` / ``_next_flex_tp_rank``) directly -- no
routing logic is re-implemented here, so a future change to those functions is
what these tests exercise.
Scope -- the CONNECTOR (decode-side) read routing. The connector never selects
the prefill instance; the proxy does and hands the connector a ``remote_host`` +
``remote_dp_rank`` per request. The connector's only fairness lever is WHICH
prefill (dp, tp) rank each read targets, and the RFC's deployments collapse to
three real connector behaviours:
symmetric TP (TP prefill + TP decode) -> decode tp_k reads prefill tp_k
flexible (TP prefill + DP decode) -> round-robin over prefill tp0..N-1
owner DP (DP prefill, any decode) -> read the owner rank, tp0
These assume the proxy delivers a FAIR request stream (each prefill instance +
owner dp-rank an equal share) and verify the connector never re-introduces a
bottleneck. That proxy contract is checked separately against the real
``flat_interleaved_dp_route`` in the toy-proxy tests (PR #46115).
A node runs one of two modes (8 GPUs each):
* TP8 -> (dp_size, tp_size) = (1, 8); MLA latent KV REPLICATED on all ranks.
* DP8EP -> (dp_size, tp_size) = (8, 1); KV PARTITIONED, one owner rank/request.
"""
from collections import Counter
from dataclasses import dataclass
from unittest.mock import MagicMock
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ReqMeta,
get_port_offset,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorWorker,
)
MODE_DIMS = {"TP8": (1, 8), "DP8EP": (8, 1)} # (dp_size, tp_size), 8 GPUs/node
@dataclass(frozen=True)
class PDConfig:
name: str
p_mode: str
d_mode: str
@property
def p_dp(self) -> int:
return MODE_DIMS[self.p_mode][0]
@property
def p_tp(self) -> int:
return MODE_DIMS[self.p_mode][1]
@property
def d_dp(self) -> int:
return MODE_DIMS[self.d_mode][0]
@property
def d_tp(self) -> int:
return MODE_DIMS[self.d_mode][1]
@property
def n_prefill_gpus(self) -> int:
return self.p_dp * self.p_tp
CONFIGS = [
PDConfig("1P_TP8:1D_TP8", "TP8", "TP8"),
PDConfig("2P_TP8:1D_DP8EP", "TP8", "DP8EP"),
PDConfig("2P_TP8:2D_TP8", "TP8", "TP8"),
PDConfig("2P_DP8EP:3D_DP8EP", "DP8EP", "DP8EP"),
PDConfig("2P_DP8EP:4D_TP8", "DP8EP", "TP8"),
]
CONFIG_IDS = [c.name for c in CONFIGS]
def make_decode_worker(*, world_size: int, tp_rank: int, dp_rank: int):
w = object.__new__(MoRIIOConnectorWorker)
w.world_size = world_size
w.tp_rank = tp_rank
w.dp_rank = dp_rank
w.use_mla = True
return w
def make_meta(*, p_tp: int, p_dp: int, remote_dp_rank: int, host: str = "phost0"):
return ReqMeta(
transfer_id="t",
local_block_ids=[1],
remote_block_ids=[2],
remote_host=host,
remote_port=1234,
remote_handshake_port=6301,
remote_notify_port=61005,
remote_engine_id=f"{host}:6301",
tp_size=p_tp,
remote_dp_size=p_dp,
remote_dp_rank=remote_dp_rank,
)
def build_decode_workers(cfg: PDConfig) -> list:
"""Decode workers that issue reads for one prefill instance. A TP decode
instance reads from every tp rank; a DP decode instance from each dp-rank
worker. Reused across requests so per-worker round-robin state advances."""
if cfg.d_tp > 1:
return [
make_decode_worker(world_size=cfg.d_tp, tp_rank=r, dp_rank=0)
for r in range(cfg.d_tp)
]
return [
make_decode_worker(world_size=1, tp_rank=0, dp_rank=d) for d in range(cfg.d_dp)
]
def prefill_target_multiset(cfg: PDConfig, rounds: int) -> Counter:
"""Drive the REAL _resolve_read_source over a fair input stream; tally which
prefill GPU each read targets. The only modelled assumption is the proxy
contract -- each owner dp-rank delivered equally (``for owner_dp in
range(p_dp)``); no proxy algorithm is reproduced."""
workers = build_decode_workers(cfg)
hits: Counter = Counter()
for _ in range(rounds):
for owner_dp in range(cfg.p_dp):
for worker in workers:
meta = make_meta(p_tp=cfg.p_tp, p_dp=cfg.p_dp, remote_dp_rank=owner_dp)
chosen_tp, _flexible = worker._resolve_read_source(meta)
target = get_port_offset(owner_dp, chosen_tp, cfg.p_tp)
assert 0 <= target < cfg.n_prefill_gpus
hits[target] += 1
return hits
@pytest.mark.parametrize("cfg", CONFIGS, ids=CONFIG_IDS)
def test_kv_read_load_is_hardware_fair(cfg: PDConfig) -> None:
# rounds a multiple of p_tp so the round-robin closes on an exact cycle.
hits = prefill_target_multiset(cfg, rounds=16)
gpus = set(range(cfg.n_prefill_gpus))
assert set(hits) == gpus, f"unused prefill GPUs: {sorted(gpus - set(hits))}"
assert max(hits.values()) == min(hits.values()), dict(sorted(hits.items()))
@pytest.mark.parametrize("cfg", CONFIGS, ids=CONFIG_IDS)
def test_flexible_gate_fires_only_for_tp_prefill_dp_decode(cfg: PDConfig) -> None:
flags = set()
for worker in build_decode_workers(cfg):
meta = make_meta(p_tp=cfg.p_tp, p_dp=cfg.p_dp, remote_dp_rank=0)
_chosen, flexible = worker._resolve_read_source(meta)
flags.add(flexible)
is_mirror = cfg.d_tp == 1 and cfg.p_dp == 1 and cfg.p_tp > 1
assert flags == {is_mirror}
def test_symmetric_tp_is_a_bijection() -> None:
targets = []
for tp_rank in range(8):
worker = make_decode_worker(world_size=8, tp_rank=tp_rank, dp_rank=0)
chosen_tp, flexible = worker._resolve_read_source(
make_meta(p_tp=8, p_dp=1, remote_dp_rank=0)
)
assert not flexible
targets.append(chosen_tp)
assert sorted(targets) == list(range(8))
def test_owner_dp_read_is_faithful_and_covers_every_rank() -> None:
worker = make_decode_worker(world_size=8, tp_rank=3, dp_rank=0)
targets = []
for owner_dp in range(8):
chosen_tp, flexible = worker._resolve_read_source(
make_meta(p_tp=1, p_dp=8, remote_dp_rank=owner_dp)
)
assert not flexible
assert chosen_tp == 0 # p_tp == 1, only tp0 exists
targets.append(get_port_offset(owner_dp, chosen_tp, 1))
assert sorted(targets) == list(range(8))
def test_flexible_round_robin_is_deterministic_uniform_and_staggered() -> None:
w0 = make_decode_worker(world_size=1, tp_rank=0, dp_rank=0)
seq = [w0._next_flex_tp_rank(8) for _ in range(64)]
assert seq[:8] == list(range(8))
assert Counter(seq) == Counter({t: 8 for t in range(8)})
first_pick = [
make_decode_worker(world_size=1, tp_rank=0, dp_rank=d)._next_flex_tp_rank(8)
for d in range(8)
]
assert sorted(first_pick) == list(range(8))
def test_read_blocks_for_req_threads_chosen_tp() -> None:
# The resolved (chosen_tp, flexible) must reach _read_blocks, which keys the
# session AND the notify port off that single value -- so a read and its
# completion notify address the same prefill rank.
worker = make_decode_worker(world_size=1, tp_rank=0, dp_rank=3)
worker._read_blocks = MagicMock()
worker._read_blocks_for_req("r", make_meta(p_tp=8, p_dp=1, remote_dp_rank=0))
kw = worker._read_blocks.call_args.kwargs
assert kw["flexible"] is True
assert kw["chosen_tp"] == 3 # first flexible pick = dp_rank seed
assert get_port_offset(0, kw["chosen_tp"], 8) == 3
@@ -1,494 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from types import SimpleNamespace
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOMode,
MoRIIOTransferAck,
TransferError,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnector,
MoRIIOConnectorWorker,
get_moriio_expected_ack_count,
get_moriio_remote_tp_rank,
resolve_moriio_transfer_ack,
validate_moriio_heterogeneous_tp_kv_heads,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWrapper,
)
def test_remote_tp_rank_same_tp_maps_to_self():
assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [
0,
1,
2,
3,
]
def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill():
assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [
0,
0,
1,
1,
2,
2,
3,
3,
]
def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair():
assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [
0,
2,
4,
6,
]
@pytest.mark.parametrize(
("local_tp_rank", "local_tp_size", "remote_tp_size"),
[
(0, 6, 4),
(0, 4, 6),
],
)
def test_remote_tp_rank_invalid_non_multiple_tp_raises(
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
):
with pytest.raises(ValueError, match="multiple"):
get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size)
@pytest.mark.parametrize(
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
[
(4, 4, 8),
(8, 4, 4),
(4, 8, 4),
],
)
def test_heterogeneous_tp_head_guard_allows_supported_layouts(
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
):
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size,
remote_tp_size,
total_num_kv_heads,
is_mla=False,
)
def test_heterogeneous_tp_head_guard_allows_mla_layouts():
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size=2,
remote_tp_size=4,
total_num_kv_heads=4,
is_mla=True,
)
@pytest.mark.parametrize(
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
[
(4, 2, 4),
(2, 4, 4),
],
)
def test_heterogeneous_tp_head_guard_rejects_split_kv_heads(
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
):
with pytest.raises(NotImplementedError, match="replicated KV heads"):
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size,
remote_tp_size,
total_num_kv_heads,
is_mla=False,
)
def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one():
assert get_moriio_expected_ack_count(4, 4) == 1
assert get_moriio_expected_ack_count(8, 4) == 1
def test_expected_ack_count_for_decode_fan_in():
assert get_moriio_expected_ack_count(4, 8) == 2
def test_expected_ack_count_rejects_non_multiple_fan_in():
with pytest.raises(ValueError, match="multiple"):
get_moriio_expected_ack_count(4, 6)
def test_plain_string_ack_is_backward_compatible_single_ack():
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
"tx-plain",
producer_tp_size=4,
live_transfer_ids={"tx-plain"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
== "tx-plain"
)
assert notification_counts == {}
assert completed_transfer_ids == {"tx-plain"}
def test_structured_release_ack_waits_for_all_expected_acks():
ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-fanin"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert notification_counts == {"tx-fanin": 1}
assert completed_transfer_ids == set()
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-fanin"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
== "tx-fanin"
)
assert notification_counts == {}
assert completed_transfer_ids == {"tx-fanin"}
def test_duplicate_ack_after_completion_does_not_resolve_twice():
ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8)
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-dup"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-dup"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
== "tx-dup"
)
assert (
resolve_moriio_transfer_ack(
ack,
producer_tp_size=4,
live_transfer_ids={"tx-dup"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert notification_counts == {}
assert completed_transfer_ids == {"tx-dup"}
def test_ack_for_non_live_transfer_is_ignored():
notification_counts: dict[str, int] = {}
completed_transfer_ids: set[str] = set()
assert (
resolve_moriio_transfer_ack(
MoRIIOTransferAck("tx-stale", consumer_tp_size=8),
producer_tp_size=4,
live_transfer_ids={"tx-live"},
notification_counts=notification_counts,
completed_transfer_ids=completed_transfer_ids,
)
is None
)
assert notification_counts == {}
assert completed_transfer_ids == set()
def test_worker_get_finished_counts_structured_release_fan_in():
class FakeWrapper:
def __init__(self):
self.batches = [
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
]
def pop_finished_req_ids(self):
return self.batches.pop(0)
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.is_producer = True
worker.mode = MoRIIOMode.READ
worker.world_size = 4
worker.moriio_wrapper = FakeWrapper()
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
worker._consumer_notification_counts = {}
worker._completed_consumer_notifications = set()
worker._pending_unmapped_acks = []
assert worker.get_finished() == (set(), set())
assert worker._consumer_notification_counts == {"tx-fanin": 1}
assert worker.get_finished() == ({"req-fanin"}, set())
assert worker._consumer_notification_counts == {}
assert worker._completed_consumer_notifications == {"tx-fanin"}
@pytest.mark.parametrize("exit_kind", ["killed", "eof"])
def test_worker_reports_unexpected_heartbeat_exit(exit_kind):
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.is_producer = True
worker.mode = MoRIIOMode.READ
worker.moriio_wrapper = SimpleNamespace(
pop_finished_req_ids=lambda: [], shutdown=lambda: None
)
worker._pending_unmapped_acks = []
worker._heartbeat = MoRIIOHeartbeat("tcp://127.0.0.1:1", {"type": "P"}, 0.02)
process = worker._heartbeat._process
try:
assert worker.get_finished() == (set(), set())
if exit_kind == "killed":
process.kill()
else:
process.stdin.close()
returncode = process.wait(timeout=5)
with pytest.raises(RuntimeError, match=f"heartbeat.*{returncode}"):
worker.get_finished()
finally:
worker.shutdown()
def test_read_completion_sends_structured_release_with_consumer_tp_size():
class DoneStatus:
def Succeeded(self):
return True
def Failed(self):
return False
class FakeWrapper:
def __init__(self):
self.lock = threading.Lock()
self.sent = []
# Exercise the real batch verdict rather than restating it here.
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
def send_notify(
self,
transfer_id,
host,
port,
message_type=None,
message_fields=None,
):
self.sent.append((transfer_id, host, port, message_type, message_fields))
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.world_size = 8
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
worker.moriio_wrapper = FakeWrapper()
# A layer maps to the list of reads posted for it (a KDA layer posts two).
worker._recving_transfers = {"req": {"layer0": [DoneStatus(), DoneStatus()]}}
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-release")
}
# Transfer-timeout reaping state consulted by _pop_done_transfers.
worker._recving_transfers_start = {}
# Load-error bookkeeping cleared alongside a completed transfer.
worker._recving_local_blocks = {}
worker._invalid_block_ids = set()
assert worker._pop_done_transfers() == {"tx-release"}
assert worker.moriio_wrapper.sent == [
(
"tx-release",
"127.0.0.1",
"7000",
"release",
{"consumer_tp_size": 8},
)
]
assert worker._recving_transfers == {}
assert worker._recving_transfers_callback_addr == {}
def test_aborted_read_without_local_blocks_only_releases_remote_blocks():
class FakeWrapper:
def __init__(self):
self.sent = []
def send_notify(
self,
transfer_id,
host,
port,
message_type=None,
message_fields=None,
):
self.sent.append((transfer_id, host, port, message_type, message_fields))
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.mode = MoRIIOMode.READ
worker.world_size = 8
worker.moriio_wrapper = FakeWrapper()
worker._read_blocks(
local_block_ids=[],
remote_block_ids=[[10, 11], [90]],
dst_engine_id="prefill",
request_id="req-aborted",
transfer_id="tx-aborted",
remote_host="127.0.0.1",
remote_notify_port=7000,
remote_tp_size=8,
remote_dp_rank=0,
chosen_tp=2,
)
assert worker.moriio_wrapper.sent == [
(
"tx-aborted",
"127.0.0.1",
"7002",
"release",
{"consumer_tp_size": 8},
)
]
def test_read_completion_waits_for_every_posted_read():
class Status:
def __init__(self, done: bool):
self.done = done
def Succeeded(self):
return self.done
def Failed(self):
return False
class FakeWrapper:
def __init__(self):
self.lock = threading.Lock()
self.sent = []
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
def send_notify(self, *args, **kwargs):
self.sent.append((args, kwargs))
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.world_size = 8
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
worker.moriio_wrapper = FakeWrapper()
worker._recving_transfers = {
"req": {"kda_layer": [Status(done=True), Status(done=False)]}
}
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-pending")
}
worker._recving_transfers_start = {"req": float("inf")}
worker._recving_local_blocks = {"req": [1, 2]}
worker._invalid_block_ids = set()
assert worker._pop_done_transfers() == set()
assert "req" in worker._recving_transfers
assert worker.moriio_wrapper.sent == []
@pytest.mark.parametrize(
("has_mamba", "expected_invalid"),
[(False, {1, 2}), (True, set())],
)
def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid):
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = has_mamba
worker._recving_local_blocks = {"req": [1, 2]}
worker._invalid_block_ids = set()
worker._record_failed_recv("req")
assert worker.get_block_ids_with_load_errors() == expected_invalid
def test_hybrid_step_barrier_fails_closed(monkeypatch):
class FailingWrapper:
def waiting_for_transfer_complete(self, _statuses):
raise TransferError("failed")
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = True
worker._reads_issued_this_step = [object()]
worker._mamba_reads_this_step = [object()]
worker.moriio_wrapper = FailingWrapper()
monkeypatch.setattr(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.get_forward_context",
lambda: SimpleNamespace(cudagraph_runtime_mode=None),
)
with pytest.raises(TransferError, match="failed"):
worker._await_reads_issued_this_step()
def test_requested_cudagraph_mode_is_never_overridden():
# The configured cudagraph mode is always honored: the barrier fires when
# the operator sets cudagraph_mode=PIECEWISE, and READ mode with full
# graphs only warns instead of silently forcing PIECEWISE.
assert (
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": True}) is False
)
assert (
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": False}) is False
)
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id.
vLLM's input_processor mutates a request_id after the connector has stored it
(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can
miss. These tests cover the stable-``transfer_id`` fallback that handles that
case (and the clean exact-match and miss paths).
The map/unmap methods touch only the two id<->id dicts, so we bind them to a
lightweight stand-in rather than constructing a full scheduler.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
)
_map = MoRIIOConnectorScheduler.map_request_id
_unmap = MoRIIOConnectorScheduler.unmap_request_id
def _sched():
return SimpleNamespace(
transfer_id_to_request_id={},
request_id_to_transfer_id={},
)
def test_exact_match_unmap_clears_both_tables():
s = _sched()
_map(s, "rid-1", "tid-1")
_unmap(s, "rid-1")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
def test_unmap_via_transfer_id_after_request_id_mutation():
s = _sched()
# Connector stores the ORIGINAL request_id at map time.
_map(s, "cmpl-abc-0", "tid-42")
# input_processor later mutates the rid; exact match would miss, so the
# stable transfer_id is used to find and clear the original entry.
_unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
def test_unmap_miss_is_noop_and_does_not_raise():
s = _sched()
_map(s, "rid-1", "tid-1")
# Unknown rid and unknown/None transfer_id: must not raise, must not touch
# the existing (unrelated) mapping.
_unmap(s, "rid-unknown", transfer_id=None)
_unmap(s, "rid-unknown", transfer_id="tid-unknown")
assert s.request_id_to_transfer_id == {"rid-1": "tid-1"}
assert s.transfer_id_to_request_id == {"tid-1": "rid-1"}
def test_exact_match_preferred_over_transfer_id_fallback():
s = _sched()
_map(s, "rid-1", "tid-1")
# Exact rid is present, so it is removed directly even if a transfer_id is
# also supplied; no stale entries left behind.
_unmap(s, "rid-1", transfer_id="tid-1")
assert s.request_id_to_transfer_id == {}
assert s.transfer_id_to_request_id == {}
@@ -1,449 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for mori-io write_done routing (#51681)."""
import threading
import time
from collections import OrderedDict
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ROLE,
LayerTransferPlan,
MoRIIOError,
MoRIIOMode,
RemoteAllocInfo,
ReqMeta,
WriteTask,
get_port_offset,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorScheduler,
MoRIIOConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWrapper,
)
from .utils import make_moriio_writer
pytestmark = pytest.mark.cpu_test
def _fake_worker(
alloc: dict[str, Any],
*,
tp_rank: int = 0,
set_remote_dp_size_local: int | None = None,
) -> Any:
wrapper = SimpleNamespace(
done_remote_allocate_req_dict=alloc,
lock=threading.Lock(),
done_req_ids=[],
sent_notifies=[],
)
wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: (
wrapper.sent_notifies.append(
(transfer_id, remote_ip, remote_port, message_type)
)
)
wrapper.waiting_for_transfer_complete = lambda statuses: None
wrapper._mark_transfer_terminal_locked = lambda transfer_id: None
wrapper._is_transfer_terminal_locked = lambda transfer_id: False
worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank)
if set_remote_dp_size_local is not None:
worker.remote_dp_size_local = set_remote_dp_size_local
return worker
def _req_meta(
*,
transfer_id: str = "tA",
remote_host: str = "10.0.0.1",
multi_pod_hosts: list[str] | None = None,
remote_dp_size: int = 16,
remote_dp_size_local: int = 8,
) -> ReqMeta:
return ReqMeta(
transfer_id=transfer_id,
local_block_ids=[[1]],
remote_block_ids=[[2]],
remote_host=remote_host,
remote_port=1,
remote_handshake_port=2,
remote_notify_port=20000,
remote_engine_id="remote-engine",
tp_size=1,
remote_dp_size=remote_dp_size,
multi_pod_hosts=list(multi_pod_hosts or []),
remote_dp_size_local=remote_dp_size_local,
)
def _write_task(
transfer_id: str,
remote_ip: str,
*,
multi_pod_hosts: list[str] | None = None,
remote_dp_size_local: int = 0,
remote_dp_size: int = 1,
request_id: str = "req",
) -> WriteTask:
return WriteTask(
request_id=request_id,
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[[1, 3]],
remote_block_ids_hint=None,
layer_name="layer0",
event=None,
remote_notify_port=20000,
remote_ip=remote_ip,
multi_pod_hosts=multi_pod_hosts or [],
remote_dp_size_local=remote_dp_size_local,
remote_dp_size=remote_dp_size,
)
def _real_wrapper() -> MoRIIOWrapper:
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
wrapper.lock = threading.Lock()
wrapper.done_req_ids = []
wrapper.done_remote_allocate_req_dict = {}
wrapper.done_write_cache_req_ids = []
wrapper.failed_write_cache_req_ids = []
wrapper._terminal_transfer_ids = OrderedDict()
return wrapper
def test_execute_write_task_ip_is_per_task_under_overwrite():
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
worker = _fake_worker(alloc)
worker.multi_pod_hosts = ["10.0.0.2"]
worker.remote_dp_size_local = 1
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
writer._execute_write_task(task_a)
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert alloc["tA"].completion_remote_ip != "10.0.0.2"
def test_deferred_task_ip_is_per_task_under_overwrite():
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
worker = _fake_worker(alloc)
worker.multi_pod_hosts = ["10.0.0.2"]
worker.remote_dp_size_local = 1
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
writer._deferred_tasks.append(task_a)
writer._process_deferred_tasks()
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert writer._deferred_tasks == []
def test_two_requests_resolve_distinct_hosts():
alloc = {
"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0),
"tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3),
}
worker = _fake_worker(alloc)
writer = make_moriio_writer(worker)
task_a = _write_task(
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
)
task_b = _write_task(
"tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1
)
worker.multi_pod_hosts = ["10.0.0.2"]
writer._execute_write_task(task_a)
writer._execute_write_task(task_b)
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
assert alloc["tB"].completion_remote_ip == "10.0.0.2"
def test_finalize_notify_port_uses_pod_local_rank():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=0)
writer = make_moriio_writer(worker)
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
writer._execute_write_task(task)
assert info.completion_remote_ip == "10.0.0.2"
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
assert worker.moriio_wrapper.sent_notifies
_tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
assert mtype == "write_done"
assert ip == "10.0.0.2"
assert port == base + get_port_offset(9 % 8, 0)
def test_finalize_port_ignores_stale_worker_dp_local():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1)
writer = make_moriio_writer(worker)
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
writer._execute_write_task(task)
assert info.completion_notify_port == base + get_port_offset(9 % 8, 0)
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
_tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
assert mtype == "write_done"
assert port == base + get_port_offset(9 % 8, 0)
assert port != base + get_port_offset(9 % 1, 0)
def test_finalize_single_pod_port_uses_global_rank():
base = 20000
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3)
alloc = {"tA": info}
worker = _fake_worker(alloc, tp_rank=2)
writer = make_moriio_writer(worker)
task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0)
writer._execute_write_task(task)
assert info.completion_remote_ip == "10.0.0.1"
assert info.completion_notify_port == base + get_port_offset(3, 2)
info.writes_expected = 1
info.writes_done = 1
writer._finalize_if_complete("tA", info)
_tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1]
assert port == base + get_port_offset(3, 2)
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = False
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
captured.update(kwargs)
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
meta = _req_meta(
remote_host="10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=8,
)
worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None)
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
assert captured["remote_dp_size_local"] == 8
assert captured["remote_ip"] == "10.0.0.1"
assert captured["local_block_ids"] == [[1]]
assert captured["remote_block_ids"] == [[2]]
assert not hasattr(worker, "multi_pod_hosts")
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = False
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
captured.update(kwargs)
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
meta = _req_meta(
remote_host="10.0.0.9",
multi_pod_hosts=[],
remote_dp_size=4,
remote_dp_size_local=0,
)
worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None)
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
assert captured["remote_dp_size_local"] == 4
assert captured["local_block_ids"] == [[1]]
assert captured["remote_block_ids"] == [[2]]
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
wrapper = _real_wrapper()
sent: list[tuple[str, int, str]] = []
wrapper.send_notify = lambda tid, ip, port, message_type: sent.append(
(ip, port, message_type)
)
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=0))
task = _write_task(
"tA",
"10.0.0.1",
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
remote_dp_size_local=2,
remote_dp_size=4,
)
task.enqueue_time = time.perf_counter() - 2 * writer._defer_timeout
writer._deferred_tasks.append(task)
writer._process_deferred_tasks()
assert writer._deferred_tasks == []
assert [ack.transfer_id for ack in wrapper.done_req_ids] == ["tA"]
# Global DP ranks 0-3, two per pod: the owning rank is unknown.
assert sent == [
(host, 20000 + get_port_offset(local_rank, 0), "write_failed")
for host in ("10.0.0.1", "10.0.0.2")
for local_rank in (0, 1)
]
# A late decode allocation must not resurrect a write into freed blocks.
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
wrapper._handle_remote_blocks_message(
{"transfer_id": "tA", "block_notify_list": [7], "decode_rank": 1}
)
assert wrapper.done_remote_allocate_req_dict == {}
def test_write_scheduler_waits_for_worker_ack_past_defer_deadline():
scheduler = SimpleNamespace(
is_producer=True,
mode=MoRIIOMode.WRITE,
_defer_timeout=60.0,
_deferred_send_deadlines={"req-1": (time.monotonic() - 1.0, "tA")},
_pending_sent_acks={},
unmap_request_id=lambda *args, **kwargs: None,
)
output = SimpleNamespace(finished_sending=set())
MoRIIOConnectorScheduler.update_connector_output(scheduler, output)
assert output.finished_sending is None
assert "req-1" in scheduler._deferred_send_deadlines
output.finished_sending = {"req-1"}
MoRIIOConnectorScheduler.update_connector_output(scheduler, output)
assert output.finished_sending == {"req-1"}
assert scheduler._deferred_send_deadlines == {}
def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
wrapper = _real_wrapper()
with patch.object(moriio_engine, "get_role", return_value=ROLE.CONSUMER):
for tid in ("tA", "tOtherRank"):
wrapper._handle_structured_message(
{"type": "write_failed", "transfer_id": tid}
)
worker = SimpleNamespace(
mode=MoRIIOMode.WRITE,
is_producer=False,
moriio_wrapper=wrapper,
transfer_id_to_request_id={"tA": "req-1"},
_unmatched_write_failures=OrderedDict(),
get_finished=lambda: (set(), set()),
)
results = MoRIIOConnectorWorker.get_transfer_results(worker)
assert results.finished_recving == {"req-1"}
assert results.failed_recving == {"req-1"}
assert wrapper.pop_finished_write_req_ids() == set()
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer():
from unittest.mock import Mock
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = True
worker.world_size = 2
worker.schedule_write_blocks = Mock()
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None)
worker.schedule_write_blocks.assert_not_called()
@pytest.mark.parametrize(
"notify,expected",
[
# Plain WRITE peers still send one flat attention list.
([7, 8], [[7, 8]]),
# Hybrid WRITE sends one list per transfer group, kept as-is even
# when the first group is empty (full attention hit, state only).
([[7], [9]], [[7], [9]]),
([[], [9]], [[], [9]]),
],
)
def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected):
wrapper = _real_wrapper()
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
wrapper._handle_remote_blocks_message(
{"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2}
)
info = wrapper.done_remote_allocate_req_dict["tA"]
assert info.block_ids == expected
assert info.decode_dp_rank == 2
def test_empty_layer_plan_posts_no_rdma_write():
from unittest.mock import Mock
wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock())
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
plan = LayerTransferPlan(
request_id="req",
transfer_id="tA",
layer_name="layer0",
sess_idx=0,
transfer_local_offsets=[],
transfer_remote_offsets=[],
transfer_sizes=[],
)
assert writer._do_layer_write(plan, ["session"]) == []
wrapper.write_remote_data.assert_not_called()
wrapper.write_remote_data_single.assert_not_called()
@pytest.mark.parametrize("local_state", [[], [1, 2]])
def test_hybrid_write_requires_exactly_one_running_state_slot(local_state):
worker = SimpleNamespace(
_mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1],
)
writer = make_moriio_writer(worker)
task = _write_task("tA", "10.0.0.1")
task.layer_name = "kda"
task.local_block_ids = [[], local_state]
with pytest.raises(MoRIIOError, match="one running-state slot"):
writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]]))
File diff suppressed because it is too large Load Diff
@@ -1,253 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the scheduler-driven heartbeat / lease-renewal system."""
import time
from unittest.mock import MagicMock
import pytest
from vllm.v1.outputs import KVConnectorOutput
from .utils import create_request, make_nixl_scheduler
_ENGINE_A = "my-engine-id"
def _sched(kv_lease_duration: int = 30):
return make_nixl_scheduler(heartbeat=True, kv_lease_duration=kv_lease_duration)
def _req(request_id: int = 1):
return create_request(request_id=request_id, do_remote_prefill=True)
def _worker_stub():
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
w = object.__new__(NixlConnectorWorker)
w._reqs_to_send = {}
w._lease_extension = 20
w._reqs_to_process = set()
w.consumer_notification_counts_by_req = {}
w.expected_consumer_notifications_by_req = {}
w.xfer_stats = MagicMock()
return w
# ===================================================================
# Scheduler: on_new_request
# ===================================================================
def test_on_new_request_tracks_and_groups():
"""Add two reqs to same engine, one to another; verify grouping."""
s = _sched()
s.on_new_request(_req(1))
s.on_new_request(_req(2))
assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-1", "prefill-2"}
info = s._heartbeat_by_engine[_ENGINE_A]
assert (info.host, info.port, info.tp_size) == ("my-host", 1234, 1)
assert s._heartbeat_req_engine["id-1"] == (_ENGINE_A, "prefill-1")
# Different engine.
r3 = _req(3)
r3.kv_transfer_params["remote_engine_id"] = "engine-b"
s.on_new_request(r3)
assert len(s._heartbeat_by_engine) == 2
@pytest.mark.parametrize(
"make_req",
[
lambda: create_request(request_id=2, do_remote_decode=True),
lambda: create_request(request_id=3), # no kv_transfer_params
],
ids=["decode", "plain"],
)
def test_on_new_request_ignores_non_prefill(make_req):
s = _sched()
s.on_new_request(make_req())
assert len(s._heartbeat_by_engine) == 0
# ===================================================================
# Scheduler: _stop_heartbeat
# ===================================================================
def test_stop_heartbeat_partial_and_full():
"""Stop one of two reqs on same engine, then stop the other."""
s = _sched()
s.on_new_request(_req(1))
s.on_new_request(_req(2))
s._stop_heartbeat("id-1")
assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-2"}
assert "id-1" not in s._heartbeat_req_engine
s._stop_heartbeat("id-2")
assert len(s._heartbeat_by_engine) == 0
assert len(s._heartbeat_req_engine) == 0
# ===================================================================
# Scheduler: build_connector_meta throttling
# ===================================================================
def test_build_connector_meta_heartbeat_throttling():
# kv_lease_duration=30 => _heartbeat_interval = 30 // 6 = 5
s = _sched(kv_lease_duration=30)
s.on_new_request(_req(1))
# Ensure the first call triggers by placing last_heartbeat far in the past.
s._last_heartbeat_time = time.perf_counter() - 10
meta1 = s.build_connector_meta(MagicMock())
assert _ENGINE_A in meta1.heartbeat_by_engine
# Immediate second call is throttled (< 5s since last).
meta2 = s.build_connector_meta(MagicMock())
assert len(meta2.heartbeat_by_engine) == 0
# ===================================================================
# Scheduler: cleanup paths (update_connector_output / request_finished)
# ===================================================================
def test_update_connector_output_stops_heartbeat():
s = _sched()
s.on_new_request(_req(1))
s.update_connector_output(
KVConnectorOutput(
finished_sending=None,
finished_recving={"id-1"},
invalid_block_ids=set(),
)
)
assert len(s._heartbeat_by_engine) == 0
assert len(s._heartbeat_req_engine) == 0
def test_request_finished_stops_heartbeat():
s = _sched()
r = _req(1)
s.on_new_request(r)
# Simulate update_state_after_alloc having consumed do_remote_prefill.
r.kv_transfer_params["do_remote_prefill"] = False
s.request_finished(r, block_ids=())
assert len(s._heartbeat_by_engine) == 0
assert len(s._heartbeat_req_engine) == 0
# ===================================================================
# Worker: _handle_heartbeat
# ===================================================================
def test_handle_heartbeat():
w = _worker_stub()
far_future = time.perf_counter() + 99999
w._reqs_to_send = {"req-a": 100.0, "req-b": far_future}
before = time.perf_counter()
w._handle_heartbeat("req-a,req-b,req-unknown")
# req-a: pushed forward to ~now+20.
assert w._reqs_to_send["req-a"] >= before + 20
# req-b: already far out, max() keeps it.
assert w._reqs_to_send["req-b"] >= far_future
# req-unknown: not added.
assert "req-unknown" not in w._reqs_to_send
def test_reaper_reclaims_expired_leases_behind_heartbeated_head(monkeypatch):
"""A heartbeated head entry must not strand expired leases behind it.
Mirrors the stdlib repro in https://github.com/vllm-project/vllm/issues/58222:
A is inserted first and kept alive via heartbeats; B and C expire and
must still be reaped. The old early-exit scan returned nothing here.
"""
w = _worker_stub()
w._reqs_to_process.update(("A", "B", "C"))
# Insertion order is A, B, C. Heartbeats will keep A at the head.
w._reqs_to_send = {"A": 30.0, "B": 32.0, "C": 34.0}
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
now = {"t": 0.0}
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"])
for t in (20.0, 40.0, 60.0, 80.0):
now["t"] = t
w._handle_heartbeat("A")
# A was renewed in place: still the dict head, expiry 100.
assert next(iter(w._reqs_to_send)) == "A"
assert w._reqs_to_send["A"] == pytest.approx(100.0)
done: set[str] = set()
now["t"] = 50.0
w._reap_expired_send_leases(done)
assert done == {"B", "C"}
assert list(w._reqs_to_send) == ["A"]
assert w._reqs_to_process == {"A"}
assert w.xfer_stats.record_kv_expired_req.call_count == 2
# now == expires is expired (now >= expires): A is due at 100.
done_eq: set[str] = set()
now["t"] = 100.0
w._reap_expired_send_leases(done_eq)
assert done_eq == {"A"}
assert not w._reqs_to_send
assert not w._reqs_to_process
assert w.xfer_stats.record_kv_expired_req.call_count == 3
def test_reaper_reclaims_when_now_equals_expiry(monkeypatch):
"""The reaper must reclaim on the now == expires boundary."""
w = _worker_stub()
w._reqs_to_process.add("eq")
w._reqs_to_send = {"eq": 50.0}
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0)
done: set[str] = set()
w._reap_expired_send_leases(done)
assert done == {"eq"}
assert not w._reqs_to_send
assert not w._reqs_to_process
w.xfer_stats.record_kv_expired_req.assert_called_once()
def test_reaper_reclaims_shorter_lease_behind_later_deadline(monkeypatch):
"""A later-inserted longer TTL must not block an earlier-expiring entry.
Delete+reinsert on heartbeat cannot restore expiry order in this case
(issue #58222), so the reaper must scan the whole map.
"""
w = _worker_stub()
w._reqs_to_process.update(("X", "Y"))
w._reqs_to_send = {"X": 55.0, "Y": 46.0}
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0)
done: set[str] = set()
w._reap_expired_send_leases(done)
assert done == {"Y"}
assert list(w._reqs_to_send) == ["X"]
assert w._reqs_to_process == {"X"}
File diff suppressed because it is too large Load Diff
@@ -1,977 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from dataclasses import replace
from unittest.mock import patch
import pytest
from vllm.v1.metrics.stats import KV_FETCH_STAGES
from vllm.v1.outputs import (
EMPTY_MODEL_RUNNER_OUTPUT,
KVConnectorOutput,
ModelRunnerOutput,
)
from vllm.v1.request import FinishReason, RequestStatus
from .utils import (
assert_scheduler_empty,
create_model_runner_output,
create_request,
create_scheduler,
create_vllm_config,
make_kv_cache_config,
)
pytestmark = pytest.mark.cpu_test
def _num_waiting_requests(scheduler) -> int:
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
def _kv_fetch_stages(scheduler) -> tuple[int, ...]:
"""(waiting_to_start, in_progress, completed_waiting) request counts."""
return tuple(scheduler._kv_fetch_counts[s] for s in KV_FETCH_STAGES)
def test_basic_lifecycle():
"""Test lifecycle of a remote prefill."""
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
# 2 Full Blocks and 1 Half Block.
BLOCK_SIZE = vllm_config.cache_config.block_size
NUM_EXTERNAL_FULL_BLOCKS = 2
NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5))
START_FREE_BLOCK_QUEUE_SIZE = (
scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks
)
request = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
do_remote_prefill=True,
)
scheduler.add_request(request)
request_id = request.request_id
assert _kv_fetch_stages(scheduler) == (1, 0, 0)
# STEP (1):
# (1a): schedule()
scheduler_output = scheduler.schedule()
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
# Nothing running and empty scheduler output.
assert len(scheduler.running) == 0
assert len(scheduler_output.scheduled_new_reqs) == 0
assert scheduler_output.scheduled_cached_reqs.num_reqs == 0
assert len(scheduler_output.num_scheduled_tokens) == 0
assert scheduler_output.total_num_scheduled_tokens == 0
# Req waiting for KVs with no computed/scheduled toks ...
assert _num_waiting_requests(scheduler) == 1
assert request in scheduler.kv_holding_waiting
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert request.num_computed_tokens == NUM_TOKENS
# ... but should have (uncached) blocks allocated to it.
block_pool = scheduler.kv_cache_manager.block_pool
assert block_pool.free_block_queue.num_free_blocks < START_FREE_BLOCK_QUEUE_SIZE
assert len(block_pool.cached_block_hash_to_block) == 0
blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks[request_id]
for block in blocks:
assert block._block_hash is None
# (1b): forward()
model_runner_output = EMPTY_MODEL_RUNNER_OUTPUT
# (1c): update_from_output()
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_runner_output
)
assert not engine_core_outputs or not engine_core_outputs[0].outputs
# STEP (2):
# (2a): schedule(): nothing happens!
scheduler_output = scheduler.schedule()
assert _num_waiting_requests(scheduler) == 1
assert len(scheduler.running) == 0
# (2b): forward(): request finishes recv.
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
model_runner_output.kv_connector_output = KVConnectorOutput(
finished_recving={request_id}
)
# (2c): update_from_output():
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_runner_output
)
assert _num_waiting_requests(scheduler) == 1
assert request_id in scheduler.finished_recving_kv_req_ids
assert _kv_fetch_stages(scheduler) == (0, 0, 1)
# STEP (3):
# (3a): schedule(): this should actually schedule.
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 1
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
# Confirm the block are actually allocated.
num_hashed_blocks = 0
blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks[request_id]
for block in blocks:
assert block.ref_cnt == 1
num_hashed_blocks += 1 if block._block_hash is not None else 0
assert num_hashed_blocks == NUM_EXTERNAL_FULL_BLOCKS
# Confirm the rest of the prompt is scheduled in this step.
scheduled_req = scheduler_output.scheduled_new_reqs[0]
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[request_id]
num_computed_tokens = scheduled_req.num_computed_tokens
total_prompt_tokens = len(scheduled_req.prompt_token_ids)
assert num_scheduled_tokens == total_prompt_tokens - num_computed_tokens
# (3b): execute_model()
model_runner_output = create_model_runner_output([request])
# (3c): update_from_output()
scheduler.update_from_output(scheduler_output, model_runner_output)
# Step (4): Hit EOS.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output([request], use_eos=True)
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_runner_output
)
scheduler.schedule()
outputs = engine_core_outputs[0].outputs
assert len(outputs) == 1
output = outputs[0]
assert output.finish_reason == FinishReason.STOP
assert_scheduler_empty(scheduler)
def test_interleaved_lifecycle():
"""Test Remote Prefills Work Well With Other Requests."""
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
# 2 Full Blocks and 1 Half Block.
BLOCK_SIZE = vllm_config.cache_config.block_size
NUM_EXTERNAL_FULL_BLOCKS = 2
NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5))
request_remote = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
do_remote_prefill=True,
)
request_local_a = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
)
request_local_b = create_request(
request_id=3,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
)
# STEP 1: Regular request is running.
scheduler.add_request(request_local_a)
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 1
model_runner_output = create_model_runner_output([request_local_a])
scheduler.update_from_output(scheduler_output, model_runner_output)
# STEP 2: Add a local and remote request.
scheduler.add_request(request_local_b)
scheduler.add_request(request_remote)
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 2
assert _num_waiting_requests(scheduler) == 1
assert len(scheduler_output.scheduled_new_reqs) == 1
assert scheduler_output.scheduled_cached_reqs.num_reqs == 1
model_runner_output = create_model_runner_output([request_local_a, request_local_b])
scheduler.update_from_output(scheduler_output, model_runner_output)
# STEP 3: continue running, KVs not arrived yet.
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 2
assert _num_waiting_requests(scheduler) == 1
assert len(scheduler_output.scheduled_new_reqs) == 0
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
model_runner_output = create_model_runner_output(
reqs=[request_local_a, request_local_b]
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 2
assert _num_waiting_requests(scheduler) == 1
assert len(scheduler_output.scheduled_new_reqs) == 0
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
# STEP 4: KVs arrive.
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 2
assert _num_waiting_requests(scheduler) == 1
assert len(scheduler_output.scheduled_new_reqs) == 0
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
model_runner_output = create_model_runner_output(
[request_local_a, request_local_b], finished_recving={request_remote.request_id}
)
scheduler.update_from_output(scheduler_output, model_runner_output)
# STEP 5: RECVed KVs are sent to ModelRunner.
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 3
assert _num_waiting_requests(scheduler) == 0
assert len(scheduler_output.scheduled_new_reqs) == 1
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
model_runner_output = create_model_runner_output(
[request_local_a, request_local_b, request_remote]
)
scheduler.update_from_output(scheduler_output, model_runner_output)
# STEP 6: Hit EOS and free.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
[request_local_a, request_local_b, request_remote],
use_eos=True,
)
scheduler.update_from_output(scheduler_output, model_runner_output)
scheduler.schedule()
assert_scheduler_empty(scheduler)
def test_no_spurious_prefix_caching():
"""With P/D, blocks can be allocated but uncomputed for
multiple engine steps. This test confirms that we do
not accidentally have cache hits against uncomputed
blocks.
"""
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
# 2 and a half full external blocks.
BLOCK_SIZE = vllm_config.cache_config.block_size
NUM_EXTERNAL_FULL_BLOCKS = 2
NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5))
# Both of these requests have prompts like [1,1,1,1,1, ...]
request_remote = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
common_prefix_len=NUM_TOKENS,
do_remote_prefill=True,
)
request_local = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
common_prefix_len=NUM_TOKENS,
do_remote_prefill=False,
)
# Schedule the remote prefill request. This should not
# cause any blocks to be cached.
scheduler.add_request(request_remote)
scheduler_output = scheduler.schedule()
scheduler.update_from_output(scheduler_output, EMPTY_MODEL_RUNNER_OUTPUT)
assert _num_waiting_requests(scheduler) == 1
# Schedule the local prefill request. This should
# cause blocks to be cached, but separately from
scheduler.add_request(request_local)
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 1
local_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks[request_local.request_id]
remote_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks[request_remote.request_id]
# Local should have cached blocks (but not all due to preallocate).
num_hashed_blocks = 0
for block in local_blocks:
assert block.ref_cnt == 1
num_hashed_blocks += 1 if block._block_hash is not None else 0
assert num_hashed_blocks > 0
# Remote blocks should not be cached.
for block in remote_blocks:
assert block.ref_cnt == 1
assert block._block_hash is None
def test_full_block_prompt():
"""Test that we handle a prompt that is the full block size."""
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
# 2 Full Blocks and 1 Half Block.
BLOCK_SIZE = vllm_config.cache_config.block_size
NUM_EXTERNAL_FULL_BLOCKS = 2
NUM_TOKENS = int(BLOCK_SIZE * NUM_EXTERNAL_FULL_BLOCKS)
request = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS,
do_remote_prefill=True,
)
scheduler.add_request(request)
request_id = request.request_id
# STEP (1): Initialize a recv.
scheduler_output = scheduler.schedule()
# All blocks should be allocated.
num_blocks = len(
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
]
)
assert num_blocks == NUM_EXTERNAL_FULL_BLOCKS
model_runner_output = EMPTY_MODEL_RUNNER_OUTPUT
scheduler.update_from_output(scheduler_output, model_runner_output)
# # STEP (2): Recv.
scheduler_output = scheduler.schedule()
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
model_runner_output.kv_connector_output = KVConnectorOutput(
finished_recving={request_id}
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert _num_waiting_requests(scheduler) == 1
assert request_id in scheduler.finished_recving_kv_req_ids
# # STEP (3): Run as usual.
scheduler_output = scheduler.schedule()
# We need to recompute the final token of the prompt to generate
# the first new token, so we should not have a new block.
num_blocks = len(
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
]
)
assert num_blocks == NUM_EXTERNAL_FULL_BLOCKS
assert scheduler_output.scheduled_new_reqs[0].num_computed_tokens == NUM_TOKENS - 1
assert scheduler_output.num_scheduled_tokens[request_id] == 1
model_runner_output = create_model_runner_output([request])
scheduler.update_from_output(scheduler_output, model_runner_output)
# # Step (4): Hit EOS.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output([request], use_eos=True)
engine_core_outputs = scheduler.update_from_output(
scheduler_output, model_runner_output
)
scheduler.schedule()
outputs = engine_core_outputs[0].outputs
assert len(outputs) == 1
output = outputs[0]
assert output.finish_reason == FinishReason.STOP
assert_scheduler_empty(scheduler)
def test_cannot_schedule_after_recv():
"""Test that we can handle no schedule after recv due to not
enough remaining KV blocks.
"""
# NOTE: the KVCacheManager will use 1 null block.
# So there are 5 total working blocks.
TOTAL_NUM_BLOCKS = 6
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config, num_blocks=TOTAL_NUM_BLOCKS)
# Prime the KVCache.
NUM_PROMPT_BLOCKS = 2
BLOCK_SIZE = vllm_config.cache_config.block_size
# Prompt will use 2 blocks + 1 block after we schedule.
NUM_TOKENS_LOCAL = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS)
NUM_TOKENS_REMOTE = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS)
request_normal = create_request(
request_id=1, block_size=BLOCK_SIZE, num_tokens=NUM_TOKENS_LOCAL
)
request_remote = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS_REMOTE,
do_remote_prefill=True,
)
# STEP 1: 3 blocks are in use (2 for prompt, 1 for decode).
scheduler.add_request(request_normal)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_normal])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 0
# Step 2: 5 blocks are in use (2 new for remote blocks).
scheduler.add_request(request_remote)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_normal])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 1
# Step 3: finish recving (5 blocks in use)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[request_normal], finished_recving={request_remote.request_id}
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 1
# Step 4: try to schedule, remote request is put to running list
# because the transfer is completed.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[request_normal, request_remote]
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 2
assert _num_waiting_requests(scheduler) == 0
# Step 5: Remote request will be put back to waiting list
# because it needs new block to hold generated token.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_normal])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 1
# Step 6: finish the request, free it.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[request_normal], use_eos=True
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 0
assert _num_waiting_requests(scheduler) == 1
# Step 7: now we can schedule (with 2 blocks computed),
# request is retrieved from preempted list.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_remote])
# V2 emits a resumed (previously preempted) request as a NewRequestData
# rather than a cached request.
if scheduler.use_v2_model_runner:
num_computed = scheduler_output.scheduled_new_reqs[0].num_computed_tokens
else:
cached = scheduler_output.scheduled_cached_reqs
num_computed = cached.num_computed_tokens[0]
assert num_computed == NUM_PROMPT_BLOCKS * BLOCK_SIZE
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 0
# Step 8: free everything.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[request_remote], use_eos=True
)
scheduler.update_from_output(scheduler_output, model_runner_output)
_ = scheduler.schedule()
assert_scheduler_empty(scheduler)
def test_cannot_recv():
"""Test that we can handle no schedule KV block transfer due to not
enough remaining KV blocks.
"""
# NOTE: the KVCacheManager will use 1 null block.
# So there are 5 total working blocks.
TOTAL_NUM_BLOCKS = 6
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config, num_blocks=TOTAL_NUM_BLOCKS)
# Prime the KVCache.
NUM_PROMPT_BLOCKS = 2
BLOCK_SIZE = vllm_config.cache_config.block_size
# Prompt will use 2 blocks + 1 block after we schedule.
NUM_TOKENS_LOCAL = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS)
NUM_TOKENS_REMOTE = int(BLOCK_SIZE * (NUM_PROMPT_BLOCKS + 0.5))
request_normal = create_request(
request_id=1, block_size=BLOCK_SIZE, num_tokens=NUM_TOKENS_LOCAL
)
request_remote = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=NUM_TOKENS_REMOTE,
do_remote_prefill=True,
)
# STEP 1: 3 blocks are in use (2 for prompt, 1 for decode).
scheduler.add_request(request_normal)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_normal])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 0
# Step 2: 3 blocks are in use,
# need 3 new for remote blocks but only 2 are available.
scheduler.add_request(request_remote)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_normal])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 1
# Should not have KV transfer in progress.
assert request_remote.status != RequestStatus.WAITING_FOR_REMOTE_KVS
# Step 3: finish the request, free it.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[request_normal], use_eos=True
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 0
assert _num_waiting_requests(scheduler) == 1
# Step 4: now we can initiate KV transfer (with 2 blocks computed).
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 0
assert _num_waiting_requests(scheduler) == 1
assert request_remote.status == RequestStatus.WAITING_FOR_REMOTE_KVS
# Step 5: finish recving (5 blocks in use)
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[], finished_recving={request_remote.request_id}
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 0
assert _num_waiting_requests(scheduler) == 1
# Step 6: schedule remote request
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(reqs=[request_remote])
scheduler.update_from_output(scheduler_output, model_runner_output)
assert len(scheduler.running) == 1
assert _num_waiting_requests(scheduler) == 0
# Step 7: free everything.
scheduler_output = scheduler.schedule()
model_runner_output = create_model_runner_output(
reqs=[request_remote], use_eos=True
)
scheduler.update_from_output(scheduler_output, model_runner_output)
_ = scheduler.schedule()
assert_scheduler_empty(scheduler)
def test_kv_fetch_stages_cleared_on_abort():
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
BLOCK_SIZE = vllm_config.cache_config.block_size
request = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=int(BLOCK_SIZE * 2.5),
do_remote_prefill=True,
)
scheduler.add_request(request)
scheduler.schedule()
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform"
)
def test_p_side_chunked_prefill_mamba(mock_platform):
"""P-side integration: Mamba N-1 truncation + chunked prefill completes.
A 64-token P-side request is truncated to 63 by the N-1 fix, then
chunked into two prefill steps (32 + 31) and finishes with
LENGTH_CAPPED because max_tokens is set to 1.
"""
mock_platform.device_type = "cpu"
BATCH_SIZE = 32
NUM_TOKENS = 64
BLOCK_SIZE = 16
vllm_config = create_vllm_config(
max_num_batched_tokens=BATCH_SIZE,
block_size=BLOCK_SIZE,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
# mamba_cache_mode "none" only runs without prefix caching.
vllm_config.cache_config.enable_prefix_caching = False
kv_cache_config = make_kv_cache_config(
block_size=BLOCK_SIZE,
mamba_enabled=True,
num_blocks=10000,
)
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
request = create_request(
num_tokens=NUM_TOKENS,
do_remote_decode=True,
block_size=BLOCK_SIZE,
)
request.max_tokens = 128
scheduler.add_request(request)
request_id = request.request_id
# ── Step 1: first chunk ──
scheduler_output = scheduler.schedule()
assert len(request.prompt_token_ids) == NUM_TOKENS - 1
assert request.max_tokens == 1
assert scheduler_output.num_scheduled_tokens[request_id] == BATCH_SIZE
assert request.num_computed_tokens == BATCH_SIZE
# Model returns no tokens for intermediate prefill chunk
intermediate_output = ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[]],
)
scheduler.update_from_output(scheduler_output, intermediate_output)
# ── Step 2: remaining chunk ──
scheduler_output = scheduler.schedule()
remaining = NUM_TOKENS - 1 - BATCH_SIZE # 31
assert scheduler_output.num_scheduled_tokens[request_id] == remaining
assert request.num_computed_tokens == NUM_TOKENS - 1
# Prefill complete: model generates 1 decode token
final_output = create_model_runner_output([request])
engine_core_outputs = scheduler.update_from_output(scheduler_output, final_output)
# max_tokens=1 → request finishes with LENGTH
outputs = engine_core_outputs[0].outputs
assert len(outputs) == 1
assert outputs[0].finish_reason == FinishReason.LENGTH
def test_async_load_reserves_blocks_for_inflight():
"""A second async KV-connector load is not admitted if its initial
allocation would consume blocks reserved for an already in-flight sequence.
req_a gets a 1-block prefix (full sequence = 4 blocks), reserving 3 more.
req_b would need a 4-block initial allocation, but only
(free - req_a's 3-block reservation) = 3 blocks are available to it, so it is
held back in WAITING (holding no blocks) rather than wedging req_a.
"""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
req_a = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 4,
do_remote_prefill=True,
num_remote_blocks=1,
)
req_b = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 5,
do_remote_prefill=True,
num_remote_blocks=1,
)
scheduler.add_request(req_a)
scheduler.add_request(req_b)
# Partial external matches: req_a loads 1 block, req_b loads 4 blocks.
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE * 4, True)],
):
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert req_b.status == RequestStatus.WAITING
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks
assert req_a.request_id in req_to_blocks
assert req_b.request_id not in req_to_blocks
def test_async_loads_both_admitted_when_pool_fits():
"""Sanity: with a pool large enough, the reservation gate admits both async
loads (it is not over-conservative)."""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=64)
reqs = [
create_request(
request_id=i,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 5,
do_remote_prefill=True,
num_remote_blocks=1,
)
for i in (1, 2)
]
for req in reqs:
scheduler.add_request(req)
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE, True)],
):
scheduler.schedule()
for req in reqs:
assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
@pytest.mark.parametrize(
("swa_enabled", "block_size", "num_spec_tokens", "num_blocks"),
[(False, 16, 3, 8), (True, 16, 3, 17), (True, 64, 31, 40)],
ids=["single-group", "multiple-groups", "different-block-sizes"],
)
def test_async_load_reserves_blocks_for_promotion_margin(
swa_enabled, block_size, num_spec_tokens, num_blocks
):
"""Async loads must leave enough blocks for promotion in every cache group.
Otherwise two loads can fill the pool and prevent the head from being
promoted, with nothing running to free blocks and break the deadlock.
"""
vllm_config = create_vllm_config(
block_size=block_size,
num_speculative_tokens=num_spec_tokens,
disable_hybrid_kv_cache_manager=False,
)
hash_block_size = 16
kv_cache_config = make_kv_cache_config(
hash_block_size, swa_enabled=swa_enabled, sw_size=4096, num_blocks=num_blocks
)
full_group = kv_cache_config.kv_cache_groups[0]
full_group.kv_cache_spec = replace(full_group.kv_cache_spec, block_size=block_size)
if swa_enabled:
sliding_group = kv_cache_config.kv_cache_groups[-1]
sliding_group.kv_cache_spec = replace(
sliding_group.kv_cache_spec,
page_size_padded=full_group.kv_cache_spec.page_size_bytes,
)
scheduler = create_scheduler(
vllm_config,
num_blocks=num_blocks,
kv_cache_config=kv_cache_config,
hash_block_size=hash_block_size,
)
req_a = create_request(
request_id=1,
block_size=hash_block_size,
num_tokens=block_size * 4,
do_remote_prefill=True,
max_tokens=1,
)
req_b = create_request(
request_id=2,
block_size=hash_block_size,
num_tokens=block_size * 3,
do_remote_prefill=True,
max_tokens=1,
)
scheduler.add_request(req_a)
scheduler.add_request(req_b)
# Both get a full external hit, so each would hold its whole prompt.
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(block_size * 4, True), (block_size * 3, True)],
):
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert req_b.status == RequestStatus.WAITING
assert not any(scheduler.kv_cache_manager.get_block_ids(req_b.request_id))
# req_a's load lands: it must be promotable and actually get scheduled.
scheduler.update_from_output(
scheduler.schedule(),
create_model_runner_output([], finished_recving={req_a.request_id}),
)
scheduler_output = scheduler.schedule()
assert req_a.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
@pytest.mark.parametrize(
("mamba_cache_mode", "mamba_block_size", "num_blocks"),
[("none", 128, 18), ("align", 16, 18)],
)
def test_async_load_does_not_double_reserve_mamba_spec_blocks(
mamba_cache_mode, mamba_block_size, num_blocks
):
"""Two hybrid loads fit when Mamba's speculative states are counted once."""
block_size = 16
num_spec_tokens = 3
vllm_config = create_vllm_config(
block_size=mamba_block_size,
max_model_len=128,
num_speculative_tokens=num_spec_tokens,
disable_hybrid_kv_cache_manager=False,
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.cache_config.enable_prefix_caching = mamba_cache_mode == "align"
kv_cache_config = make_kv_cache_config(
block_size,
mamba_enabled=True,
mamba_cache_mode=mamba_cache_mode,
num_blocks=num_blocks,
)
mamba_group = kv_cache_config.kv_cache_groups[-1]
mamba_group.kv_cache_spec = replace(
mamba_group.kv_cache_spec,
block_size=mamba_block_size,
num_speculative_blocks=num_spec_tokens,
)
scheduler = create_scheduler(
vllm_config,
num_blocks=num_blocks,
kv_cache_config=kv_cache_config,
hash_block_size=block_size,
)
reqs = [
create_request(
request_id=i,
block_size=block_size,
num_tokens=num_tokens,
do_remote_prefill=True,
max_tokens=1,
)
for i, num_tokens in enumerate((64, 48), 1)
]
for req in reqs:
scheduler.add_request(req)
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(req.num_tokens, True) for req in reqs],
):
scheduler.schedule()
assert all(req.status == RequestStatus.WAITING_FOR_REMOTE_KVS for req in reqs)
scheduler.update_from_output(
scheduler.schedule(),
create_model_runner_output(
[], finished_recving={req.request_id for req in reqs}
),
)
output = scheduler.schedule()
for req in reqs:
assert req.status == RequestStatus.RUNNING
assert output.num_scheduled_tokens[req.request_id] > 0
def test_deferred_lookup_does_not_block_parked_load():
"""A request with a deferred connector lookup must not strand an async
load queued behind it.
req_a's lookup is deferred (connector returns None), so it is parked in
waiting holding no blocks; req_b is then admitted as an async
load holding most of the pool. Once req_a's lookup resolves it cannot
allocate, and the waiting scan stops at it every step. The parked load
must still be promotable: it lives in the KV-holding queue
that is drained first, so it runs, finishes, and frees its blocks for
req_a.
"""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
req_a = create_request(
request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1
)
req_b = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 5,
do_remote_prefill=True,
max_tokens=1,
)
scheduler.add_request(req_a)
scheduler.add_request(req_b)
deferrals = [(None, False)]
def lookup(request, num_computed_tokens):
if request is req_b:
return BLOCK_SIZE * 5, True
return deferrals.pop() if deferrals else (0, False)
with patch.object(
scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup
):
# Step 1: req_a's lookup is deferred; req_b is admitted as an async
# load holding 5 of 7 usable blocks.
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING
assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS
# req_a's lookup resolves (no hit) while req_b's transfer is in
# flight; req_a needs 4 blocks but only 2 are free, so the waiting
# scan stops at it. req_b's transfer then finishes.
scheduler.update_from_output(
scheduler.schedule(),
create_model_runner_output([], finished_recving={req_b.request_id}),
)
# req_b must still be reached, promoted, and scheduled despite the
# scan stopping at req_a.
scheduler_output = scheduler.schedule()
assert req_b.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0
# req_b finishes and frees its blocks; now req_a fits and runs.
scheduler.update_from_output(
scheduler_output, create_model_runner_output([req_b])
)
scheduler_output = scheduler.schedule()
assert req_a.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SimpleCPUOffloadConnector-level behavior."""
from __future__ import annotations
import pytest
from tests.v1.kv_connector.unit.utils import create_vllm_config
from tests.v1.simple_kv_offload.test_scheduler import (
_BYTES_PER_BLOCK,
BLOCK_SIZE,
DTYPE,
HEAD_SIZE,
NUM_KV_HEADS,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
def _make_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
fa_layers = ["layer0"]
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
)
groups = [KVCacheGroupSpec(fa_layers, spec)]
size = _BYTES_PER_BLOCK * num_blocks
tensors = [
KVCacheTensor(
size=size,
layers=fa_layers,
layer_stride=size,
block_stride=_BYTES_PER_BLOCK,
)
]
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=tensors,
kv_cache_groups=groups,
)
def _make_connector(
extra_config: dict | None = None,
num_gpu_blocks: int = 16,
cpu_bytes: int = _BYTES_PER_BLOCK * 8,
) -> SimpleCPUOffloadConnector:
kv_cache_config = _make_kv_cache_config(num_gpu_blocks)
config_dict = {
"cpu_bytes_to_use": cpu_bytes,
}
if extra_config:
config_dict.update(extra_config)
vllm_config = create_vllm_config(
kv_connector="SimpleCPUOffloadConnector",
kv_role="kv_both",
kv_connector_extra_config=config_dict,
)
connector = SimpleCPUOffloadConnector(
vllm_config=vllm_config,
role=KVConnectorRole.SCHEDULER,
kv_cache_config=kv_cache_config,
)
return connector
def test_disk_mode_rejects_non_positive_capacity() -> None:
"""Disk mode with disk_capacity_bytes <= 0 raises ValueError.
Without this guard, the worker operates in disk mode but the scheduler
classifies events as MEDIUM_CPU instead of MEDIUM_STORAGE, silently
mislabeling events.
"""
with pytest.raises(ValueError, match="disk_capacity_bytes > 0"):
_make_connector(
extra_config={
"kv_offload_backend": "disk",
"disk_path": "/tmp/fake",
"disk_capacity_bytes": 0,
}
)
def _make_ring_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
"""A paged full-attention group beside a per-request ring group."""
full = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_SIZE,
dtype=DTYPE,
)
ring = CircularBufferSpec(
block_size=6, num_kv_heads=1, head_size=32, head_size_v=0, dtype=DTYPE
)
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=_BYTES_PER_BLOCK * num_blocks,
layers=["attn"],
layer_stride=_BYTES_PER_BLOCK * num_blocks,
block_stride=_BYTES_PER_BLOCK,
),
KVCacheTensor(
size=ring.page_size_bytes * num_blocks,
layers=["ring"],
layer_stride=ring.page_size_bytes * num_blocks,
block_stride=ring.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["attn"], full),
KVCacheGroupSpec(["ring"], ring),
],
)
def test_ring_scratch_group_is_accepted_and_never_offloaded() -> None:
"""The ring group's capacity is not a token granularity, it holds no
hashed blocks, and it never participates in CPU hits."""
vllm_config = create_vllm_config(
kv_connector="SimpleCPUOffloadConnector",
kv_role="kv_both",
kv_connector_extra_config={"cpu_bytes_to_use": _BYTES_PER_BLOCK * 8},
)
connector = SimpleCPUOffloadConnector(
vllm_config=vllm_config,
role=KVConnectorRole.SCHEDULER,
kv_cache_config=_make_ring_kv_cache_config(),
)
scheduler = connector.scheduler_manager
assert scheduler.fa_gidx == 0
cpu_groups = scheduler.cpu_kv_cache_config.kv_cache_groups
assert [g.kv_cache_spec.prefix_cacheable for g in cpu_groups] == [True, False]
# Only the paged group takes part in CPU-side hit lookup.
attention_groups = scheduler.cpu_coordinator.attention_groups
assert [g.group_ids for g in attention_groups] == [[0]]
@@ -1,466 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SimpleCPUOffloadConnector metrics."""
from __future__ import annotations
import dataclasses
from types import SimpleNamespace
from typing import Any
import pytest
from prometheus_client import Counter, Gauge, Histogram
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
_make_connector,
)
from tests.v1.simple_kv_offload.test_scheduler import (
make_request,
make_scheduler,
make_scheduler_output,
)
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
LoadRequestState,
TransferMeta,
)
from vllm.v1.simple_kv_offload.metrics import (
INFO_LABELS,
OUTCOME_TO_FIELD,
MetricName,
SimpleCPUOffloadStats,
_MetricType,
_StatsKey,
)
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
USED_BLOCKS = MetricName.USED_BLOCKS
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
INFO = MetricName.INFO
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.children: list[_FakeMetric] = []
self.observed: list[int | float] = []
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.labelvalues: tuple[object, ...] = ()
def labels(self, *labelvalues: object) -> _FakeMetric:
child = _FakeMetric(**self.kwargs)
child.labelvalues = labelvalues
self.children.append(child)
return child
def observe(self, value: int | float) -> None:
self.observed.append(value)
def inc(self, value: int | float) -> None:
self.increments.append(value)
def set(self, value: int | float) -> None:
self.set_values.append(value)
def _fake_metric_types() -> dict[type, type]:
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
return SimpleCPUOffloadPromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
)
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
return stats.data[_StatsKey.DATA][name]
# ---------------------------------------------------------------------------
# Stats class
# ---------------------------------------------------------------------------
def test_build_kv_connector_stats_none_and_empty() -> None:
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
assert isinstance(stats, SimpleCPUOffloadStats)
assert stats.is_empty()
def test_stats_round_trip_through_serialized_dict() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.increase_counter(LOAD_BLOCKS, 4)
stats.set_gauge(USED_BLOCKS, 7)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
assert isinstance(rebuilt, SimpleCPUOffloadStats)
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
stats1.set_gauge(USED_BLOCKS, 7)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
stats2.set_gauge(USED_BLOCKS, 2)
result = stats1.aggregate(stats2)
assert result is stats1
outcomes = _values(stats1, SAVE_OUTCOMES)
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
assert _values(stats1, USED_BLOCKS) == {(): 2}
def test_stats_aggregate_merges_types_from_other() -> None:
stats1 = SimpleCPUOffloadStats()
stats1.set_gauge(USED_BLOCKS, 1)
stats2 = SimpleCPUOffloadStats()
stats2.increase_counter(LOAD_BLOCKS, 2)
stats1.aggregate(stats2)
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
def test_stats_reduce_and_reset() -> None:
stats = SimpleCPUOffloadStats()
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
stats.increase_counter(LOAD_BLOCKS, 9)
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
reduced = stats.reduce()
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
assert reduced[PENDING_STORE_BLOCKS] == 4
assert reduced[LOAD_BLOCKS] == 9
assert not any(key.startswith(INFO) for key in reduced)
stats.reset()
assert stats.is_empty()
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
boundary_fields = {
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
}
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
# ---------------------------------------------------------------------------
# Prom metrics class
# ---------------------------------------------------------------------------
def test_prom_metrics_registers_tier1_metrics() -> None:
prom = _make_prom_metrics()
defs = prom._defs
assert set(defs) == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
# The prom class registers literal names (required by the docs
# generator); they must stay in sync with the emission-side constants.
assert {m.kwargs["name"] for m in defs.values()} == {
SAVE_OUTCOMES,
LOAD_BLOCKS,
USED_BLOCKS,
PENDING_STORE_BLOCKS,
INFO,
}
for metric in defs.values():
assert metric.kwargs["documentation"]
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
"model_name",
"engine",
"outcome",
]
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
assert defs[INFO].kwargs["labelnames"] == [
"model_name",
"engine",
*INFO_LABELS,
]
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
# mostrecent: multiprocess scrapes expose one series per label set, not
# one pid-tagged series per writer process.
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
def test_prom_metrics_observe_routes_each_series() -> None:
prom = _make_prom_metrics()
prom.observe(
_stats_payload(
types={
SAVE_OUTCOMES: _MetricType.COUNTER,
LOAD_BLOCKS: _MetricType.COUNTER,
USED_BLOCKS: _MetricType.GAUGE,
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
INFO: _MetricType.GAUGE,
},
data={
SAVE_OUTCOMES: {("stored",): 3},
LOAD_BLOCKS: {(): 4},
USED_BLOCKS: {(): 5},
PENDING_STORE_BLOCKS: {(): 1},
INFO: {("cpu", "false", "false", "64"): 1},
},
)
)
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
"model",
"0",
"stored",
)
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
"model",
"0",
"cpu",
"false",
"false",
"64",
)
def test_prom_metrics_reuses_bound_children() -> None:
prom = _make_prom_metrics()
payload = _stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored",): 1}},
)
prom.observe(payload)
prom.observe(payload)
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
assert child.increments == [1, 1]
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
def test_prom_metrics_rejects_unknown_metric() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="Unknown"):
prom.observe(
_stats_payload(
types={"vllm:not_a_metric": _MetricType.COUNTER},
data={"vllm:not_a_metric": {(): 1}},
)
)
def test_prom_metrics_rejects_wrong_label_count() -> None:
prom = _make_prom_metrics()
with pytest.raises(AssertionError, match="labels"):
prom.observe(
_stats_payload(
types={SAVE_OUTCOMES: _MetricType.COUNTER},
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
)
)
def test_prom_metrics_routes_per_engine() -> None:
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
prom.observe(
_stats_payload(
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
),
engine_idx=1,
)
assert (0, USED_BLOCKS, ()) not in prom._metrics
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
def test_connector_build_prom_metrics() -> None:
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
prom = SimpleCPUOffloadConnector.build_prom_metrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
metric_types=_fake_metric_types(),
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"]},
)
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
# ---------------------------------------------------------------------------
# Manager: interval stats
# ---------------------------------------------------------------------------
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.boundary_store_stats.dropped_cpu_full += 2
sched.boundary_store_stats.skipped_in_flight += 1
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {
("stored",): 3,
("dropped_cpu_full",): 2,
("skipped_in_flight",): 1,
}
# Deltas are per-interval: a second drain reports no new outcomes.
stats2 = sched.get_stats()
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
fixture = make_scheduler()
sched = fixture.scheduler
sched.boundary_store_stats.stored += 3
sched.reset()
stats = sched.get_stats()
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
def test_manager_get_stats_gauges_and_info_labels() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
sched._pending_finished_stores.append(TransferMeta([12], [13]))
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
stats = sched.get_stats()
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
used = _values(stats, USED_BLOCKS)[()]
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
info = _values(stats, INFO)
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
# Gauges are re-reported every drain.
stats2 = sched.get_stats()
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
def test_manager_info_labels_reflect_disk_mode() -> None:
connector = _make_connector(
extra_config={
"kv_offload_backend": "disk",
"disk_path": "/tmp/fake-disk-metrics",
"disk_capacity_bytes": 1024**3,
"use_page_cache": True,
}
)
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[0] == "disk"
assert sched._info_labelvalues[1] == "true"
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
connector = _make_connector(extra_config={"use_page_cache": True})
sched = connector.scheduler_manager
assert sched is not None
assert sched._info_labelvalues[:2] == ("cpu", "false")
stats = sched.get_stats()
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
def test_manager_counts_load_blocks_completed() -> None:
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
"""Loads abandoned by reset() that still finish on the worker are
counted as loaded blocks.
"""
fixture = make_scheduler(num_cpu_blocks=8)
sched = fixture.scheduler
request = make_request(num_blocks=2)
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
sched._reqs_to_load[request.request_id] = LoadRequestState(
request=request,
transfer_meta=TransferMeta(
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
),
)
sched.build_connector_meta(make_scheduler_output({}))
stats = sched.get_stats()
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
assert sched.reset() is False
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
sched.update_connector_output(
KVConnectorOutput(finished_recving={request.request_id})
)
assert sched._abandoned_reqs_to_load == {}
stats = sched.get_stats()
assert _values(stats, LOAD_BLOCKS) == {(): 2}
@@ -1,316 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for TP mapping and transfer plan utilities.
These tests verify that TP mapping produces correct outputs
(source ranks, split handles, desc IDs).
No GPU or NIXL required.
"""
from __future__ import annotations
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
compute_tp_mapping,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
# ======================================================================
# Test fixtures / helpers
# ======================================================================
def _compute_mapping(
tp_rank: int = 0,
tp_size: int = 1,
remote_tp_size: int = 1,
is_mla: bool = False,
num_kv_heads: int = 8,
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
dcp_size: int = 1,
remote_dcp_size: int = 1,
) -> TPMapping:
transfer_topology = object.__new__(TransferTopology)
transfer_topology.tp_rank = tp_rank
transfer_topology.tp_size = tp_size
transfer_topology.is_mla = is_mla
transfer_topology.total_num_kv_heads = num_kv_heads
transfer_topology.dcp_size = dcp_size
return compute_tp_mapping(
transfer_topology=transfer_topology,
remote_tp_size=remote_tp_size,
group_spec_types=group_spec_types,
remote_dcp_size=remote_dcp_size,
)
# ======================================================================
# TP mapping structure tests
# ======================================================================
class TestTPMappingStructure:
def test_source_ranks_homogeneous(self):
m = _compute_mapping(tp_size=2, tp_rank=1, remote_tp_size=2)
assert m.all_source_ranks == (1,)
def test_source_ranks_d_gt_p(self):
m = _compute_mapping(tp_size=4, tp_rank=2, remote_tp_size=2)
assert m.all_source_ranks == (1,)
def test_source_ranks_p_gt_d(self):
m = _compute_mapping(tp_size=1, tp_rank=0, remote_tp_size=2)
assert m.all_source_ranks == (0, 1)
@pytest.mark.parametrize(
"tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks",
[
(0, 1, 8, 1, 8, tuple(range(8))),
(0, 4, 4, 1, 4, (0, 1, 2, 3)),
(2, 4, 4, 4, 4, (2,)),
(0, 2, 4, 2, 4, (0, 2)),
(3, 4, 2, 4, 2, (1,)),
],
)
def test_mla_dcp_source_ranks(
tp_rank,
tp_size,
remote_tp_size,
dcp_size,
remote_dcp_size,
expected_ranks,
):
mapping = _compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=True,
num_kv_heads=1,
dcp_size=dcp_size,
remote_dcp_size=remote_dcp_size,
)
assert mapping.all_source_ranks == expected_ranks
@pytest.mark.parametrize(
"tp_size,remote_tp_size,dcp_size,remote_dcp_size",
[
(4, 2, 4, 2),
(4, 4, 4, 4),
(4, 1, 4, 1),
(4, 4, 1, 4),
(2, 4, 2, 4),
],
)
def test_dcp_consumer_count_matches_readers(
tp_size, remote_tp_size, dcp_size, remote_dcp_size
):
"""Each producer waits for exactly the local ranks that read from it."""
mappings = [
_compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=True,
num_kv_heads=1,
dcp_size=dcp_size,
remote_dcp_size=remote_dcp_size,
)
for tp_rank in range(tp_size)
]
for remote_rank in range(remote_tp_size):
readers = [m for m in mappings if remote_rank in m.all_source_ranks]
for mapping in readers:
assert mapping.local_consumers == len(readers)
# ======================================================================
# Split handle tests
# ======================================================================
def _make_mock_worker_for_splits(group_spec_types):
"""Build a mock NixlConnectorWorker with _group_spec_types for split tests.
No per-region replicate flags are configured (``block_len_per_layer`` empty
and ``num_regions == 0``), so ``_build_local_splits_from_plan`` treats every
FA descriptor as SPLIT, matching the legacy behavior these tests assert.
"""
worker = object.__new__(NixlConnectorWorker)
worker._group_spec_types = group_spec_types
worker.transfer_topo = SimpleNamespace(virtually_split_kv_in_blocks=False)
worker.block_len_per_layer = []
worker.num_regions = 0
worker._region_is_mla = []
worker._conv_decomp = SimpleNamespace(local_conv_offsets=())
worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else []
worker._ple_region_index = None
return worker
class TestBuildSrcSplitHandles:
@pytest.mark.parametrize("remote_tp_size", [2, 4])
def test_build_src_split_handles(self, remote_tp_size):
tp_rank = 0
tp_size = 1
plan = _compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
)
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
# One region of 8 dense 1024-byte blocks as a single strided run.
src_blocks_data = np.array([(0x2000, 1024, 0, 1024, 8)], dtype=np.uint64)
num_descs = 8
splits = list(
worker._build_local_splits_from_plan(
plan,
src_blocks_data,
num_descs,
)
)
assert len(splits) == remote_tp_size
for handle in splits:
assert handle.shape == src_blocks_data.shape
# The head slice shrinks the length; block stride and count are kept.
assert handle[:, 1].tolist() == [1024 // remote_tp_size]
assert handle[:, 3:].tolist() == [[1024, 8]]
class TestMambaPlanSplitHandles:
"""Verify split handles for Mamba with FA/SSM distinction."""
def test_fa_and_ssm_different_split_factors(self):
"""Section 0 split by num_attn_reads, section 1 by abs_tp."""
fa_readers = (0,)
ssm_readers = (0, 1)
plan = TPMapping(
source_ranks_per_group=(fa_readers, ssm_readers),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
# 2 FA runs + 1 SSM run of one block each: (addr, len, dev, stride, count)
src_blocks_data = np.array(
[
(1000, 200, 0, 200, 1), # FA desc 0
(2000, 200, 0, 200, 1), # FA desc 1
(3000, 400, 0, 400, 1), # SSM desc 0
],
dtype=np.uint64,
)
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2))
assert len(splits) == 2 # 2 source ranks
# Rank 0 (FA source, p_idx=0):
# FA: chunk=200//1=200, slot=0 → (1000, 200), (2000, 200)
# SSM: chunk=400//2=200, idx=0 → (3000, 200); block stride stays 400
assert splits[0].tolist() == [
[1000, 200, 0, 200, 1],
[2000, 200, 0, 200, 1],
[3000, 200, 0, 400, 1],
]
# Rank 1 (not FA source, p_idx=1):
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200), (2000, 200)
# SSM: chunk=400//2=200, idx=1 → (3200, 200); block stride stays 400
assert splits[1].tolist() == [
[1000, 200, 0, 200, 1],
[2000, 200, 0, 200, 1],
[3200, 200, 0, 400, 1],
]
def test_hetero_block_size_splits(self):
"""With a block-size ratio, single-source FA sub-block descs pass
through whole; SSM descs are unexpanded and split per source."""
plan = TPMapping(
source_ranks_per_group=((0,), (0, 1)),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
# 2 FA blocks x ratio 2 sub-blocks (one run of 2 per block) + 1 SSM desc
# (never expanded).
src_blocks_data = np.array(
[
(1000, 100, 0, 100, 2),
(2000, 100, 0, 100, 2),
(3000, 400, 0, 400, 1),
],
dtype=np.uint64,
)
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 4, 2))
assert len(splits) == 2
fa_passthrough = [[1000, 100, 0, 100, 2], [2000, 100, 0, 100, 2]]
assert splits[0].tolist() == fa_passthrough + [[3000, 200, 0, 400, 1]]
assert splits[1].tolist() == fa_passthrough + [[3200, 200, 0, 400, 1]]
def test_hetero_block_size_head_sharded_asserts(self):
"""Head-sharded FA reads (multiple FA sources) are incompatible with
a block-size mismatch and must fail loudly."""
plan = TPMapping(
source_ranks_per_group=((0, 1), (0, 1)),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 1},
rank_offset_factor=0,
)
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
src_blocks_data = np.array(
[(1000, 100, 0, 100, 2), (3000, 400, 0, 400, 1)],
dtype=np.uint64,
)
with pytest.raises(AssertionError, match="Head-sharded"):
list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2))
@pytest.mark.parametrize(
("total_kv_heads", "local_tp", "remote_tp", "compatible"),
[
# 2 heads: TP1 packs both heads into a page, TP >= 2 packs one.
(2, 1, 2, False),
(2, 2, 1, False),
(2, 1, 1, True),
(2, 2, 4, True),
(2, 4, 2, True),
# 4 heads: the boundary moves with the head count.
(4, 2, 4, False),
(4, 2, 2, True),
(4, 4, 8, True),
],
)
def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible):
worker = object.__new__(NixlConnectorWorker)
worker.world_size = local_tp
worker._is_csa_linear = True
worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads)
if compatible:
worker._validate_csa_linear_tp_layout(remote_tp)
else:
with pytest.raises(ValueError, match="KV-head sharding boundary"):
worker._validate_csa_linear_tp_layout(remote_tp)
-148
View File
@@ -537,154 +537,6 @@ def make_kv_cache_config(
)
def make_nixl_scheduler(
has_mamba: bool = False,
is_hma_required: bool = False,
bounded_replay: bool = False,
heartbeat: bool = False,
kv_lease_duration: int = 30,
):
"""Create a NixlConnectorScheduler via __new__ (skipping __init__).
Only sets the flags needed by the tests. When *heartbeat=True* the
scheduler-side heartbeat bookkeeping fields are also initialised.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
NixlConnectorScheduler,
)
sched = object.__new__(NixlConnectorScheduler)
sched._has_mamba = has_mamba
sched._bounded_replay = bounded_replay
sched._is_hma_required = is_hma_required
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0)
if heartbeat:
sched._heartbeat_by_engine = {}
sched._heartbeat_req_engine = {}
sched._last_heartbeat_time = 0.0
sched._kv_lease_duration = kv_lease_duration
sched._heartbeat_interval = kv_lease_duration // 6
# Fields touched by build_connector_meta / request_finished:
sched._reqs_need_recv = {}
sched._hisparse_host_blocks_to_recv = {}
sched._reqs_need_send = {}
sched._reqs_in_batch = set()
sched._reqs_not_processed = set()
sched._reqs_need_save = {}
sched.use_host_buffer = False
sched.engine_id = "test-engine"
sched.transfer_tp_size = 1
sched.side_channel_host = "localhost"
sched.side_channel_port = 5555
sched.blocks_per_sw = []
sched.is_bidirectional_kv_xfer_enabled = False
return sched
def make_nixl_push_scheduler(
*,
decoder_kv_blocks_ttl: float = 30.0,
push_registration_timeout: float | None = None,
is_bidirectional_kv_xfer_enabled: bool = False,
has_mamba: bool = False,
):
"""Create a NixlPushConnectorScheduler via __new__ (skipping __init__).
The push scheduler can't reuse :func:`make_nixl_scheduler` because it
is a different class (``NixlPushConnectorScheduler`` vs
``NixlConnectorScheduler``) and carries push-specific state. Only the
fields touched by the unit tests are populated.
"""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
NixlPushConnectorScheduler,
)
sched = object.__new__(NixlPushConnectorScheduler)
# Base scheduler fields (shared with pull / heartbeat path).
sched._reqs_need_recv = {}
sched._reqs_need_send = {}
sched._reqs_in_batch = set()
sched._reqs_not_processed = set()
sched._reqs_need_save = {}
sched._kv_lease_duration = 30
sched.decoder_kv_blocks_ttl = decoder_kv_blocks_ttl
sched.use_host_buffer = False
sched.engine_id = "decode-engine"
sched.transfer_tp_size = 1
sched.side_channel_host = "127.0.0.1"
sched.side_channel_port = 5600
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
sched._has_mamba = has_mamba
sched._bounded_replay = False
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
# vllm_config is consulted for parallel_config.tensor_parallel_size, and by
# `_prefill_backoff` on both the P and D prefill paths.
vllm_config = MagicMock()
vllm_config.parallel_config.tensor_parallel_size = 1
vllm_config.num_prefill_lookahead_tokens = 0
sched.vllm_config = vllm_config
# Push-specific state.
sched._push_pending_registrations = {}
sched._push_registration_deadlines = {}
sched._finished_request_blocks = {}
sched._newly_finished_push_blocks = {}
sched._push_registration_timeout = (
push_registration_timeout
if push_registration_timeout is not None
else decoder_kv_blocks_ttl
)
# Heartbeat fields touched by base request_finished /
# update_connector_output.
sched._heartbeat_by_engine = {}
sched._heartbeat_req_engine = {}
sched._last_heartbeat_time = 0.0
sched.blocks_per_sw = []
return sched
def make_moriio_writer(fake_worker: Any) -> Any:
"""Build a MoRIIOWriter with internals stubbed for unit tests.
Bypasses ``__init__`` and wires only the write/finalize state the tests
touch, including the deferred-task fields used by the routing suite.
"""
import threading
from queue import Queue
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWriter,
)
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
writer._write_task_q = Queue()
writer._write_state_lock = threading.Lock()
writer._scheduled_writes = defaultdict(int)
writer._scheduled_layers = defaultdict(set)
writer._sealed_writes = {}
writer._deferred_tasks = []
writer._defer_timeout = 60.0
writer.ensure_worker_started = lambda: None
return writer
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
File diff suppressed because it is too large Load Diff