[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,775 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for packed tensor broadcasting functionality.
|
||||
|
||||
Unit tests for packed_nccl_broadcast_producer and packed_nccl_broadcast_consumer.
|
||||
These utilities enable efficient batched tensor transfer over NCCL.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
pack_tensors,
|
||||
packed_ipc_consumer,
|
||||
packed_ipc_producer,
|
||||
packed_nccl_broadcast_consumer,
|
||||
packed_nccl_broadcast_producer,
|
||||
unpack_tensor,
|
||||
)
|
||||
|
||||
|
||||
class MockCommunicationGroup:
|
||||
"""Mock communication group for testing producer broadcast operations."""
|
||||
|
||||
def __init__(self):
|
||||
self.broadcasted_tensors: list[torch.Tensor] = []
|
||||
self.broadcast_count = 0
|
||||
self.device = torch.device("cuda:0")
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
"""Mock broadcast that stores the tensor for later verification."""
|
||||
self.broadcasted_tensors.append(tensor.clone())
|
||||
self.broadcast_count += 1
|
||||
|
||||
|
||||
class MockConsumerCommunicationGroup:
|
||||
"""Mock communication group for consumer that returns pre-stored tensors."""
|
||||
|
||||
def __init__(self, tensors_to_return: list[torch.Tensor]):
|
||||
self.tensors_to_return = tensors_to_return
|
||||
self.current_index = 0
|
||||
self.device = torch.device("cuda:0")
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
"""Mock broadcast that fills the tensor with pre-stored data."""
|
||||
if self.current_index < len(self.tensors_to_return):
|
||||
tensor.copy_(self.tensors_to_return[self.current_index])
|
||||
self.current_index += 1
|
||||
|
||||
|
||||
def create_mock_model_params(
|
||||
num_layers: int = 3,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Create mock model parameters for testing."""
|
||||
params = []
|
||||
for i in range(num_layers):
|
||||
params.append((f"layer{i}.weight", torch.randn(10, 20, dtype=dtype)))
|
||||
params.append((f"layer{i}.bias", torch.randn(10, dtype=dtype)))
|
||||
return params
|
||||
|
||||
|
||||
def create_state_dict_info(
|
||||
params: list[tuple[str, torch.Tensor]],
|
||||
) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
|
||||
"""Create state dict info (name -> (shape, dtype)) from params."""
|
||||
return {name: (tuple(tensor.shape), tensor.dtype) for name, tensor in params}
|
||||
|
||||
|
||||
# --- Unit Tests: packed_nccl_broadcast_producer ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastProducer:
|
||||
"""Test packed_nccl_broadcast_producer function."""
|
||||
|
||||
def test_producer_empty_iterator(self):
|
||||
"""Test producer handles empty iterator gracefully."""
|
||||
mock_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter([]),
|
||||
group=mock_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
|
||||
# No broadcasts for empty iterator
|
||||
assert mock_group.broadcast_count == 0
|
||||
|
||||
|
||||
# --- Integration Tests: Producer-Consumer Roundtrip ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastRoundtrip:
|
||||
"""Test producer-consumer roundtrip behavior."""
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_roundtrip_different_dtypes(self, dtype):
|
||||
"""Test roundtrip with different data types."""
|
||||
params = create_mock_model_params(num_layers=2, dtype=dtype)
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
buffer_size = 1000
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify roundtrip preserves data
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.dtype == dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_roundtrip_mixed_dtypes(self):
|
||||
"""Test roundtrip with mixed data types."""
|
||||
# Create params with mixed dtypes
|
||||
params = [
|
||||
("layer1.weight", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("layer1.bias", torch.randn(10, dtype=torch.float16).cuda()),
|
||||
("layer2.weight", torch.randn(20, 30, dtype=torch.bfloat16).cuda()),
|
||||
]
|
||||
|
||||
buffer_size = 500
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all params roundtrip correctly with correct dtypes
|
||||
for name, original_tensor in params:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
@pytest.mark.parametrize("target_size", [100, 100000])
|
||||
def test_roundtrip_different_batch_sizes(self, target_size):
|
||||
"""Test roundtrip with different target batch sizes."""
|
||||
params = create_mock_model_params(num_layers=5)
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=target_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=target_size,
|
||||
)
|
||||
|
||||
# Verify all params roundtrip correctly
|
||||
assert len(unpacked_tensors) == len(params)
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
assert torch.allclose(
|
||||
unpacked_tensors[name], original_tensor, rtol=1e-5, atol=1e-7
|
||||
)
|
||||
|
||||
def test_roundtrip_non_contiguous_tensors(self):
|
||||
"""Test roundtrip with non-contiguous tensors from the trainer."""
|
||||
# Create non-contiguous tensors (simulating trainer outputs)
|
||||
# Transposed tensors are non-contiguous
|
||||
weight1 = torch.randn(20, 10, dtype=torch.float32).cuda().T
|
||||
# Sliced tensors with step are non-contiguous
|
||||
weight2 = torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]
|
||||
# Permuted tensors are non-contiguous
|
||||
weight3 = torch.randn(5, 10, 15, dtype=torch.bfloat16).cuda().permute(2, 0, 1)
|
||||
|
||||
params = [
|
||||
("layer1.weight", weight1),
|
||||
("layer2.weight", weight2),
|
||||
("layer3.weight", weight3),
|
||||
]
|
||||
|
||||
# Verify tensors are indeed non-contiguous
|
||||
for name, tensor in params:
|
||||
assert not tensor.is_contiguous(), f"{name} should be non-contiguous"
|
||||
|
||||
buffer_size = 500
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all non-contiguous params roundtrip correctly
|
||||
for name, original_tensor in params:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
|
||||
# --- Unit Tests: unpack_tensor ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestUnpackTensor:
|
||||
"""Test the shared unpack_tensor function."""
|
||||
|
||||
def test_unpack_produces_independent_copies(self):
|
||||
"""Verify unpacked tensors don't share memory with packed buffer."""
|
||||
original = torch.randn(10, dtype=torch.float32).cuda()
|
||||
packed = original.contiguous().view(torch.uint8).view(-1)
|
||||
|
||||
result = unpack_tensor(
|
||||
packed,
|
||||
names=["w"],
|
||||
shapes=[[10]],
|
||||
dtypes=[torch.float32],
|
||||
tensor_sizes=[packed.numel()],
|
||||
)
|
||||
|
||||
# Mutate the packed buffer
|
||||
packed.zero_()
|
||||
|
||||
# Unpacked tensor should be unaffected
|
||||
assert torch.allclose(result[0][1], original)
|
||||
|
||||
|
||||
# --- Unit Tests: pack_tensors ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackTensors:
|
||||
"""Test the shared pack_tensors function."""
|
||||
|
||||
def test_pack_basic(self):
|
||||
"""Test packing a few tensors into one buffer."""
|
||||
params = [
|
||||
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("w2", torch.randn(5, dtype=torch.float16).cuda()),
|
||||
]
|
||||
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
|
||||
assert chunk is not None
|
||||
assert len(chunk.names) == 2
|
||||
assert chunk.names == ["w1", "w2"]
|
||||
assert chunk.shapes == [[10, 20], [5]]
|
||||
assert chunk.dtypes == [torch.float32, torch.float16]
|
||||
assert chunk.packed_tensor.dtype == torch.uint8
|
||||
|
||||
def test_pack_respects_buffer_limit(self):
|
||||
"""Test that packing stops when buffer_size_bytes is exceeded."""
|
||||
params = [
|
||||
(f"w{i}", torch.randn(100, 100, dtype=torch.float32).cuda())
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=50_000,
|
||||
)
|
||||
|
||||
assert chunk is not None
|
||||
assert len(chunk.names) < 10
|
||||
|
||||
def test_pack_empty_iterator(self):
|
||||
"""Test that an empty iterator returns None."""
|
||||
chunk = pack_tensors(
|
||||
iterator=iter([]),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
assert chunk is None
|
||||
|
||||
def test_pack_single_tensor_larger_than_buffer_warns(self):
|
||||
"""Test that a tensor exceeding buffer_size_bytes emits a warning."""
|
||||
big = torch.randn(1000, 1000, dtype=torch.float32).cuda()
|
||||
params = [("big", big)]
|
||||
|
||||
import warnings
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always")
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=100,
|
||||
)
|
||||
assert chunk is not None
|
||||
assert len(chunk.names) == 1
|
||||
assert any("exceeds buffer_size_bytes" in str(wi.message) for wi in w)
|
||||
|
||||
def test_pack_unpack_roundtrip(self):
|
||||
"""Test pack then unpack produces identical tensors."""
|
||||
params = [
|
||||
("a", torch.randn(8, 16, dtype=torch.float32).cuda()),
|
||||
("b", torch.randn(4, dtype=torch.float16).cuda()),
|
||||
("c", torch.randn(3, 5, 7, dtype=torch.bfloat16).cuda()),
|
||||
]
|
||||
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
|
||||
assert chunk is not None
|
||||
result = unpack_tensor(
|
||||
chunk.packed_tensor,
|
||||
chunk.names,
|
||||
chunk.shapes,
|
||||
chunk.dtypes,
|
||||
chunk.tensor_sizes,
|
||||
)
|
||||
|
||||
assert len(result) == len(params)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.shape == orig_tensor.shape
|
||||
assert res_tensor.dtype == orig_tensor.dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_pack_multiple_chunks(self):
|
||||
"""Test consuming an iterator across multiple pack_tensors calls."""
|
||||
params = [
|
||||
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
|
||||
]
|
||||
it = iter(params)
|
||||
|
||||
all_names = []
|
||||
chunks = []
|
||||
while True:
|
||||
chunk = pack_tensors(it, lambda x: x[1], buffer_size_bytes=12_000)
|
||||
if chunk is None:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
all_names.extend(chunk.names)
|
||||
|
||||
assert len(chunks) > 1
|
||||
assert all_names == [f"w{i}" for i in range(6)]
|
||||
|
||||
|
||||
# --- Unit Tests: packed_ipc_producer ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedIpcProducer:
|
||||
"""Test the packed_ipc_producer generator."""
|
||||
|
||||
def test_producer_yields_chunks(self):
|
||||
"""Test that the producer yields PackedIpcChunk objects."""
|
||||
params = [
|
||||
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
|
||||
]
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid="test-uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=12_000,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(chunks) > 1
|
||||
|
||||
def test_producer_ipc_handle_has_uuid(self):
|
||||
"""Test that each chunk's ipc_handle is keyed by the given UUID."""
|
||||
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid="my-gpu-uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
assert "my-gpu-uuid" in chunks[0].ipc_handle
|
||||
|
||||
def test_producer_dtype_names_are_strings(self):
|
||||
"""Test that dtype_names are string representations."""
|
||||
params = [
|
||||
("a", torch.randn(10, dtype=torch.float32).cuda()),
|
||||
("b", torch.randn(10, dtype=torch.float16).cuda()),
|
||||
]
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid="uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
assert chunks[0].dtype_names == ["float32", "float16"]
|
||||
|
||||
def test_producer_empty_iterator(self):
|
||||
"""Test producer with empty iterator yields nothing."""
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter([]),
|
||||
gpu_uuid="uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
)
|
||||
assert len(chunks) == 0
|
||||
|
||||
|
||||
# --- Integration Tests: IPC Producer-Consumer Roundtrip ---
|
||||
|
||||
|
||||
def _ipc_consumer_worker(cmd_q, ack_q, result_q, done_event, device_index):
|
||||
"""Worker that consumes chunks streamed one at a time from the parent.
|
||||
|
||||
CUDA IPC requires the consumer to be in a separate process from the
|
||||
producer. The producer reuses a single IPC buffer between chunks, so
|
||||
the parent must wait for our ack (sent after we copy the chunk to
|
||||
CPU) before advancing the producer.
|
||||
"""
|
||||
try:
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
all_results = []
|
||||
while True:
|
||||
cd = cmd_q.get()
|
||||
if cd is None:
|
||||
break
|
||||
result = packed_ipc_consumer(
|
||||
ipc_handle=cd["ipc_handle"],
|
||||
names=cd["names"],
|
||||
shapes=cd["shapes"],
|
||||
dtype_names=cd["dtype_names"],
|
||||
tensor_sizes=cd["tensor_sizes"],
|
||||
device_index=device_index,
|
||||
)
|
||||
# .cpu() forces a GPU→CPU copy off the shared IPC buffer, so
|
||||
# the producer is free to overwrite it once we ack.
|
||||
all_results.extend([(name, tensor.cpu()) for name, tensor in result])
|
||||
del result
|
||||
ack_q.put("ack")
|
||||
result_q.put(("ok", all_results))
|
||||
except Exception as e:
|
||||
result_q.put(("error", str(e)))
|
||||
# Keep the process alive until the parent has finished reading from
|
||||
# the result queue — torch serializes CPU tensors via fd sharing,
|
||||
# which requires this process's resource-sharer server to be running.
|
||||
done_event.wait(timeout=60)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedIpcRoundtrip:
|
||||
"""Test IPC producer-consumer roundtrip using real CUDA IPC.
|
||||
|
||||
These tests spawn a child process for the consumer because
|
||||
rebuild_cuda_tensor requires a separate process from the one that
|
||||
called reduce_tensor.
|
||||
"""
|
||||
|
||||
def _get_gpu_uuid(self) -> str:
|
||||
device_index = torch.cuda.current_device()
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
return str(props.uuid)
|
||||
|
||||
def _run_roundtrip(self, chunk_iter, device_index, timeout=30):
|
||||
"""Stream chunks through a child consumer one at a time.
|
||||
|
||||
``packed_ipc_producer`` reuses a single IPC buffer for every
|
||||
chunk, so the producer must not be advanced until the consumer
|
||||
has finished reading the current chunk. We enforce that with an
|
||||
ack queue: the consumer puts ``"ack"`` after it has copied the
|
||||
chunk to CPU, and only then do we pull the next chunk from the
|
||||
generator.
|
||||
|
||||
Returns ``(num_chunks, results)``.
|
||||
"""
|
||||
import multiprocessing as mp
|
||||
|
||||
ctx = mp.get_context("spawn")
|
||||
cmd_q = ctx.Queue()
|
||||
ack_q = ctx.Queue()
|
||||
result_q = ctx.Queue()
|
||||
done_event = ctx.Event()
|
||||
proc = ctx.Process(
|
||||
target=_ipc_consumer_worker,
|
||||
args=(cmd_q, ack_q, result_q, done_event, device_index),
|
||||
)
|
||||
proc.start()
|
||||
|
||||
num_chunks = 0
|
||||
try:
|
||||
for chunk in chunk_iter:
|
||||
cmd_q.put(
|
||||
{
|
||||
"ipc_handle": chunk.ipc_handle,
|
||||
"names": chunk.names,
|
||||
"shapes": chunk.shapes,
|
||||
"dtype_names": chunk.dtype_names,
|
||||
"tensor_sizes": chunk.tensor_sizes,
|
||||
}
|
||||
)
|
||||
if ack_q.get(timeout=timeout) != "ack":
|
||||
raise RuntimeError("Consumer did not ack chunk")
|
||||
num_chunks += 1
|
||||
cmd_q.put(None)
|
||||
status, payload = result_q.get(timeout=timeout)
|
||||
finally:
|
||||
done_event.set()
|
||||
proc.join(timeout=10)
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
|
||||
if status == "error":
|
||||
raise RuntimeError(f"Consumer process failed: {payload}")
|
||||
# Reclaim IPC-shared memory now that the child has released it
|
||||
torch.cuda.ipc_collect()
|
||||
return num_chunks, payload
|
||||
|
||||
def test_roundtrip_basic(self):
|
||||
"""Test basic IPC producer -> consumer roundtrip."""
|
||||
params = [
|
||||
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("scale", torch.tensor(1.5, dtype=torch.float32).cuda()),
|
||||
("w2", torch.randn(5, dtype=torch.float16).cuda()),
|
||||
]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
num_chunks, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
assert num_chunks == 1
|
||||
assert len(result) == len(params)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.shape == orig_tensor.shape
|
||||
assert res_tensor.dtype == orig_tensor.dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_roundtrip_dtypes(self, dtype):
|
||||
"""Test IPC roundtrip with different dtypes."""
|
||||
params = create_mock_model_params(num_layers=2, dtype=dtype)
|
||||
params_cuda = [(n, t.cuda()) for n, t in params]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
_, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params_cuda),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
assert len(result) == len(params_cuda)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(
|
||||
params_cuda, result
|
||||
):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.dtype == dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_roundtrip_multiple_chunks(self):
|
||||
"""Test IPC roundtrip across multiple chunks."""
|
||||
params = [
|
||||
(f"layer{i}.weight", torch.randn(100, 100, dtype=torch.float32).cuda())
|
||||
for i in range(8)
|
||||
]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
num_chunks, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=50_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
assert num_chunks > 1
|
||||
assert len(result) == len(params)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-5, atol=1e-7)
|
||||
|
||||
def test_roundtrip_non_contiguous(self):
|
||||
"""Test IPC roundtrip with non-contiguous tensors."""
|
||||
params = [
|
||||
("transposed", torch.randn(20, 10, dtype=torch.float32).cuda().T),
|
||||
("sliced", torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]),
|
||||
]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
for _, t in params:
|
||||
assert not t.is_contiguous()
|
||||
|
||||
_, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.shape == orig_tensor.shape
|
||||
assert res_tensor.dtype == orig_tensor.dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_consumer_wrong_uuid_raises(self):
|
||||
"""Test that consumer raises ValueError for unknown GPU UUID."""
|
||||
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
c = chunks[0]
|
||||
fake_handle = {"fake-uuid-12345": c.ipc_handle[gpu_uuid]}
|
||||
|
||||
with pytest.raises(ValueError, match="IPC handle not found"):
|
||||
packed_ipc_consumer(
|
||||
ipc_handle=fake_handle,
|
||||
names=c.names,
|
||||
shapes=c.shapes,
|
||||
dtype_names=c.dtype_names,
|
||||
tensor_sizes=c.tensor_sizes,
|
||||
device_index=torch.cuda.current_device(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_pack_unpack_scalar(dtype):
|
||||
"""Preserve scalar shape, dtype and value alongside a matrix on CPU."""
|
||||
params = [
|
||||
("weight", torch.randn(2, 3, dtype=dtype)),
|
||||
("scale", torch.tensor(1.5, dtype=dtype)),
|
||||
]
|
||||
|
||||
chunk = pack_tensors(iter(params), lambda item: item[1], buffer_size_bytes=1024)
|
||||
assert chunk is not None
|
||||
|
||||
result = unpack_tensor(
|
||||
chunk.packed_tensor,
|
||||
chunk.names,
|
||||
chunk.shapes,
|
||||
chunk.dtypes,
|
||||
chunk.tensor_sizes,
|
||||
)
|
||||
|
||||
for (expected_name, expected), (name, actual) in zip(params, result, strict=True):
|
||||
assert name == expected_name
|
||||
assert actual.shape == expected.shape
|
||||
assert actual.dtype == expected.dtype
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
@@ -1,628 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the sharded-RDT trainer-side engine.
|
||||
|
||||
The trainer engine's own surface: init info and factory registration, the
|
||||
send_weights round trip against an in-process fake producer server, the
|
||||
ownership table it resolves and ships to the workers, and the producer's packed
|
||||
destination-view cache.
|
||||
|
||||
The plan/replay half (including `RdtRouter`, which is consumer-side) lives in
|
||||
test_sharded_rdt_plan.py; the producer server's protocol in
|
||||
test_sharded_rdt_producer.py.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.distributed.test_weight_transfer import RecordingClient
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
ParamMeta,
|
||||
WeightSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import layerwise_groups
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
ShardedRDTTrainerInitInfo,
|
||||
ShardedRDTTrainerWeightTransferEngine,
|
||||
)
|
||||
|
||||
|
||||
class _ListSource(WeightSource):
|
||||
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
||||
sharded-RDT group/order logic can be tested without a real model."""
|
||||
|
||||
def __init__(self, pairs):
|
||||
self._pairs = list(pairs)
|
||||
|
||||
def metadata(self):
|
||||
return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs]
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._pairs)
|
||||
|
||||
|
||||
class _FakeProducerServer:
|
||||
"""In-process stand-in for the _RDTProducerServer Ray actor. Records the
|
||||
engine->server call sequence and, by default, frees each group as soon as
|
||||
it is published (simulating the consumers' free_group barrier) so the
|
||||
gather loop's credit gate never parks. Mirrors the real server's
|
||||
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
||||
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
||||
groups flow back only through wait_freed / end_sync (see
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
|
||||
|
||||
def __init__(self, auto_free=True):
|
||||
self.order: list[str] = []
|
||||
self.published: list[int] = []
|
||||
self.live_count = 1
|
||||
self._inflight_groups: list[int] = []
|
||||
self.auto_free = auto_free
|
||||
self.free_counts: dict[int, int] = {}
|
||||
self._pending_freed: list[int] = []
|
||||
|
||||
# Alias for tests that inspect in-flight groups.
|
||||
@property
|
||||
def inflight(self):
|
||||
return self._inflight_groups
|
||||
|
||||
def begin_sync(self, live_count, live_consumer_ids=None):
|
||||
self.order.append("begin")
|
||||
self.live_count = max(1, int(live_count))
|
||||
self.live_ids = live_consumer_ids
|
||||
|
||||
def publish_group(self, group_idx, entries):
|
||||
self.order.append("publish")
|
||||
self.published.append(group_idx)
|
||||
self._inflight_groups.append(group_idx)
|
||||
if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count:
|
||||
self._inflight_groups.remove(group_idx)
|
||||
self._pending_freed.append(group_idx)
|
||||
|
||||
def free_group(self, group_idx):
|
||||
"""Consumer back-edge; may arrive before the group's publish."""
|
||||
self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1
|
||||
if (
|
||||
group_idx in self._inflight_groups
|
||||
and self.free_counts[group_idx] >= self.live_count
|
||||
):
|
||||
self._inflight_groups.remove(group_idx)
|
||||
self._pending_freed.append(group_idx)
|
||||
|
||||
def wait_freed(self):
|
||||
"""The engine's credit gate. The real server blocks here; the fake
|
||||
must already have a banked credit when the gate asks (auto_free, or a
|
||||
test's own free_group) — anything else is the deadlock the real
|
||||
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
||||
pacing, not a lifecycle milestone."""
|
||||
assert self._pending_freed, (
|
||||
"wait_freed with nothing freed: the gather loop would deadlock"
|
||||
)
|
||||
freed = self._pending_freed
|
||||
self._pending_freed = []
|
||||
return freed
|
||||
|
||||
def end_sync(self):
|
||||
self.order.append("end")
|
||||
freed = self._pending_freed
|
||||
self._pending_freed = []
|
||||
return freed
|
||||
|
||||
def set_gather_error(self, message):
|
||||
self.order.append("error")
|
||||
|
||||
|
||||
def _rdt_engine_with_fake_server(
|
||||
source, *, is_sender, client, server, monkeypatch, fleet_owned=None
|
||||
):
|
||||
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
||||
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
||||
group-major metadata, and route _rpc to the fake."""
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
||||
|
||||
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
||||
monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",)))
|
||||
|
||||
init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1)
|
||||
engine = ShardedRDTTrainerWeightTransferEngine(
|
||||
client=client, source=source, is_sender=is_sender, init_info=init_info
|
||||
)
|
||||
engine._meta = list(source.metadata())
|
||||
names = [m.name for m in engine._meta]
|
||||
engine._groups = layerwise_groups(names)
|
||||
engine._server = server
|
||||
engine._rpc = lambda method, *args: getattr(server, method)(*args)
|
||||
# What trainer_init resolves from the source's held names + the fleet's
|
||||
# all-gather. ``fleet_owned`` stands in for that all-gather so a partial-
|
||||
# ownership rank can be tested without a real process group; the fleet must
|
||||
# cover every name or the transpose rejects it (nothing would serve the rest).
|
||||
if fleet_owned is None:
|
||||
engine._resolve_ownership(1, 0)
|
||||
else:
|
||||
# Stands in for the (metadata digest, held-name bitmask) all-gather:
|
||||
# each rank's group list becomes the mask its held names would set.
|
||||
names = [m.name for m in engine._meta]
|
||||
groups = engine._groups
|
||||
|
||||
def _fake_gather(_world, mine):
|
||||
out = []
|
||||
for owned in fleet_owned:
|
||||
held = {n for gi in owned for n in groups[gi]}
|
||||
mask = bytearray((len(names) + 7) // 8)
|
||||
for i, n in enumerate(names):
|
||||
if n in held:
|
||||
mask[i >> 3] |= 1 << (i & 7)
|
||||
out.append((mine[0], bytes(mask)))
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather)
|
||||
engine._resolve_ownership(len(fleet_owned), 0)
|
||||
return engine
|
||||
|
||||
|
||||
def _rdt_source_two_layers():
|
||||
return _ListSource(
|
||||
[
|
||||
("embed.weight", torch.zeros(2)),
|
||||
("model.layers.0.w", torch.zeros(2)),
|
||||
("model.layers.1.w", torch.zeros(2)),
|
||||
("norm.weight", torch.zeros(2)),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TestShardedRDTTrainerInitInfo:
|
||||
def test_declares_backend(self):
|
||||
assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt"
|
||||
|
||||
def test_rank_is_keyword_only_and_drives_is_sender(self):
|
||||
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True
|
||||
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False
|
||||
with pytest.raises(TypeError):
|
||||
# rank is keyword-only.
|
||||
ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc]
|
||||
|
||||
def test_registered_in_trainer_factory(self):
|
||||
cls = WeightTransferTrainerFactory._registry["sharded_rdt"]()
|
||||
assert cls is ShardedRDTTrainerWeightTransferEngine
|
||||
|
||||
|
||||
def test_sharded_rdt_trainer_init_requires_source():
|
||||
with pytest.raises(ValueError, match="requires a WeightSource"):
|
||||
ShardedRDTTrainerWeightTransferEngine.trainer_init(
|
||||
ShardedRDTTrainerInitInfo(num_consumers=1, rank=0),
|
||||
client=RecordingClient(),
|
||||
source=None,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch):
|
||||
source = _rdt_source_two_layers()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
source,
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
worker_init = engine._build_worker_init_info(["srv_rk0"])
|
||||
# 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1.
|
||||
assert worker_init.names == [
|
||||
"embed.weight",
|
||||
"model.layers.0.w",
|
||||
"model.layers.1.w",
|
||||
"norm.weight",
|
||||
]
|
||||
assert worker_init.group_lens == [1, 1, 1, 1]
|
||||
assert worker_init.trainer_actor_names == ["srv_rk0"]
|
||||
assert worker_init.produce_method_name == "rdt_produce_weights_batched"
|
||||
assert sum(worker_init.group_lens) == len(worker_init.names)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch):
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
client = RecordingClient()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=True,
|
||||
client=client,
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights()
|
||||
|
||||
assert client.order == ["start", "update", "finish"]
|
||||
# begin, one publish per group (4), end.
|
||||
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
|
||||
assert len(server.published) == 4
|
||||
# every group freed -> no engine-held refs remain.
|
||||
assert engine._inflight == {}
|
||||
|
||||
|
||||
def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch):
|
||||
# Source whose iteration order disagrees with its metadata order.
|
||||
class _BadSource(_ListSource):
|
||||
def __iter__(self):
|
||||
reordered = list(self._pairs)
|
||||
reordered[0], reordered[1] = reordered[1], reordered[0]
|
||||
return iter(reordered)
|
||||
|
||||
server = _FakeProducerServer()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_BadSource(_rdt_source_two_layers()._pairs),
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
engine.send_weights()
|
||||
assert "error" in server.order # gather error propagated to the server
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_non_sender_skips_client(monkeypatch):
|
||||
class _RaisingClient(RecordingClient):
|
||||
def start_weight_update(self):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
def update_weights(self, update_info):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
client = _RaisingClient()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=False,
|
||||
client=client,
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights() # gathers only; must not raise
|
||||
assert client.order == []
|
||||
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
|
||||
class _FailingUpdateClient(RecordingClient):
|
||||
def update_weights(self, update_info):
|
||||
self.order.append("update")
|
||||
raise RuntimeError("inference side rejected update")
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=True,
|
||||
client=_FailingUpdateClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="inference side rejected update"):
|
||||
engine.send_weights()
|
||||
|
||||
|
||||
class _OwnedSource(_ListSource):
|
||||
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
||||
Takes group indices for convenience and declares the names inside them."""
|
||||
|
||||
def __init__(self, pairs, owned_group_idx):
|
||||
super().__init__(pairs)
|
||||
self._owned = list(owned_group_idx)
|
||||
groups = layerwise_groups([n for n, _ in pairs])
|
||||
self._owned_names = [n for gi in self._owned for n in groups[gi]]
|
||||
|
||||
def held_names(self):
|
||||
return list(self._owned_names)
|
||||
|
||||
def __iter__(self):
|
||||
by_name = dict(self._pairs)
|
||||
return iter([(n, by_name[n]) for n in self._owned_names])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch):
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest
|
||||
)
|
||||
engine.send_weights()
|
||||
|
||||
assert server.published == [1, 2]
|
||||
assert server.order == ["begin", "publish", "publish", "end"]
|
||||
assert engine._inflight == {}
|
||||
# embed + norm on rank 1, the two layers here: two distinct owner sets,
|
||||
# numbered by first appearance in metadata order.
|
||||
assert engine._owner_sets == [[1], [0]]
|
||||
assert engine._name_owner_class == [0, 1, 1, 0]
|
||||
|
||||
|
||||
def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
|
||||
class _MisorderedOwned(_OwnedSource):
|
||||
def __iter__(self):
|
||||
return iter(list(super().__iter__())[::-1])
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]],
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
engine.send_weights()
|
||||
assert "error" in server.order
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
"""The free barrier's target is one integer per sync: the live consumer
|
||||
count, defaulting to the whole provisioned fleet."""
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights()
|
||||
assert server.live_count == 1 # num_consumers=1 in the harness
|
||||
engine.send_weights(live_consumer_ids=[0])
|
||||
assert server.live_count == 1
|
||||
|
||||
|
||||
class TestLiveCountPlumbing:
|
||||
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
||||
provisioned geometry is frozen; a degraded sync only lowers the target and
|
||||
narrows each slot-sharing group's rendezvous."""
|
||||
|
||||
@staticmethod
|
||||
def _engine(num_consumers, world=2, rank=0):
|
||||
engine = ShardedRDTTrainerWeightTransferEngine.__new__(
|
||||
ShardedRDTTrainerWeightTransferEngine
|
||||
)
|
||||
engine._init_info = ShardedRDTTrainerInitInfo(
|
||||
rank=rank, num_consumers=num_consumers
|
||||
)
|
||||
engine.source = object() # send_weights asserts a source is present
|
||||
received: list = []
|
||||
engine._send_weights_inner = lambda count, ids: received.append((count, ids))
|
||||
return engine, received
|
||||
|
||||
def test_none_counts_the_whole_provisioned_fleet(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(None)
|
||||
assert got == [(8, list(range(8)))]
|
||||
|
||||
def test_a_live_set_counts_its_distinct_members(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([0, 1, 4, 5, 5])
|
||||
assert got == [(4, [0, 1, 4, 5])]
|
||||
|
||||
def test_a_full_live_set_matches_the_provisioned_count(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(list(range(8)))
|
||||
engine.send_weights(None)
|
||||
assert got == [(8, list(range(8)))] * 2
|
||||
|
||||
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
||||
"""The whole point of the barrier: no routed per-producer targets, so
|
||||
the identity of the dead consumer is irrelevant to the producers."""
|
||||
counts = []
|
||||
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(live)
|
||||
counts += [c for c, _ids in got]
|
||||
assert counts == [4, 4, 4]
|
||||
|
||||
def test_the_live_ids_travel_with_the_count(self):
|
||||
"""The count sizes the free barrier, the ids size the slot-sharing
|
||||
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
||||
is still waiting for. They must describe the same set."""
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([6, 0, 2, 2])
|
||||
((count, ids),) = got
|
||||
assert ids == [0, 2, 6] and count == len(ids)
|
||||
|
||||
|
||||
def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch):
|
||||
import json
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[0, 1], [2, 3]],
|
||||
)
|
||||
worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"])
|
||||
assert worker_init.owner_sets == [[0], [1]]
|
||||
assert worker_init.name_owner_class == [0, 0, 1, 1]
|
||||
assert len(worker_init.name_owner_class) == len(worker_init.names)
|
||||
# The payload crosses the control plane as JSON: nested lists must survive.
|
||||
round_tripped = json.loads(json.dumps(asdict(worker_init)))
|
||||
assert round_tripped["owner_sets"] == [[0], [1]]
|
||||
assert round_tripped["name_owner_class"] == [0, 0, 1, 1]
|
||||
|
||||
|
||||
def test_weight_source_holds_everything_by_default():
|
||||
"""The contract's default: a source that says nothing holds the whole model."""
|
||||
src = _rdt_source_two_layers()
|
||||
assert src.held_names() is None
|
||||
assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch):
|
||||
class _BadHeld(_OwnedSource):
|
||||
def held_names(self):
|
||||
return ["embed.weight", "not.a.real.name"]
|
||||
|
||||
with pytest.raises(ValueError, match="not"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_BadHeld(_rdt_source_two_layers()._pairs, [0]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
|
||||
class _HoldsNothing(_ListSource):
|
||||
def held_names(self):
|
||||
return []
|
||||
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_HoldsNothing(_rdt_source_two_layers()._pairs),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
"""Every name must be held somewhere or it can never be served — caught
|
||||
when the holdings are transposed, naming the orphan."""
|
||||
with pytest.raises(ValueError, match="no trainer rank holds"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[0, 1], [2]], # nobody holds group 3
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
||||
"""Only the sender's metadata reaches the consumers, so a rank describing
|
||||
just its own share must fail loudly rather than silently drop the rest."""
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(auto_free=True),
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds
|
||||
)
|
||||
# Now rank 1 reports a DIFFERENT metadata digest for the same model.
|
||||
monkeypatch.setattr(
|
||||
engine,
|
||||
"_all_gather_owned",
|
||||
lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])],
|
||||
)
|
||||
with pytest.raises(ValueError, match="disagrees across trainer ranks"):
|
||||
engine._resolve_ownership(2, 0)
|
||||
|
||||
|
||||
def _serve_ring_server(src_name, src):
|
||||
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
||||
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
||||
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
||||
what puts the destination-view cache, and only it, under test."""
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
_RDTProducerServer,
|
||||
)
|
||||
|
||||
srv = _RDTProducerServer(
|
||||
num_rdt_buffers=2,
|
||||
buffer_presize_gb=0.0,
|
||||
gather_lookahead=2,
|
||||
)
|
||||
srv._cache[src_name] = src
|
||||
srv._serve_rings[0] = [
|
||||
torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2)
|
||||
]
|
||||
|
||||
# The slot is ``seq % nring``, so stepping seq by nring keeps every pull in
|
||||
# slot 0 while still making each one its own generation -- reusing a seq
|
||||
# would rendezvous with the finished pack and return it unchanged.
|
||||
seqs = itertools.count(0, srv._nring)
|
||||
|
||||
def serve(chain):
|
||||
return srv.rdt_produce_weights_batched(
|
||||
[(src_name, chain)], consumer_id=0, seq=next(seqs)
|
||||
)[0]
|
||||
|
||||
return srv, serve
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
|
||||
)
|
||||
def test_serve_does_not_reuse_packed_views_of_another_shape():
|
||||
"""Two requests can share a name yet pack different slices of it.
|
||||
|
||||
The producer caches the destination views it carves into a serve ring slot.
|
||||
Keyed by name alone, the second request is packed through the first's views.
|
||||
Reachable when one name's copies split across owner-class chunks.
|
||||
"""
|
||||
name = "model.layers.0.w"
|
||||
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
|
||||
_srv, serve = _serve_ring_server(name, src)
|
||||
|
||||
serve((("narrow", (0, 0, 2), ()),)) # 2 rows
|
||||
wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot
|
||||
blob = serve((("narrow", (0, 0, 6), ()),))
|
||||
|
||||
got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8)
|
||||
assert torch.equal(got, wide)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
|
||||
)
|
||||
def test_serve_does_not_reuse_packed_views_of_another_dtype():
|
||||
"""The SILENT case of the same cache: two requests whose slices have the same
|
||||
name and the same shape but different dtypes pack at identical offsets, so
|
||||
reusing the stale views raises nothing — ``copy_`` just casts, and the blob
|
||||
carries the wrong bytes with no check downstream.
|
||||
|
||||
``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under
|
||||
``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte
|
||||
length identical, which is exactly what makes the stale view fit.
|
||||
"""
|
||||
name = "model.layers.0.w"
|
||||
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
|
||||
_srv, serve = _serve_ring_server(name, src)
|
||||
|
||||
serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16
|
||||
blob = serve(()) # same name and shape, bf16
|
||||
|
||||
got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8)
|
||||
assert torch.equal(got, src), "packed through a cached view of the wrong dtype"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,465 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous.
|
||||
|
||||
The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX
|
||||
trainer) initialize NCCL weight transfer from a unique id minted out of band.
|
||||
|
||||
* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL
|
||||
init infos.
|
||||
* Two-GPU integration tests driving the real worker engine against a peer: one
|
||||
via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer
|
||||
(cupy buffer + raw ``ncclBroadcast``).
|
||||
|
||||
The shared Ray helpers come from the sibling module (unchanged); the worker task
|
||||
is local so the sibling's TCP test and its worker stay untouched.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from tests.distributed.test_weight_transfer import (
|
||||
_init_ray_for_weight_transfer,
|
||||
_set_ray_assigned_device,
|
||||
create_mock_vllm_config,
|
||||
)
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerInitInfo,
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
)
|
||||
|
||||
# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId.
|
||||
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
|
||||
|
||||
|
||||
class TestNCCLRendezvousValidation:
|
||||
"""Test the xor-of-rendezvous-modes validation on the NCCL init infos."""
|
||||
|
||||
def test_worker_init_info_is_keyword_only(self):
|
||||
# kw_only so a stale positional call fails loudly rather than silently
|
||||
# swapping fields once the rendezvous fields became optional.
|
||||
with pytest.raises(TypeError):
|
||||
NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg]
|
||||
|
||||
def test_trainer_init_info_accepts_positional_fields(self):
|
||||
# NCCLTrainerInitInfo keeps positional subclass fields; only `rank`
|
||||
# (from TrainerInitInfo) is keyword-only.
|
||||
trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0)
|
||||
trainer_keyword = NCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
world_size=3,
|
||||
rank=0,
|
||||
)
|
||||
assert trainer_positional == trainer_keyword
|
||||
|
||||
def test_tcp_mode_valid(self):
|
||||
info = NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
)
|
||||
assert info.master_address == "127.0.0.1"
|
||||
assert info.nccl_unique_id_b64 is None
|
||||
|
||||
def test_uid_mode_valid(self):
|
||||
info = NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64=VALID_UID_B64,
|
||||
)
|
||||
assert info.nccl_unique_id_b64 == VALID_UID_B64
|
||||
assert info.master_address is None
|
||||
assert info.master_port is None
|
||||
assert VALID_UID_B64 not in repr(info)
|
||||
|
||||
def test_both_modes_raises(self):
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
nccl_unique_id_b64=VALID_UID_B64,
|
||||
)
|
||||
|
||||
def test_neither_mode_raises(self):
|
||||
with pytest.raises(ValueError, match="need nccl_unique_id_b64"):
|
||||
NCCLWeightTransferInitInfo(rank_offset=1, world_size=3)
|
||||
|
||||
def test_half_specified_tcp_raises(self):
|
||||
with pytest.raises(ValueError, match="master_port"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
)
|
||||
|
||||
def test_bad_base64_uid_raises(self):
|
||||
with pytest.raises(ValueError, match="not valid standard base64"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64="!!!!",
|
||||
)
|
||||
|
||||
def test_wrong_length_uid_raises(self):
|
||||
short_uid = base64.b64encode(b"\x00" * 64).decode()
|
||||
with pytest.raises(ValueError, match="128"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64=short_uid,
|
||||
)
|
||||
|
||||
def test_parse_init_info_uid_valid(self):
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
init_info = engine.parse_init_info(
|
||||
{
|
||||
"nccl_unique_id_b64": VALID_UID_B64,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
assert isinstance(init_info, NCCLWeightTransferInitInfo)
|
||||
assert init_info.nccl_unique_id_b64 == VALID_UID_B64
|
||||
assert init_info.master_address is None
|
||||
|
||||
def test_parse_init_info_both_modes_raises(self):
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
"master_port": 12345,
|
||||
"nccl_unique_id_b64": VALID_UID_B64,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("unique_id_bytes", "rank", "world_size", "match"),
|
||||
[
|
||||
(b"\x00" * 64, 0, 2, "128-byte"),
|
||||
(b"\x00" * 128, -1, 2, "out of range"),
|
||||
(b"\x00" * 128, 2, 2, "out of range"),
|
||||
],
|
||||
)
|
||||
def test_from_unique_id_bytes_validation(
|
||||
self, unique_id_bytes, rank, world_size, match
|
||||
):
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
PyNcclCommunicator.from_unique_id_bytes(
|
||||
unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=0,
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def inference_receive_tensor(
|
||||
init_info_dict: dict,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Worker that joins via the given init dict and records the tensor it gets.
|
||||
|
||||
Local to this module (rather than the sibling's TCP-only worker) so it can
|
||||
take a UID init payload -- the same dict boundary an external trainer POSTs.
|
||||
"""
|
||||
import contextlib
|
||||
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
class Recorder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
for name, tensor in weights:
|
||||
self.received.append((name, tensor.clone()))
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
recorder = Recorder()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, vllm_config, torch.device("cuda"), recorder
|
||||
)
|
||||
# Transport-only test: bypass the set_current_vllm_config context that
|
||||
# receive_weights enters, since vllm_config here is a mock.
|
||||
import vllm.config as _vllm_config_mod
|
||||
|
||||
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
|
||||
|
||||
try:
|
||||
engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict))
|
||||
engine.receive_weights(
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=[tensor_dtype],
|
||||
shapes=[tensor_shape],
|
||||
)
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
received_shape = None
|
||||
received_sum = None
|
||||
if len(recorder.received) == 1:
|
||||
_name, tensor = recorder.received[0]
|
||||
received_shape = list(tensor.shape)
|
||||
received_sum = tensor.sum().item()
|
||||
|
||||
return {
|
||||
"success": received_shape == tensor_shape
|
||||
and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01,
|
||||
"received_shape": received_shape,
|
||||
"received_sum": received_sum,
|
||||
}
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor_uid(
|
||||
uid_b64: str,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Drive rank 0 via vLLM's own communicator from a pre-shared unique id.
|
||||
|
||||
A real torch-free trainer would mint the id and drive rank 0 itself; here we
|
||||
reuse the real send protocol by injecting the communicator into a sender
|
||||
engine, exercising the worker-side UID path end to end.
|
||||
"""
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
|
||||
from vllm.distributed.weight_transfer import ModuleSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerWeightTransferEngine,
|
||||
)
|
||||
|
||||
class NoopClient:
|
||||
def init_weight_transfer_engine(self, init_info):
|
||||
pass
|
||||
|
||||
def start_weight_update(self):
|
||||
pass
|
||||
|
||||
def update_weights(self, update_info):
|
||||
pass
|
||||
|
||||
def finish_weight_update(self, weight_version=None):
|
||||
pass
|
||||
|
||||
dtype = getattr(torch, tensor_dtype)
|
||||
model = torch.nn.Module()
|
||||
test_module = torch.nn.Module()
|
||||
test_module.register_parameter(
|
||||
"weight",
|
||||
torch.nn.Parameter(
|
||||
torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False
|
||||
),
|
||||
)
|
||||
model.add_module("test", test_module)
|
||||
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=NoopClient(),
|
||||
source=ModuleSource(model),
|
||||
is_sender=True,
|
||||
packed=False,
|
||||
)
|
||||
engine.model_update_group = uid_init_process_group(
|
||||
base64.b64decode(uid_b64),
|
||||
rank=0,
|
||||
world_size=world_size,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
try:
|
||||
engine.send_weights()
|
||||
torch.accelerator.synchronize()
|
||||
return {"success": True}
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor_torch_free(
|
||||
uid_b64: str,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in).
|
||||
|
||||
Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it
|
||||
exercises interop with a foreign NCCL peer rather than vLLM against itself.
|
||||
``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right
|
||||
after ``ncclCommInitRank``; that is a collective, so this peer must issue the
|
||||
matching all_reduce before the broadcast or both ranks deadlock.
|
||||
"""
|
||||
import cupy as cp
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
|
||||
device = _set_ray_assigned_device()
|
||||
cp.cuda.Device(device.index).use()
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCLLibrary,
|
||||
buffer_type,
|
||||
cudaStream_t,
|
||||
ncclDataTypeEnum,
|
||||
ncclRedOpTypeEnum,
|
||||
)
|
||||
|
||||
assert tensor_dtype == "float32", "harness only wires float32"
|
||||
# Fill on the host and copy up (a plain H2D memcpy) so we neither use torch
|
||||
# nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets
|
||||
# the worker verify by sum == numel.
|
||||
sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32))
|
||||
warmup = cp.asarray(np.zeros(1, dtype=np.float32))
|
||||
numel = int(sendbuff.size)
|
||||
# The copies above run on the default stream; drain them before the NCCL
|
||||
# collectives read the buffers on our own stream.
|
||||
cp.cuda.Device(device.index).synchronize()
|
||||
stream = cp.cuda.Stream()
|
||||
|
||||
nccl = NCCLLibrary()
|
||||
comm = nccl.ncclCommInitRank(
|
||||
world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0
|
||||
)
|
||||
try:
|
||||
# Mirror the worker's init-time warm-up all_reduce (rank 0 must join it).
|
||||
nccl.ncclAllReduce(
|
||||
buffer_type(warmup.data.ptr),
|
||||
buffer_type(warmup.data.ptr),
|
||||
1,
|
||||
ncclDataTypeEnum.ncclFloat32,
|
||||
ncclRedOpTypeEnum.ncclSum,
|
||||
comm,
|
||||
cudaStream_t(stream.ptr),
|
||||
)
|
||||
nccl.ncclBroadcast(
|
||||
buffer_type(sendbuff.data.ptr),
|
||||
buffer_type(sendbuff.data.ptr),
|
||||
numel,
|
||||
ncclDataTypeEnum.ncclFloat32,
|
||||
0, # root
|
||||
comm,
|
||||
cudaStream_t(stream.ptr),
|
||||
)
|
||||
stream.synchronize()
|
||||
return {"success": True}
|
||||
finally:
|
||||
# Abort (not the collective destroy) for an uncoordinated teardown.
|
||||
nccl.ncclCommAbort(comm)
|
||||
|
||||
|
||||
def _run_uid_transfer(trainer_task) -> None:
|
||||
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
||||
and assert the tensor round-trips. Both ranks must enter init together --
|
||||
there is no store barrier on the UID path."""
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
|
||||
_init_ray_for_weight_transfer()
|
||||
|
||||
nccl = NCCLLibrary()
|
||||
uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode()
|
||||
|
||||
world_size = 2 # 1 trainer + 1 inference worker
|
||||
tensor_shape = [100, 100]
|
||||
tensor_dtype = "float32"
|
||||
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
{
|
||||
"rank_offset": 1,
|
||||
"world_size": world_size,
|
||||
"nccl_unique_id_b64": uid_b64,
|
||||
"packed": False,
|
||||
},
|
||||
tensor_shape,
|
||||
tensor_dtype,
|
||||
)
|
||||
trainer_future = trainer_task.remote(
|
||||
uid_b64, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
try:
|
||||
trainer_result, result = ray.get(
|
||||
[trainer_future, inference_future], timeout=120
|
||||
)
|
||||
finally:
|
||||
ray.shutdown()
|
||||
|
||||
assert trainer_result["success"], "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
f"Weight transfer failed. "
|
||||
f"Received shape: {result['received_shape']}, "
|
||||
f"Received sum: {result['received_sum']}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes_uid():
|
||||
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
||||
ranks on vLLM's ``PyNcclCommunicator``."""
|
||||
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_torch_free_trainer():
|
||||
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
|
||||
pytest.importorskip("cupy")
|
||||
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight-transfer metrics scraped from a real two-API-server deployment."""
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.rl.conftest import MODEL_NAME
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||
SYNCS = 8
|
||||
|
||||
|
||||
def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||
text = requests.get(f"{url}/metrics", timeout=30).text
|
||||
return {
|
||||
(sample.name, sample.labels["operation"]): sample.value
|
||||
for family in text_string_to_metric_families(text)
|
||||
for sample in family.samples
|
||||
if sample.name in (f"{DURATION}_count", IN_FLIGHT)
|
||||
}
|
||||
|
||||
|
||||
def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
||||
args = [
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--gpu-memory-utilization",
|
||||
"0.5",
|
||||
"--api-server-count",
|
||||
"2",
|
||||
"--weight-transfer-config",
|
||||
'{"backend": "ipc"}',
|
||||
]
|
||||
env = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION": "1",
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(tmp_path),
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server:
|
||||
url = server.url_root
|
||||
trainer = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer),
|
||||
)
|
||||
for _ in range(SYNCS):
|
||||
engine.send_weights()
|
||||
metrics = scrape(url)
|
||||
|
||||
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
|
||||
for operation, count in calls.items():
|
||||
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
||||
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
||||
@@ -1,234 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch.testing import assert_close
|
||||
|
||||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||||
|
||||
|
||||
def test_pack_seq_basic_fp8():
|
||||
"""Test basic functionality of pack_seq_triton with fp8 and 3D tensors."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
|
||||
# Test cases with 3D tensors (N, H, D)
|
||||
test_cases = [
|
||||
(6, 8, 4, 2, [3, 3]), # (6, 8, 4) -> (2, 3, 8, 4)
|
||||
(10, 4, 8, 3, [2, 4, 4]), # (10, 4, 8) -> (3, 4, 4, 8)
|
||||
(20, 16, 32, 4, [5, 5, 5, 5]), # (20, 16, 32) -> (4, 5, 16, 32)
|
||||
]
|
||||
|
||||
for N, H, D, B, lengths_list in test_cases:
|
||||
# Create input tensor with small values for fp8
|
||||
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor(lengths_list, device=device)
|
||||
|
||||
# Pack the data
|
||||
packed = pack_seq_triton(x, lengths)
|
||||
|
||||
# Check output shape and properties
|
||||
expected_shape = (B, max(lengths_list), H, D)
|
||||
assert packed.shape == expected_shape
|
||||
assert packed.dtype == dtype
|
||||
assert packed.device == x.device
|
||||
|
||||
# Check that valid data is preserved (within fp8 precision)
|
||||
for b in range(B):
|
||||
start_idx = sum(lengths_list[:b])
|
||||
seq_len = lengths_list[b]
|
||||
|
||||
expected_data = x[start_idx : start_idx + seq_len].to(torch.float32)
|
||||
actual_data = packed[b, :seq_len].to(torch.float32)
|
||||
|
||||
assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2)
|
||||
|
||||
|
||||
def test_pack_seq_custom_padding_fp8():
|
||||
"""Test pack_seq_triton with custom padding values for fp8."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
N, H, D, B = 20, 8, 16, 2
|
||||
lengths = torch.tensor([10, 10], device=device)
|
||||
|
||||
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
|
||||
# Test with different padding values
|
||||
for pad_value in [-100.0, -10.0, 0.0, 10.0, 100.0]:
|
||||
result = pack_seq_triton(x, lengths, pad_value=pad_value)
|
||||
|
||||
# Check valid data
|
||||
for b in range(B):
|
||||
start_idx = b * 10
|
||||
expected_data = x[start_idx : start_idx + 10].to(torch.float32)
|
||||
actual_data = result[b, :10].to(torch.float32)
|
||||
assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2)
|
||||
|
||||
# Check padding (fp8 has limited range, so check for large values)
|
||||
padded_data = result[:, 10:].to(torch.float32)
|
||||
if pad_value < 0:
|
||||
assert torch.all(padded_data < -50) # Large negative values
|
||||
elif pad_value > 0:
|
||||
assert torch.all(padded_data > 50) # Large positive values
|
||||
else:
|
||||
assert torch.allclose(padded_data, torch.zeros_like(padded_data), atol=1e-2)
|
||||
|
||||
|
||||
def test_pack_seq_default_negative_inf_padding_fp8():
|
||||
"""Test that pack_seq_triton uses -inf padding by default for fp8."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
# B = 2
|
||||
N, H, D = 20, 8, 16
|
||||
lengths = torch.tensor([10, 10], device=device)
|
||||
|
||||
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
result = pack_seq_triton(x, lengths)
|
||||
|
||||
# Check that padding is large negative values (fp8 representation of -inf)
|
||||
padded_data = result[:, 10:].to(torch.float32)
|
||||
assert torch.all(
|
||||
padded_data < -100
|
||||
) # fp8 -inf is represented as large negative number
|
||||
|
||||
|
||||
def test_pack_seq_edge_cases_fp8():
|
||||
"""Test pack_seq_triton with edge cases for fp8."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
|
||||
# Test with single batch element
|
||||
x = torch.randn(10, 8, 16, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor([10], device=device)
|
||||
result = pack_seq_triton(x, lengths)
|
||||
assert result.shape == (1, 10, 8, 16)
|
||||
|
||||
# Test with very short sequences
|
||||
x = torch.randn(20, 4, 8, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor([1, 1, 1], device=device)
|
||||
result = pack_seq_triton(x, lengths)
|
||||
assert result.shape == (3, 1, 4, 8)
|
||||
|
||||
# Test with different sequence lengths
|
||||
x = torch.randn(15, 8, 16, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor([5, 7, 3], device=device)
|
||||
result = pack_seq_triton(x, lengths)
|
||||
assert result.shape == (3, 7, 8, 16)
|
||||
|
||||
|
||||
def test_pack_seq_different_block_sizes_fp8():
|
||||
"""Test pack_seq_triton with different block sizes for fp8."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
N, H, D, B = 100, 16, 32, 4
|
||||
lengths = torch.tensor([25, 25, 25, 25], device=device)
|
||||
|
||||
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
|
||||
# Test different block sizes
|
||||
for block_t, block_d in [(32, 32), (64, 64), (128, 128)]:
|
||||
result = pack_seq_triton(x, lengths, block_t=block_t, block_d=block_d)
|
||||
|
||||
assert result.shape == (B, 25, H, D)
|
||||
|
||||
# Check that valid data is preserved (within fp8 precision)
|
||||
for b in range(B):
|
||||
start_idx = b * 25
|
||||
expected_data = x[start_idx : start_idx + 25].to(torch.float32)
|
||||
actual_data = result[b, :25].to(torch.float32)
|
||||
assert_close(actual_data, expected_data, rtol=1e-1, atol=1e-2)
|
||||
|
||||
|
||||
def test_pack_seq_shape_consistency():
|
||||
"""Test that pack_seq_triton maintains shape consistency."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
N, H, D, B = 20, 8, 16, 2
|
||||
lengths = torch.tensor([10, 10], device=device)
|
||||
|
||||
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
|
||||
result = pack_seq_triton(x, lengths)
|
||||
|
||||
# Check shape consistency
|
||||
assert result.shape[0] == B # Batch dimension
|
||||
assert result.shape[1] == lengths.max().item() # Max sequence length
|
||||
assert result.shape[2:] == x.shape[1:] # Feature dimensions preserved
|
||||
|
||||
|
||||
def test_pack_unpack_roundtrip_fp8():
|
||||
"""Test that pack -> unpack gives us back the original data for fp8."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
|
||||
# Test cases with 3D tensors
|
||||
test_cases = [
|
||||
(6, 8, 4, 2, [3, 3]),
|
||||
(10, 4, 8, 3, [2, 4, 4]),
|
||||
(20, 16, 32, 4, [5, 5, 5, 5]),
|
||||
(15, 8, 16, 3, [7, 5, 3]),
|
||||
]
|
||||
|
||||
for N, H, D, B, lengths_list in test_cases:
|
||||
# Create input tensor with small values for fp8
|
||||
x = torch.randn(N, H, D, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor(lengths_list, device=device)
|
||||
|
||||
# Pack the data
|
||||
packed = pack_seq_triton(x, lengths)
|
||||
|
||||
# Unpack the data
|
||||
unpacked = unpack_seq_triton(packed, lengths)
|
||||
|
||||
# Check that we get back the original data (within fp8 precision)
|
||||
assert unpacked.shape == x.shape
|
||||
x_f32 = x.to(torch.float32)
|
||||
unpacked_f32 = unpacked.to(torch.float32)
|
||||
assert_close(x_f32, unpacked_f32, rtol=1e-3, atol=1e-3)
|
||||
|
||||
# Unpack without explicit start locations (computed in kernel)
|
||||
unpacked_with_loc = unpack_seq_triton(packed, lengths)
|
||||
assert_close(x_f32, unpacked_with_loc.to(torch.float32), rtol=1e-3, atol=1e-2)
|
||||
|
||||
|
||||
def test_unpack_seq_triton_edge_cases_fp8():
|
||||
"""Test unpack function with edge cases for fp8."""
|
||||
device = "cuda"
|
||||
dtype = torch.float8_e4m3fn
|
||||
|
||||
# Test with single batch element
|
||||
x = torch.randn(10, 8, 16, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor([10], device=device)
|
||||
packed = pack_seq_triton(x, lengths)
|
||||
unpacked = unpack_seq_triton(packed, lengths)
|
||||
assert unpacked.shape == x.shape
|
||||
assert_close(x.to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2)
|
||||
|
||||
# Test with very short sequences
|
||||
x = torch.randn(20, 4, 8, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor([1, 1, 1], device=device)
|
||||
packed = pack_seq_triton(x, lengths)
|
||||
unpacked = unpack_seq_triton(packed, lengths)
|
||||
# Only compare the first 3 elements that were actually packed
|
||||
assert_close(
|
||||
x[:3].to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2
|
||||
)
|
||||
|
||||
x = torch.randn(15, 8, 16, dtype=torch.float32, device=device) * 0.1
|
||||
x = x.to(dtype=dtype)
|
||||
lengths = torch.tensor([5, 7, 3], device=device)
|
||||
packed = pack_seq_triton(x, lengths)
|
||||
unpacked = unpack_seq_triton(packed, lengths)
|
||||
assert unpacked.shape == x.shape
|
||||
assert_close(x.to(torch.float32), unpacked.to(torch.float32), rtol=1e-1, atol=1e-2)
|
||||
@@ -250,8 +250,8 @@ def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
return BatchFeature(data=inputs, tensor_type="pt")
|
||||
|
||||
hf_model.processor = processor
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language.model.embed_tokens
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language.model.embed_tokens
|
||||
)
|
||||
return hf_model
|
||||
|
||||
@@ -353,8 +353,8 @@ def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
)
|
||||
|
||||
hf_model.processor = processor
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.transformer.output_layer
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.transformer.output_layer
|
||||
)
|
||||
return hf_model
|
||||
|
||||
@@ -442,8 +442,8 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = H2OVLProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
@@ -678,8 +678,8 @@ def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = SkyworkR1VProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
@@ -787,8 +787,8 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = InternVLProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
@@ -941,8 +941,8 @@ def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.llm.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.llm.get_output_embeddings()
|
||||
)
|
||||
|
||||
def processor(*args, text="", images=None, **kwargs):
|
||||
@@ -977,8 +977,8 @@ def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patches and returns an instance of the HfRunner to use for Ovis2."""
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.llm.get_output_embeddings()
|
||||
hf_model.model.get_output_embeddings = lambda: (
|
||||
hf_model.model.llm.get_output_embeddings()
|
||||
)
|
||||
|
||||
def processor(*args, text="", images=None, videos=None, **kwargs):
|
||||
@@ -1209,221 +1209,3 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
||||
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
|
||||
hf_model.model.generate = patched_generate # type: ignore[method-assign]
|
||||
return hf_model
|
||||
|
||||
|
||||
def moondream3_processor(model: str):
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
return Moondream3Processor.from_pretrained(model, trust_remote_code=True)
|
||||
|
||||
|
||||
def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patch HfRunner for Moondream3."""
|
||||
moondream_processor = hf_model.processor
|
||||
|
||||
def processor(*args, text="", images=None, **kwargs):
|
||||
if images is None:
|
||||
return moondream_processor(text=text, **kwargs)
|
||||
|
||||
images_list = [images] if isinstance(images, Image) else images
|
||||
return moondream_processor(images=images_list, text=text, **kwargs)
|
||||
|
||||
hf_model.processor = processor
|
||||
|
||||
# Expose the LM head for logprob extraction.
|
||||
hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head
|
||||
|
||||
native_model = hf_model.model.model # MoondreamModel instance
|
||||
|
||||
from torch.nn import functional as F
|
||||
|
||||
from vllm.model_executor.models.moondream3 import reconstruct_from_crops
|
||||
|
||||
# Resolve the placeholder tokens from the tokenizer instead of hard-coding.
|
||||
image_placeholder_ids = moondream_processor.tokenizer.encode(
|
||||
"<image>", add_special_tokens=False
|
||||
)
|
||||
|
||||
def _normalize_tiling(tilings):
|
||||
"""Extract (h, w) tuple from various tiling container formats."""
|
||||
tiling = tilings
|
||||
if isinstance(tiling, torch.Tensor):
|
||||
tiling = tuple(tiling.squeeze().tolist())
|
||||
elif isinstance(tiling, (list, tuple)):
|
||||
t0 = tiling[0]
|
||||
if isinstance(t0, torch.Tensor):
|
||||
tiling = tuple(t0.tolist())
|
||||
elif isinstance(t0, (list, tuple)):
|
||||
tiling = tuple(t0)
|
||||
return tiling
|
||||
|
||||
def _encode_vision(pixel_values, tilings):
|
||||
"""Run preprocessed crops through vision encoder + projection."""
|
||||
device = native_model.device
|
||||
dtype = native_model.vision.pos_emb.dtype
|
||||
config = native_model.config
|
||||
|
||||
pv = pixel_values
|
||||
while pv.dim() > 4:
|
||||
pv = pv.squeeze(0)
|
||||
pv = pv.to(device=device, dtype=dtype)
|
||||
|
||||
features = native_model._vis_enc(pv)
|
||||
grid_size = config.vision.crop_size // config.vision.enc_patch_size
|
||||
global_feat = features[0]
|
||||
|
||||
if features.shape[0] > 1 and tilings is not None:
|
||||
tiling = _normalize_tiling(tilings)
|
||||
local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim)
|
||||
reconstructed = reconstruct_from_crops(
|
||||
local,
|
||||
tiling,
|
||||
config.vision.overlap_margin,
|
||||
patch_size=1,
|
||||
)
|
||||
else:
|
||||
reconstructed = global_feat.view(
|
||||
grid_size, grid_size, config.vision.enc_dim
|
||||
)
|
||||
|
||||
return native_model._vis_proj(global_feat, reconstructed)
|
||||
|
||||
def _find_subsequence(seq, subseq):
|
||||
"""Find start index of subseq in seq, or None."""
|
||||
n = len(subseq)
|
||||
for i in range(len(seq) - n + 1):
|
||||
if seq[i : i + n] == subseq:
|
||||
return i
|
||||
return None
|
||||
|
||||
def _generate(
|
||||
self,
|
||||
input_ids=None,
|
||||
pixel_values=None,
|
||||
tilings=None,
|
||||
attention_mask=None,
|
||||
**kwargs,
|
||||
):
|
||||
max_new_tokens = kwargs.get("max_new_tokens", 128)
|
||||
return_dict = kwargs.get("return_dict_in_generate", False)
|
||||
output_hs = kwargs.get("output_hidden_states", False)
|
||||
|
||||
if pixel_values is None:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
# Processor may return lists; extract the single element.
|
||||
if isinstance(pixel_values, (list, tuple)):
|
||||
pixel_values = pixel_values[0]
|
||||
if (
|
||||
isinstance(tilings, (list, tuple))
|
||||
and tilings
|
||||
and not isinstance(tilings[0], int)
|
||||
):
|
||||
tilings = tilings[0]
|
||||
|
||||
hf_model.model._setup_caches()
|
||||
native_model.use_flex_decoding = False
|
||||
|
||||
device = native_model.device
|
||||
config = native_model.config
|
||||
|
||||
with torch.inference_mode():
|
||||
for block in native_model.text.blocks:
|
||||
block.kv_cache.k_cache.zero_()
|
||||
block.kv_cache.v_cache.zero_()
|
||||
|
||||
img_emb = _encode_vision(pixel_values, tilings)
|
||||
|
||||
bos_emb = F.embedding(
|
||||
torch.tensor([[config.tokenizer.bos_id]], device=device),
|
||||
native_model.text.wte,
|
||||
)
|
||||
img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1)
|
||||
prefix_len = img_input.size(1)
|
||||
|
||||
mask = native_model.attn_mask[:, :, :prefix_len, :]
|
||||
pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device)
|
||||
native_model._prefill(img_input, mask, pos_ids, None)
|
||||
|
||||
ids = input_ids.squeeze(0).tolist()
|
||||
img_start = _find_subsequence(ids, image_placeholder_ids)
|
||||
|
||||
if img_start is None:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
prompt_tokens = ids[img_start + len(image_placeholder_ids) :]
|
||||
|
||||
if not prompt_tokens:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
prompt_tensor = torch.tensor([prompt_tokens], device=device)
|
||||
prompt_emb = F.embedding(prompt_tensor, native_model.text.wte)
|
||||
prompt_len = prompt_emb.size(1)
|
||||
|
||||
mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :]
|
||||
pos_ids = torch.arange(
|
||||
prefix_len,
|
||||
prefix_len + prompt_len,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
hidden = native_model._prefill(prompt_emb, mask, pos_ids, None)
|
||||
pos = prefix_len + prompt_len
|
||||
|
||||
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
|
||||
logits = native_model.text.lm_head(hidden_last.squeeze(1))
|
||||
|
||||
generated = []
|
||||
all_hidden_states = []
|
||||
# Record the hidden state that predicted each generated token.
|
||||
prev_hs = hidden_last
|
||||
for _ in range(max_new_tokens):
|
||||
next_token = logits.argmax(dim=-1).item()
|
||||
if next_token == 0:
|
||||
break
|
||||
generated.append(next_token)
|
||||
if output_hs:
|
||||
all_hidden_states.append((prev_hs,))
|
||||
|
||||
next_emb = F.embedding(
|
||||
torch.tensor([[next_token]], device=device),
|
||||
native_model.text.wte,
|
||||
)
|
||||
mask = native_model.attn_mask[:, :, pos : pos + 1, :]
|
||||
pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device)
|
||||
hidden = native_model._prefill(next_emb, mask, pos_ids_step, None)
|
||||
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
|
||||
prev_hs = hidden_last
|
||||
logits = native_model.text.lm_head(hidden_last.squeeze(1))
|
||||
pos += 1
|
||||
|
||||
result_ids = ids + generated
|
||||
sequences = torch.tensor([result_ids], device=device)
|
||||
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=tuple(all_hidden_states) if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
@@ -15,7 +15,6 @@ from safetensors import safe_open
|
||||
from safetensors.torch import load, load_file, save_file
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
@@ -31,10 +30,6 @@ from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer
|
||||
from vllm.models.qwen4_exp.amd.ple_layer import (
|
||||
Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.ple import (
|
||||
PLEShardOverlap,
|
||||
compute_ple_shard_overlap,
|
||||
@@ -51,7 +46,6 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ple_layer import Qwen4ExpPLELayer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import weak_ref_tensor
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionMetadata,
|
||||
@@ -732,10 +726,6 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
@@ -2289,162 +2279,6 @@ def _make_conv_metadata(
|
||||
return metadata, token_offset
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused conv needs CUDA")
|
||||
@pytest.mark.parametrize("state_layout", ["SD", "DS"])
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
pytest.param(
|
||||
_ConvBatchCase(num_decodes=1, channels=512),
|
||||
id="decode-single",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
num_decodes=33,
|
||||
channels=2048,
|
||||
kernel_size=5,
|
||||
dilation=1,
|
||||
),
|
||||
id="decode-padded",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(prefill_query_lens=(37, 0, 5, 128)),
|
||||
id="prefill",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
prefill_query_lens=(16, 676),
|
||||
channels=10240,
|
||||
state_index_stride=4,
|
||||
include_null_state=False,
|
||||
),
|
||||
id="prefill-strided-state-indices",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
spec_query_lens=(1, 4, 4, 0),
|
||||
num_accepted=(1, 2, 4, 1),
|
||||
spec_query_len=4,
|
||||
graph_padding=4,
|
||||
),
|
||||
id="spec-graph-padded",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
spec_query_lens=(3, 4),
|
||||
num_accepted=(2, 4),
|
||||
num_decodes=1,
|
||||
prefill_query_lens=(5,),
|
||||
spec_query_len=4,
|
||||
),
|
||||
id="mixed",
|
||||
),
|
||||
pytest.param(
|
||||
_ConvBatchCase(
|
||||
spec_query_lens=(1, 2),
|
||||
num_accepted=(0, 1),
|
||||
num_decodes=2,
|
||||
prefill_query_lens=(0, 8, 3),
|
||||
channels=640,
|
||||
kernel_size=3,
|
||||
dilation=2,
|
||||
spec_query_len=3,
|
||||
),
|
||||
id="mixed-varied",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("layer_cls", "fuse_residual"),
|
||||
[
|
||||
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
|
||||
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
|
||||
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
|
||||
],
|
||||
)
|
||||
def test_fused_conv_correctness(
|
||||
case: _ConvBatchCase,
|
||||
state_layout: str,
|
||||
layer_cls: type[nn.Module],
|
||||
fuse_residual: bool,
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
metadata, num_real_tokens = _make_conv_metadata(case, device)
|
||||
if case.state_index_stride > 1:
|
||||
assert metadata.state_indices_tensor.stride(0) == case.state_index_stride
|
||||
# layer_cls is type[nn.Module], so layer_cls.__new__ is type.__new__.
|
||||
module = object.__new__(layer_cls)
|
||||
nn.Module.__init__(module)
|
||||
module.conv_state_len = (case.kernel_size - 1) * case.dilation
|
||||
module.short_conv_dilation = case.dilation
|
||||
|
||||
rng, state_reference, conv_state, weights = _make_conv_case(
|
||||
device,
|
||||
seed=num_real_tokens + case.channels,
|
||||
channels=case.channels,
|
||||
kernel_size=case.kernel_size,
|
||||
dilation=case.dilation,
|
||||
state_layout=state_layout,
|
||||
spec_query_len=case.spec_query_len,
|
||||
)
|
||||
inputs = torch.randn(
|
||||
metadata.num_actual_tokens,
|
||||
case.channels,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
residual = torch.randn(
|
||||
inputs.shape,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
outer_residual = torch.randn(
|
||||
inputs.shape,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
null_state = conv_state[NULL_BLOCK_ID].clone()
|
||||
residual_kernel = residual.clone()
|
||||
# The unfused kernel must overwrite the buffer without reading its contents.
|
||||
residual_reference = (
|
||||
residual.clone() if fuse_residual else torch.zeros_like(residual)
|
||||
)
|
||||
|
||||
module._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual_kernel,
|
||||
outer_residual=outer_residual if fuse_residual else None,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=weights,
|
||||
)
|
||||
_short_conv_dilated_dispatch_pytorch(
|
||||
inputs=inputs,
|
||||
residual=residual_reference,
|
||||
metadata=metadata,
|
||||
conv_state=state_reference,
|
||||
conv_weights=weights,
|
||||
conv_state_len=module.conv_state_len,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
if fuse_residual:
|
||||
residual_reference = outer_residual + residual_reference
|
||||
|
||||
assert torch.equal(
|
||||
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
|
||||
)
|
||||
assert torch.equal(conv_state, state_reference)
|
||||
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
|
||||
if case.graph_padding:
|
||||
assert torch.equal(
|
||||
residual_kernel[num_real_tokens:],
|
||||
residual_reference[num_real_tokens:],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="fused gate needs CUDA")
|
||||
@pytest.mark.parametrize(("num_tokens", "strided_kv"), [(1, False), (64, True)])
|
||||
def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
|
||||
@@ -2509,273 +2343,3 @@ def test_fused_gate_correctness(num_tokens: int, strided_kv: bool) -> None:
|
||||
expected_normed = grouped_norm(expected_gated, norm_conv)
|
||||
assert torch.equal(gated, expected_gated)
|
||||
torch.testing.assert_close(normed, expected_normed, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _build_amd_ngram_embedding(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
device: str,
|
||||
cpu_offload: bool,
|
||||
fp8_checkpoint: bool,
|
||||
) -> tuple[amd_ple_layer.Qwen4ExpNGramEmbedding, torch.Tensor]:
|
||||
"""Construct and load a small AMD embedding through its checkpoint loader."""
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
amd_ple_layer,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(engram_config=SimpleNamespace(cpu_offload=cpu_offload)),
|
||||
)
|
||||
config = Qwen4ExpTextConfig(
|
||||
ngram_size=3,
|
||||
heads_per_ngram=1,
|
||||
ngram_vocab_size_base=5,
|
||||
make_ngram_vocab_size_divisible_by=8,
|
||||
split_ngram_parts=2,
|
||||
ple_embed_dim=6,
|
||||
eos_token_id=0,
|
||||
vocab_size=64,
|
||||
)
|
||||
with torch.device(device):
|
||||
module = amd_ple_layer.Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
embedding_dim=config.ple_embed_dim,
|
||||
ple_dense_layer_id=0,
|
||||
max_total_tokens=4,
|
||||
prefix="test.ple_embedding",
|
||||
layer_name="test.ple",
|
||||
quant_config=(
|
||||
Fp8Config(is_checkpoint_fp8_serialized=True) if fp8_checkpoint else None
|
||||
),
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
# Only the outer owner is a stub; the embedding constructor and loader run.
|
||||
layer = Qwen4ExpPLELayerAMD.__new__(Qwen4ExpPLELayerAMD)
|
||||
nn.Module.__init__(layer)
|
||||
layer.ple_embedding = module
|
||||
|
||||
monkeypatch.setattr(
|
||||
amd_ple_layer,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(no_compile_layers={module.layer_name: layer}),
|
||||
)
|
||||
|
||||
embedding = module.ngram_embedding
|
||||
loaded_weight = (
|
||||
torch.arange(embedding.org_vocab_size * embedding.embedding_dim)
|
||||
.reshape(embedding.org_vocab_size, embedding.embedding_dim)
|
||||
.to(embedding.weight.dtype)
|
||||
)
|
||||
shard_size = (embedding.org_vocab_size + 1) // 2
|
||||
weights = [
|
||||
(f"ngram_embedding.shard_{i}.weight", shard)
|
||||
for i, shard in enumerate(loaded_weight.split(shard_size))
|
||||
]
|
||||
expected_loaded = {"ngram_embedding.weight"}
|
||||
if fp8_checkpoint:
|
||||
weights.append(("ngram_embedding.weight_scale", torch.tensor([0.25])))
|
||||
expected_loaded.add("ngram_embedding.weight_scale")
|
||||
assert module.load_weights(weights) == expected_loaded
|
||||
embedding.quant_method.process_weights_after_loading(embedding)
|
||||
return module, loaded_weight
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device,cpu_offload",
|
||||
[
|
||||
pytest.param("cpu", False, id="cpu-resident"),
|
||||
pytest.param(
|
||||
"cuda:0",
|
||||
False,
|
||||
id="gpu-resident",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
"cuda:0",
|
||||
True,
|
||||
id="gpu-offloaded",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA or ROCm"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
device: str,
|
||||
cpu_offload: bool,
|
||||
) -> None:
|
||||
"""AMD checkpoint loading must accept and apply the global FP8 scale."""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device=device, cpu_offload=cpu_offload, fp8_checkpoint=True
|
||||
)
|
||||
embedding = module.ngram_embedding
|
||||
assert embedding.weight.dtype == torch.float8_e4m3fn
|
||||
assert embedding.weight.device == torch.device("cpu" if cpu_offload else device)
|
||||
if cpu_offload:
|
||||
assert embedding.weight.is_pinned()
|
||||
assert embedding.weight_scale.device == torch.device(device)
|
||||
torch.testing.assert_close(embedding.weight.float().cpu(), loaded_weight.float())
|
||||
dequantized = embedding.dequantize(embedding.weight.to(device), torch.bfloat16)
|
||||
torch.testing.assert_close(
|
||||
dequantized.cpu(), loaded_weight.to(torch.bfloat16) * 0.25, rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
|
||||
|
||||
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
|
||||
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
|
||||
first ``start_prefetch`` — the eager profile run, always before cudagraph
|
||||
capture — allocates them lazily.
|
||||
"""
|
||||
module, _ = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
|
||||
)
|
||||
embedding = module.ngram_embedding
|
||||
assert embedding._prefetch_buffer is None
|
||||
assert embedding._prefetch_stream is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
|
||||
def test_amd_pinned_embedding_output_written_under_compile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
"""The AMD pinned lookup op must survive aot_autograd/Inductor.
|
||||
|
||||
The op writes via the declared-mutated ``output``. Without
|
||||
``mutates_args=["output"]`` the call is dead-code-eliminated, the UVA lookup
|
||||
never runs, and the returned ``output`` stays untouched.
|
||||
"""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = torch.zeros(
|
||||
ngram_ids.shape[0],
|
||||
module.embedding_dim,
|
||||
dtype=module.ngram_embedding.weight.dtype,
|
||||
device="cuda:0",
|
||||
)
|
||||
|
||||
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
ids, out, module.layer_name
|
||||
)
|
||||
return out
|
||||
|
||||
torch.compile(lookup, fullgraph=True)(ngram_ids, output)
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
|
||||
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
|
||||
def test_amd_pinned_embedding_output_written_under_cudagraph_capture(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_checkpoint: bool,
|
||||
) -> None:
|
||||
"""The AMD pinned lookup op must be capture-safe.
|
||||
|
||||
It launches a single kernel on the current stream, so a cudagraph capture
|
||||
reproduces the lookup into ``output`` without cross-stream work.
|
||||
"""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=fp8_checkpoint
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
output = torch.zeros(
|
||||
ngram_ids.shape[0],
|
||||
module.embedding_dim,
|
||||
dtype=module.ngram_embedding.weight.dtype,
|
||||
device="cuda:0",
|
||||
)
|
||||
|
||||
def lookup(ids: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
ids, out, module.layer_name
|
||||
)
|
||||
return out
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
warmup_stream = torch.cuda.Stream(device="cuda:0")
|
||||
warmup_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(warmup_stream):
|
||||
for _ in range(3):
|
||||
lookup(ngram_ids, output)
|
||||
torch.cuda.current_stream().wait_stream(warmup_stream)
|
||||
|
||||
with torch.cuda.graph(graph):
|
||||
lookup(ngram_ids, output)
|
||||
for _ in range(2):
|
||||
ngram_ids.copy_(ngram_ids.flip(-1) + 3)
|
||||
output.fill_(float("nan"))
|
||||
graph.replay()
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="only the ROCm backend runs the n-gram embedding under torch.compile",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("query_lens", "contexts"),
|
||||
[([3, 1], [[5, 6], [0, 7]]), ([1, 0, 2], [[1, 2], [3, 4], [0, 0]])],
|
||||
ids=["two-requests", "empty-request"],
|
||||
)
|
||||
def test_amd_ngram_embedding_matches_reference_under_compile(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
query_lens: list[int],
|
||||
contexts: list[list[int]],
|
||||
) -> None:
|
||||
"""The AMD n-gram ID op must run under Dynamo with a symbolic request count."""
|
||||
module, loaded_weight = _build_amd_ngram_embedding(
|
||||
monkeypatch, device="cuda:0", cpu_offload=False, fp8_checkpoint=False
|
||||
)
|
||||
device = torch.device("cuda:0")
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *accumulate(query_lens)], dtype=torch.int32, device=device
|
||||
)
|
||||
input_ids = torch.tensor(
|
||||
[9, 0, 11, 12][: sum(query_lens)], dtype=torch.int32, device=device
|
||||
)
|
||||
ngram_context = torch.tensor(contexts, dtype=torch.int32, device=device)
|
||||
|
||||
compiled = torch.compile(module, fullgraph=True, dynamic=True)
|
||||
output = compiled(input_ids, query_start_loc, ngram_context)
|
||||
|
||||
ngram_ids = _reference_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
module.layer_multipliers,
|
||||
module.ngram_heads_vocab_sizes,
|
||||
module.ngram_heads_offsets,
|
||||
module.eos_token_id,
|
||||
module.heads_per_ngram,
|
||||
)
|
||||
expected = loaded_weight[ngram_ids.cpu()].flatten(-2).to(device)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers modeling backend's MLA fuser.
|
||||
|
||||
The fuser must discover every MLA submodule structurally (never by assuming the
|
||||
Transformers attribute names) and, when the query is low-rank, merge the checkpoint's
|
||||
separate `q_a_proj`/`kv_a_proj_with_mqa` weights into the single fused down-projection.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers import MLAFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
|
||||
|
||||
|
||||
def _attention(q_lora_rank: int | None):
|
||||
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
|
||||
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
|
||||
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
|
||||
DeepseekV2Config,
|
||||
)
|
||||
from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
|
||||
DeepseekV2Attention,
|
||||
)
|
||||
|
||||
cfg = DeepseekV2Config(
|
||||
hidden_size=256,
|
||||
num_attention_heads=16,
|
||||
kv_lora_rank=128,
|
||||
qk_rope_head_dim=32,
|
||||
qk_nope_head_dim=32,
|
||||
v_head_dim=32,
|
||||
q_lora_rank=q_lora_rank,
|
||||
num_hidden_layers=1,
|
||||
)
|
||||
with torch.device("meta"):
|
||||
return DeepseekV2Attention(cfg, layer_idx=0)
|
||||
|
||||
|
||||
def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
|
||||
attn = _attention(q_lora_rank)
|
||||
return MLAFuser.match(trace(attn), attn)
|
||||
|
||||
|
||||
def test_discovers_modules_without_q_lora():
|
||||
fuser = _match(q_lora_rank=None)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
assert fuser.q_proj_name == "q_proj"
|
||||
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
|
||||
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
|
||||
assert fuser.kv_b_proj_name == "kv_b_proj"
|
||||
assert fuser.o_proj_name == "o_proj"
|
||||
assert fuser.q_a_proj_name is None
|
||||
# Nothing is stacked without a query LoRA.
|
||||
assert fuser.packed_modules_mapping == {}
|
||||
assert fuser.orig_to_new_stacked("model.layers.0.self_attn") == {}
|
||||
|
||||
|
||||
def test_discovers_modules_with_q_lora():
|
||||
fuser = _match(q_lora_rank=64)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
assert fuser.q_a_proj_name == "q_a_proj"
|
||||
assert fuser.q_a_layernorm_name == "q_a_layernorm"
|
||||
assert fuser.q_b_proj_name == "q_b_proj"
|
||||
assert fuser.kv_a_proj_name == "kv_a_proj_with_mqa"
|
||||
assert fuser.kv_a_layernorm_name == "kv_a_layernorm"
|
||||
assert fuser.kv_b_proj_name == "kv_b_proj"
|
||||
assert fuser.o_proj_name == "o_proj"
|
||||
assert fuser.q_proj_name is None
|
||||
|
||||
|
||||
def test_q_lora_stacks_qkv_a_proj():
|
||||
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
|
||||
down-projection, so both checkpoint weights must remap into it."""
|
||||
fuser = _match(q_lora_rank=64)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
prefix = "model.layers.0.self_attn"
|
||||
merged = f"{prefix}.{_FUSED_QKV_A_PROJ}"
|
||||
assert fuser.packed_modules_mapping == {
|
||||
_FUSED_QKV_A_PROJ: ["q_a_proj", "kv_a_proj_with_mqa"]
|
||||
}
|
||||
assert fuser.orig_to_new_stacked(prefix) == {
|
||||
f"{prefix}.q_a_proj": (merged, 0),
|
||||
f"{prefix}.kv_a_proj_with_mqa": (merged, 1),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("q_lora_rank", [64, None])
|
||||
def test_update_forward_rewrites_real_attention(q_lora_rank):
|
||||
"""The source rewrite must survive the input-stability check.
|
||||
|
||||
The q-LoRA path hoists the merged down-projection to a top-level statement
|
||||
while its calls sit inside the `q_lora_rank` branches, so the region it
|
||||
spans is wide; a check that over-approximates aliasing would silently refuse
|
||||
here and cost the fusion rather than fail loudly."""
|
||||
fuser = _match(q_lora_rank)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
fuser.update_forward(_attention(q_lora_rank))
|
||||
names = set(fuser.fused_forward.__code__.co_names)
|
||||
if fuser.q_a_proj_name is not None:
|
||||
assert _FUSED_QKV_A_PROJ in names
|
||||
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
|
||||
|
||||
|
||||
class _Norm(nn.Module):
|
||||
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""
|
||||
|
||||
def __init__(self, n: int):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(n))
|
||||
|
||||
def forward(self, x):
|
||||
variance = x.pow(2).mean(-1, keepdim=True)
|
||||
return self.weight * (x * torch.rsqrt(variance + 1e-6))
|
||||
|
||||
|
||||
class RenamedMLA(nn.Module):
|
||||
"""An MLA-shaped attention whose children have non-standard names, proving
|
||||
discovery is by structure and not attribute name."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
heads, kv_lora, rope, nope, v, hidden = 4, 32, 8, 8, 16, 64
|
||||
self.alpha = nn.Linear(hidden, heads * (nope + rope)) # q_proj
|
||||
self.beta = nn.Linear(hidden, kv_lora + rope) # kv_a_proj_with_mqa
|
||||
self.gamma = _Norm(kv_lora) # kv_a_layernorm
|
||||
self.delta = nn.Linear(kv_lora, heads * (nope + v)) # kv_b_proj
|
||||
self.omega = nn.Linear(heads * v, hidden) # o_proj (uncalled)
|
||||
self.kv_lora, self.rope = kv_lora, rope
|
||||
|
||||
def forward(self, hidden_states):
|
||||
q = self.alpha(hidden_states)
|
||||
kv_lora, k_pe = torch.split(
|
||||
self.beta(hidden_states), [self.kv_lora, self.rope], dim=-1
|
||||
)
|
||||
expanded = self.delta(self.gamma(kv_lora))
|
||||
# Stand-in for the attention interface; `match` finds `o_proj` (omega) from
|
||||
# the source as the Linear producing the returned value.
|
||||
attn_output = expanded.sum() + q.sum() + k_pe.sum()
|
||||
attn_output = self.omega(attn_output)
|
||||
return attn_output
|
||||
|
||||
|
||||
def test_discovers_modules_under_arbitrary_names():
|
||||
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
|
||||
names, and `match` still locates each projection by dataflow."""
|
||||
with torch.device("meta"):
|
||||
module = RenamedMLA()
|
||||
fuser = MLAFuser.match(trace(module), module)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
assert fuser.q_a_proj_name is None
|
||||
assert fuser.q_proj_name == "alpha"
|
||||
assert fuser.kv_a_proj_name == "beta"
|
||||
assert fuser.kv_a_layernorm_name == "gamma"
|
||||
assert fuser.kv_b_proj_name == "delta"
|
||||
assert fuser.o_proj_name == "omega"
|
||||
|
||||
|
||||
class GLU(nn.Module):
|
||||
"""A gated MLP: matches the GLU fuser, never MLA."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = nn.Linear(16, 16)
|
||||
self.up = nn.Linear(16, 16)
|
||||
|
||||
def forward(self, x):
|
||||
return self.up(F.silu(self.gate(x)))
|
||||
|
||||
|
||||
def test_non_mla_is_not_matched():
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(GLU(), MLAFuser) is None
|
||||
@@ -1,611 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers modeling backend's MoE fuser."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fusers import MoEBlockFuser
|
||||
|
||||
from .test_linear import GLUMLP
|
||||
|
||||
|
||||
class TopKRouter(nn.Module):
|
||||
"""HF v5 top-k router: `linear -> softmax -> topk (-> renorm)`."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16, top_k=2, sigmoid=False):
|
||||
super().__init__()
|
||||
self.top_k = top_k
|
||||
self.sigmoid = sigmoid
|
||||
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits) if self.sigmoid else F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value / value.sum(dim=-1, keepdim=True)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class SelectionScoredRouter(TopKRouter):
|
||||
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight).float()
|
||||
value, index = torch.topk(logits, self.top_k, dim=-1)
|
||||
value = (
|
||||
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
|
||||
).type_as(hidden_states)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class EpsilonRenormRouter(TopKRouter):
|
||||
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
|
||||
value /= denominator
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class UntraceableRouter(TopKRouter):
|
||||
"""Scores and selects, then loses the trace before the renormalization."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
if value.sum() > 0: # data-dependent branch: tracing stops here
|
||||
value = value / value.sum(dim=-1, keepdim=True)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ScaledRouter(TopKRouter):
|
||||
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value * 16.0
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class Fp32Router(TopKRouter):
|
||||
"""Router that computes its logits in fp32 (DeepSeek/GLM style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(
|
||||
hidden_states.type(torch.float32), self.weight.type(torch.float32)
|
||||
)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
return logits, value.to(hidden_states.dtype), index
|
||||
|
||||
|
||||
class GroupedRouter(TopKRouter):
|
||||
"""Group-limited router (DeepSeek `group_limited_greedy`), scaled weights."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
group_scores = scores.view(-1, 4, 2).max(dim=-1).values
|
||||
group_idx = torch.topk(group_scores, k=2, dim=-1)[1]
|
||||
group_mask = torch.zeros_like(group_scores)
|
||||
group_mask.scatter_(1, group_idx, 1)
|
||||
score_mask = group_mask.unsqueeze(-1).expand(-1, 4, 2).reshape(-1, 8)
|
||||
scores = scores.masked_fill(~score_mask.bool(), 0.0)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
value = value * 16.0
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class CorrectionRouter(nn.Module):
|
||||
"""Router with a score-correction bias buffer (DeepSeek-V3 noaux) -> matched."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(num_experts, hidden))
|
||||
self.register_buffer("e_score_correction_bias", torch.zeros(num_experts))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits) + self.e_score_correction_bias
|
||||
_, index = torch.topk(scores, 2, dim=-1)
|
||||
return logits, scores, index
|
||||
|
||||
|
||||
class BiasedRouter(TopKRouter):
|
||||
"""A valid top-k router but not `weight`-only (extra `bias` param) -> declined."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.bias = nn.Parameter(torch.zeros(8))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight) + self.bias
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class DisconnectedRouter(TopKRouter):
|
||||
"""linear+softmax+top-k present but top-k ignores the logits -> not a router."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
_ = F.softmax(logits, dim=-1) # scored, but not consumed by top-k
|
||||
value, index = torch.topk(hidden_states, self.top_k, dim=-1)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ExtraInputRouter(TopKRouter):
|
||||
"""Router taking a block-owned correction bias as a second input (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states, bias=0.0):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
|
||||
value = logits.sigmoid().gather(1, index)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class MoEExperts(nn.Module):
|
||||
"""Packed experts (3D weights); only its name (`experts`) matters here."""
|
||||
|
||||
def __init__(self, num_experts=8, hidden=16, inter=32):
|
||||
super().__init__()
|
||||
self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * inter, hidden))
|
||||
self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden, inter))
|
||||
|
||||
def forward(self, hidden_states, index, weights):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class MoEBlock(nn.Module):
|
||||
"""Single-tensor MoE block (Qwen3-style); subclasses override `_shared`."""
|
||||
|
||||
def __init__(self, router_cls=TopKRouter):
|
||||
super().__init__()
|
||||
self.experts = MoEExperts()
|
||||
self.gate = router_cls()
|
||||
|
||||
def _shared(self, x, logits):
|
||||
"""The term added to the experts' output (none for a plain block)."""
|
||||
return None
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
logits, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights)
|
||||
if (shared := self._shared(x, logits)) is not None:
|
||||
out = out + shared
|
||||
return out.reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class MoEBlockNoShared(MoEBlock):
|
||||
"""No shared-expert child but a gate-derived add -> declined, not misread."""
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return logits.sum()
|
||||
|
||||
|
||||
class MoEBlockShared(MoEBlock):
|
||||
"""A block with a shared expert and its sigmoid gate (Qwen2-style)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_expert = GLUMLP()
|
||||
self.shared_expert_gate = nn.Linear(16, 1, bias=False)
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return torch.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockSharedNoGate(MoEBlock):
|
||||
"""A block with an ungated shared expert -> native, shared passed through."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_expert = GLUMLP()
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockTuple(MoEBlock):
|
||||
"""A tuple-returning block (gpt-oss-style) -> must decline."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights), index
|
||||
|
||||
|
||||
class MoEBlockTupleVar(MoEBlock):
|
||||
"""Returns a name bound to a tuple, not a literal tuple -> must still decline."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
result = self.experts(x, index, weights), index
|
||||
return result
|
||||
|
||||
|
||||
class MoEBlockNestedTupleReturn(MoEBlock):
|
||||
"""Tuple `return` in a nested helper; block returns one tensor -> still fuses."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
def keep(a, b):
|
||||
return a, b
|
||||
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out, _ = keep(self.experts(x, index, weights), index)
|
||||
return out.reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class PlainMLP(nn.Module):
|
||||
"""A non-GLU FFN: `down(act(up(x)))`, no gating multiply."""
|
||||
|
||||
def __init__(self, hidden: int = 16, inter: int = 32):
|
||||
super().__init__()
|
||||
self.up_proj = nn.Linear(hidden, inter, bias=False)
|
||||
self.down_proj = nn.Linear(inter, hidden, bias=False)
|
||||
self.act_fn = nn.SiLU()
|
||||
|
||||
def forward(self, x):
|
||||
return self.down_proj(self.act_fn(self.up_proj(x)))
|
||||
|
||||
|
||||
class MoEBlockSharedNonGLU(MoEBlock):
|
||||
"""A non-GLU shared expert -> detected by dataflow (no gate/up merge)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_expert = PlainMLP()
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockUnaccounted(MoEBlock):
|
||||
"""A weight-bearing child outside the fused dataflow (pre-router) -> declined."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.extra = nn.Linear(16, 16, bias=False)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = self.extra(hidden_states.reshape(-1, hidden_states.shape[-1]))
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights).reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class BufferScale(nn.Module):
|
||||
"""A stateful child carrying only a buffer (no parameters)."""
|
||||
|
||||
def __init__(self, hidden: int = 16):
|
||||
super().__init__()
|
||||
self.register_buffer("scale", torch.ones(hidden))
|
||||
|
||||
def forward(self, x):
|
||||
return x * self.scale
|
||||
|
||||
|
||||
class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
|
||||
"""Like `MoEBlockUnaccounted`, but the extra child holds only a buffer."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.extra = BufferScale()
|
||||
|
||||
|
||||
class MoEBlockExpertsKeywords(MoEBlock):
|
||||
"""Passes the experts' routing by keyword -> still fuses."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, weights=weights, index=index)
|
||||
|
||||
|
||||
class ScaledExperts(MoEExperts):
|
||||
def forward(self, hidden_states, index, weights, scale=1.0):
|
||||
return hidden_states * scale
|
||||
|
||||
|
||||
class MoEBlockExpertsExtraArg(MoEBlock):
|
||||
"""Passes the experts an extra argument the fused forward would drop."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = ScaledExperts()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights, scale=2.0)
|
||||
|
||||
|
||||
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
|
||||
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
_, weights, index = self.gate(hidden_states)
|
||||
x = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
out = self.experts(x, index, weights).view(*hidden_states.shape)
|
||||
return out + self.shared_expert(hidden_states)
|
||||
|
||||
|
||||
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
|
||||
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return super().forward(hidden_states) / 2
|
||||
|
||||
|
||||
class MoEBlockScaled(MoEBlockSharedNoGate):
|
||||
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
|
||||
"""An unrecognized router with a scaled experts' output."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
|
||||
"""An unrecognized router with a shared expert."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockGateExtraInput(MoEBlock):
|
||||
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(ExtraInputRouter)
|
||||
self.register_buffer("e_score_correction_bias", torch.zeros(8))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x, self.e_score_correction_bias)
|
||||
return self.experts(x, index, weights)
|
||||
|
||||
|
||||
class MoEBlockCastCombine(MoEBlockSharedNoGate):
|
||||
"""Combines the outputs in fp32 (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
|
||||
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fc1_latent_proj = nn.Identity()
|
||||
self.fc2_latent_proj = nn.Identity()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.fc2_latent_proj(
|
||||
self.experts(self.fc1_latent_proj(x), index, weights)
|
||||
)
|
||||
return out + self.shared_expert(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_detects_router(sigmoid):
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(lambda: TopKRouter(sigmoid=sigmoid))
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.gate_name == "gate"
|
||||
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
|
||||
"""Scoring the selection rather than all logits still allows the fuser to match."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.gate_name == "gate"
|
||||
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
|
||||
assert fuser.renormalize is (not sigmoid)
|
||||
assert fuser.router_dtype == torch.float32
|
||||
|
||||
|
||||
def test_moe_fuser_matches_scaled_router():
|
||||
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(ScaledRouter)
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_cls,expected",
|
||||
[
|
||||
(TopKRouter, True), # divides by the sum of the selection
|
||||
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
|
||||
(ScaledRouter, False), # scales the weights instead of renormalizing
|
||||
(UntraceableRouter, None), # trace stops before it could say
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
|
||||
"""The router's graph, not the config, decides whether vLLM renormalizes: it
|
||||
is the code being replaced, and it already bakes in `norm_topk_prob` because
|
||||
the flag is a Python bool that fx resolves at trace time. Absence of the
|
||||
division only means "no" on a complete trace."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(router_cls)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.renormalize is expected
|
||||
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(GroupedRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.scoring_func == "softmax"
|
||||
|
||||
|
||||
def test_moe_fuser_matches_correction_router():
|
||||
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(CorrectionRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.scoring_func == "sigmoid"
|
||||
|
||||
|
||||
def test_moe_fuser_reads_router_dtype_from_the_gate():
|
||||
"""A router that computes its logits in fp32 must keep routing in fp32 when
|
||||
rebuilt, even though no config field names the dtype. The cast back to the
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype."""
|
||||
with torch.device("meta"):
|
||||
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
|
||||
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
|
||||
assert fp32.router_dtype == torch.float32
|
||||
assert default.router_dtype is None
|
||||
|
||||
|
||||
def test_moe_fuser_detects_shared_experts():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockShared()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
assert fuser.shared_gate_name == "shared_expert_gate"
|
||||
|
||||
|
||||
def test_moe_fuser_shared_without_gate():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedNoGate()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
assert fuser.shared_gate_name is None
|
||||
|
||||
|
||||
def test_moe_fuser_detects_non_glu_shared_expert():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedNonGLU()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
# Recognised by dataflow (added to the experts' output), though not a GLU.
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
assert fuser.shared_gate_name is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block_cls",
|
||||
[
|
||||
lambda: MoEBlock(BiasedRouter), # router with an unrecognized extra param
|
||||
MoEBlockTuple, # tuple-returning block (e.g. gpt-oss)
|
||||
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
|
||||
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
|
||||
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
|
||||
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
|
||||
MoEBlockNoShared, # gate-derived term added to the experts' output
|
||||
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
|
||||
MoEBlockGateExtraInput, # gate input the fused forward drops
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_declines_unsupported(block_cls):
|
||||
with torch.device("meta"):
|
||||
block = block_cls()
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_ignores_nested_returns():
|
||||
"""A tuple `return` inside a nested helper must not decline a block whose own
|
||||
forward returns a single tensor."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNestedTupleReturn()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_router_requires_connected_dataflow():
|
||||
"""A gate with linear + softmax + top-k present but not wired as a router
|
||||
(top-k selects over the input, not the scored logits) is not detected."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(DisconnectedRouter)
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_unrecognized_router_moves_shared_expert():
|
||||
"""Without vLLM routing, the shared expert still moves to vLLM."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedBiasedRouter()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert fuser.scoring_func is None
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block_cls", "scoring_func"),
|
||||
[
|
||||
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
|
||||
(MoEBlock, "softmax"), # no shared output to divide
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
|
||||
assert fuser.scoring_func == scoring_func
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_needs_vllm_routing():
|
||||
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockScaledBiasedRouter()
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_matches_experts_called_by_keyword():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockExpertsKeywords()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_with_vllm_routing():
|
||||
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
|
||||
assert fuser.scoring_func == "softmax"
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
|
||||
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts")
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
@@ -1,409 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers modeling backend's RMSNorm fuser."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers import RMSNormFuser
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
"""The canonical HF RMSNorm: `weight * x * rsqrt(mean(x**2) + eps)`."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5, weight: bool = True):
|
||||
super().__init__()
|
||||
if weight:
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def _rms(self, x):
|
||||
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon)
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
|
||||
|
||||
class GemmaRMSNorm(RMSNorm):
|
||||
"""Zero-centered weight: `(1 + weight) * normalized`."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps)
|
||||
self.weight = nn.Parameter(torch.zeros(hidden))
|
||||
|
||||
def forward(self, x):
|
||||
return (1.0 + self.weight) * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
|
||||
|
||||
class WeightlessRMSNorm(RMSNorm):
|
||||
"""No scale parameter (e.g. Gemma3n `with_scale=False`)."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps, weight=False)
|
||||
|
||||
def forward(self, x):
|
||||
return self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
|
||||
|
||||
class Gemma4RMSNorm(RMSNorm):
|
||||
"""`pow(v, -0.5)` where the others write `rsqrt(v)` (HF `Gemma4RMSNorm._norm`).
|
||||
|
||||
Gemma 4 spells the reciprocal square root as `torch.pow(..., -0.5)` to keep
|
||||
Torch and JAX in agreement, and -- unlike Gemma 1-3 -- its weight is *not*
|
||||
zero-centered, so this fuses to a plain `RMSNorm`.
|
||||
"""
|
||||
|
||||
def _rms(self, x):
|
||||
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
|
||||
return x * torch.pow(mean_squared, -0.5)
|
||||
|
||||
def forward(self, x):
|
||||
return (self._rms(x.float()) * self.weight.float()).type_as(x)
|
||||
|
||||
|
||||
class PowOperatorRMSNorm(Gemma4RMSNorm):
|
||||
"""The `v ** -0.5` operator spelling of the same reciprocal square root."""
|
||||
|
||||
def _rms(self, x):
|
||||
return x * (x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon) ** -0.5
|
||||
|
||||
|
||||
class KwargPowRMSNorm(Gemma4RMSNorm):
|
||||
"""The keyword spelling `torch.pow(v, exponent=-0.5)` of the same rsqrt."""
|
||||
|
||||
def _rms(self, x):
|
||||
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
|
||||
return x * torch.pow(mean_squared, exponent=-0.5)
|
||||
|
||||
|
||||
class KwargBasePowRMSNorm(Gemma4RMSNorm):
|
||||
"""The fully functional spelling `torch.pow(input=v, exponent=-0.5)`."""
|
||||
|
||||
def _rms(self, x):
|
||||
mean_squared = x.pow(2).mean(-1, keepdim=True) + self.variance_epsilon
|
||||
return x * torch.pow(input=mean_squared, exponent=-0.5)
|
||||
|
||||
|
||||
class WeightlessGemma4RMSNorm(Gemma4RMSNorm):
|
||||
"""Gemma 4 with `with_scale=False`: the `pow` spelling and no scale parameter."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps, weight=False)
|
||||
|
||||
def forward(self, x):
|
||||
return self._rms(x.float()).type_as(x)
|
||||
|
||||
|
||||
class LayerNorm(RMSNorm):
|
||||
"""An RMSNorm not named `*RMSNorm`, keeping the input dtype (no upcast)."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps)
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * self._rms(x)
|
||||
|
||||
|
||||
class NamedEpsRMSNorm(nn.Module):
|
||||
"""Holds eps under `attr`, with no `variance_epsilon` to find instead."""
|
||||
|
||||
attr = "eps"
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
setattr(self, self.attr, eps)
|
||||
|
||||
def forward(self, x):
|
||||
eps = getattr(self, self.attr)
|
||||
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps))
|
||||
|
||||
|
||||
class ToleranceRMSNorm(NamedEpsRMSNorm):
|
||||
"""Names eps something other than `eps`, so only its value identifies it."""
|
||||
|
||||
attr = "tolerance"
|
||||
|
||||
|
||||
class LiteralEpsRMSNorm(RMSNorm):
|
||||
"""eps is a literal in the source, and an unrelated attribute happens to
|
||||
hold the same value: matching on the value alone would bind to it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__(hidden, eps)
|
||||
self.epsilon_lookalike = 1e-4
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4))
|
||||
|
||||
|
||||
class AmbiguousEpsRMSNorm(nn.Module):
|
||||
"""Two attributes hold the eps value, and the forward reads the second, so
|
||||
their order cannot pick it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.decoy_eps = eps
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, x):
|
||||
var = x.pow(2).mean(-1, True)
|
||||
return self.weight * (x * torch.rsqrt(var + self.variance_epsilon))
|
||||
|
||||
|
||||
class NotAnRMSNorm(RMSNorm):
|
||||
"""Mean-subtracting LayerNorm-like math -> not an RMSNorm."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__(hidden, eps)
|
||||
|
||||
def forward(self, x):
|
||||
x = x - x.mean(-1, keepdim=True)
|
||||
variance = x.var(-1, keepdim=True)
|
||||
return self.weight * x / torch.sqrt(variance + self.variance_epsilon)
|
||||
|
||||
|
||||
class GatedRMSNorm(RMSNorm):
|
||||
"""Second input and tail compute -> not an RMSNorm."""
|
||||
|
||||
def forward(self, x, gate=None):
|
||||
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
return normed * F.silu(gate)
|
||||
|
||||
|
||||
class GatedFusedRMSNorm(nn.Module):
|
||||
"""Same as GatedRMSNorm, but built on the fused `rms_norm` op -> not an RMSNorm."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, x, gate=None):
|
||||
return F.rms_norm(x, (x.shape[-1],), self.weight, self.eps) * F.silu(gate)
|
||||
|
||||
|
||||
class UntraceableGatedRMSNorm(RMSNorm):
|
||||
"""Tracer can't see tail compute in forward, but still has a second input (gate)."""
|
||||
|
||||
def forward(self, x, gate=None):
|
||||
normed = self.weight * self._rms(x.to(torch.float32)).to(x.dtype)
|
||||
if gate.sum() > 0: # untraceable -> partial graph, no visible tail
|
||||
normed = normed * F.silu(gate)
|
||||
return normed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls,eps,zero_centered",
|
||||
[
|
||||
(RMSNorm, 1e-5, False),
|
||||
(GemmaRMSNorm, 1e-6, True),
|
||||
(WeightlessRMSNorm, 1e-6, False),
|
||||
(Gemma4RMSNorm, 1e-6, False),
|
||||
(PowOperatorRMSNorm, 1e-6, False),
|
||||
(KwargPowRMSNorm, 1e-6, False),
|
||||
(KwargBasePowRMSNorm, 1e-6, False),
|
||||
(WeightlessGemma4RMSNorm, 1e-6, False),
|
||||
(LayerNorm, 1e-6, False),
|
||||
(torch.nn.RMSNorm, 1e-5, False), # fused `F.rms_norm` op
|
||||
],
|
||||
)
|
||||
def test_detects_rms_norm_variants(cls, eps, zero_centered):
|
||||
with torch.device("meta"):
|
||||
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
|
||||
assert isinstance(fuser, RMSNormFuser)
|
||||
assert fuser.zero_centered == zero_centered
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [NotAnRMSNorm, nn.LayerNorm, nn.SiLU])
|
||||
def test_non_rms_norms_are_not_matched(cls):
|
||||
with torch.device("meta"):
|
||||
module = cls(16) if cls is nn.LayerNorm else cls()
|
||||
assert get_fuser(module, RMSNormFuser) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls", [GatedRMSNorm, GatedFusedRMSNorm, UntraceableGatedRMSNorm]
|
||||
)
|
||||
def test_gated_rms_norm_is_not_fused(cls):
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(cls(), RMSNormFuser) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls,expected,zero_centered",
|
||||
[
|
||||
(RMSNorm, "RMSNorm", False),
|
||||
(GemmaRMSNorm, "GemmaRMSNorm", True),
|
||||
(WeightlessRMSNorm, "RMSNorm", False),
|
||||
(Gemma4RMSNorm, "RMSNorm", False),
|
||||
(KwargPowRMSNorm, "RMSNorm", False),
|
||||
(KwargBasePowRMSNorm, "RMSNorm", False),
|
||||
(WeightlessGemma4RMSNorm, "RMSNorm", False),
|
||||
],
|
||||
)
|
||||
def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_config):
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as VLLMGemmaRMSNorm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
|
||||
|
||||
with torch.device("meta"):
|
||||
module = cls()
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import (
|
||||
TPAwareNormMixin,
|
||||
)
|
||||
|
||||
types_by_name = {"RMSNorm": VLLMRMSNorm, "GemmaRMSNorm": VLLMGemmaRMSNorm}
|
||||
assert isinstance(built, types_by_name[expected])
|
||||
assert isinstance(built, TPAwareNormMixin)
|
||||
assert built.variance_epsilon == module.variance_epsilon
|
||||
# The weight states the hidden size (and is what the TP check reads); a norm
|
||||
# without one has none to state.
|
||||
weight = getattr(module, "weight", None)
|
||||
assert isinstance(built.weight, nn.Parameter) == (weight is not None)
|
||||
assert built.weight.shape[0] == (weight.size(0) if weight is not None else 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls", [Gemma4RMSNorm, PowOperatorRMSNorm, KwargPowRMSNorm, KwargBasePowRMSNorm]
|
||||
)
|
||||
def test_pow_spelled_norm_fuses_to_equivalent_math(cls, default_vllm_config):
|
||||
"""Matching `pow(v, -0.5)` is only sound if it really is the reciprocal square
|
||||
root: the fused norm must reproduce the unfused forward, not just replace it.
|
||||
`pow` and `rsqrt` take different paths to the same value, so compare at float32
|
||||
tolerance rather than bit-for-bit."""
|
||||
torch.manual_seed(0)
|
||||
module = cls(16)
|
||||
with torch.no_grad():
|
||||
module.weight.copy_(torch.randn(16))
|
||||
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
|
||||
with torch.no_grad():
|
||||
built.weight.copy_(module.weight)
|
||||
x = torch.randn(4, 16)
|
||||
torch.testing.assert_close(built(x), module(x), rtol=1e-5, atol=1e-6)
|
||||
|
||||
|
||||
def test_weightless_norm_has_no_hidden_size(default_vllm_config):
|
||||
"""A weightless norm states no hidden size, and the model's (LM) hidden size
|
||||
would be wrong for one on a sub-dimension: Llama 4's `qk_norm` normalizes
|
||||
head_dim, so sizing it at hidden_size made the TP gather reject its input.
|
||||
It fuses with none, and so normalizes the width it is given at any TP size,
|
||||
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
|
||||
"""
|
||||
module = WeightlessRMSNorm(72)
|
||||
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
|
||||
assert built.hidden_size == 0
|
||||
|
||||
built.tp_size = 2 # emulate TP=2 without a real process group
|
||||
x = torch.randn(4, 72)
|
||||
torch.testing.assert_close(built(x), module(x))
|
||||
|
||||
|
||||
def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
|
||||
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
|
||||
|
||||
with torch.device("meta"):
|
||||
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
assert isinstance(fuser, RMSNormFuser)
|
||||
assert not fuser.zero_centered
|
||||
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
|
||||
built = fuser.fuse(module, "norm", vllm_config)
|
||||
assert isinstance(built, VLLMRMSNorm)
|
||||
assert built.variance_epsilon == torch.finfo(torch.float32).eps
|
||||
|
||||
|
||||
def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
"""Two instances of the same norm class with different eps must fuse to their
|
||||
own eps: the type-cached fuser holds only structure, not this value."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = RMSNorm(16, eps=eps)
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
|
||||
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
"""The eps attribute is identified by holding the traced value, so a norm
|
||||
stays per-instance correct whatever it names it."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = cls(16, eps=eps)
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
assert fuser.eps_attr == cls.attr
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
|
||||
"""A literal eps is recognised as coming from no attribute, even when one
|
||||
holds the same value, and is taken from the traced source instead."""
|
||||
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
|
||||
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
|
||||
module = LiteralEpsRMSNorm()
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert fuser.eps_attr is None
|
||||
assert built.variance_epsilon == 1e-4
|
||||
assert "does not hold its eps" in caplog.text
|
||||
|
||||
|
||||
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
"""When several attributes hold the eps value, the one the forward actually
|
||||
reads is identified, and they are all left as they were found."""
|
||||
with torch.device("meta"):
|
||||
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
|
||||
before = dict(vars(module))
|
||||
fuser = get_fuser(module, RMSNormFuser)
|
||||
assert fuser.eps_attr == "variance_epsilon"
|
||||
assert vars(module) == before
|
||||
|
||||
other = AmbiguousEpsRMSNorm(16, eps=1e-5)
|
||||
other.decoy_eps = 1e-3
|
||||
built = fuser.fuse(other, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == 1e-5
|
||||
|
||||
|
||||
def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
|
||||
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
|
||||
plumbing. A full-width input skips the gather and equals a plain norm."""
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(4, 16)
|
||||
for gathered_cls, plain_cls in [
|
||||
(rms_norm.TPAwareRMSNorm, RMSNorm),
|
||||
(rms_norm.TPAwareGemmaRMSNorm, GemmaRMSNorm),
|
||||
]:
|
||||
gathered = gathered_cls(hidden_size=16, eps=1e-6)
|
||||
assert isinstance(gathered, rms_norm.TPAwareNormMixin)
|
||||
plain = plain_cls(hidden_size=16, eps=1e-6)
|
||||
with torch.no_grad():
|
||||
weight = torch.randn(16)
|
||||
gathered.weight.copy_(weight)
|
||||
plain.weight.copy_(weight)
|
||||
torch.testing.assert_close(gathered(x), plain(x))
|
||||
|
||||
|
||||
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
|
||||
"""A sharded input (narrower than the full-width weight) that does not tile
|
||||
the weight evenly across ranks is rejected before any collective."""
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
|
||||
norm.tp_size = 2 # emulate TP=2 without a real process group
|
||||
with pytest.raises(ValueError, match="does not tile it evenly"):
|
||||
norm(torch.randn(2, 3)) # 3 * 2 != 8
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for skipping dynamic rope's device sync where it can never rescale."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import HunYuanDenseV1Config, LlamaConfig
|
||||
from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import (
|
||||
HunYuanDenseV1RotaryEmbedding,
|
||||
)
|
||||
from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding
|
||||
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
)
|
||||
|
||||
MAX_POSITION_EMBEDDINGS = 128
|
||||
CONFIG_KWARGS = dict(
|
||||
hidden_size=64,
|
||||
num_attention_heads=4,
|
||||
num_hidden_layers=1,
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
)
|
||||
DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4}
|
||||
|
||||
|
||||
def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace:
|
||||
model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len)
|
||||
return SimpleNamespace(model_config=model_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rope_parameters,max_model_len,compilable",
|
||||
[
|
||||
(DYNAMIC, MAX_POSITION_EMBEDDINGS, True),
|
||||
(DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False),
|
||||
# The longrope layer type still syncs
|
||||
(
|
||||
{"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}},
|
||||
MAX_POSITION_EMBEDDINGS,
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable):
|
||||
"""Dynamic rope only blocks compilation when it can still rescale."""
|
||||
config = SimpleNamespace(
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
config.get_text_config = lambda: config
|
||||
vllm_config = make_vllm_config(config, max_model_len)
|
||||
assert can_enable_torch_compile(vllm_config) is compilable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config,rotary_cls",
|
||||
[
|
||||
(LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding),
|
||||
# Hunyuan's dynamic init also applies `alpha`, which must be kept
|
||||
(
|
||||
HunYuanDenseV1Config(
|
||||
**CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3}
|
||||
),
|
||||
HunYuanDenseV1RotaryEmbedding,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls):
|
||||
"""The update no longer syncs and the frequencies are unchanged."""
|
||||
inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :]
|
||||
rotary_emb = rotary_cls(config)
|
||||
expected = rotary_emb(*inputs)
|
||||
|
||||
vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS)
|
||||
disable_noop_dynamic_rope(rotary_emb, vllm_config)
|
||||
|
||||
def no_sync(*args, **kwargs):
|
||||
raise AssertionError("dynamic rope update still runs")
|
||||
|
||||
monkeypatch.setattr(torch, "max", no_sync)
|
||||
actual = rotary_emb(*inputs)
|
||||
assert all(torch.equal(e, a) for e, a in zip(expected, actual))
|
||||
@@ -1,80 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Transformers backend's hw-agnostic layer resolution.
|
||||
|
||||
`layers._resolve` imports a layer symbol from
|
||||
`vllm.model_executor.hw_agnostic.layers.<module>` when `VLLM_USE_HW_AGNOSTIC`
|
||||
is set and the symbol exists, and otherwise falls back to
|
||||
`vllm.model_executor.layers.<module>`. These tests pin that contract and the
|
||||
logging that reports which source was used.
|
||||
|
||||
CPU-only: no engine is built. The end-to-end counterpart that serves a model
|
||||
through `vllm_runner` lives in `test_layer_registry_e2e.py` (GPU job).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.transformers import layers
|
||||
|
||||
HW_MODULE = "vllm.model_executor.hw_agnostic.layers.layernorm"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_hw_layernorm(monkeypatch):
|
||||
"""Inject a hw-agnostic `layernorm` module exposing a sentinel `RMSNorm`.
|
||||
|
||||
A `SimpleNamespace` stands in for the module: `importlib.import_module`
|
||||
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
|
||||
attributes are set at construction (no `ModuleType` attribute-set that mypy
|
||||
rejects, no constant `setattr` that ruff rejects)."""
|
||||
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, module)
|
||||
return module
|
||||
|
||||
|
||||
def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm):
|
||||
"""Disabled: the vLLM class is used even if a hw-agnostic one exists."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
|
||||
|
||||
assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm
|
||||
|
||||
|
||||
def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog_vllm):
|
||||
"""Enabled and available: the hw-agnostic class is used and logged."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
with caplog_vllm.at_level(logging.INFO):
|
||||
resolved = layers._resolve("layernorm", "RMSNorm")
|
||||
assert resolved is fake_hw_layernorm.RMSNorm
|
||||
assert "Using hw-agnostic layer: RMSNorm" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_falls_back_when_symbol_missing(monkeypatch, caplog_vllm):
|
||||
"""Enabled but the symbol is not ported: fall back to vLLM and warn."""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
# A hw-agnostic module without the requested attribute triggers fallback.
|
||||
empty = types.ModuleType(HW_MODULE)
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, empty)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm
|
||||
|
||||
with caplog_vllm.at_level(logging.WARNING):
|
||||
resolved = layers._resolve("layernorm", "RMSNorm")
|
||||
assert resolved is VllmRMSNorm
|
||||
assert "falling back to default" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_act_and_mul_falls_back_for_unknown_activation(
|
||||
monkeypatch, default_vllm_config
|
||||
):
|
||||
"""An activation with no hw-agnostic equivalent falls back to vLLM's.
|
||||
|
||||
`default_vllm_config` supplies the config context the CustomOp needs.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
from vllm.model_executor.layers.activation import GeluAndMul
|
||||
|
||||
assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul)
|
||||
@@ -418,20 +418,6 @@ def test_allowed_image_formats_exclude_external_renderer_plugins():
|
||||
assert denied.isdisjoint(ALLOWED_IMAGE_FORMATS)
|
||||
|
||||
|
||||
def test_kimi_fused_vision_rejects_postscript_payload():
|
||||
"""Kimi fused vision must use the same image format allowlist."""
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import _to_pil
|
||||
|
||||
eps = b"""%!PS-Adobe-3.0 EPSF-3.0
|
||||
%%BoundingBox: 0 0 10 10
|
||||
%%EndComments
|
||||
0 0 moveto 10 10 lineto stroke
|
||||
showpage
|
||||
"""
|
||||
with pytest.raises(Image.UnidentifiedImageError):
|
||||
_to_pil(eps)
|
||||
|
||||
|
||||
def test_open_image_does_not_accept_format_override():
|
||||
"""The allowlist helper must not expose a formats override."""
|
||||
assert "formats" not in inspect.signature(open_image).parameters
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Citation source resolution and the streamed citation wire shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from vllm.parser.cohere_command import (
|
||||
_melody_citations_to_vllm,
|
||||
_melody_sources_to_vllm,
|
||||
)
|
||||
|
||||
from .utils import make_parser, stream_parser
|
||||
|
||||
|
||||
class TestMelodySourceResolution:
|
||||
"""Pin the parser-side source resolution (see
|
||||
``_melody_sources_to_vllm``). The parser receives melody's numeric
|
||||
``(tool_call_index, tool_result_indices)`` addressing and resolves
|
||||
it against a ``position_to_source`` map handed in by
|
||||
``CohereServingChatV2._apply_cohere_template_kwargs`` -- the same
|
||||
map that would otherwise live in the serving layer's resolver.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fake_melody_source(bucket: int, indices: list[int]) -> Any:
|
||||
return SimpleNamespace(tool_call_index=bucket, tool_result_indices=indices)
|
||||
|
||||
def test_multi_index_source_fans_out(self):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
|
||||
position_map: dict[tuple[int, int], CitationSource] = {
|
||||
(0, 0): CitationSource(type="document", id="d0", document={"id": "d0"}),
|
||||
(0, 1): CitationSource(type="document", id="d1", document={"id": "d1"}),
|
||||
}
|
||||
raw = [self._fake_melody_source(0, [0, 1])]
|
||||
out = _melody_sources_to_vllm(raw, position_map)
|
||||
assert [s.id for s in out] == ["d0", "d1"]
|
||||
# Verify type / payload were plumbed through, not just the id.
|
||||
assert out[0].type == "document"
|
||||
assert out[0].document == {"id": "d0"}
|
||||
|
||||
def test_unresolvable_position_skipped(self):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
|
||||
position_map: dict[tuple[int, int], CitationSource] = {
|
||||
(0, 0): CitationSource(type="document", id="d0"),
|
||||
}
|
||||
raw = [self._fake_melody_source(9, [0])]
|
||||
assert _melody_sources_to_vllm(raw, position_map) == []
|
||||
|
||||
def test_missing_position_map_drops_all_sources(self):
|
||||
# A parser instance without a position map (parser wired
|
||||
# outside of ``CohereServingChatV2``) can't attribute anything,
|
||||
# so every source is dropped. Callers downstream will see the
|
||||
# citation with empty ``sources`` and drop it entirely.
|
||||
raw = [self._fake_melody_source(0, [0])]
|
||||
assert _melody_sources_to_vllm(raw, None) == []
|
||||
|
||||
def test_citations_pass_through_is_thinking_tag(self):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
|
||||
position_map: dict[tuple[int, int], CitationSource] = {
|
||||
(0, 0): CitationSource(type="document", id="d0"),
|
||||
}
|
||||
raw = [
|
||||
SimpleNamespace(
|
||||
start_index=0,
|
||||
end_index=5,
|
||||
text="hello",
|
||||
is_thinking=True,
|
||||
sources=[self._fake_melody_source(0, [0])],
|
||||
)
|
||||
]
|
||||
out = _melody_citations_to_vllm(raw, position_map)
|
||||
assert out is not None
|
||||
assert out[0].type == "THINKING_CONTENT"
|
||||
assert out[0].sources[0].id == "d0"
|
||||
|
||||
|
||||
class TestParserStreamingEndToEnd:
|
||||
"""End-to-end streaming shape: raw model output text is tokenized
|
||||
with the mock byte-level tokenizer and fed one token at a time
|
||||
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
|
||||
We collect the ordered sequence of ``DeltaMessage``s the parser
|
||||
hands back -- i.e. what the client would receive on the wire --
|
||||
and pin the exact shape.
|
||||
|
||||
Combines the three flavors that reach ``delta``: a thinking block
|
||||
(populates ``reasoning``), a text block (populates ``content``),
|
||||
and a resolved citation (populates ``citations``) referencing
|
||||
``(start, end)`` offsets into the already-streamed content.
|
||||
"""
|
||||
|
||||
_MODEL_OUTPUT = (
|
||||
"<|START_THINKING|>Let me check.<|END_THINKING|>"
|
||||
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
|
||||
)
|
||||
|
||||
def test_full_wire_stream_shape(self, tokenizer, request_obj):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
|
||||
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
|
||||
# installs on a request whose tool_call_index=0 tool result at
|
||||
# index 1 is the "France" document.
|
||||
position_to_source = {
|
||||
(0, 1): CitationSource(
|
||||
type="document",
|
||||
id="doc-paris",
|
||||
document={"id": "doc-paris", "title": "France"},
|
||||
),
|
||||
}
|
||||
|
||||
parser = make_parser(
|
||||
tokenizer,
|
||||
"cohere_command3",
|
||||
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
|
||||
)
|
||||
deltas = stream_parser(parser, request_obj, tokenizer, self._MODEL_OUTPUT)
|
||||
|
||||
# -- 1. Reasoning and content stream separately and reach
|
||||
# the wire only as their real payload. Framing tokens
|
||||
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
|
||||
# state machine and never produce a delta.
|
||||
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
|
||||
content_stream = [d.content for d in deltas if d.content is not None]
|
||||
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
|
||||
|
||||
assert "".join(reasoning_stream) == "Let me check."
|
||||
assert "".join(content_stream) == "The capital is Paris."
|
||||
|
||||
# -- 2. Exactly one citation reaches the wire, resolved to the
|
||||
# source from ``position_to_source`` (not the raw
|
||||
# ``(bucket, idx)`` coordinates).
|
||||
assert len(citation_deltas) == 1
|
||||
(cite_delta,) = citation_deltas
|
||||
assert cite_delta.citations is not None
|
||||
assert len(cite_delta.citations) == 1
|
||||
cite = cite_delta.citations[0]
|
||||
assert cite.text == "Paris"
|
||||
# Char offsets are into the accumulated ``content``: "The
|
||||
# capital is " is 15 chars, "Paris" is 5.
|
||||
assert cite.start == 15
|
||||
assert cite.end == 20
|
||||
assert len(cite.sources) == 1
|
||||
assert cite.sources[0].id == "doc-paris"
|
||||
assert cite.sources[0].type == "document"
|
||||
|
||||
# -- 3. The citation's ``(start, end)`` offsets point into
|
||||
# bytes that were already emitted as content-deltas earlier
|
||||
# in the stream -- i.e. the citation is anchored to real
|
||||
# already-shipped content.
|
||||
cite_delta_idx = deltas.index(cite_delta)
|
||||
earlier_content = "".join(
|
||||
d.content for d in deltas[:cite_delta_idx] if d.content is not None
|
||||
)
|
||||
assert earlier_content[cite.start : cite.end] == "Paris"
|
||||
|
||||
# Everything after the citation delta is the trailing period.
|
||||
later_content = "".join(
|
||||
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
|
||||
)
|
||||
assert later_content == "."
|
||||
|
||||
# -- 4. Reasoning and content are never mixed on the same
|
||||
# delta: the parser flips modes on the ``<|END_THINKING|>``
|
||||
# boundary and never emits a delta with both fields set.
|
||||
for d in deltas:
|
||||
assert not (d.reasoning is not None and d.content is not None)
|
||||
@@ -1,476 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``adjust_request``: folding tools and JSON schemas into Cohere structural tags."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import UserDict
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
JsonSchemaResponseFormat,
|
||||
ResponseFormat,
|
||||
StructuralTagResponseFormat,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.cohere_command import (
|
||||
CohereCommandParser,
|
||||
CohereNormalizedTool,
|
||||
_has_effective_tools,
|
||||
_response_format_type,
|
||||
_schema_dict_from_structured_outputs,
|
||||
collect_tool_schema,
|
||||
convert_schema_to_structural_tags,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
from .utils import MockCohereTokenizer, make_parser
|
||||
|
||||
SCHEMA_A = {"type": "object", "properties": {"a": {"type": "string"}}}
|
||||
SCHEMA_B = {"type": "object", "properties": {"b": {"type": "number"}}}
|
||||
GET_WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
NESTED_DEFS_PARAMETERS = {
|
||||
"$defs": {
|
||||
"Location": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"country": {"type": "string"},
|
||||
},
|
||||
"required": ["city"],
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"properties": {"location": {"$ref": "#/$defs/Location"}},
|
||||
"required": ["location"],
|
||||
}
|
||||
NESTED_DEFS_TOOL = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": NESTED_DEFS_PARAMETERS},
|
||||
}
|
||||
RECURSIVE_DEFS_PARAMETERS = {
|
||||
"$defs": {
|
||||
"Finding": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"note": {"type": "string"},
|
||||
"related": {"type": "array", "items": {"$ref": "#/$defs/Finding"}},
|
||||
},
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"properties": {"finding": {"$ref": "#/$defs/Finding"}},
|
||||
}
|
||||
DRAFT07_DEFINITIONS_PARAMETERS = {
|
||||
"definitions": {
|
||||
"Location": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"properties": {"location": {"$ref": "#/definitions/Location"}},
|
||||
}
|
||||
VALID_STRUCTURAL_TAG = {
|
||||
"type": "structural_tag",
|
||||
"format": {
|
||||
"type": "triggered_tags",
|
||||
"tags": [
|
||||
{
|
||||
"begin": "<tool>",
|
||||
"content": {"type": "any_text"},
|
||||
"end": "</tool>",
|
||||
}
|
||||
],
|
||||
"triggers": ["<tool>"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _model_config(arch: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
architecture=arch,
|
||||
architectures=[arch],
|
||||
hf_text_config=SimpleNamespace(architectures=[arch]),
|
||||
)
|
||||
|
||||
|
||||
def _make_chat_request(**kwargs) -> ChatCompletionRequest:
|
||||
data = {"messages": [{"role": "user", "content": "hi"}], "model": "m"}
|
||||
data.update(kwargs)
|
||||
return ChatCompletionRequest.model_validate(data)
|
||||
|
||||
|
||||
def _first_json_schema(tag_json: str) -> dict | None:
|
||||
outer = json.loads(tag_json)
|
||||
for t in (outer.get("format") or {}).get("tags") or []:
|
||||
c = t.get("content") or {}
|
||||
if c.get("type") == "json_schema":
|
||||
js = c.get("json_schema")
|
||||
return js if isinstance(js, dict) else None
|
||||
return None
|
||||
|
||||
|
||||
def _content_types(tag_json: str) -> set[str]:
|
||||
outer = json.loads(tag_json)
|
||||
out: set[str] = set()
|
||||
for t in (outer.get("format") or {}).get("tags") or []:
|
||||
ty = (t.get("content") or {}).get("type")
|
||||
if isinstance(ty, str):
|
||||
out.add(ty)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
|
||||
"""Parser configured with a supported Cohere architecture."""
|
||||
return make_parser(
|
||||
tokenizer, "cohere_command4", model_config=_model_config("Cohere2ForCausalLM")
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_no_model_config(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
|
||||
"""Parser with no ``model_config`` (cannot resolve architecture)."""
|
||||
return make_parser(tokenizer, "cohere_command4")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_unsupported_arch(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
|
||||
"""Parser configured with an architecture that has no structural tag style."""
|
||||
return make_parser(
|
||||
tokenizer, "cohere_command4", model_config=_model_config("LlamaForCausalLM")
|
||||
)
|
||||
|
||||
|
||||
class TestAdjustRequestPassthrough:
|
||||
def test_structured_outputs_structural_tag_not_modified(self, parser) -> None:
|
||||
tag = json.dumps(VALID_STRUCTURAL_TAG)
|
||||
r = _make_chat_request(structured_outputs={"structural_tag": tag})
|
||||
o = parser.adjust_request(r)
|
||||
assert o.structured_outputs.structural_tag == tag
|
||||
|
||||
def test_response_format_structural_tag_short_circuit(self, parser) -> None:
|
||||
# ``ChatCompletionRequest`` validates ``response_format`` as a union;
|
||||
# bare ``{"type": "structural_tag"}`` is invalid (use pydantic model).
|
||||
rf = StructuralTagResponseFormat(
|
||||
type="structural_tag",
|
||||
format=VALID_STRUCTURAL_TAG["format"],
|
||||
)
|
||||
r = _make_chat_request(response_format=rf)
|
||||
o = parser.adjust_request(r)
|
||||
assert _response_format_type(o.response_format) == "structural_tag"
|
||||
assert o.structured_outputs is None
|
||||
|
||||
|
||||
class TestAdjustRequestNoOp:
|
||||
def test_no_schema_no_tools(self, parser) -> None:
|
||||
o = parser.adjust_request(_make_chat_request())
|
||||
assert o.structured_outputs is None
|
||||
assert o.response_format is None
|
||||
|
||||
def test_no_model_config(self, parser_no_model_config) -> None:
|
||||
inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A)
|
||||
r = _make_chat_request(
|
||||
response_format=ResponseFormat(type="json_schema", json_schema=inner),
|
||||
)
|
||||
o = parser_no_model_config.adjust_request(r)
|
||||
assert o.response_format is not None
|
||||
assert o.structured_outputs is None
|
||||
|
||||
|
||||
class TestAdjustRequestUnsupportedArchitecture:
|
||||
def test_json_schema_raises(self, parser_unsupported_arch) -> None:
|
||||
inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A)
|
||||
r = _make_chat_request(
|
||||
response_format=ResponseFormat(type="json_schema", json_schema=inner),
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not support"):
|
||||
parser_unsupported_arch.adjust_request(r)
|
||||
|
||||
|
||||
class TestAdjustRequestFoldFromResponseFormat:
|
||||
@pytest.mark.parametrize(
|
||||
"response_format, expected_schema",
|
||||
[
|
||||
pytest.param(
|
||||
ResponseFormat(
|
||||
type="json_schema",
|
||||
json_schema=JsonSchemaResponseFormat(
|
||||
name="n", json_schema=SCHEMA_A
|
||||
),
|
||||
),
|
||||
SCHEMA_A,
|
||||
id="json_schema_pydantic",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "n", "schema": SCHEMA_A},
|
||||
},
|
||||
SCHEMA_A,
|
||||
id="json_schema_dict",
|
||||
),
|
||||
pytest.param(
|
||||
{"type": "json_object"},
|
||||
{"type": "object"},
|
||||
id="json_object",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_response_format_cleared(
|
||||
self, parser, response_format, expected_schema
|
||||
) -> None:
|
||||
r = _make_chat_request(response_format=response_format)
|
||||
o = parser.adjust_request(r)
|
||||
assert o.response_format is None
|
||||
assert (
|
||||
_first_json_schema(o.structured_outputs.structural_tag) == expected_schema
|
||||
)
|
||||
|
||||
|
||||
class TestHasEffectiveTools:
|
||||
@pytest.mark.parametrize(
|
||||
"tools, expected",
|
||||
[
|
||||
pytest.param(None, False, id="none"),
|
||||
pytest.param([], False, id="empty_list"),
|
||||
pytest.param(" ", False, id="blank_str"),
|
||||
pytest.param(
|
||||
[{"type": "function", "function": {"name": "f"}}],
|
||||
True,
|
||||
id="non_empty_list",
|
||||
),
|
||||
pytest.param('{"x": 1}', True, id="non_empty_str"),
|
||||
],
|
||||
)
|
||||
def test_has_effective_tools(self, tools, expected) -> None:
|
||||
assert _has_effective_tools(tools) is expected
|
||||
|
||||
def test_convert_schema_json_only_with_empty_tools_list(self) -> None:
|
||||
tag = convert_schema_to_structural_tags(
|
||||
schema=SCHEMA_B,
|
||||
tools=[],
|
||||
model_architecture="Cohere2ForCausalLM",
|
||||
)
|
||||
assert tag is not None
|
||||
assert _first_json_schema(tag) == SCHEMA_B
|
||||
|
||||
|
||||
class TestAdjustRequestFoldFromStructuredOutputs:
|
||||
@pytest.mark.parametrize(
|
||||
"structured_outputs, expected_schema",
|
||||
[
|
||||
pytest.param({"json": SCHEMA_B}, SCHEMA_B, id="json_dict"),
|
||||
pytest.param({"json": json.dumps(SCHEMA_B)}, SCHEMA_B, id="json_string"),
|
||||
pytest.param(
|
||||
{"json_object": True}, {"type": "object"}, id="json_object_flag"
|
||||
),
|
||||
pytest.param(
|
||||
StructuredOutputsParams(json=SCHEMA_B),
|
||||
SCHEMA_B,
|
||||
id="structured_outputs_dataclass",
|
||||
),
|
||||
pytest.param(
|
||||
{"json": {"name": "n", "schema": SCHEMA_A}},
|
||||
SCHEMA_A,
|
||||
id="openai_wrapper_dict_unwrapped",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_structured_outputs_folded(
|
||||
self, parser, structured_outputs, expected_schema
|
||||
) -> None:
|
||||
o = parser.adjust_request(
|
||||
_make_chat_request(structured_outputs=structured_outputs),
|
||||
)
|
||||
assert (
|
||||
_first_json_schema(o.structured_outputs.structural_tag) == expected_schema
|
||||
)
|
||||
|
||||
def test_responses_request_default_empty_tools(self, parser) -> None:
|
||||
"""``ResponsesRequest.tools`` defaults to ``[]``, not ``None``."""
|
||||
r = ResponsesRequest.model_validate(
|
||||
{
|
||||
"input": "hi",
|
||||
"model": "m",
|
||||
"structured_outputs": {"json": SCHEMA_B},
|
||||
}
|
||||
)
|
||||
assert r.tools == []
|
||||
o = parser.adjust_request(r)
|
||||
assert _first_json_schema(o.structured_outputs.structural_tag) == SCHEMA_B
|
||||
|
||||
def test_json_userdict_mapping_unwrapped(self) -> None:
|
||||
inner = {"type": "object", "properties": {"u": {"type": "number"}}}
|
||||
so = StructuredOutputsParams(json=UserDict(inner))
|
||||
assert _schema_dict_from_structured_outputs(so) == inner
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"json_value, match",
|
||||
[
|
||||
pytest.param("{not json}", "valid JSON", id="invalid_json_string"),
|
||||
pytest.param(
|
||||
json.dumps(["a", "b"]), "JSON object", id="non_object_json_string"
|
||||
),
|
||||
pytest.param(" ", "empty", id="empty_json_string"),
|
||||
],
|
||||
)
|
||||
def test_structured_outputs_json_string_raises(
|
||||
self, parser, json_value, match
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match=match):
|
||||
parser.adjust_request(
|
||||
_make_chat_request(structured_outputs={"json": json_value}),
|
||||
)
|
||||
|
||||
def test_deeply_nested_json_string_rejected(self, parser) -> None:
|
||||
"""json.loads raises RecursionError on a string nested this deeply."""
|
||||
schema = '{"type": "array", "items": ' * 20_000 + "{}" + "}" * 20_000
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
parser.adjust_request(
|
||||
_make_chat_request(structured_outputs={"json": schema}),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"construct",
|
||||
[
|
||||
pytest.param(
|
||||
lambda: _make_chat_request(structured_outputs={"json": [1, 2, 3]}),
|
||||
id="chat_completion_request",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: StructuredOutputsParams(json=[1, 2, 3]),
|
||||
id="structured_outputs_params",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_json_wrong_type_raises(self, construct) -> None:
|
||||
"""Non-str / non-dict ``json`` fails at Pydantic validation."""
|
||||
with pytest.raises(ValidationError):
|
||||
construct()
|
||||
|
||||
|
||||
class TestAdjustRequestPrecedence:
|
||||
def test_response_format_over_structured_outputs_json(self, parser) -> None:
|
||||
s_rf = {"type": "object", "properties": {"rf": {"type": "string"}}}
|
||||
s_so = {"type": "object", "properties": {"so": {"type": "number"}}}
|
||||
inner = JsonSchemaResponseFormat(name="n", json_schema=s_rf)
|
||||
r = _make_chat_request(
|
||||
response_format=ResponseFormat(type="json_schema", json_schema=inner),
|
||||
structured_outputs={"json": s_so},
|
||||
)
|
||||
o = parser.adjust_request(r)
|
||||
assert _first_json_schema(o.structured_outputs.structural_tag) == s_rf
|
||||
|
||||
|
||||
class TestAdjustRequestTextPlusStructuredOutputs:
|
||||
def test_text_response_format_preserved(self, parser) -> None:
|
||||
sch = {"type": "object", "properties": {"k": {"type": "string"}}}
|
||||
r = _make_chat_request(
|
||||
response_format=ResponseFormat(type="text"),
|
||||
structured_outputs={"json": sch},
|
||||
)
|
||||
o = parser.adjust_request(r)
|
||||
assert o.response_format is not None
|
||||
assert o.response_format.type == "text"
|
||||
assert _first_json_schema(o.structured_outputs.structural_tag) == sch
|
||||
|
||||
|
||||
class TestAdjustRequestTools:
|
||||
def test_tools_only_command_a_grammar(self, parser) -> None:
|
||||
o = parser.adjust_request(
|
||||
_make_chat_request(tools=[GET_WEATHER_TOOL], tool_choice="auto"),
|
||||
)
|
||||
assert "grammar" in _content_types(o.structured_outputs.structural_tag)
|
||||
|
||||
def test_tools_plus_json_schema_both_kinds(self, parser) -> None:
|
||||
inner = JsonSchemaResponseFormat(
|
||||
name="n",
|
||||
json_schema={"type": "object", "properties": {"r": {"type": "string"}}},
|
||||
)
|
||||
r = _make_chat_request(
|
||||
response_format=ResponseFormat(type="json_schema", json_schema=inner),
|
||||
tools=[GET_WEATHER_TOOL],
|
||||
tool_choice="auto",
|
||||
)
|
||||
o = parser.adjust_request(r)
|
||||
types = _content_types(o.structured_outputs.structural_tag)
|
||||
assert "grammar" in types
|
||||
assert "json_schema" in types
|
||||
|
||||
|
||||
class TestCollectToolSchemaDefs:
|
||||
def test_nested_defs_tool_adjust_request(self, parser) -> None:
|
||||
o = parser.adjust_request(
|
||||
_make_chat_request(tools=[NESTED_DEFS_TOOL], tool_choice="auto"),
|
||||
)
|
||||
assert "grammar" in _content_types(o.structured_outputs.structural_tag)
|
||||
|
||||
@staticmethod
|
||||
def _tool_calls_json(name: str, parameters: dict) -> str:
|
||||
return json.dumps(
|
||||
[{"tool_call_id": "0", "tool_name": name, "parameters": parameters}]
|
||||
)
|
||||
|
||||
def test_recursive_defs(self) -> None:
|
||||
grammar = collect_tool_schema(
|
||||
[CohereNormalizedTool(name="report", parameters=RECURSIVE_DEFS_PARAMETERS)]
|
||||
)
|
||||
nested = {"note": "a", "related": [{"note": "b", "related": []}]}
|
||||
assert _is_grammar_accept_string(
|
||||
grammar, self._tool_calls_json("report", {"finding": nested})
|
||||
)
|
||||
bad = {"note": 1, "related": []}
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, self._tool_calls_json("report", {"finding": bad})
|
||||
)
|
||||
|
||||
def test_draft07_definitions(self) -> None:
|
||||
grammar = collect_tool_schema(
|
||||
[
|
||||
CohereNormalizedTool(
|
||||
name="get_weather", parameters=DRAFT07_DEFINITIONS_PARAMETERS
|
||||
)
|
||||
]
|
||||
)
|
||||
assert _is_grammar_accept_string(
|
||||
grammar,
|
||||
self._tool_calls_json("get_weather", {"location": {"city": "Prague"}}),
|
||||
)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar,
|
||||
self._tool_calls_json("get_weather", {"location": {"city": 1}}),
|
||||
)
|
||||
|
||||
def test_deeply_nested_parameters_rejected(self) -> None:
|
||||
"""Tool schemas are compiled here, before request validation, and end up
|
||||
inside an EBNF grammar that the nesting check cannot see later."""
|
||||
parameters = json.loads('{"type": "array", "items": ' * 200 + "{}" + "}" * 200)
|
||||
with pytest.raises(VLLMValidationError, match="nested too deeply"):
|
||||
collect_tool_schema(
|
||||
[CohereNormalizedTool(name="deep", parameters=parameters)]
|
||||
)
|
||||
|
||||
def test_input_parameters_not_mutated(self) -> None:
|
||||
params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS))
|
||||
collect_tool_schema(
|
||||
[CohereNormalizedTool(name="get_weather", parameters=params)]
|
||||
)
|
||||
assert params == NESTED_DEFS_PARAMETERS
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Replay tests for DelegatingParser with engine adapters.
|
||||
|
||||
Exercises DelegatingParser in engine-adapter mode to verify that delegated
|
||||
routing produces correct output across chunk sizes.
|
||||
See test_replay.py for tests that target engine parsers directly.
|
||||
|
||||
Parser discovery is automatic: any engine parser in ``registered_adapters``
|
||||
that has both tool and reasoning adapters and a builder in
|
||||
``trace_builder._BUILDERS`` is picked up with zero manual wiring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
CHUNK_SIZES,
|
||||
DUMMY_TOOLS,
|
||||
MockTokenizer,
|
||||
_test_request,
|
||||
assert_no_terminal_leakage,
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
parse_non_streaming,
|
||||
replay_streaming,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser, Parser
|
||||
from vllm.parser.engine import registered_adapters as _adapters_mod
|
||||
from vllm.parser.engine.adapters import (
|
||||
ParserEngineReasoningAdapter,
|
||||
ParserEngineToolAdapter,
|
||||
)
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
|
||||
|
||||
# ── Pairing discovery ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _PairingInfo(NamedTuple):
|
||||
parser_cls: type[Parser]
|
||||
name: str
|
||||
samples: tuple
|
||||
|
||||
|
||||
def _discover_pairings() -> list[_PairingInfo]:
|
||||
"""Discover valid delegating pairings from registered engine adapters.
|
||||
|
||||
Groups tool and reasoning adapters by their engine class, then builds
|
||||
a DelegatingParser subclass for each engine that has both adapters
|
||||
and a test builder.
|
||||
"""
|
||||
bare_tok = MockTokenizer(vocab={}, tokens=[])
|
||||
engines: dict[type, dict[str, type]] = {}
|
||||
for obj in vars(_adapters_mod).values():
|
||||
if not isinstance(obj, type):
|
||||
continue
|
||||
if (
|
||||
issubclass(obj, ParserEngineToolAdapter)
|
||||
and obj is not ParserEngineToolAdapter
|
||||
):
|
||||
tool_adapter: type[ParserEngineToolAdapter] = obj
|
||||
engines.setdefault(tool_adapter._parser_engine_cls, {})["tool"] = obj
|
||||
elif (
|
||||
issubclass(obj, ParserEngineReasoningAdapter)
|
||||
and obj is not ParserEngineReasoningAdapter
|
||||
):
|
||||
reasoning_adapter: type[ParserEngineReasoningAdapter] = obj
|
||||
engines.setdefault(reasoning_adapter._parser_engine_cls, {})[
|
||||
"reasoning"
|
||||
] = obj
|
||||
|
||||
found: list[_PairingInfo] = []
|
||||
missing_builders: list[str] = []
|
||||
for engine_cls, adapters in engines.items():
|
||||
if "tool" not in adapters or "reasoning" not in adapters:
|
||||
continue
|
||||
if engine_cls is MistralParser:
|
||||
# Mistral uses brace-balanced JSON tool args with no TOOL_END
|
||||
# token, so it does not fit this TOOL_END-based replay harness.
|
||||
# It is covered by tests/parser/mistral/ instead.
|
||||
continue
|
||||
cfg = engine_cls(bare_tok, None).parser_engine_config
|
||||
if cfg.name not in _BUILDERS:
|
||||
missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
|
||||
continue
|
||||
if cfg.name == "inkling":
|
||||
# Inkling uses typed structural blocks and opts out of token-id
|
||||
# terminal matching; combined-parser replay coverage lives in
|
||||
# test_inkling.py.
|
||||
continue
|
||||
if cfg.name == "granite":
|
||||
# Granite has a JSON-array tool body with no TOOL_END terminal;
|
||||
# its replay coverage lives in test_granite.py.
|
||||
continue
|
||||
|
||||
parser_cls = type(
|
||||
f"_Delegating{engine_cls.__name__}",
|
||||
(DelegatingParser,),
|
||||
{
|
||||
"reasoning_parser_cls": adapters["reasoning"],
|
||||
"tool_parser_cls": adapters["tool"],
|
||||
},
|
||||
)
|
||||
found.append(
|
||||
_PairingInfo(
|
||||
parser_cls=parser_cls,
|
||||
name=cfg.name,
|
||||
samples=build_samples(cfg.name),
|
||||
)
|
||||
)
|
||||
if missing_builders:
|
||||
raise RuntimeError(
|
||||
f"Engine adapters in registered_adapters have no test builder "
|
||||
f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
|
||||
f"Add a builder to _BUILDERS for each new parser."
|
||||
)
|
||||
found.sort(key=lambda p: p.name)
|
||||
return found
|
||||
|
||||
|
||||
_PAIRINGS = _discover_pairings()
|
||||
|
||||
_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample",
|
||||
_ALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else "",
|
||||
)
|
||||
def test_delegating_replay(parser_cls, sample, chunk_size):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
validated_tools = (
|
||||
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
|
||||
)
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
validated_tools,
|
||||
chat_template_kwargs=sample.chat_template_kwargs,
|
||||
)
|
||||
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=True,
|
||||
tools=sample.tools,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
|
||||
_TOOL_CALL_SAMPLES = [
|
||||
(p.parser_cls, p.name, s)
|
||||
for p in _PAIRINGS
|
||||
for s in p.samples
|
||||
if s.expected_tool_calls
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,parser_name,sample",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else "",
|
||||
)
|
||||
def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
|
||||
"""Non-streaming parse() with tool_choice='none' via DelegatingParser
|
||||
must not leak special tokens into content."""
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
validated_tools = (
|
||||
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
|
||||
)
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
validated_tools,
|
||||
chat_template_kwargs=sample.chat_template_kwargs,
|
||||
)
|
||||
|
||||
request = _test_request(tools=DUMMY_TOOLS)
|
||||
request.tool_choice = "none"
|
||||
|
||||
output = parse_non_streaming(parser, sample, request)
|
||||
|
||||
assert output.tool_calls == [], (
|
||||
f"Expected no tool calls but got {output.tool_calls}"
|
||||
)
|
||||
|
||||
cfg = parser._tool_parser._parser_engine.parser_engine_config
|
||||
terminals = sorted(
|
||||
v
|
||||
for v in cfg.terminal_literals | set(cfg.token_id_terminals.values())
|
||||
if len(v) > 1
|
||||
)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
terminals,
|
||||
context=f"parser={parser_name}",
|
||||
)
|
||||
@@ -1,632 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
|
||||
|
||||
Replays dynamically built token sequences at different chunk sizes and
|
||||
holdback depths to verify chunk-size invariance and terminal-token hygiene.
|
||||
|
||||
Parser discovery is automatic: any ``ParserEngine`` subclass registered in
|
||||
``registered_adapters`` that also has a builder in ``trace_builder._BUILDERS``
|
||||
is picked up with zero manual wiring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
DUMMY_TOOLS,
|
||||
MockTokenizer,
|
||||
Sample,
|
||||
_test_request,
|
||||
assert_no_terminal_leakage,
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
parse_non_streaming,
|
||||
replay_streaming,
|
||||
replay_with_text_holdback,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
|
||||
from vllm.parser.engine import registered_adapters as _adapters_mod
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import ParserState
|
||||
from vllm.parser.mistral import MistralParser
|
||||
|
||||
# ── Parser discovery ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _ParserInfo(NamedTuple):
|
||||
parser_cls: type[ParserEngine]
|
||||
name: str
|
||||
samples: tuple
|
||||
terminals: list[str]
|
||||
tool_end: str
|
||||
think_end: str
|
||||
tool_start: str
|
||||
|
||||
|
||||
def _discover_parsers() -> list[_ParserInfo]:
|
||||
"""Discover engine parsers from registered_adapters that have test builders.
|
||||
|
||||
Returns one ``_ParserInfo`` per parser, sorted by config name.
|
||||
Raises ``RuntimeError`` if any registered parser lacks a builder.
|
||||
"""
|
||||
bare_tok = MockTokenizer(vocab={}, tokens=[])
|
||||
found: list[_ParserInfo] = []
|
||||
missing_builders: list[str] = []
|
||||
for obj in vars(_adapters_mod).values():
|
||||
if not (
|
||||
isinstance(obj, type)
|
||||
and issubclass(obj, ParserEngine)
|
||||
and obj is not ParserEngine
|
||||
):
|
||||
continue
|
||||
if obj is MistralParser:
|
||||
# Mistral uses brace-balanced JSON tool args with no TOOL_END
|
||||
# token, so it does not fit this TOOL_END-based replay harness.
|
||||
# It is covered by tests/parser/mistral/ instead.
|
||||
continue
|
||||
cfg = obj(bare_tok, None).parser_engine_config
|
||||
if cfg.name not in _BUILDERS:
|
||||
missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
|
||||
continue
|
||||
if cfg.name in ("inkling", "plamo3", "minimax_m3"):
|
||||
# Inkling uses typed structural blocks. PLaMo and MiniMax M3
|
||||
# markers can span several token IDs. None fits this TOOL_END-token
|
||||
# harness; their direct and delegating coverage lives in dedicated
|
||||
# tests.
|
||||
continue
|
||||
if cfg.name == "granite":
|
||||
# Granite has a JSON-array tool body with no TOOL_END terminal, so
|
||||
# it does not fit this token-terminal harness; its replay coverage
|
||||
# lives in test_granite.py.
|
||||
continue
|
||||
tool_end = cfg.token_id_terminals.get("TOOL_END")
|
||||
if not tool_end:
|
||||
raise RuntimeError(
|
||||
f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals"
|
||||
)
|
||||
all_vals = cfg.terminal_literals | set(cfg.token_id_terminals.values())
|
||||
found.append(
|
||||
_ParserInfo(
|
||||
parser_cls=obj,
|
||||
name=cfg.name,
|
||||
samples=build_samples(cfg.name),
|
||||
terminals=sorted(v for v in all_vals if len(v) > 1),
|
||||
tool_end=tool_end,
|
||||
think_end=cfg.terminal_literal("THINK_END") or "",
|
||||
tool_start=(
|
||||
cfg.terminal_literal("TOOL_SECTION_START")
|
||||
if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions
|
||||
else cfg.terminal_literal("TOOL_START")
|
||||
)
|
||||
or "",
|
||||
)
|
||||
)
|
||||
if missing_builders:
|
||||
raise RuntimeError(
|
||||
f"Engine parsers in registered_adapters have no test builder "
|
||||
f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
|
||||
f"Add a builder to _BUILDERS for each new parser."
|
||||
)
|
||||
found.sort(key=lambda p: p.name)
|
||||
return found
|
||||
|
||||
|
||||
_PARSERS = _discover_parsers()
|
||||
|
||||
|
||||
def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra):
|
||||
kwargs = dict(extra)
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
return parser_cls(tokenizer, sample.tools, **kwargs)
|
||||
|
||||
|
||||
_ENGINE_PARSERS: dict[str, type[ParserEngine]] = {
|
||||
f"{p.name}_engine": p.parser_cls for p in _PARSERS
|
||||
}
|
||||
|
||||
# ── Parametrize sample lists ─────────────────────────────────────────
|
||||
|
||||
HOLDBACK_CONFIGS = [6, 12, 24]
|
||||
|
||||
_REPLAY_SAMPLES = [(p.parser_cls, s, p.terminals) for p in _PARSERS for s in p.samples]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
|
||||
@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,terminals",
|
||||
_REPLAY_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else "",
|
||||
)
|
||||
class TestReplayWithHoldback:
|
||||
"""Replay all parsers with simulated detokenizer holdback."""
|
||||
|
||||
def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = _make_parser(parser_cls, tokenizer, sample)
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
holdback_chars=holdback,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert_parse_output(output, sample)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
terminals,
|
||||
context=f"chunk_size={chunk_size}, holdback={holdback}",
|
||||
)
|
||||
|
||||
|
||||
TEXT_HOLDBACK_DELAYS = [1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,terminals",
|
||||
_REPLAY_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else "",
|
||||
)
|
||||
class TestTextHoldback:
|
||||
"""Replay with production-like text/token-ID misalignment.
|
||||
|
||||
In production the detokenizer sends token IDs immediately but holds
|
||||
back text by N tokens. This exercises the TokenIDScanner deferred
|
||||
terminal path that aligned-holdback tests do not cover.
|
||||
"""
|
||||
|
||||
def test_replay(self, parser_cls, sample, terminals, delay):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = _make_parser(parser_cls, tokenizer, sample)
|
||||
deltas = replay_with_text_holdback(
|
||||
parser,
|
||||
sample.tokens,
|
||||
text_delay=delay,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert_parse_output(output, sample)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
terminals,
|
||||
context=f"text_delay={delay}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"chunk_size", [1, 2, 3, 5, 10, 19, 20, None], ids=lambda c: f"chunk{c}"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,terminals",
|
||||
_REPLAY_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else "",
|
||||
)
|
||||
class TestReplay:
|
||||
"""Replay all parsers at varied chunk sizes without holdback."""
|
||||
|
||||
def test_replay(self, parser_cls, sample, terminals, chunk_size):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = _make_parser(parser_cls, tokenizer, sample)
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert_parse_output(output, sample)
|
||||
assert_no_terminal_leakage(output, terminals)
|
||||
|
||||
|
||||
_DEFERRAL_SAMPLES = [
|
||||
(p.parser_cls, s, p.tool_end)
|
||||
for p in _PARSERS
|
||||
for s in p.samples
|
||||
if s.expected_tool_calls
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,tool_end_text",
|
||||
_DEFERRAL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestDeferralFinish:
|
||||
"""Test that parse_delta(finished=True) resolves deferred scanner state.
|
||||
|
||||
Simulates a production failure where delta_text is missing the
|
||||
tool-call-end text but delta_token_ids has the token, causing the
|
||||
scanner to defer it. Without finish(), the deferred state is lost
|
||||
and tool call arguments are empty.
|
||||
"""
|
||||
|
||||
def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = _make_parser(parser_cls, tokenizer, sample)
|
||||
|
||||
request = _test_request()
|
||||
|
||||
all_ids = [tid for tid, _ in sample.tokens]
|
||||
all_texts = [text for _, text in sample.tokens]
|
||||
|
||||
tool_end_id = sample.vocab.get(tool_end_text)
|
||||
split_idx = None
|
||||
for i in range(len(all_ids) - 1, -1, -1):
|
||||
if all_ids[i] == tool_end_id:
|
||||
split_idx = i
|
||||
break
|
||||
|
||||
if split_idx is None:
|
||||
pytest.skip(f"no {tool_end_text} token found")
|
||||
|
||||
first_ids = all_ids[:split_idx]
|
||||
first_text = "".join(all_texts[:split_idx])
|
||||
|
||||
last_ids = all_ids[split_idx:]
|
||||
last_text_missing = "".join(all_texts[split_idx:]).replace(tool_end_text, "")
|
||||
|
||||
result1 = parser.parse_delta(
|
||||
first_text,
|
||||
first_ids,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
finished=False,
|
||||
)
|
||||
result2 = parser.parse_delta(
|
||||
last_text_missing, last_ids, request, finished=True
|
||||
)
|
||||
|
||||
output = collect_output([result1, result2])
|
||||
|
||||
tool_calls_only = dataclasses.replace(
|
||||
sample, expected_reasoning=None, expected_content=None
|
||||
)
|
||||
assert_parse_output(output, tool_calls_only)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample",
|
||||
[(p.parser_cls, p.samples[0]) for p in _PARSERS],
|
||||
ids=[p.name for p in _PARSERS],
|
||||
)
|
||||
class TestParserEngineAdjustRequest:
|
||||
"""Verify ParserEngine and its adapters set skip_special_tokens=False."""
|
||||
|
||||
def test_adjust_request_disables_skip_special_tokens(self, parser_cls, sample):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = parser_cls(tokenizer, sample.tools)
|
||||
request = _test_request()
|
||||
assert request.skip_special_tokens is True
|
||||
adjusted = parser.adjust_request(request)
|
||||
assert adjusted.skip_special_tokens is False
|
||||
|
||||
|
||||
_TOOL_CALL_SAMPLES = [
|
||||
(p.parser_cls, s, p.think_end, p.tool_start)
|
||||
for p in _PARSERS
|
||||
for s in p.samples
|
||||
if s.expected_tool_calls
|
||||
and s.expected_reasoning
|
||||
# Inkling's typed-block format (structural role/kind tokens, shared
|
||||
# block-end token) doesn't fit the generic reasoning/tool split
|
||||
# below; its filtering modes are covered in test_inkling.py instead.
|
||||
and p.name != "inkling"
|
||||
]
|
||||
|
||||
|
||||
def _tool_suppression_expectations(
|
||||
sample, think_end: str, tool_start: str, *, include_tool_block: bool
|
||||
) -> tuple[str, str]:
|
||||
reasoning, content = _raw_tool_suppression_expectations(
|
||||
sample, think_end, tool_start, include_tool_block=include_tool_block
|
||||
)
|
||||
if sample.content_lstrip:
|
||||
content = content.lstrip(sample.content_lstrip)
|
||||
return reasoning, content
|
||||
|
||||
|
||||
def _raw_tool_suppression_expectations(
|
||||
sample, think_end: str, tool_start: str, *, include_tool_block: bool
|
||||
) -> tuple[str, str]:
|
||||
"""Expected (reasoning, content) from the raw token text.
|
||||
|
||||
With ``include_tool_block=True`` (skip_tool_parsing / reasoning
|
||||
adapter first pass), tool terminal text is preserved as content so
|
||||
a second-pass parser can see it.
|
||||
|
||||
With ``include_tool_block=False`` (_suppress_tool_calls /
|
||||
tool_choice='none'), the state machine consumes tool blocks and
|
||||
only non-tool content survives.
|
||||
"""
|
||||
full_text = "".join(text for _, text in sample.tokens)
|
||||
reasoning = sample.expected_reasoning
|
||||
idx = full_text.find(reasoning)
|
||||
if idx < 0:
|
||||
return (full_text, "")
|
||||
after_reasoning = full_text[idx + len(reasoning) :]
|
||||
if think_end:
|
||||
pos = after_reasoning.find(think_end)
|
||||
if pos >= 0:
|
||||
if include_tool_block:
|
||||
return (reasoning, after_reasoning[pos + len(think_end) :])
|
||||
after_reasoning = after_reasoning[pos + len(think_end) :]
|
||||
if tool_start:
|
||||
pos = after_reasoning.find(tool_start)
|
||||
if pos >= 0:
|
||||
if include_tool_block:
|
||||
return (reasoning, after_reasoning[pos:])
|
||||
return (reasoning, after_reasoning[:pos])
|
||||
if include_tool_block:
|
||||
return (full_text, "")
|
||||
return (reasoning, after_reasoning)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
["skip_tool_parsing", "suppress_tool_calls"],
|
||||
ids=["skip_tool_parsing", "suppress_tool_calls"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,think_end,tool_start",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestToolCallFilteringReplay:
|
||||
"""Replay with tool calls not extracted, in both filtering modes.
|
||||
|
||||
``skip_tool_parsing`` (reasoning adapter first pass): tool terminal
|
||||
text is preserved as content for a second-pass tool parser.
|
||||
|
||||
``suppress_tool_calls`` (tool_choice='none'): tool call blocks are
|
||||
consumed by the state machine and do not leak into content.
|
||||
"""
|
||||
|
||||
def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
parser = parser_cls(tokenizer, **kwargs)
|
||||
|
||||
request = _test_request()
|
||||
request.tools = DUMMY_TOOLS
|
||||
if mode == "skip_tool_parsing":
|
||||
parser.skip_tool_parsing = True
|
||||
else:
|
||||
request.tool_choice = "none"
|
||||
|
||||
all_ids = [tid for tid, _ in sample.tokens]
|
||||
all_texts = [text for _, text in sample.tokens]
|
||||
if chunk_size is None:
|
||||
chunk_size = len(all_ids)
|
||||
|
||||
results = []
|
||||
chunks = list(range(0, len(all_ids), chunk_size))
|
||||
for i, start in enumerate(chunks):
|
||||
end = min(start + chunk_size, len(all_ids))
|
||||
is_last = i == len(chunks) - 1
|
||||
result = parser.parse_delta(
|
||||
"".join(all_texts[start:end]),
|
||||
all_ids[start:end],
|
||||
request,
|
||||
prompt_token_ids=(sample.prompt_token_ids or [])
|
||||
if start == 0
|
||||
else None,
|
||||
finished=is_last,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
output = collect_output(results)
|
||||
|
||||
include_block = mode == "skip_tool_parsing"
|
||||
expected_reasoning, expected_content = _tool_suppression_expectations(
|
||||
sample, think_end, tool_start, include_tool_block=include_block
|
||||
)
|
||||
|
||||
assert output.reasoning == expected_reasoning, (
|
||||
f"Reasoning mismatch (mode={mode}):\n"
|
||||
f" expected: {expected_reasoning!r}\n"
|
||||
f" actual: {output.reasoning!r}"
|
||||
)
|
||||
assert output.tool_calls == [], (
|
||||
f"Expected no tool calls (mode={mode}) but got {output.tool_calls}"
|
||||
)
|
||||
assert output.content == expected_content, (
|
||||
f"Content mismatch (mode={mode}):\n"
|
||||
f" expected: {expected_content!r}\n"
|
||||
f" actual: {output.content!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,think_end,tool_start",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestToolCallFilteringNonStreaming:
|
||||
"""Non-streaming parse() with tool_choice='none' must suppress tool
|
||||
calls and not leak special tokens into content."""
|
||||
|
||||
def test_parse(self, parser_cls, sample, think_end, tool_start):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
parser = parser_cls(tokenizer, **kwargs)
|
||||
|
||||
request = _test_request()
|
||||
request.tools = DUMMY_TOOLS
|
||||
request.tool_choice = "none"
|
||||
|
||||
output = parse_non_streaming(parser, sample, request)
|
||||
|
||||
expected_reasoning, expected_content = _tool_suppression_expectations(
|
||||
sample, think_end, tool_start, include_tool_block=False
|
||||
)
|
||||
assert output.reasoning == expected_reasoning, (
|
||||
f"Reasoning mismatch:\n"
|
||||
f" expected: {expected_reasoning!r}\n"
|
||||
f" actual: {output.reasoning!r}"
|
||||
)
|
||||
assert output.tool_calls == [], (
|
||||
f"Expected no tool calls but got {output.tool_calls}"
|
||||
)
|
||||
assert output.content == expected_content, (
|
||||
f"Content mismatch:\n"
|
||||
f" expected: {expected_content!r}\n"
|
||||
f" actual: {output.content!r}"
|
||||
)
|
||||
|
||||
|
||||
_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample",
|
||||
_WS_TOOL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestToolChoiceNoneStreamingParity:
|
||||
"""Streaming and non-streaming must return the same content
|
||||
when tool_choice='none' suppresses tool calls."""
|
||||
|
||||
def test_content_matches(self, parser_cls, sample):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
request = _test_request()
|
||||
request.tools = DUMMY_TOOLS
|
||||
request.tool_choice = "none"
|
||||
|
||||
ns_output = parse_non_streaming(
|
||||
parser_cls(tokenizer, **kwargs),
|
||||
sample,
|
||||
request,
|
||||
)
|
||||
|
||||
s_parser = parser_cls(tokenizer, **kwargs)
|
||||
results = []
|
||||
for i, (tid, text) in enumerate(sample.tokens):
|
||||
is_last = i == len(sample.tokens) - 1
|
||||
results.append(
|
||||
s_parser.parse_delta(
|
||||
text,
|
||||
[tid],
|
||||
request,
|
||||
prompt_token_ids=(sample.prompt_token_ids or [])
|
||||
if i == 0
|
||||
else None,
|
||||
finished=is_last,
|
||||
)
|
||||
)
|
||||
s_output = collect_output(results)
|
||||
|
||||
assert ns_output.content == s_output.content, (
|
||||
f"Streaming/non-streaming content mismatch:\n"
|
||||
f" streaming: {s_output.content!r}\n"
|
||||
f" non-streaming: {ns_output.content!r}"
|
||||
)
|
||||
|
||||
|
||||
_DROP_TOKENS = {"<bos>": 99990, "<eos>": 99991}
|
||||
|
||||
|
||||
def _inject_drop_tokens(sample):
|
||||
"""Insert <bos> at stream start and <eos> between the first two tokens."""
|
||||
new_vocab = {**sample.vocab, **_DROP_TOKENS}
|
||||
tokens = list(sample.tokens)
|
||||
tokens.insert(0, (99990, "<bos>"))
|
||||
if len(tokens) >= 3:
|
||||
tokens.insert(2, (99991, "<eos>"))
|
||||
else:
|
||||
tokens.append((99991, "<eos>"))
|
||||
return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens)
|
||||
|
||||
|
||||
class TestDropTokenReplay:
|
||||
"""Verify unconfigured special tokens are silently dropped across
|
||||
all parsers and chunk sizes."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_info",
|
||||
_PARSERS,
|
||||
ids=[p.name for p in _PARSERS],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, None])
|
||||
def test_drop_tokens_removed_from_output(self, parser_info, chunk_size):
|
||||
for sample in parser_info.samples:
|
||||
injected = _inject_drop_tokens(sample)
|
||||
tokenizer = make_mock_tokenizer(injected)
|
||||
parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
|
||||
|
||||
results = replay_streaming(
|
||||
parser,
|
||||
injected.tokens,
|
||||
chunk_size=chunk_size,
|
||||
tools=sample.tools,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(results)
|
||||
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
list(_DROP_TOKENS.keys()),
|
||||
context=f"parser={parser_info.name}, chunk={chunk_size}",
|
||||
)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
|
||||
class TestDropTokenNonStreaming:
|
||||
"""Non-streaming parse() must also strip unconfigured special tokens."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_info",
|
||||
_PARSERS,
|
||||
ids=[p.name for p in _PARSERS],
|
||||
)
|
||||
def test_drop_tokens_removed_from_output(self, parser_info):
|
||||
for sample in parser_info.samples:
|
||||
injected = _inject_drop_tokens(sample)
|
||||
tokenizer = make_mock_tokenizer(injected)
|
||||
parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
|
||||
|
||||
request = _test_request(tools=sample.tools)
|
||||
output = parse_non_streaming(parser, injected, request)
|
||||
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
list(_DROP_TOKENS.keys()),
|
||||
context=f"parser={parser_info.name}",
|
||||
)
|
||||
|
||||
|
||||
class TestAdapterReferences:
|
||||
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
|
||||
parser classes so the serving layer finds them and calls adjust_request."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_name",
|
||||
list(_ENGINE_PARSERS.keys()),
|
||||
)
|
||||
def test_adapter_cls_refs_set(self, parser_name):
|
||||
parser_cls = _ENGINE_PARSERS[parser_name]
|
||||
assert parser_cls.reasoning_parser_cls is not None, (
|
||||
f"{parser_name}: reasoning_parser_cls is None"
|
||||
)
|
||||
assert parser_cls.tool_parser_cls is not None, (
|
||||
f"{parser_name}: tool_parser_cls is None"
|
||||
)
|
||||
@@ -23,8 +23,6 @@ from tests.parser.engine.replay_harness import (
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Glm47MoeParserReasoningAdapter,
|
||||
Glm47MoeParserToolAdapter,
|
||||
Qwen3ParserReasoningAdapter,
|
||||
Qwen3ParserToolAdapter,
|
||||
)
|
||||
@@ -62,11 +60,6 @@ class ByteFallbackMockTokenizer(MockTokenizer):
|
||||
# ── Model-specific DelegatingParser subclasses ───────────────────────
|
||||
|
||||
|
||||
class _Glm47Delegating(DelegatingParser):
|
||||
reasoning_parser_cls = Glm47MoeParserReasoningAdapter
|
||||
tool_parser_cls = Glm47MoeParserToolAdapter
|
||||
|
||||
|
||||
class _Qwen3Delegating(DelegatingParser):
|
||||
reasoning_parser_cls = Qwen3ParserReasoningAdapter
|
||||
tool_parser_cls = Qwen3ParserToolAdapter
|
||||
@@ -92,20 +85,6 @@ EXPECTED_REASONING = "Let me think about Samsung."
|
||||
EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다."
|
||||
|
||||
_MODEL_CONFIGS = [
|
||||
pytest.param(
|
||||
{
|
||||
"<think>": 50,
|
||||
"</think>": 51,
|
||||
"<tool_call>": 60,
|
||||
"</tool_call>": 61,
|
||||
"<arg_key>": 62,
|
||||
"</arg_key>": 63,
|
||||
"<arg_value>": 64,
|
||||
"</arg_value>": 65,
|
||||
},
|
||||
_Glm47Delegating,
|
||||
id="glm47",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"<think>": 50,
|
||||
@@ -165,7 +144,7 @@ class TestUfffdReasoningTransition:
|
||||
vocab = dict(_MODEL_CONFIGS[0].values[0])
|
||||
|
||||
tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids)
|
||||
parser = _Glm47Delegating(tokenizer)
|
||||
parser = _Qwen3Delegating(tokenizer)
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,224 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, KVTransferConfig, VllmConfig
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls, vllm_config: VllmConfig
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
state_dtype = {
|
||||
"float16": torch.float16,
|
||||
"bfloat16": torch.bfloat16,
|
||||
}.get(vllm_config.cache_config.mamba_ssm_cache_dtype, torch.float32)
|
||||
return torch.float32, state_dtype
|
||||
|
||||
|
||||
class _FakeRegistry:
|
||||
@staticmethod
|
||||
def resolve_model_cls(
|
||||
architecture: str,
|
||||
model_config: SimpleNamespace,
|
||||
) -> tuple[type[_FakeModel], str]:
|
||||
return _FakeModel, architecture
|
||||
|
||||
|
||||
def _cpu_config(
|
||||
cache_config: CacheConfig,
|
||||
*,
|
||||
model_type: str,
|
||||
resolved_dtype: str,
|
||||
architecture: str | None = None,
|
||||
layer_types: tuple[str, ...] = ("linear_attention",),
|
||||
) -> SimpleNamespace:
|
||||
model_config = SimpleNamespace(
|
||||
architecture=architecture or model_type,
|
||||
has_inner_state=True,
|
||||
use_mla=False,
|
||||
hf_config=SimpleNamespace(model_type=model_type),
|
||||
hf_text_config=SimpleNamespace(
|
||||
model_type=model_type,
|
||||
layer_types=layer_types,
|
||||
),
|
||||
registry=_FakeRegistry,
|
||||
)
|
||||
config = VllmConfig(cache_config=cache_config)
|
||||
config.model_config = model_config
|
||||
cache_config.mamba_ssm_cache_dtype = resolved_dtype
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"model_type",
|
||||
"requested_dtype",
|
||||
"resolved_dtype",
|
||||
"layer_types",
|
||||
"expected_dtype",
|
||||
),
|
||||
[
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"bfloat16",
|
||||
"bfloat16",
|
||||
("linear_attention",),
|
||||
"bfloat16",
|
||||
id="gdn-explicit-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"float16",
|
||||
"float16",
|
||||
("linear_attention",),
|
||||
"float16",
|
||||
id="gdn-explicit-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"auto",
|
||||
"bfloat16",
|
||||
("linear_attention",),
|
||||
"bfloat16",
|
||||
id="gdn-model-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"qwen3_5",
|
||||
"auto",
|
||||
"float16",
|
||||
("linear_attention",),
|
||||
"float16",
|
||||
id="gdn-model-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"auto",
|
||||
"bfloat16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-model-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"auto",
|
||||
"float16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-model-fp16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"bfloat16",
|
||||
"bfloat16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-explicit-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
"nemotron_h",
|
||||
"float16",
|
||||
"float16",
|
||||
("mamba",),
|
||||
"float32",
|
||||
id="unsupported-explicit-fp16",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_cpu_accelerated_gdn_dtype_policy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_type: str,
|
||||
requested_dtype: str,
|
||||
resolved_dtype: str,
|
||||
layer_types: tuple[str, ...],
|
||||
expected_dtype: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr("torch.cpu._is_avx512_bf16_supported", lambda: True)
|
||||
cache_config = CacheConfig(mamba_ssm_cache_dtype=requested_dtype)
|
||||
config = _cpu_config(
|
||||
cache_config,
|
||||
model_type=model_type,
|
||||
resolved_dtype=resolved_dtype,
|
||||
layer_types=layer_types,
|
||||
)
|
||||
|
||||
CpuPlatform.check_and_update_config(config)
|
||||
assert cache_config.mamba_ssm_cache_dtype == expected_dtype
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"connector",
|
||||
"extra_config",
|
||||
"explicit_layout",
|
||||
"avx512_bf16_supported",
|
||||
"expected_layout",
|
||||
),
|
||||
[
|
||||
pytest.param("NixlConnector", None, None, True, "DS", id="nixl"),
|
||||
pytest.param("NixlPullConnector", None, None, True, "DS", id="nixl-pull"),
|
||||
pytest.param("NixlPushConnector", None, None, True, "DS", id="nixl-push"),
|
||||
pytest.param(
|
||||
"MultiConnector",
|
||||
{
|
||||
"connectors": [
|
||||
{
|
||||
"kv_connector": "ExampleConnector",
|
||||
"kv_connector_extra_config": {},
|
||||
},
|
||||
{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_connector_extra_config": {},
|
||||
},
|
||||
]
|
||||
},
|
||||
None,
|
||||
True,
|
||||
"DS",
|
||||
id="multi-nixl",
|
||||
),
|
||||
pytest.param("NixlConnector", None, "SD", True, "SD", id="explicit-override"),
|
||||
pytest.param("OffloadingConnector", None, None, True, "SD", id="non-nixl"),
|
||||
pytest.param(
|
||||
"OffloadingConnector", None, None, False, None, id="non-nixl-no-avx"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_cpu_conv_state_layout_selection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
connector: str,
|
||||
extra_config: dict | None,
|
||||
explicit_layout: str | None,
|
||||
avx512_bf16_supported: bool,
|
||||
expected_layout: str | None,
|
||||
) -> None:
|
||||
layout_env = "VLLM_SSM_CONV_STATE_LAYOUT"
|
||||
monkeypatch.setattr(
|
||||
"torch.cpu._is_avx512_bf16_supported",
|
||||
lambda: avx512_bf16_supported,
|
||||
)
|
||||
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector=connector,
|
||||
kv_connector_extra_config=extra_config or {},
|
||||
kv_role="kv_both",
|
||||
)
|
||||
config = _cpu_config(
|
||||
CacheConfig(mamba_ssm_cache_dtype="float32"),
|
||||
model_type="qwen3_5",
|
||||
resolved_dtype="float32",
|
||||
)
|
||||
config.kv_transfer_config = kv_transfer_config
|
||||
monkeypatch.delenv(layout_env, raising=False)
|
||||
if explicit_layout is not None:
|
||||
monkeypatch.setenv(layout_env, explicit_layout)
|
||||
|
||||
CpuPlatform.check_and_update_config(config)
|
||||
assert os.environ.get(layout_env) == expected_layout
|
||||
@@ -1,436 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerBase
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
|
||||
|
||||
# Create a concrete test implementation of BaseThinkingReasoningParser
|
||||
class TestThinkingReasoningParser(BaseThinkingReasoningParser):
|
||||
"""Test implementation of BaseThinkingReasoningParser."""
|
||||
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<test:think>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "</test:think>"
|
||||
|
||||
|
||||
class TestThinkingReasoningParserAlt(BaseThinkingReasoningParser):
|
||||
"""Alternative test implementation with different tokens."""
|
||||
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<alt:start>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "<alt:end>"
|
||||
|
||||
|
||||
# Use a test model
|
||||
REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_tokenizer() -> PreTrainedTokenizerBase:
|
||||
tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(
|
||||
REASONING_MODEL_NAME
|
||||
)
|
||||
# Add custom test tokens
|
||||
test_tokens = ["<test:think>", "</test:think>", "<alt:start>", "<alt:end>"]
|
||||
existing_tokens = set(tokenizer.get_vocab().keys())
|
||||
new_tokens = [token for token in test_tokens if token not in existing_tokens]
|
||||
if new_tokens:
|
||||
tokenizer.add_tokens(new_tokens)
|
||||
return tokenizer
|
||||
|
||||
|
||||
class TestBaseThinkingReasoningParserInit:
|
||||
"""Test initialization and basic properties of
|
||||
BaseThinkingReasoningParser.
|
||||
"""
|
||||
|
||||
def test_successful_initialization(self, test_tokenizer):
|
||||
"""Test successful initialization with valid tokens."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
assert parser.start_token == "<test:think>"
|
||||
assert parser.end_token == "</test:think>"
|
||||
assert parser.start_token_id is not None
|
||||
assert parser.end_token_id is not None
|
||||
|
||||
def test_initialization_with_missing_tokenizer(self):
|
||||
"""Test that initialization fails without tokenizer."""
|
||||
with pytest.raises(ValueError, match="model tokenizer must be passed"):
|
||||
TestThinkingReasoningParser(None)
|
||||
|
||||
def test_initialization_with_missing_tokens(self, test_tokenizer):
|
||||
"""Test that initialization fails when tokens are not in vocabulary."""
|
||||
|
||||
# Create a parser with tokens not in vocabulary
|
||||
class MissingTokenParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<missing:start>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "<missing:end>"
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError, match="could not locate think start/end tokens"
|
||||
):
|
||||
MissingTokenParser(test_tokenizer)
|
||||
|
||||
def test_initialization_with_empty_tokens(self, test_tokenizer):
|
||||
"""Test that initialization fails with empty token strings."""
|
||||
|
||||
class EmptyTokenParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return ""
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return ""
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="start_token and end_token must be defined"
|
||||
):
|
||||
EmptyTokenParser(test_tokenizer)
|
||||
|
||||
|
||||
class TestBaseThinkingReasoningParserMethods:
|
||||
"""Test the methods of BaseThinkingReasoningParser."""
|
||||
|
||||
def test_is_reasoning_end(self, test_tokenizer):
|
||||
"""Test the is_reasoning_end method."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
end_token_id = parser.end_token_id
|
||||
start_token_id = parser.start_token_id
|
||||
# Test with end token present
|
||||
assert parser.is_reasoning_end([1, 2, end_token_id, 4]) is True
|
||||
|
||||
# Test without end token
|
||||
assert parser.is_reasoning_end([1, 2, 3, 4]) is False
|
||||
|
||||
# Test with empty list
|
||||
assert parser.is_reasoning_end([]) is False
|
||||
|
||||
# Test with interleaved thinking
|
||||
assert parser.is_reasoning_end([1, start_token_id, 2, end_token_id]) is True
|
||||
assert parser.is_reasoning_end([1, start_token_id, 2, 3]) is False
|
||||
assert (
|
||||
parser.is_reasoning_end(
|
||||
[1, start_token_id, 2, end_token_id, 2, 2, start_token_id]
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
def test_is_reasoning_end_streaming(self, test_tokenizer):
|
||||
"""Test the is_reasoning_end_streaming method."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
end_token_id = parser.end_token_id
|
||||
start_token_id = parser.start_token_id
|
||||
|
||||
assert (
|
||||
parser.is_reasoning_end_streaming([1, 2, end_token_id], [end_token_id])
|
||||
is True
|
||||
)
|
||||
assert parser.is_reasoning_end_streaming([1, 2, 3, 4], [4]) is False
|
||||
assert parser.is_reasoning_end_streaming([], []) is False
|
||||
assert (
|
||||
parser.is_reasoning_end_streaming(
|
||||
[1, start_token_id, 2, end_token_id], [end_token_id]
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
parser.is_reasoning_end_streaming([1, start_token_id, 2, 3], [3]) is False
|
||||
)
|
||||
assert (
|
||||
parser.is_reasoning_end_streaming(
|
||||
[1, start_token_id, 2, end_token_id, 2, start_token_id, 2],
|
||||
[2],
|
||||
)
|
||||
is False
|
||||
)
|
||||
assert (
|
||||
parser.is_reasoning_end_streaming(
|
||||
[1, start_token_id, 2, end_token_id, 2, 2], [2]
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
def test_count_reasoning_tokens(self, test_tokenizer):
|
||||
"""Count tokens between start/end markers."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
start = parser.start_token_id
|
||||
end = parser.end_token_id
|
||||
token_ids = [0, start, 11, 12, end, 99]
|
||||
assert parser.count_reasoning_tokens(token_ids) == 2
|
||||
|
||||
def test_count_reasoning_tokens_nested(self, test_tokenizer):
|
||||
"""Ensure nested thinking spans count all inner tokens safely."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
s = parser.start_token_id
|
||||
e = parser.end_token_id
|
||||
token_ids = [s, 1, s, 2, e, 3, e]
|
||||
# Tokens 1,2,3 are inside reasoning (depth>0) => 3 tokens
|
||||
assert parser.count_reasoning_tokens(token_ids) == 3
|
||||
|
||||
def test_extract_content_ids(self, test_tokenizer):
|
||||
"""Test the extract_content_ids method."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
end_token_id = parser.end_token_id
|
||||
|
||||
# Test with end token in the middle
|
||||
input_ids = [1, 2, end_token_id, 4, 5]
|
||||
content_ids = parser.extract_content_ids(input_ids)
|
||||
assert content_ids == [4, 5]
|
||||
|
||||
# Test with end token at the end
|
||||
input_ids = [1, 2, 3, end_token_id]
|
||||
content_ids = parser.extract_content_ids(input_ids)
|
||||
assert content_ids == []
|
||||
|
||||
# Test without end token
|
||||
input_ids = [1, 2, 3, 4]
|
||||
content_ids = parser.extract_content_ids(input_ids)
|
||||
assert content_ids == []
|
||||
|
||||
# Test with end token as last element (should not extract)
|
||||
input_ids = [1, 2, 3, end_token_id]
|
||||
content_ids = parser.extract_content_ids(input_ids)
|
||||
assert content_ids == []
|
||||
|
||||
|
||||
class TestBaseThinkingReasoningParserExtraction:
|
||||
"""Test reasoning content extraction methods."""
|
||||
|
||||
def test_extract_reasoning_with_both_tokens(self, test_tokenizer):
|
||||
"""Test extraction when both start and end tokens are present."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
model_output = "<test:think>This is reasoning</test:think>This is content"
|
||||
reasoning, content = parser.extract_reasoning(model_output, request)
|
||||
|
||||
assert reasoning == "This is reasoning"
|
||||
assert content == "This is content"
|
||||
|
||||
def test_extract_reasoning_only_end_token(self, test_tokenizer):
|
||||
"""Test extraction when only end token is present."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
model_output = "This is reasoning</test:think>This is content"
|
||||
reasoning, content = parser.extract_reasoning(model_output, request)
|
||||
|
||||
assert reasoning == "This is reasoning"
|
||||
assert content == "This is content"
|
||||
|
||||
def test_extract_reasoning_no_end_token(self, test_tokenizer):
|
||||
"""Test extraction when no end token is present."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
model_output = "This is just content"
|
||||
reasoning, content = parser.extract_reasoning(model_output, request)
|
||||
|
||||
assert reasoning == "This is just content"
|
||||
assert content is None
|
||||
|
||||
def test_extract_reasoning_empty_output(self, test_tokenizer):
|
||||
"""Test extraction with empty output."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
model_output = ""
|
||||
reasoning, content = parser.extract_reasoning(model_output, request)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content is None
|
||||
|
||||
def test_extract_reasoning_only_tokens(self, test_tokenizer):
|
||||
"""Test extraction with only tokens and no content."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
model_output = "<test:think></test:think>"
|
||||
reasoning, content = parser.extract_reasoning(model_output, request)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content is None
|
||||
|
||||
|
||||
class TestBaseThinkingReasoningParserStreaming:
|
||||
"""Test streaming functionality of BaseThinkingReasoningParser."""
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_simple_reasoning_extraction(self, test_tokenizer, streaming):
|
||||
"""Test basic reasoning extraction in both
|
||||
streaming and non-streaming modes.
|
||||
"""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
model_output = [
|
||||
"<test:think>",
|
||||
"Some ",
|
||||
"reasoning ",
|
||||
"content",
|
||||
"</test:think>",
|
||||
"Final ",
|
||||
"answer",
|
||||
]
|
||||
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert reasoning == "Some reasoning content"
|
||||
assert content == "Final answer"
|
||||
|
||||
def test_streaming_with_incremental_deltas(self, test_tokenizer):
|
||||
"""Test streaming processing with small incremental deltas."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
deltas = [
|
||||
"<test:think>",
|
||||
"Some ",
|
||||
"reasoning ",
|
||||
"content",
|
||||
"</test:think>",
|
||||
"Final ",
|
||||
"answer",
|
||||
]
|
||||
|
||||
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
|
||||
|
||||
assert reasoning == "Some reasoning content"
|
||||
assert content == "Final answer"
|
||||
|
||||
def test_streaming_with_start_token(self, test_tokenizer):
|
||||
"""Test streaming with start token included."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
deltas = [
|
||||
"<test:think>",
|
||||
"Some ",
|
||||
"reasoning",
|
||||
"</test:think>",
|
||||
"Answer",
|
||||
]
|
||||
|
||||
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
|
||||
|
||||
assert reasoning == "Some reasoning"
|
||||
assert content == "Answer"
|
||||
|
||||
def test_streaming_no_end_token(self, test_tokenizer):
|
||||
"""Test streaming when no end token is encountered."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
deltas = [
|
||||
"<test:think>",
|
||||
"Some ",
|
||||
"reasoning ",
|
||||
"without ",
|
||||
"end",
|
||||
]
|
||||
|
||||
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
|
||||
|
||||
assert reasoning == "Some reasoning without end"
|
||||
assert content is None
|
||||
|
||||
def test_streaming_only_end_token(self, test_tokenizer):
|
||||
"""Test streaming when only end token appears."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
deltas = [
|
||||
"<test:think>",
|
||||
"Reasoning ",
|
||||
"content",
|
||||
"</test:think>",
|
||||
"Final",
|
||||
]
|
||||
|
||||
reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True)
|
||||
|
||||
assert reasoning == "Reasoning content"
|
||||
assert content == "Final"
|
||||
|
||||
|
||||
class TestBaseThinkingReasoningParserMultipleImplementations:
|
||||
"""Test that multiple implementations of
|
||||
BaseThinkingReasoningParser work correctly.
|
||||
"""
|
||||
|
||||
def test_different_token_implementations(self, test_tokenizer):
|
||||
"""Test that different implementations
|
||||
with different tokens work independently.
|
||||
"""
|
||||
parser1 = TestThinkingReasoningParser(test_tokenizer)
|
||||
parser2 = TestThinkingReasoningParserAlt(test_tokenizer)
|
||||
|
||||
# Test parser1
|
||||
model_output1 = "Reasoning1</test:think>Content1"
|
||||
reasoning1, content1 = run_reasoning_extraction(parser1, [model_output1])
|
||||
assert reasoning1 == "Reasoning1"
|
||||
assert content1 == "Content1"
|
||||
|
||||
# Test parser2
|
||||
model_output2 = "Reasoning2<alt:end>Content2"
|
||||
reasoning2, content2 = run_reasoning_extraction(parser2, [model_output2])
|
||||
assert reasoning2 == "Reasoning2"
|
||||
assert content2 == "Content2"
|
||||
|
||||
# Verify tokens are different
|
||||
assert parser1.start_token != parser2.start_token
|
||||
assert parser1.end_token != parser2.end_token
|
||||
assert parser1.start_token_id != parser2.start_token_id
|
||||
assert parser1.end_token_id != parser2.end_token_id
|
||||
|
||||
|
||||
class TestBaseThinkingReasoningParserEdgeCases:
|
||||
"""Test edge cases and error conditions."""
|
||||
|
||||
def test_multiple_end_tokens(self, test_tokenizer):
|
||||
"""Test behavior with multiple end tokens."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
model_output = "First</test:think>Middle</test:think>Last"
|
||||
reasoning, content = run_reasoning_extraction(parser, [model_output])
|
||||
|
||||
# Should stop at first end token
|
||||
assert reasoning == "First"
|
||||
assert content == "Middle</test:think>Last"
|
||||
|
||||
def test_nested_tokens(self, test_tokenizer):
|
||||
"""Test behavior with nested-like token patterns."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
model_output = "<test:think>Outer<test:think>Inner</test:think>Content"
|
||||
reasoning, content = run_reasoning_extraction(parser, [model_output])
|
||||
|
||||
# Should process normally, start from first start token
|
||||
assert reasoning == "Outer<test:think>Inner"
|
||||
assert content == "Content"
|
||||
|
||||
def test_malformed_tokens(self, test_tokenizer):
|
||||
"""Test behavior with malformed token-like strings."""
|
||||
parser = TestThinkingReasoningParser(test_tokenizer)
|
||||
|
||||
model_output = "<test:thinking>Not a real token</test:thinking>Content"
|
||||
reasoning, content = run_reasoning_extraction(parser, [model_output])
|
||||
|
||||
# Should treat as regular content since tokens don't match exactly
|
||||
assert reasoning == ("<test:thinking>Not a real token</test:thinking>Content")
|
||||
assert content is None
|
||||
@@ -1,161 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.device_communicators.all2all import AgRsAll2AllManager
|
||||
from vllm.forward_context import DPMetadata
|
||||
from vllm.v1.attention.ops.pcp import maybe_gather_mla_latent_cache_inputs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pcp_size,sp_size,enable_ep,expected",
|
||||
[
|
||||
(1, 1, True, [5, 7]),
|
||||
(1, 2, True, [3, 3, 4, 4]),
|
||||
(2, 1, False, [10, 14]),
|
||||
(2, 1, True, [5, 5, 7, 7]),
|
||||
(2, 2, True, [3, 3, 3, 3, 4, 4, 4, 4]),
|
||||
],
|
||||
)
|
||||
def test_dispatch_sizes_expand_pcp_before_tp(pcp_size, sp_size, enable_ep, expected):
|
||||
config = ParallelConfig(
|
||||
distributed_executor_backend="mp",
|
||||
data_parallel_size=2,
|
||||
prefill_context_parallel_size=pcp_size,
|
||||
tensor_parallel_size=sp_size,
|
||||
enable_expert_parallel=enable_ep,
|
||||
)
|
||||
metadata = DPMetadata.make(config, 5, torch.tensor([5, 7]))
|
||||
with metadata.sp_local_sizes(sp_size, pcp_size=pcp_size, use_ep=enable_ep) as sizes:
|
||||
assert sizes == expected
|
||||
assert metadata.local_sizes is None
|
||||
assert metadata.num_tokens_across_dp_cpu.tolist() == [5, 7]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dp_size,pcp_size,tp_size,use_ep,is_sp,expected",
|
||||
[
|
||||
(2, 2, 1, True, False, "ep"),
|
||||
(2, 2, 2, True, True, "ep"),
|
||||
(2, 2, 2, False, False, "dp"),
|
||||
(2, 1, 2, True, False, "dp"),
|
||||
(1, 2, 2, True, False, "pcp"),
|
||||
(2, 2, 2, True, False, None),
|
||||
],
|
||||
)
|
||||
def test_dispatch_reuses_existing_groups(
|
||||
dp_size, pcp_size, tp_size, use_ep, is_sp, expected, monkeypatch
|
||||
):
|
||||
groups = {
|
||||
"dp": SimpleNamespace(world_size=dp_size),
|
||||
"pcp": SimpleNamespace(world_size=pcp_size),
|
||||
"ep": object(),
|
||||
}
|
||||
for name, group in groups.items():
|
||||
monkeypatch.setattr(
|
||||
f"vllm.distributed.device_communicators.all2all.get_{name}_group",
|
||||
lambda group=group: group,
|
||||
)
|
||||
manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager)
|
||||
manager.dp_world_size = dp_size
|
||||
manager.tp_group = SimpleNamespace(world_size=tp_size)
|
||||
manager.use_ep = use_ep
|
||||
if expected is None:
|
||||
with pytest.raises(AssertionError, match="requires sequence-parallel MoE"):
|
||||
manager._get_comm_group(is_sp)
|
||||
else:
|
||||
assert manager._get_comm_group(is_sp) is groups[expected]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_ep", [False, True])
|
||||
def test_ag_rs_dispatch_and_combine_use_dp_pcp_sizes(monkeypatch, enable_ep):
|
||||
calls = []
|
||||
local_tokens = [20, 21] if enable_ep else [20, 21, 22, 23]
|
||||
|
||||
class FakeGroup:
|
||||
world_size = 4 if enable_ep else 2
|
||||
rank_in_group = 2 if enable_ep else 1
|
||||
|
||||
def all_gatherv(self, tensors, dim, sizes):
|
||||
calls.append(("gather", sizes))
|
||||
return [torch.tensor([10, 11, 20, 21, 22, 23]) for _ in tensors]
|
||||
|
||||
def reduce_scatterv(self, tensor, dim, sizes):
|
||||
calls.append(("scatter", sizes))
|
||||
return tensor[2 : 2 + len(local_tokens)]
|
||||
|
||||
config = ParallelConfig(
|
||||
distributed_executor_backend="mp",
|
||||
data_parallel_size=2,
|
||||
data_parallel_rank=1,
|
||||
prefill_context_parallel_size=2,
|
||||
enable_expert_parallel=enable_ep,
|
||||
)
|
||||
metadata = DPMetadata.make(config, 2, torch.tensor([1, 2]))
|
||||
manager = AgRsAll2AllManager.__new__(AgRsAll2AllManager)
|
||||
manager.dp_world_size = 2
|
||||
manager.use_ep = enable_ep
|
||||
|
||||
manager.tp_group = SimpleNamespace(world_size=1)
|
||||
for name in ("dp", "ep"):
|
||||
monkeypatch.setattr(
|
||||
f"vllm.distributed.device_communicators.all2all.get_{name}_group",
|
||||
FakeGroup,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.device_communicators.all2all.get_pcp_group",
|
||||
lambda: SimpleNamespace(world_size=2),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.device_communicators.all2all.get_forward_context",
|
||||
lambda: SimpleNamespace(dp_metadata=metadata),
|
||||
)
|
||||
with metadata.sp_local_sizes(1, pcp_size=2, use_ep=enable_ep):
|
||||
hidden_states, _, _ = manager.dispatch(
|
||||
torch.tensor(local_tokens),
|
||||
torch.ones(len(local_tokens)),
|
||||
torch.zeros(len(local_tokens)),
|
||||
)
|
||||
combined = manager.combine(hidden_states)
|
||||
assert combined.tolist() == local_tokens
|
||||
sizes = [1, 1, 2, 2] if enable_ep else [2, 4]
|
||||
assert calls == [("gather", sizes), ("scatter", sizes)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slots", [[3, 4], [3, 4, -1, -1, -1, -1]])
|
||||
def test_decode_cache_write_ignores_dp_padding(slots):
|
||||
kv = torch.arange(12).reshape(6, 2)
|
||||
pe = torch.arange(6).reshape(6, 1, 1)
|
||||
slots = torch.tensor(slots)
|
||||
cache_kv, cache_pe, cache_slots = maybe_gather_mla_latent_cache_inputs(
|
||||
kv, pe, slots, num_decode_tokens=2, use_pcp=True
|
||||
)
|
||||
torch.testing.assert_close(cache_kv, kv[:2])
|
||||
torch.testing.assert_close(cache_pe, pe[:2])
|
||||
torch.testing.assert_close(cache_slots, slots[:2])
|
||||
|
||||
|
||||
def test_expanded_slot_mapping_keeps_pcp_prefill_padding(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def all_gather(tensor, dim):
|
||||
calls.append(tensor.shape[0])
|
||||
return torch.cat((tensor, tensor), dim=dim)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.ops.pcp.get_pcp_group",
|
||||
lambda: SimpleNamespace(world_size=2, all_gather=all_gather),
|
||||
)
|
||||
kv = torch.zeros(3, 2) # Two decodes followed by one PCP padding row.
|
||||
pe = torch.zeros(3, 1, 1)
|
||||
slots = torch.tensor([3, 4, 8, 3, 4, -1])
|
||||
cache_kv, _, cache_slots = maybe_gather_mla_latent_cache_inputs(
|
||||
kv, pe, slots, num_decode_tokens=2, use_pcp=True
|
||||
)
|
||||
assert calls == [1, 1]
|
||||
assert cache_kv.shape == (4, 2)
|
||||
assert cache_slots.tolist() == [3, 4, 8, -1]
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import transformers.image_utils
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.pixtral import MistralCommonImageProcessor
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_processor() -> MistralCommonImageProcessor:
|
||||
return MistralCommonImageProcessor(mm_encoder=None)
|
||||
|
||||
|
||||
def test_fetch_images_passes_through_decoded_image(
|
||||
image_processor: MistralCommonImageProcessor,
|
||||
):
|
||||
image = Image.new("RGB", (4, 4))
|
||||
result = image_processor.fetch_images(image)
|
||||
assert result is image
|
||||
|
||||
|
||||
def test_fetch_images_recurses_over_list(
|
||||
image_processor: MistralCommonImageProcessor,
|
||||
):
|
||||
a = Image.new("RGB", (4, 4))
|
||||
b = Image.new("RGB", (8, 8))
|
||||
result = image_processor.fetch_images([a, b])
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
assert result[0] is a
|
||||
assert result[1] is b
|
||||
|
||||
|
||||
def test_fetch_images_recurses_over_nested_list(
|
||||
image_processor: MistralCommonImageProcessor,
|
||||
):
|
||||
a = Image.new("RGB", (4, 4))
|
||||
b = Image.new("RGB", (8, 8))
|
||||
result = image_processor.fetch_images([[a], [b]])
|
||||
assert result == [[a], [b]]
|
||||
|
||||
|
||||
def test_fetch_images_str_delegates_to_load_image(
|
||||
monkeypatch, image_processor: MistralCommonImageProcessor
|
||||
):
|
||||
sentinel = Image.new("RGB", (2, 2))
|
||||
received: dict[str, object] = {}
|
||||
|
||||
def fake_load_image(path):
|
||||
received["path"] = path
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(transformers.image_utils, "load_image", fake_load_image)
|
||||
|
||||
result = image_processor.fetch_images("/tmp/fake.png")
|
||||
assert result is sentinel
|
||||
assert received["path"] == "/tmp/fake.png"
|
||||
|
||||
|
||||
def test_fetch_images_rejects_unsupported_type(
|
||||
image_processor: MistralCommonImageProcessor,
|
||||
):
|
||||
with pytest.raises(TypeError, match="only a single or a list"):
|
||||
image_processor.fetch_images(42)
|
||||
@@ -1,127 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``MistralCommonFeatureExtractor.fetch_audio``.
|
||||
|
||||
``transformers>=5.10`` adds a ``ProcessorMixin.prepare_inputs_layout`` helper
|
||||
that calls ``self.feature_extractor.fetch_audio(...)`` unconditionally. The
|
||||
duck-typed :class:`MistralCommonFeatureExtractor` previously did not implement
|
||||
that method, so loading any voxtral model under transformers 5.10.x raised
|
||||
``AttributeError: 'MistralCommonFeatureExtractor' object has no attribute
|
||||
'fetch_audio'``. These tests pin the new ``fetch_audio`` method to the same
|
||||
contract as ``transformers.SequenceFeatureExtractor.fetch_audio``.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.transformers_utils.processors.voxtral import (
|
||||
MistralCommonFeatureExtractor,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def feature_extractor() -> MistralCommonFeatureExtractor:
|
||||
tokenizer = MistralTokenizer.from_pretrained("mistralai/Voxtral-Mini-3B-2507")
|
||||
return MistralCommonFeatureExtractor(tokenizer.instruct.audio_encoder)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"audio",
|
||||
[
|
||||
np.zeros(1024, dtype=np.float32),
|
||||
torch.zeros(1024),
|
||||
[0.0, 1.0, 2.0],
|
||||
],
|
||||
ids=["numpy_array", "torch_tensor", "list_of_floats"],
|
||||
)
|
||||
def test_fetch_audio_passes_through(
|
||||
feature_extractor: MistralCommonFeatureExtractor, audio
|
||||
):
|
||||
result = feature_extractor.fetch_audio(audio)
|
||||
assert result is audio
|
||||
|
||||
|
||||
def test_fetch_audio_recurses_over_list_of_arrays(
|
||||
feature_extractor: MistralCommonFeatureExtractor,
|
||||
):
|
||||
a = np.zeros(8, dtype=np.float32)
|
||||
b = np.ones(8, dtype=np.float32)
|
||||
result = feature_extractor.fetch_audio([a, b])
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
assert result[0] is a
|
||||
assert result[1] is b
|
||||
|
||||
|
||||
def test_fetch_audio_uses_self_sampling_rate_when_none(
|
||||
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
|
||||
):
|
||||
"""If ``sampling_rate`` is None, ``self.sampling_rate`` must be used.
|
||||
|
||||
Verified indirectly via the recursion path: when we pass a list of arrays
|
||||
without sampling_rate, recursive calls receive the resolved rate.
|
||||
"""
|
||||
captured: list[int | None] = []
|
||||
original = feature_extractor.fetch_audio
|
||||
|
||||
def spy(audio, sampling_rate=None):
|
||||
captured.append(sampling_rate)
|
||||
return original(audio, sampling_rate=sampling_rate)
|
||||
|
||||
monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
|
||||
feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)])
|
||||
# Top-level call has sampling_rate=None; inner recursive call sees the
|
||||
# resolved rate from self.sampling_rate.
|
||||
assert captured[0] is None
|
||||
assert captured[1] == 16000
|
||||
|
||||
|
||||
def test_fetch_audio_explicit_sampling_rate_propagates(
|
||||
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
|
||||
):
|
||||
captured: list[int | None] = []
|
||||
original = feature_extractor.fetch_audio
|
||||
|
||||
def spy(audio, sampling_rate=None):
|
||||
captured.append(sampling_rate)
|
||||
return original(audio, sampling_rate=sampling_rate)
|
||||
|
||||
monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
|
||||
feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)], sampling_rate=8000)
|
||||
assert captured[0] == 8000
|
||||
assert captured[1] == 8000
|
||||
|
||||
|
||||
def test_fetch_audio_rejects_unsupported_type(
|
||||
feature_extractor: MistralCommonFeatureExtractor,
|
||||
):
|
||||
with pytest.raises(TypeError, match="only a numpy array"):
|
||||
feature_extractor.fetch_audio(42)
|
||||
|
||||
|
||||
def test_fetch_audio_str_delegates_to_load_audio(
|
||||
monkeypatch, feature_extractor: MistralCommonFeatureExtractor
|
||||
):
|
||||
"""A str input must round-trip through ``transformers.audio_utils.load_audio``.
|
||||
|
||||
We monkey-patch ``load_audio`` so the test stays offline (no real URL/path
|
||||
fetched) and still asserts the delegation contract.
|
||||
"""
|
||||
sentinel = np.array([0.5, -0.5], dtype=np.float32)
|
||||
received: dict[str, object] = {}
|
||||
|
||||
def fake_load_audio(path, sampling_rate=None):
|
||||
received["path"] = path
|
||||
received["sampling_rate"] = sampling_rate
|
||||
return sentinel
|
||||
|
||||
import transformers.audio_utils
|
||||
|
||||
monkeypatch.setattr(transformers.audio_utils, "load_audio", fake_load_audio)
|
||||
|
||||
result = feature_extractor.fetch_audio("/tmp/fake.wav")
|
||||
assert result is sentinel
|
||||
assert received["path"] == "/tmp/fake.wav"
|
||||
assert received["sampling_rate"] == 16000
|
||||
@@ -1,159 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from transformers import AutoConfig
|
||||
|
||||
from vllm.transformers_utils.config import get_config, uses_mrope
|
||||
from vllm.transformers_utils.configs.bailing_moe_v3_vl import (
|
||||
BailingMoeV3TextConfig,
|
||||
BailingMoeV3VisionConfig,
|
||||
BailingMoeV3VLConfig,
|
||||
)
|
||||
|
||||
|
||||
def _bailing_v3_vl_config() -> dict[str, Any]:
|
||||
return {
|
||||
"architectures": ["BailingMoeV3VLForConditionalGeneration"],
|
||||
"auto_map": {
|
||||
"AutoConfig": ("configuration_bailing_moe_v3_vl.BailingMoeV3VLConfig")
|
||||
},
|
||||
"model_type": "bailing_moe_v3_vl",
|
||||
"mrope_section": [8, 12, 12],
|
||||
"text_config": {
|
||||
"num_hidden_layers": 42,
|
||||
"layer_group_size": 6,
|
||||
"rope_theta": 6_000_000,
|
||||
},
|
||||
"vision_config": {"disable_merger_proj": True},
|
||||
}
|
||||
|
||||
|
||||
def _write_config(path: Path, config: dict[str, Any]) -> None:
|
||||
path.mkdir()
|
||||
(path / "config.json").write_text(json.dumps(config), encoding="utf-8")
|
||||
|
||||
|
||||
def test_bailing_v3_vl_config_loads_without_remote_code(tmp_path: Path):
|
||||
model_path = tmp_path / "model"
|
||||
_write_config(model_path, _bailing_v3_vl_config())
|
||||
|
||||
config = get_config(model_path, trust_remote_code=False)
|
||||
|
||||
assert isinstance(config, BailingMoeV3VLConfig)
|
||||
assert isinstance(config.text_config, BailingMoeV3TextConfig)
|
||||
assert isinstance(config.vision_config, BailingMoeV3VisionConfig)
|
||||
assert isinstance(
|
||||
AutoConfig.from_pretrained(model_path, trust_remote_code=False),
|
||||
BailingMoeV3VLConfig,
|
||||
)
|
||||
assert config.mrope_section == [8, 12, 12]
|
||||
assert uses_mrope(config)
|
||||
|
||||
text_config = config.text_config
|
||||
assert text_config.model_type == "bailing_hybrid"
|
||||
assert text_config.architectures == ["BailingMoeV3ForCausalLM"]
|
||||
assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
|
||||
assert text_config.rope_parameters["rope_theta"] == 6_000_000
|
||||
assert text_config.layer_types.count("linear_attention") == 35
|
||||
assert [
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(text_config.layer_types)
|
||||
if layer_type == "full_attention"
|
||||
] == [5, 11, 17, 23, 29, 35, 41]
|
||||
assert config.vision_config.disable_merger_proj is True
|
||||
|
||||
normalized = config.to_dict()
|
||||
restored = BailingMoeV3VLConfig.from_dict(normalized)
|
||||
assert isinstance(restored.text_config, BailingMoeV3TextConfig)
|
||||
assert isinstance(restored.vision_config, BailingMoeV3VisionConfig)
|
||||
assert restored.to_dict() == normalized
|
||||
|
||||
|
||||
def test_bailing_v3_vl_config_preserves_explicit_nested_values():
|
||||
config_dict = _bailing_v3_vl_config()
|
||||
config_dict["architectures"] = ["CustomBailingVLForConditionalGeneration"]
|
||||
text_config = config_dict["text_config"]
|
||||
text_config["architectures"] = ["CustomBailingForCausalLM"]
|
||||
text_config["layer_types"] = (["linear_attention"] * 5 + ["full_attention"]) * 7
|
||||
text_config["rope_theta"] = 1_000_000
|
||||
text_config["rope_parameters"] = {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 1_000_000,
|
||||
"mrope_section": [8, 12, 12],
|
||||
}
|
||||
original = copy.deepcopy(config_dict)
|
||||
|
||||
config = BailingMoeV3VLConfig(**config_dict)
|
||||
|
||||
assert config_dict == original
|
||||
assert config.architectures == ["CustomBailingVLForConditionalGeneration"]
|
||||
assert config.text_config.architectures == ["CustomBailingForCausalLM"]
|
||||
assert config.text_config.layer_types == text_config["layer_types"]
|
||||
assert config.text_config.rope_parameters["rope_theta"] == 1_000_000
|
||||
assert config.text_config.rope_parameters["mrope_section"] == [8, 12, 12]
|
||||
|
||||
|
||||
def test_bailing_v3_vl_config_normalizes_config_instance():
|
||||
text_config = BailingMoeV3TextConfig()
|
||||
|
||||
config = BailingMoeV3VLConfig(
|
||||
text_config=text_config,
|
||||
mrope_section=[8, 12, 12],
|
||||
)
|
||||
|
||||
assert config.text_config is text_config
|
||||
assert text_config.rope_parameters["mrope_section"] == [8, 12, 12]
|
||||
|
||||
|
||||
def test_bailing_v3_text_config_trailing_and_legacy_layers():
|
||||
expected = ["linear_attention", "linear_attention", "full_attention"] * 2 + [
|
||||
"full_attention"
|
||||
] * 2
|
||||
text_config = BailingMoeV3TextConfig(
|
||||
num_hidden_layers=8,
|
||||
layer_group_size=3,
|
||||
)
|
||||
assert text_config.layer_types == expected
|
||||
|
||||
text_config = BailingMoeV3TextConfig(
|
||||
num_hidden_layers=8,
|
||||
layer_group_size=3,
|
||||
layers_block_type=["mamba", "mamba", "attention"] * 2 + ["attention"] * 2,
|
||||
)
|
||||
assert text_config.layer_types == expected
|
||||
|
||||
|
||||
def test_bailing_v3_text_config_accepts_legacy_rope_type():
|
||||
text_config = BailingMoeV3TextConfig(rope_scaling={"type": "linear", "factor": 2.0})
|
||||
|
||||
assert text_config.rope_parameters["type"] == "linear"
|
||||
assert text_config.rope_parameters["rope_type"] == "linear"
|
||||
|
||||
|
||||
def test_bailing_v3_vl_config_rejects_invalid_values():
|
||||
with pytest.raises(ValueError, match="layer_group_size must be positive"):
|
||||
BailingMoeV3TextConfig(layer_group_size=0)
|
||||
|
||||
with pytest.raises(ValueError, match="must match the layer_group_size schedule"):
|
||||
BailingMoeV3TextConfig(
|
||||
num_hidden_layers=8,
|
||||
layer_group_size=3,
|
||||
layer_types=["full_attention"] * 8,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="conflicting M-RoPE sections"):
|
||||
BailingMoeV3VLConfig(
|
||||
text_config={
|
||||
"rope_parameters": {
|
||||
"rope_type": "default",
|
||||
"mrope_section": [4, 14, 14],
|
||||
}
|
||||
},
|
||||
mrope_section=[8, 12, 12],
|
||||
)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.transformers_utils.configs.k3_dspark import K3DSparkConfig
|
||||
|
||||
|
||||
def _write_dspark_config(path, **overrides):
|
||||
path.mkdir()
|
||||
config = {
|
||||
"architectures": ["K3DSparkModel"],
|
||||
"model_type": "k3_dspark",
|
||||
"hidden_size": 7168,
|
||||
"intermediate_size": 14336,
|
||||
"num_hidden_layers": 5,
|
||||
"num_attention_heads": 96,
|
||||
"num_key_value_heads": 96,
|
||||
"q_lora_rank": 1536,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"vocab_size": 163840,
|
||||
"rms_norm_eps": 1e-5,
|
||||
"max_position_embeddings": 32768,
|
||||
"rope_theta": 50000.0,
|
||||
"num_target_layers": 5,
|
||||
"target_hidden_size": 7168,
|
||||
"target_num_hidden_layers": 93,
|
||||
"target_layer_ids": [2, 23, 47, 71, 89],
|
||||
"markov_rank": 256,
|
||||
"draft_vocab_size": 163840,
|
||||
"torch_dtype": "bfloat16",
|
||||
}
|
||||
config.update(overrides)
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
def _write_target_config(path):
|
||||
path.mkdir()
|
||||
config = {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": 7168,
|
||||
"intermediate_size": 14336,
|
||||
"num_hidden_layers": 93,
|
||||
"num_attention_heads": 56,
|
||||
"num_key_value_heads": 8,
|
||||
"vocab_size": 163840,
|
||||
"max_position_embeddings": 32768,
|
||||
"torch_dtype": "bfloat16",
|
||||
}
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
def test_dspark_mla_config_loads_from_local_json(tmp_path):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path)
|
||||
|
||||
config = get_config(draft_path, trust_remote_code=False)
|
||||
|
||||
assert isinstance(config, K3DSparkConfig)
|
||||
assert config.model_type == "k3_dspark"
|
||||
assert config.architectures == ["K3DSparkModel"]
|
||||
assert config.hidden_act == "silu"
|
||||
assert config.rope_parameters == {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 50000.0,
|
||||
}
|
||||
assert config.n_routed_experts == 0
|
||||
assert config.draft_vocab_size == config.vocab_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"mla_use_nope": True},
|
||||
{"mla_use_output_gate": True},
|
||||
{"mla_use_qk_norm": True},
|
||||
{"dspark_bonus_anchor": True},
|
||||
{"q_lora_rank": None},
|
||||
{"draft_vocab_size": 8192},
|
||||
{"target_layer_ids": []},
|
||||
{"num_target_layers": 4},
|
||||
],
|
||||
)
|
||||
def test_dspark_mla_rejects_unsupported_checkpoint_options(tmp_path, overrides):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(draft_path, **overrides)
|
||||
|
||||
with pytest.raises(ValueError, match="MLA DSpark"):
|
||||
get_config(draft_path, trust_remote_code=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_attention_heads", "expected_local_heads"),
|
||||
[(64, 8), (96, 12)],
|
||||
ids=["released-64-head", "block5-96-head"],
|
||||
)
|
||||
def test_dspark_mla_uses_latent_kv_geometry(
|
||||
tmp_path, num_attention_heads, expected_local_heads
|
||||
):
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_dspark_config(
|
||||
draft_path,
|
||||
num_attention_heads=num_attention_heads,
|
||||
num_key_value_heads=num_attention_heads,
|
||||
)
|
||||
model_config = ModelConfig(
|
||||
model=str(draft_path),
|
||||
tokenizer_mode="skip",
|
||||
runner="draft",
|
||||
max_model_len=32768,
|
||||
)
|
||||
|
||||
assert model_config.is_deepseek_mla
|
||||
assert model_config.use_mla
|
||||
assert model_config.get_head_size() == 576
|
||||
# external_launcher skips ParallelConfig's local-GPU-count check so the
|
||||
# config logic can be exercised at TP8 on a single-GPU test node.
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=8, distributed_executor_backend="external_launcher"
|
||||
)
|
||||
assert model_config.get_num_kv_heads(parallel_config) == 1
|
||||
assert model_config.get_num_attention_heads(parallel_config) == expected_local_heads
|
||||
assert model_config.get_num_experts() == 0
|
||||
|
||||
|
||||
def test_dspark_mla_speculative_config_preserves_architecture(tmp_path):
|
||||
target_path = tmp_path / "target"
|
||||
draft_path = tmp_path / "draft"
|
||||
_write_target_config(target_path)
|
||||
_write_dspark_config(draft_path)
|
||||
target_config = ModelConfig(
|
||||
model=str(target_path), tokenizer_mode="skip", max_model_len=32768
|
||||
)
|
||||
speculative_config = SpeculativeConfig(
|
||||
model=str(draft_path),
|
||||
method="dspark",
|
||||
num_speculative_tokens=8,
|
||||
target_model_config=target_config,
|
||||
target_parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
assert speculative_config.parallel_drafting
|
||||
assert speculative_config.draft_model_config.architectures == ["K3DSparkModel"]
|
||||
assert speculative_config.draft_model_config.hf_config.model_type == "k3_dspark"
|
||||
assert speculative_config.draft_model_config.use_mla
|
||||
File diff suppressed because it is too large
Load Diff
@@ -24,12 +24,7 @@ from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
HiSparseConnectorScheduler,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalKwargsItem,
|
||||
@@ -49,7 +44,6 @@ from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.engine.core import EngineCore
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
@@ -219,30 +213,6 @@ def test_max_num_active_seqs_caps_admission():
|
||||
assert scheduler.max_num_running_reqs == 16
|
||||
|
||||
|
||||
def _bind_hisparse_connector(scheduler):
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
|
||||
|
||||
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
|
||||
connector.connector_scheduler.bind_coordinator(coordinator)
|
||||
composite = object.__new__(MultiConnector)
|
||||
composite._connectors = (connector,)
|
||||
scheduler.connector = composite
|
||||
return coordinator
|
||||
|
||||
|
||||
def test_pending_hisparse_spill_keeps_scheduler_alive():
|
||||
"""A final host spill must complete after the last request finishes."""
|
||||
scheduler = create_scheduler()
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
pending = coordinator.pending_spills
|
||||
pending[0] = Mock()
|
||||
assert scheduler.has_requests()
|
||||
pending.clear()
|
||||
assert not scheduler.has_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_prefix_caching, prompt_logprobs",
|
||||
[
|
||||
@@ -1776,45 +1746,6 @@ def test_preempt_during_execution():
|
||||
assert requests[1].output_token_ids[0] == 42
|
||||
|
||||
|
||||
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
|
||||
"""Pending reclamation must stall rather than preempt its owning request."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[0]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
|
||||
)
|
||||
coordinator = _bind_hisparse_connector(scheduler)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "build_connector_meta", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
coordinator,
|
||||
"has_pending_reclamation",
|
||||
Mock(return_value=True),
|
||||
)
|
||||
|
||||
deferred_output = scheduler.schedule()
|
||||
|
||||
assert not deferred_output.num_scheduled_tokens
|
||||
assert scheduler.running == [request]
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
assert request.num_preemptions == 0
|
||||
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,283 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for resource management in hf3fs_client.py: constructor failure cleanup
|
||||
and idempotent close(). Tests use mock to replace real I/O operations
|
||||
(hf3fs_fuse.io, SharedMemory, os, CUDA).
|
||||
Requires hf3fs_fuse.io to be installed; skipped otherwise.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
HF3FS_AVAILABLE = True
|
||||
try:
|
||||
from hf3fs_fuse.io import ( # noqa: F401
|
||||
deregister_fd,
|
||||
extract_mount_point,
|
||||
make_ioring,
|
||||
make_iovec,
|
||||
register_fd,
|
||||
)
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client import (
|
||||
Hf3fsClient,
|
||||
)
|
||||
except Exception:
|
||||
HF3FS_AVAILABLE = False
|
||||
|
||||
requires_hf3fs = pytest.mark.skipif(
|
||||
not HF3FS_AVAILABLE,
|
||||
reason="hf3fs_fuse.io is not available on this machine",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeShm:
|
||||
"""Shared-memory stub matching the multiprocessing.shared_memory.SharedMemory
|
||||
interface used by Hf3fsClient:
|
||||
|
||||
Attributes accessed by the constructor:
|
||||
.buf – memoryview / buffer-protocol object consumed by torch.frombuffer
|
||||
Methods called during normal lifetime:
|
||||
.unlink() – called right after the iovec is set up
|
||||
.close() – called in _release_resources()
|
||||
"""
|
||||
|
||||
def __init__(self, size: int = 1024):
|
||||
self._data = bytearray(size)
|
||||
self.buf = memoryview(self._data)
|
||||
self.closed = False
|
||||
self.close_call_count = 0
|
||||
self.unlink_call_count = 0
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
self.close_call_count += 1
|
||||
|
||||
def unlink(self):
|
||||
self.unlink_call_count += 1
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestHf3fsClientResourceManagement
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@requires_hf3fs
|
||||
class TestHf3fsClientResourceManagement:
|
||||
"""Tests for constructor failure cleanup and idempotent close()."""
|
||||
|
||||
_MOD = "vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_client"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helper: build a minimal Hf3fsClient bypassing all real I/O so that
|
||||
# we can fully control its internal state.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _make_client(self, tmp_path):
|
||||
"""Return a fully-mocked Hf3fsClient with controllable internals."""
|
||||
fake_shm_r = _FakeShm()
|
||||
fake_shm_w = _FakeShm()
|
||||
|
||||
patcher_list: list[Any] = [
|
||||
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
|
||||
patch(f"{self._MOD}.register_fd"),
|
||||
patch(f"{self._MOD}.deregister_fd"),
|
||||
patch(f"{self._MOD}.extract_mount_point", return_value="/mnt/hf3fs"),
|
||||
patch(f"{self._MOD}.make_ioring", return_value=MagicMock()),
|
||||
patch(f"{self._MOD}.make_iovec", return_value=MagicMock()),
|
||||
patch(
|
||||
"multiprocessing.shared_memory.SharedMemory",
|
||||
side_effect=[fake_shm_r, fake_shm_w],
|
||||
),
|
||||
patch("os.open", return_value=99),
|
||||
patch("os.ftruncate"),
|
||||
patch("os.close"),
|
||||
patch("os.fsync"),
|
||||
patch("torch.cuda.Stream", return_value=MagicMock()),
|
||||
patch("torch.frombuffer", return_value=MagicMock()),
|
||||
patch("torch.empty", return_value=MagicMock()),
|
||||
]
|
||||
for p in patcher_list:
|
||||
p.start()
|
||||
|
||||
try:
|
||||
client = Hf3fsClient(
|
||||
path=str(tmp_path / "test.bin"),
|
||||
size=1024,
|
||||
bytes_per_page=256,
|
||||
entries=4,
|
||||
)
|
||||
finally:
|
||||
for p in patcher_list:
|
||||
p.stop()
|
||||
|
||||
# Manually point internal handles to our controllable fakes so that
|
||||
# assertions after close() can inspect them directly.
|
||||
client.shm_r = fake_shm_r
|
||||
client.shm_w = fake_shm_w
|
||||
client.file = 99
|
||||
return client, fake_shm_r, fake_shm_w
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# close() idempotency
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_close_idempotent_and_handles_cleared(self, tmp_path):
|
||||
"""Multiple close() calls must not raise; deregister_fd called exactly
|
||||
once, all handles set to None, shm.close() invoked."""
|
||||
client, shm_r, shm_w = self._make_client(tmp_path)
|
||||
|
||||
with (
|
||||
patch(f"{self._MOD}.deregister_fd") as mock_dereg,
|
||||
patch("os.close"),
|
||||
):
|
||||
client.close() # first close
|
||||
client.close() # second close — must be no-op
|
||||
client.close() # third close — must be no-op
|
||||
|
||||
assert client._closed is True
|
||||
assert mock_dereg.call_count == 1, (
|
||||
f"deregister_fd called {mock_dereg.call_count} times; expected 1"
|
||||
)
|
||||
for attr in ("iov_r", "iov_w", "ior_r", "ior_w", "shm_r", "shm_w", "file"):
|
||||
assert getattr(client, attr) is None, f"{attr} should be None after close()"
|
||||
assert shm_r.closed is True
|
||||
assert shm_w.closed is True
|
||||
|
||||
def test_flush_after_close_is_noop(self, tmp_path):
|
||||
"""flush() after close() must silently do nothing (no fsync call)."""
|
||||
client, _, _ = self._make_client(tmp_path)
|
||||
|
||||
with (
|
||||
patch(f"{self._MOD}.deregister_fd"),
|
||||
patch("os.close"),
|
||||
patch("os.fsync") as mock_fsync,
|
||||
):
|
||||
client.close()
|
||||
client.flush()
|
||||
|
||||
mock_fsync.assert_not_called()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Constructor failure leaves no leaked resources
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_constructor_failure_after_file_open_cleans_file(self, tmp_path):
|
||||
"""If the constructor raises after os.open(), the fd must be closed."""
|
||||
with (
|
||||
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
|
||||
patch(f"{self._MOD}.register_fd"),
|
||||
patch(f"{self._MOD}.deregister_fd"),
|
||||
patch(
|
||||
f"{self._MOD}.extract_mount_point",
|
||||
side_effect=RuntimeError("mount point not found"),
|
||||
),
|
||||
patch("os.open", return_value=55),
|
||||
patch("os.ftruncate"),
|
||||
patch("os.close") as mock_os_close,
|
||||
patch("torch.cuda.Stream", return_value=MagicMock()),
|
||||
pytest.raises(RuntimeError, match="mount point not found"),
|
||||
):
|
||||
Hf3fsClient(
|
||||
path=str(tmp_path / "fail.bin"),
|
||||
size=1024,
|
||||
bytes_per_page=256,
|
||||
entries=4,
|
||||
)
|
||||
|
||||
mock_os_close.assert_called_once_with(55)
|
||||
|
||||
def test_constructor_failure_after_shm_alloc_closes_shm(self, tmp_path):
|
||||
"""Constructor raises after SharedMemory creation → both shm objects closed."""
|
||||
fake_shm_r = _FakeShm()
|
||||
fake_shm_w = _FakeShm()
|
||||
|
||||
with (
|
||||
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
|
||||
patch(f"{self._MOD}.register_fd"),
|
||||
patch(f"{self._MOD}.deregister_fd"),
|
||||
patch(f"{self._MOD}.extract_mount_point", return_value="/mnt/hf3fs"),
|
||||
patch(
|
||||
"multiprocessing.shared_memory.SharedMemory",
|
||||
side_effect=[fake_shm_r, fake_shm_w],
|
||||
),
|
||||
patch("os.open", return_value=66),
|
||||
patch("os.ftruncate"),
|
||||
patch("os.close"),
|
||||
patch("torch.frombuffer", return_value=MagicMock()),
|
||||
patch("torch.empty", return_value=MagicMock()),
|
||||
patch(
|
||||
f"{self._MOD}.make_ioring",
|
||||
side_effect=RuntimeError("ioring init failed"),
|
||||
),
|
||||
patch(f"{self._MOD}.make_iovec", return_value=MagicMock()),
|
||||
patch("torch.cuda.Stream", return_value=MagicMock()),
|
||||
pytest.raises(RuntimeError, match="ioring init failed"),
|
||||
):
|
||||
Hf3fsClient(
|
||||
path=str(tmp_path / "fail2.bin"),
|
||||
size=1024,
|
||||
bytes_per_page=256,
|
||||
entries=4,
|
||||
)
|
||||
|
||||
assert fake_shm_r.closed is True, (
|
||||
"shm_r was not closed after constructor failure"
|
||||
)
|
||||
assert fake_shm_w.closed is True, (
|
||||
"shm_w was not closed after constructor failure"
|
||||
)
|
||||
|
||||
def test_constructor_failure_does_not_close_unallocated_shm(self, tmp_path):
|
||||
"""Failure before SharedMemory is created must not raise AttributeError
|
||||
or TypeError from cleanup."""
|
||||
with (
|
||||
patch(f"{self._MOD}.HF3FS_AVAILABLE", True),
|
||||
patch(f"{self._MOD}.register_fd"),
|
||||
patch(f"{self._MOD}.deregister_fd"),
|
||||
patch(
|
||||
f"{self._MOD}.extract_mount_point",
|
||||
side_effect=RuntimeError("early failure"),
|
||||
),
|
||||
patch("os.open", return_value=77),
|
||||
patch("os.ftruncate"),
|
||||
patch("os.close"),
|
||||
patch("torch.cuda.Stream", return_value=MagicMock()),
|
||||
pytest.raises(RuntimeError, match="early failure"),
|
||||
):
|
||||
Hf3fsClient(
|
||||
path=str(tmp_path / "early_fail.bin"),
|
||||
size=1024,
|
||||
bytes_per_page=256,
|
||||
entries=4,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# _release_resources on already-cleared state must be a no-op
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_release_resources_on_empty_state_is_safe(self, tmp_path):
|
||||
"""_release_resources() on a fully-cleared client must not raise."""
|
||||
client, _, _ = self._make_client(tmp_path)
|
||||
|
||||
with (
|
||||
patch(f"{self._MOD}.deregister_fd"),
|
||||
patch("os.close"),
|
||||
):
|
||||
client.close() # clears all handles
|
||||
|
||||
with (
|
||||
patch(f"{self._MOD}.deregister_fd") as mock_dereg2,
|
||||
patch("os.close") as mock_os_close2,
|
||||
):
|
||||
client._release_resources() # must not raise
|
||||
|
||||
mock_dereg2.assert_not_called()
|
||||
mock_os_close2.assert_not_called()
|
||||
@@ -1,233 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for HF3FS KV Connector high-level components:
|
||||
- TestHf3fsMockClient : file-backed mock client I/O correctness
|
||||
- TestHF3FSKVConnectorStats: metric collection, aggregation, serialisation
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_connector import (
|
||||
HF3FSKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.utils.hf3fs_mock_client import (
|
||||
Hf3fsClient as MockHf3fsClient,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hf3fs_stats():
|
||||
"""Fresh HF3FSKVConnectorStats instance."""
|
||||
return HF3FSKVConnectorStats()
|
||||
|
||||
|
||||
def _make_cuda_event():
|
||||
"""Return a real CUDA event when available, otherwise a MagicMock."""
|
||||
if torch.cuda.is_available():
|
||||
return torch.cuda.Event()
|
||||
return MagicMock()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestHf3fsMockClient
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestHf3fsMockClient:
|
||||
"""Tests for hf3fs_mock_client.Hf3fsClient (file-backend mock)."""
|
||||
|
||||
def test_init_creates_file(self, tmp_path):
|
||||
"""Initializing the client should create the backing file."""
|
||||
path = str(tmp_path / "test_file")
|
||||
client = MockHf3fsClient(path=path, size=4096, bytes_per_page=512, entries=4)
|
||||
assert os.path.exists(path), "Backing file should be created on init"
|
||||
assert os.path.getsize(path) == 4096
|
||||
client.close()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype, bytes_per_page",
|
||||
[
|
||||
(torch.float32, 512),
|
||||
(torch.float16, 256),
|
||||
(torch.bfloat16, 256),
|
||||
],
|
||||
ids=["float32", "float16", "bfloat16"],
|
||||
)
|
||||
def test_batch_write_and_read_dtype(self, tmp_path, dtype, bytes_per_page):
|
||||
"""Write a tensor of the given dtype and verify round-trip correctness."""
|
||||
path = str(tmp_path / f"rw_{dtype}")
|
||||
client = MockHf3fsClient(
|
||||
path=path, size=bytes_per_page * 8, bytes_per_page=bytes_per_page, entries=4
|
||||
)
|
||||
elem_size = torch.tensor([], dtype=dtype).element_size()
|
||||
numel = bytes_per_page // elem_size
|
||||
tensor_write = torch.arange(numel, dtype=dtype)
|
||||
event = _make_cuda_event()
|
||||
|
||||
results = client.batch_write([0], [tensor_write], event)
|
||||
assert results == [bytes_per_page], f"Write should succeed, got {results}"
|
||||
|
||||
tensor_read = torch.zeros(numel, dtype=dtype)
|
||||
results = client.batch_read([0], [tensor_read])
|
||||
assert results == [bytes_per_page], f"Read should succeed, got {results}"
|
||||
assert torch.equal(tensor_write, tensor_read), (
|
||||
"Read tensor should match written tensor"
|
||||
)
|
||||
client.close()
|
||||
|
||||
def test_batch_read_empty_file_returns_error(self, tmp_path):
|
||||
"""Reading out-of-bounds offset should return -1."""
|
||||
bytes_per_page = 128
|
||||
size = bytes_per_page * 4
|
||||
path = str(tmp_path / "empty_read")
|
||||
client = MockHf3fsClient(
|
||||
path=path, size=size, bytes_per_page=bytes_per_page, entries=4
|
||||
)
|
||||
numel = bytes_per_page // 4
|
||||
tensor_read = torch.zeros(numel, dtype=torch.float32)
|
||||
results = client.batch_read([size], [tensor_read]) # offset == size => OOB
|
||||
assert results[0] == -1, "Out-of-bounds read should return -1"
|
||||
client.close()
|
||||
|
||||
def test_batch_write_out_of_bounds_returns_error(self, tmp_path):
|
||||
"""Writing at an offset beyond file size should return -1."""
|
||||
bytes_per_page = 128
|
||||
size = bytes_per_page * 4
|
||||
path = str(tmp_path / "oob_write")
|
||||
client = MockHf3fsClient(
|
||||
path=path, size=size, bytes_per_page=bytes_per_page, entries=4
|
||||
)
|
||||
numel = bytes_per_page // 4
|
||||
tensor = torch.ones(numel, dtype=torch.float32)
|
||||
event = _make_cuda_event()
|
||||
results = client.batch_write([size], [tensor], event) # OOB offset
|
||||
assert results[0] == -1, "Out-of-bounds write should return -1"
|
||||
client.close()
|
||||
|
||||
def test_multiple_tensors_rw(self, tmp_path):
|
||||
"""Write multiple tensors at different offsets, then read all back."""
|
||||
bytes_per_page = 128
|
||||
n = 4
|
||||
path = str(tmp_path / "multi_rw")
|
||||
client = MockHf3fsClient(
|
||||
path=path,
|
||||
size=bytes_per_page * n * 2,
|
||||
bytes_per_page=bytes_per_page,
|
||||
entries=8,
|
||||
)
|
||||
tensors_write = [
|
||||
torch.full((bytes_per_page // 4,), float(i), dtype=torch.float32)
|
||||
for i in range(n)
|
||||
]
|
||||
offsets = [i * bytes_per_page for i in range(n)]
|
||||
event = _make_cuda_event()
|
||||
|
||||
results = client.batch_write(offsets, tensors_write, event)
|
||||
assert all(r == bytes_per_page for r in results)
|
||||
|
||||
tensors_read = [
|
||||
torch.zeros(bytes_per_page // 4, dtype=torch.float32) for _ in range(n)
|
||||
]
|
||||
results = client.batch_read(offsets, tensors_read)
|
||||
assert all(r == bytes_per_page for r in results)
|
||||
|
||||
for i, (tw, tr) in enumerate(zip(tensors_write, tensors_read)):
|
||||
assert torch.allclose(tw, tr), f"Tensor {i} mismatch after round-trip"
|
||||
client.close()
|
||||
|
||||
def test_flush_and_close_no_error(self, tmp_path):
|
||||
"""flush() and close() should not raise exceptions."""
|
||||
path = str(tmp_path / "flush_close")
|
||||
client = MockHf3fsClient(path=path, size=1024, bytes_per_page=128, entries=4)
|
||||
client.flush()
|
||||
client.close()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestHF3FSKVConnectorStats
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestHF3FSKVConnectorStats:
|
||||
"""Tests for HF3FSKVConnectorStats metric collection and aggregation."""
|
||||
|
||||
def test_initial_is_empty(self, hf3fs_stats):
|
||||
"""Fresh stats object should report is_empty() == True."""
|
||||
assert hf3fs_stats.is_empty() is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"task_type, duration_key",
|
||||
[
|
||||
("Saved", "save_duration"),
|
||||
("Loaded", "load_duration"),
|
||||
],
|
||||
ids=["save", "load"],
|
||||
)
|
||||
def test_record_success_duration(self, hf3fs_stats, task_type, duration_key):
|
||||
"""Recording a successful task should update duration list and total count."""
|
||||
hf3fs_stats.record_success_task_duration(task_type, 0.5)
|
||||
assert not hf3fs_stats.is_empty()
|
||||
assert len(hf3fs_stats.data[duration_key]) == 1
|
||||
assert hf3fs_stats.data[duration_key][0] == pytest.approx(0.5)
|
||||
assert hf3fs_stats.data["num_transfer_task"] == 1
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"task_type, failed_key",
|
||||
[
|
||||
("Saved", "num_failed_save"),
|
||||
("Loaded", "num_failed_load"),
|
||||
],
|
||||
ids=["save", "load"],
|
||||
)
|
||||
def test_record_failed_task(self, hf3fs_stats, task_type, failed_key):
|
||||
"""Recording a failed task should increment the corresponding counter."""
|
||||
hf3fs_stats.record_failed_task_count(task_type)
|
||||
assert hf3fs_stats.data[failed_key] == 1
|
||||
assert hf3fs_stats.data["num_transfer_task"] == 1
|
||||
|
||||
def test_aggregate_two_stats(self):
|
||||
"""aggregate() should merge save/load duration lists and sum counters."""
|
||||
stats1 = HF3FSKVConnectorStats()
|
||||
stats1.record_success_task_duration("Saved", 0.1)
|
||||
stats1.record_success_task_duration("Loaded", 0.2)
|
||||
|
||||
stats2 = HF3FSKVConnectorStats()
|
||||
stats2.record_success_task_duration("Saved", 0.3)
|
||||
stats2.record_failed_task_count("Loaded")
|
||||
|
||||
stats1.aggregate(stats2)
|
||||
assert stats1.data["save_duration"] == pytest.approx([0.1, 0.3])
|
||||
assert stats1.data["load_duration"] == pytest.approx([0.2])
|
||||
assert stats1.data["num_failed_load"] == 1
|
||||
assert stats1.data["num_transfer_task"] == 4
|
||||
|
||||
def test_reduce_with_data(self):
|
||||
"""reduce() computes correct averages when data is present."""
|
||||
stats = HF3FSKVConnectorStats()
|
||||
stats.record_success_task_duration("Saved", 1.0)
|
||||
stats.record_success_task_duration("Saved", 3.0)
|
||||
result = stats.reduce()
|
||||
assert result["Num save task success"] == pytest.approx(2.0, rel=0.01)
|
||||
assert result["Num save task failed"] == pytest.approx(0.0, rel=0.01)
|
||||
assert result["Avg save duration (ms)"] == pytest.approx(2000.0, rel=0.01)
|
||||
# Reduced values must be plain Python scalars so CLI logging renders
|
||||
# them without numpy reprs (eg np.float64(...)).
|
||||
assert all(not isinstance(v, np.generic) for v in result.values())
|
||||
|
||||
def test_clone_and_reset(self, hf3fs_stats):
|
||||
"""clone_and_reset() returns a copy with data and resets the original."""
|
||||
hf3fs_stats.record_success_task_duration("Saved", 0.7)
|
||||
hf3fs_stats.record_success_task_duration("Loaded", 0.4)
|
||||
|
||||
clone = hf3fs_stats.clone_and_reset()
|
||||
assert clone.data["num_transfer_task"] == 2
|
||||
assert hf3fs_stats.is_empty()
|
||||
@@ -1,192 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for HF3FS metadata server data structures and allocation logic:
|
||||
- RankFileMetadata : page allocation / release primitives
|
||||
- KeyMetadata : per-key rank-page tracking and completion detection
|
||||
- GlobalMetadataState : coordinated allocation with cache-hit semantics
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hf3fs.hf3fs_metadata_server import (
|
||||
GlobalMetadataState,
|
||||
KeyMetadata,
|
||||
RankFileMetadata,
|
||||
)
|
||||
|
||||
# ===========================================================================
|
||||
# TestRankFileMetadata
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestRankFileMetadata:
|
||||
"""Unit tests for RankFileMetadata page allocation primitives."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"alloc_count, expected_pages",
|
||||
[(3, 3), (5, 0)],
|
||||
ids=["alloc_partial", "alloc_exceeds"],
|
||||
)
|
||||
def test_allocate_pages(self, alloc_count, expected_pages):
|
||||
"""allocate_pages returns correct pages or empty list when insufficient."""
|
||||
rank_meta = RankFileMetadata(rank_id=0, num_pages=3, free_pages=list(range(3)))
|
||||
pages = rank_meta.allocate_pages(alloc_count)
|
||||
assert len(pages) == expected_pages
|
||||
if expected_pages > 0:
|
||||
rank_meta.release_pages(pages)
|
||||
assert rank_meta.get_free_page_count() == 3
|
||||
|
||||
def test_release_pages_restores_count(self):
|
||||
"""Releasing allocated pages returns them to the free pool."""
|
||||
rank_meta = RankFileMetadata(rank_id=0, num_pages=4, free_pages=list(range(4)))
|
||||
pages = rank_meta.allocate_pages(2)
|
||||
assert rank_meta.get_free_page_count() == 2
|
||||
rank_meta.release_pages(pages)
|
||||
assert rank_meta.get_free_page_count() == 4
|
||||
|
||||
def test_release_pages_no_duplicates(self):
|
||||
"""Releasing the same page twice must not create duplicates."""
|
||||
rank_meta = RankFileMetadata(rank_id=0, num_pages=3, free_pages=list(range(3)))
|
||||
rank_meta.allocate_pages(1) # takes page 0
|
||||
rank_meta.release_pages([0])
|
||||
rank_meta.release_pages([0]) # second release of the same page
|
||||
assert rank_meta.get_free_page_count() == 3
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestKeyMetadata
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestKeyMetadata:
|
||||
"""Unit tests for KeyMetadata completion tracking."""
|
||||
|
||||
def test_is_complete_false_until_all_ranks(self):
|
||||
"""is_complete() returns True only when all ranks confirmed."""
|
||||
key_meta = KeyMetadata(key="k", rank_to_page={}, tp_world_size=2)
|
||||
assert key_meta.is_complete() is False
|
||||
key_meta.add_rank_page(0, 5)
|
||||
assert key_meta.is_complete() is False
|
||||
key_meta.add_rank_page(1, 10)
|
||||
assert key_meta.is_complete() is True
|
||||
|
||||
def test_get_rank_page_returns_none_for_missing_rank(self):
|
||||
"""get_rank_page() returns None when the rank has no entry."""
|
||||
key_meta = KeyMetadata(key="k", rank_to_page={0: 3}, tp_world_size=2)
|
||||
assert key_meta.get_rank_page(0) == 3
|
||||
assert key_meta.get_rank_page(1) is None
|
||||
|
||||
def test_get_all_pages(self):
|
||||
"""get_all_pages() returns all (rank, page) pairs."""
|
||||
key_meta = KeyMetadata(key="k", rank_to_page={0: 1, 1: 2}, tp_world_size=2)
|
||||
pairs = key_meta.get_all_pages()
|
||||
assert set(pairs) == {(0, 1), (1, 2)}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestGlobalMetadataStateAllocation
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestGlobalMetadataStateAllocation:
|
||||
"""Tests for GlobalMetadataState allocation and cache-hit semantics."""
|
||||
|
||||
def test_uninitialized_rank_raises_on_allocate(self):
|
||||
"""allocate_pages_for_keys raises ValueError for unknown rank."""
|
||||
state = GlobalMetadataState()
|
||||
with pytest.raises((ValueError, Exception)):
|
||||
state.allocate_pages_for_keys(99, [("key", "")])
|
||||
|
||||
def test_uninitialized_rank_raises_on_get_locations(self):
|
||||
"""get_key_locations raises ValueError for unknown rank."""
|
||||
state = GlobalMetadataState()
|
||||
with pytest.raises((ValueError, Exception)):
|
||||
state.get_key_locations(99, ["any_key"])
|
||||
|
||||
def test_basic_allocation_and_confirm(self):
|
||||
"""Allocating a page and confirming it marks the key as complete."""
|
||||
state = GlobalMetadataState()
|
||||
state.initialize_rank(0, 4)
|
||||
|
||||
results = state.allocate_pages_for_keys(0, [("K", "")])
|
||||
assert results["K"] >= 0
|
||||
|
||||
state.confirm_write_for_keys(0, [("K", results["K"])])
|
||||
assert state.batch_key_exists(["K"]) == [True]
|
||||
locations = state.get_key_locations(0, ["K"])
|
||||
assert locations == [results["K"]]
|
||||
|
||||
def test_allocate_pages_cache_hit_does_not_leak_pages(self):
|
||||
"""Cache-hit key must not consume a page from the free pool;
|
||||
the pre-allocated slot must be returned before reusing the existing page.
|
||||
"""
|
||||
state = GlobalMetadataState()
|
||||
state.initialize_rank(0, 5) # 5 free pages: [0,1,2,3,4]
|
||||
|
||||
# Simulate a key that has already been fully written and confirmed.
|
||||
state.key_metadata["K_cached"] = KeyMetadata(
|
||||
key="K_cached", rank_to_page={0: 2}, tp_world_size=1
|
||||
)
|
||||
|
||||
free_before = state.rank_metadata[0].get_free_page_count() # 5
|
||||
|
||||
results = state.allocate_pages_for_keys(0, [("K_cached", ""), ("K_new", "")])
|
||||
|
||||
free_after = state.rank_metadata[0].get_free_page_count()
|
||||
|
||||
# Cache-hit key must reuse its existing page.
|
||||
assert results["K_cached"] == 2, (
|
||||
f"Cache-hit key should reuse page 2, got {results['K_cached']}"
|
||||
)
|
||||
# New key must receive a valid page.
|
||||
assert results["K_new"] >= 0, (
|
||||
f"New key should get a valid page, got {results['K_new']}"
|
||||
)
|
||||
# Exactly one page consumed from the free pool.
|
||||
assert free_before - free_after == 1, (
|
||||
f"Expected 1 page consumed, got delta={free_before - free_after}"
|
||||
)
|
||||
|
||||
def test_allocate_pages_all_cache_hits_frees_all_slots(self):
|
||||
"""When every key in the batch is a cache hit, no pages are consumed."""
|
||||
state = GlobalMetadataState()
|
||||
state.initialize_rank(0, 5)
|
||||
|
||||
for key, page in (("K1", 0), ("K2", 1)):
|
||||
state.key_metadata[key] = KeyMetadata(
|
||||
key=key, rank_to_page={0: page}, tp_world_size=1
|
||||
)
|
||||
|
||||
free_before = state.rank_metadata[0].get_free_page_count()
|
||||
results = state.allocate_pages_for_keys(0, [("K1", ""), ("K2", "")])
|
||||
free_after = state.rank_metadata[0].get_free_page_count()
|
||||
|
||||
assert results["K1"] == 0
|
||||
assert results["K2"] == 1
|
||||
assert free_after == free_before, (
|
||||
f"All-cache-hit batch must not consume free pages; "
|
||||
f"before={free_before}, after={free_after}"
|
||||
)
|
||||
|
||||
def test_allocate_returns_minus_one_when_pool_exhausted(self):
|
||||
"""If the free pool is exhausted, all new keys receive -1."""
|
||||
state = GlobalMetadataState()
|
||||
state.initialize_rank(0, 1) # only 1 free page
|
||||
|
||||
results = state.allocate_pages_for_keys(0, [("K1", ""), ("K2", "")])
|
||||
# allocate_pages uses all-or-nothing: 2 needed but only 1 available → []
|
||||
assert all(v == -1 for v in results.values()), f"Expected all -1, got {results}"
|
||||
|
||||
def test_confirm_write_releases_pages(self):
|
||||
"""confirm_write_for_keys with pages_to_release returns them to pool."""
|
||||
state = GlobalMetadataState()
|
||||
state.initialize_rank(0, 3)
|
||||
|
||||
results = state.allocate_pages_for_keys(0, [("K", "")])
|
||||
page = results["K"]
|
||||
free_after_alloc = state.rank_metadata[0].get_free_page_count()
|
||||
|
||||
state.confirm_write_for_keys(0, [("K", page)], pages_to_release=[page])
|
||||
free_after_release = state.rank_metadata[0].get_free_page_count()
|
||||
|
||||
assert free_after_release == free_after_alloc + 1
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparseKVConnectorStats,
|
||||
)
|
||||
|
||||
|
||||
def test_is_empty_on_fresh_stats():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_record_snapshot_and_reduce():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
stats.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
|
||||
assert not stats.is_empty()
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced["HiSparse hot-buffer hits"] == 12
|
||||
assert reduced["HiSparse hot-buffer misses"] == 4
|
||||
assert reduced["HiSparse host-to-device bytes"] == 64
|
||||
|
||||
|
||||
def test_record_host_usage_reports_latest_level():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_host_usage(usage=0.4, pending_page_transfers=2)
|
||||
stats.record_host_usage(usage=0.8, pending_page_transfers=0)
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced["HiSparse host KV cache usage %"] == 80.0
|
||||
|
||||
|
||||
def test_aggregate_keeps_latest_host_usage_level():
|
||||
first = HiSparseKVConnectorStats()
|
||||
first.record_host_usage(usage=0.4, pending_page_transfers=2)
|
||||
second = HiSparseKVConnectorStats()
|
||||
second.record_host_usage(usage=0.8, pending_page_transfers=0)
|
||||
second.record_host_usage(usage=0.6, pending_page_transfers=1)
|
||||
|
||||
first.aggregate(second)
|
||||
|
||||
assert first.data["host_cache_usage_perc"] == [0.4, 0.6]
|
||||
assert first.data["pending_page_transfers"] == [2, 1]
|
||||
|
||||
|
||||
def test_aggregate_extends_snapshot_deltas():
|
||||
first = HiSparseKVConnectorStats()
|
||||
first.record_snapshot(hits=7, misses=3, host_to_device_bytes=48)
|
||||
second = HiSparseKVConnectorStats()
|
||||
second.record_snapshot(hits=5, misses=1, host_to_device_bytes=16)
|
||||
|
||||
first.aggregate(second)
|
||||
|
||||
assert first.data["cache_hits"] == [7, 5]
|
||||
assert first.data["cache_misses"] == [3, 1]
|
||||
assert first.data["host_to_device_bytes"] == [48, 16]
|
||||
|
||||
|
||||
def test_aggregate_skips_empty_stats():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=2, misses=1, host_to_device_bytes=16)
|
||||
|
||||
stats.aggregate(HiSparseKVConnectorStats())
|
||||
|
||||
assert stats.data["cache_hits"] == [2]
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_round_trip():
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=12, misses=4, host_to_device_bytes=64)
|
||||
payload = stats.to_dict()
|
||||
|
||||
rebuilt = HiSparseConnector.build_kv_connector_stats(data=payload)
|
||||
|
||||
assert rebuilt is not None
|
||||
assert isinstance(rebuilt, HiSparseKVConnectorStats)
|
||||
assert rebuilt.reduce() == stats.reduce()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_metrics", [True, False])
|
||||
def test_prom_metrics_observe_host_metrics(kv_cache_metrics: bool):
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
from vllm.v1.metrics.stats import KVCacheEvictionEvent
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.observed: list[float] = []
|
||||
|
||||
def labels(self, *labelvalues: object) -> "_FakeMetric":
|
||||
return self
|
||||
|
||||
def inc(self, value: int | float) -> None:
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
def observe(self, value: float) -> None:
|
||||
self.observed.append(value)
|
||||
|
||||
created: dict[str, _FakeMetric] = {}
|
||||
|
||||
class _NamedFake(_FakeMetric):
|
||||
def __init__(self, **kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
created[kwargs["name"]] = self
|
||||
|
||||
prom = HiSparsePromMetrics(
|
||||
vllm_config=SimpleNamespace(
|
||||
kv_transfer_config=None,
|
||||
observability_config=SimpleNamespace(
|
||||
kv_cache_metrics=kv_cache_metrics, custom_histogram_buckets=None
|
||||
),
|
||||
),
|
||||
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
|
||||
labelnames=["model_name"],
|
||||
per_engine_labelvalues={0: ["model"]},
|
||||
)
|
||||
|
||||
stats = HiSparseKVConnectorStats()
|
||||
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
|
||||
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
|
||||
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
|
||||
stats.record_host_evictions([KVCacheEvictionEvent(5.0, 2.0, (1.0, 2.0))])
|
||||
later = HiSparseKVConnectorStats()
|
||||
later.record_host_evictions([KVCacheEvictionEvent(3.0, 3.0, ())])
|
||||
prom.observe(stats.aggregate(later).to_dict())
|
||||
|
||||
assert created["vllm:hisparse_cache_hits"].increments == [3]
|
||||
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
|
||||
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
|
||||
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
|
||||
if not kv_cache_metrics:
|
||||
assert not any("host_block" in name for name in created)
|
||||
return
|
||||
assert created["vllm:hisparse_host_block_lifetime_seconds"].observed == [5.0, 3.0]
|
||||
assert created["vllm:hisparse_host_block_idle_before_evict_seconds"].observed == [
|
||||
2.0,
|
||||
3.0,
|
||||
]
|
||||
assert created["vllm:hisparse_host_block_reuse_gap_seconds"].observed == [1.0, 2.0]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,335 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Hardware-fair request routing in the MoRIIO toy P/D proxy.
|
||||
|
||||
Exercises the REAL ``flat_interleaved_dp_route`` from the toy proxy (loaded from
|
||||
its ``examples/`` path with heavy deps stubbed) — no routing logic is copied
|
||||
here. This is the request-distribution half of end-to-end fairness: the proxy
|
||||
must hand every prefill/decode (instance, dp_rank) slot an equal share so no GPU
|
||||
is starved. The connector-side read routing is covered in
|
||||
``test_moriio_routing_fairness.py``.
|
||||
|
||||
Regression target: the previous scheme derived ``instance = req % n_instances``
|
||||
and ``dp_rank = req % dp_size`` from the same counter, so when
|
||||
``n_instances | dp_size`` each instance was locked to a stride-``n`` subset of
|
||||
its ranks — e.g. 2 prefill instances x DP8 stranded 4 of every node's 8 GPUs.
|
||||
``flat_interleaved_dp_route`` walks ONE counter over the full
|
||||
``(instance, dp_rank)`` slot space, so the two selections can never alias.
|
||||
|
||||
Role shapes below are exactly those in the RFC (#46107) deployments:
|
||||
(1, 1) 1P/1D TP8 (2, 1) 2P/2D TP8 (4, 1) 4D TP8
|
||||
(1, 8) 1D DP8EP (2, 8) 2P DP8EP (3, 8) 3D DP8EP
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import ctypes
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import msgpack
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
PROXY_REL = "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
|
||||
|
||||
|
||||
def _module(name, **attrs):
|
||||
module = types.ModuleType(name)
|
||||
for attr, value in attrs.items():
|
||||
setattr(module, attr, value)
|
||||
return module
|
||||
|
||||
|
||||
def _package(name):
|
||||
module = _module(name)
|
||||
module.__path__ = []
|
||||
return module
|
||||
|
||||
|
||||
class _QuartStub:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def route(self, *a, **k):
|
||||
return lambda fn: fn
|
||||
|
||||
def post(self, *a, **k):
|
||||
return self.route()
|
||||
|
||||
|
||||
async def _make_response_stub(value):
|
||||
return value
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _proxy_import_stubs():
|
||||
"""Stub the proxy's external deps so the module imports for a pure unit test.
|
||||
|
||||
Only third-party/vllm imports are stubbed; the routing function under test
|
||||
is executed as-is from the real module.
|
||||
"""
|
||||
common = "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
|
||||
|
||||
class _MoRIIOConstants:
|
||||
TRANSFER_PREFIX = "moriio-transfer"
|
||||
|
||||
stubs = {
|
||||
"aiohttp": _module("aiohttp"),
|
||||
"msgpack": _module("msgpack"),
|
||||
"zmq": _module("zmq"),
|
||||
"quart": _module(
|
||||
"quart",
|
||||
Quart=_QuartStub,
|
||||
Request=object,
|
||||
make_response=_make_response_stub,
|
||||
request=object(),
|
||||
),
|
||||
"vllm": _package("vllm"),
|
||||
"vllm.distributed": _package("vllm.distributed"),
|
||||
"vllm.distributed.kv_transfer": _package("vllm.distributed.kv_transfer"),
|
||||
"vllm.distributed.kv_transfer.kv_connector": _package(
|
||||
"vllm.distributed.kv_transfer.kv_connector"
|
||||
),
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1": _package(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1"
|
||||
),
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio": _package(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio"
|
||||
),
|
||||
common: _module(common, MoRIIOConstants=_MoRIIOConstants),
|
||||
}
|
||||
saved = {}
|
||||
for name, module in stubs.items():
|
||||
if name not in sys.modules:
|
||||
saved[name] = _MISSING
|
||||
sys.modules[name] = module
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for name, previous in saved.items():
|
||||
if previous is _MISSING:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = cast(types.ModuleType, previous)
|
||||
|
||||
|
||||
def _load_proxy_module():
|
||||
path = Path(__file__).parents[4] / PROXY_REL
|
||||
spec = importlib.util.spec_from_file_location("moriio_proxy_under_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
with _proxy_import_stubs():
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def route():
|
||||
return _load_proxy_module().flat_interleaved_dp_route
|
||||
|
||||
|
||||
def _instances(n: int, dp_size: int):
|
||||
# Only dp_size is read by the router; tp_size carried for realism.
|
||||
return [{"dp_size": dp_size, "tp_size": 8 // dp_size} for _ in range(n)]
|
||||
|
||||
|
||||
# (n_instances, dp_size) for every distinct P/D role shape in the RFC configs.
|
||||
ROLE_SHAPES = [(1, 1), (2, 1), (4, 1), (1, 8), (2, 8), (3, 8)]
|
||||
SHAPE_IDS = [f"n{n}_dp{dp}" for n, dp in ROLE_SHAPES]
|
||||
|
||||
|
||||
def _route_n(route, instances, count):
|
||||
# Proxy uses 1-indexed request numbers (slot = (request_number - 1) % ...).
|
||||
return [route(rn, instances) for rn in range(1, count + 1)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
|
||||
def test_full_slot_space_is_covered_uniformly(route, n, dp):
|
||||
instances = _instances(n, dp)
|
||||
period = n * dp
|
||||
# Three full cycles -> every (instance, dp_rank) slot must be hit the same
|
||||
# number of times (exactly uniform, no starved slot, no aliasing).
|
||||
hits = Counter(_route_n(route, instances, period * 3))
|
||||
|
||||
expected: set[tuple[int, int | None]]
|
||||
if dp == 1:
|
||||
expected = {(inst, None) for inst in range(n)}
|
||||
else:
|
||||
expected = {(inst, r) for inst in range(n) for r in range(dp)}
|
||||
assert set(hits) == expected, f"missing slots: {expected - set(hits)}"
|
||||
assert max(hits.values()) == min(hits.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "dp"), ROLE_SHAPES, ids=SHAPE_IDS)
|
||||
def test_instance_and_dp_rank_marginals_are_balanced(route, n, dp):
|
||||
instances = _instances(n, dp)
|
||||
routed = _route_n(route, instances, n * dp * 5)
|
||||
|
||||
inst_counts = Counter(inst for inst, _ in routed)
|
||||
assert set(inst_counts) == set(range(n))
|
||||
assert max(inst_counts.values()) == min(inst_counts.values())
|
||||
|
||||
dp_counts = Counter(r for _, r in routed)
|
||||
if dp == 1:
|
||||
assert set(dp_counts) == {None}
|
||||
else:
|
||||
assert set(dp_counts) == set(range(dp))
|
||||
assert max(dp_counts.values()) == min(dp_counts.values())
|
||||
|
||||
|
||||
def test_tp_instance_forwards_no_dp_rank(route):
|
||||
# dp_size == 1 (a TP instance) must yield dp_rank None so the proxy never
|
||||
# forwards an out-of-range data-parallel rank.
|
||||
assert all(r is None for _, r in _route_n(route, _instances(2, 1), 8))
|
||||
|
||||
|
||||
def test_two_instance_dp8_gives_every_node_all_ranks(route):
|
||||
# Direct regression for the stranded-GPU bug: with 2 prefill instances x DP8
|
||||
# each instance must receive ALL 8 dp ranks (16 distinct slots), not a
|
||||
# stride-2 subset of 4.
|
||||
routed = _route_n(route, _instances(2, 8), 2 * 8)
|
||||
per_instance: dict[int, set] = {0: set(), 1: set()}
|
||||
for inst, dp_rank in routed:
|
||||
per_instance[inst].add(dp_rank)
|
||||
assert per_instance[0] == set(range(8))
|
||||
assert per_instance[1] == set(range(8))
|
||||
|
||||
|
||||
def test_consecutive_requests_alternate_instances(route):
|
||||
# Interleaved order spreads consecutive requests across instances rather
|
||||
# than filling one instance's ranks before moving on.
|
||||
insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)]
|
||||
assert insts == [0, 1, 2]
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep")
|
||||
def test_discovery_heartbeat_progresses_while_worker_holds_gil():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
|
||||
MoRIIOHeartbeat,
|
||||
)
|
||||
|
||||
payload = {
|
||||
"type": "P",
|
||||
"http_address": "127.0.0.1:8000",
|
||||
"zmq_address": "host:127.0.0.1,handshake:6301,notify:61005",
|
||||
"dp_size": 1,
|
||||
"tp_size": 8,
|
||||
"transfer_mode": "READ",
|
||||
}
|
||||
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
|
||||
port = socket.bind_to_random_port("tcp://127.0.0.1")
|
||||
heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02)
|
||||
try:
|
||||
assert socket.poll(5000), "Discovery did not register"
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
# PyDLL retains the GIL throughout the native call.
|
||||
ctypes.PyDLL(None).usleep(300_000)
|
||||
received = 0
|
||||
while socket.poll(0):
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
received += 1
|
||||
assert received >= 3, "Discovery stalled while the worker held the GIL"
|
||||
finally:
|
||||
heartbeat.shutdown()
|
||||
heartbeat.shutdown()
|
||||
while socket.poll(0):
|
||||
socket.recv_multipart()
|
||||
assert not socket.poll(100), "Heartbeats continued after shutdown"
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs")
|
||||
def test_discovery_heartbeat_exits_when_worker_dies():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat
|
||||
|
||||
owner_code = """
|
||||
import runpy
|
||||
import sys
|
||||
Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"]
|
||||
heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02)
|
||||
print(heartbeat._process.pid, flush=True)
|
||||
sys.stdin.read()
|
||||
"""
|
||||
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
|
||||
port = socket.bind_to_random_port("tcp://127.0.0.1")
|
||||
owner = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
owner_code,
|
||||
moriio_heartbeat.__file__,
|
||||
f"tcp://127.0.0.1:{port}",
|
||||
],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
child_pid = None
|
||||
child_exited = False
|
||||
try:
|
||||
assert socket.poll(5000), "Discovery did not register"
|
||||
assert owner.stdout is not None
|
||||
child_pid = int(owner.stdout.readline())
|
||||
owner.kill()
|
||||
owner.wait(timeout=5)
|
||||
stat = Path(f"/proc/{child_pid}/stat")
|
||||
deadline = time.monotonic() + 5
|
||||
while stat.exists():
|
||||
# A terminated child may await reaping by the container's init.
|
||||
try:
|
||||
if stat.read_text().split(") ")[1].startswith("Z "):
|
||||
break
|
||||
except FileNotFoundError:
|
||||
break
|
||||
assert time.monotonic() < deadline, "Orphan heartbeat survived worker"
|
||||
time.sleep(0.01)
|
||||
child_exited = True
|
||||
finally:
|
||||
if owner.poll() is None:
|
||||
owner.kill()
|
||||
owner.communicate(timeout=5)
|
||||
if child_pid is not None and not child_exited:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(child_pid, 9)
|
||||
|
||||
|
||||
def test_discovery_heartbeat_recovers_after_router_outage(capfd):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
|
||||
MoRIIOHeartbeat,
|
||||
)
|
||||
|
||||
payload = {"type": "P", "http_address": "127.0.0.1:8000"}
|
||||
with zmq.Context() as context:
|
||||
with context.socket(zmq.ROUTER) as socket:
|
||||
port = socket.bind_to_random_port("tcp://127.0.0.1")
|
||||
endpoint = f"tcp://127.0.0.1:{port}"
|
||||
heartbeat = MoRIIOHeartbeat(endpoint, payload, 0.001, max_retries=1)
|
||||
try:
|
||||
assert socket.poll(5000), "Discovery did not register"
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
except BaseException:
|
||||
heartbeat.shutdown()
|
||||
raise
|
||||
try:
|
||||
# Fill the real DEALER queue while discovery is unavailable.
|
||||
deadline = time.monotonic() + 10
|
||||
errors = ""
|
||||
while "heartbeat send failed" not in errors:
|
||||
errors += capfd.readouterr().err
|
||||
assert time.monotonic() < deadline, "No send timeout during outage"
|
||||
time.sleep(0.01)
|
||||
with context.socket(zmq.ROUTER) as socket:
|
||||
socket.bind(endpoint)
|
||||
assert socket.poll(5000), "Discovery did not recover after outage"
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
assert heartbeat._process.poll() is None
|
||||
finally:
|
||||
heartbeat.shutdown()
|
||||
@@ -1,98 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MoRIIO router-authoritative DP-rank routing.
|
||||
|
||||
These lock in the redesign that addresses the PR #45043 review feedback
|
||||
(njhill): the connector must NOT self-derive a DP rank, and the prefill leg
|
||||
must *return* the rank it actually ran on via ``request_finished`` so the
|
||||
decode->prefill notify is routed by pure propagation (READ / serial WRITE),
|
||||
not by an independent hash on each side.
|
||||
|
||||
Following test_moriio_unmap.py, we bind the unbound method to a lightweight
|
||||
stand-in rather than constructing a full scheduler.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
_request_finished = MoRIIOConnectorScheduler.request_finished
|
||||
_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks
|
||||
|
||||
|
||||
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
|
||||
s = SimpleNamespace(
|
||||
is_producer=True,
|
||||
_is_hma_required=False,
|
||||
_has_mamba=False,
|
||||
blocks_per_sw=[0],
|
||||
_global_dp_rank=global_dp_rank,
|
||||
engine_id="engine-abc",
|
||||
host_ip="10.0.0.1",
|
||||
handshake_port=6300,
|
||||
side_notify_port=7300,
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=dp_size,
|
||||
data_parallel_size_local=dp_size_local,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
),
|
||||
)
|
||||
s.get_exchange_clipped_blocks = _clip_blocks.__get__(s)
|
||||
return s
|
||||
|
||||
|
||||
def _req(
|
||||
request_id: str, params: dict | None, status=RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
):
|
||||
return SimpleNamespace(
|
||||
request_id=request_id,
|
||||
kv_transfer_params=params,
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def test_request_finished_returns_prefill_rank_for_propagation():
|
||||
# 2P2D: global rank 11 lives on pod 1 (ranks 8..15), local size 8.
|
||||
s = _producer(global_dp_rank=11, dp_size=16, dp_size_local=8)
|
||||
req = _req("r1", {"do_remote_decode": True, "transfer_id": "t1"})
|
||||
|
||||
delay_free, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
# The prefill leg returns the rank IT actually ran on, marked authoritative,
|
||||
# so the router forwards it to decode and the notify propagates (no hashing).
|
||||
assert params is not None
|
||||
assert params["remote_dp_rank"] == 11
|
||||
assert params["remote_dp_rank_override"] is True
|
||||
assert params["remote_dp_size"] == 16
|
||||
assert params["remote_dp_size_local"] == 8
|
||||
assert params["transfer_id"] == "t1"
|
||||
# Empty block_ids => nothing deferred, blocks freed immediately.
|
||||
assert delay_free is False
|
||||
|
||||
|
||||
def test_request_finished_rank_tracks_the_actual_rank():
|
||||
# A different rank must be echoed verbatim -- the value is the producer's
|
||||
# own rank, never a recomputation of the request id.
|
||||
s = _producer(global_dp_rank=4, dp_size=8, dp_size_local=8)
|
||||
req = _req("r2", {"do_remote_decode": True, "transfer_id": "t2"})
|
||||
|
||||
_, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
assert params["remote_dp_rank"] == 4
|
||||
assert params["remote_dp_size_local"] == 8
|
||||
|
||||
|
||||
def test_request_finished_no_kv_params_is_noop():
|
||||
s = _producer(global_dp_rank=0, dp_size=16, dp_size_local=8)
|
||||
req = _req("r3", None)
|
||||
|
||||
delay_free, params = _request_finished(s, req, block_ids=[])
|
||||
|
||||
assert delay_free is False
|
||||
assert params is None
|
||||
@@ -1,210 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Hardware-fair KV-read routing across MoRIIO heterogeneous P/D configs.
|
||||
|
||||
Dependency-light (no GPU / ROCm / mori): builds bare ``MoRIIOConnectorWorker``
|
||||
instances via ``object.__new__`` and drives the REAL read-source routing
|
||||
decision (``_resolve_read_source`` / ``_next_flex_tp_rank``) directly -- no
|
||||
routing logic is re-implemented here, so a future change to those functions is
|
||||
what these tests exercise.
|
||||
|
||||
Scope -- the CONNECTOR (decode-side) read routing. The connector never selects
|
||||
the prefill instance; the proxy does and hands the connector a ``remote_host`` +
|
||||
``remote_dp_rank`` per request. The connector's only fairness lever is WHICH
|
||||
prefill (dp, tp) rank each read targets, and the RFC's deployments collapse to
|
||||
three real connector behaviours:
|
||||
|
||||
symmetric TP (TP prefill + TP decode) -> decode tp_k reads prefill tp_k
|
||||
flexible (TP prefill + DP decode) -> round-robin over prefill tp0..N-1
|
||||
owner DP (DP prefill, any decode) -> read the owner rank, tp0
|
||||
|
||||
These assume the proxy delivers a FAIR request stream (each prefill instance +
|
||||
owner dp-rank an equal share) and verify the connector never re-introduces a
|
||||
bottleneck. That proxy contract is checked separately against the real
|
||||
``flat_interleaved_dp_route`` in the toy-proxy tests (PR #46115).
|
||||
|
||||
A node runs one of two modes (8 GPUs each):
|
||||
* TP8 -> (dp_size, tp_size) = (1, 8); MLA latent KV REPLICATED on all ranks.
|
||||
* DP8EP -> (dp_size, tp_size) = (8, 1); KV PARTITIONED, one owner rank/request.
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ReqMeta,
|
||||
get_port_offset,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
|
||||
MODE_DIMS = {"TP8": (1, 8), "DP8EP": (8, 1)} # (dp_size, tp_size), 8 GPUs/node
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PDConfig:
|
||||
name: str
|
||||
p_mode: str
|
||||
d_mode: str
|
||||
|
||||
@property
|
||||
def p_dp(self) -> int:
|
||||
return MODE_DIMS[self.p_mode][0]
|
||||
|
||||
@property
|
||||
def p_tp(self) -> int:
|
||||
return MODE_DIMS[self.p_mode][1]
|
||||
|
||||
@property
|
||||
def d_dp(self) -> int:
|
||||
return MODE_DIMS[self.d_mode][0]
|
||||
|
||||
@property
|
||||
def d_tp(self) -> int:
|
||||
return MODE_DIMS[self.d_mode][1]
|
||||
|
||||
@property
|
||||
def n_prefill_gpus(self) -> int:
|
||||
return self.p_dp * self.p_tp
|
||||
|
||||
|
||||
CONFIGS = [
|
||||
PDConfig("1P_TP8:1D_TP8", "TP8", "TP8"),
|
||||
PDConfig("2P_TP8:1D_DP8EP", "TP8", "DP8EP"),
|
||||
PDConfig("2P_TP8:2D_TP8", "TP8", "TP8"),
|
||||
PDConfig("2P_DP8EP:3D_DP8EP", "DP8EP", "DP8EP"),
|
||||
PDConfig("2P_DP8EP:4D_TP8", "DP8EP", "TP8"),
|
||||
]
|
||||
CONFIG_IDS = [c.name for c in CONFIGS]
|
||||
|
||||
|
||||
def make_decode_worker(*, world_size: int, tp_rank: int, dp_rank: int):
|
||||
w = object.__new__(MoRIIOConnectorWorker)
|
||||
w.world_size = world_size
|
||||
w.tp_rank = tp_rank
|
||||
w.dp_rank = dp_rank
|
||||
w.use_mla = True
|
||||
return w
|
||||
|
||||
|
||||
def make_meta(*, p_tp: int, p_dp: int, remote_dp_rank: int, host: str = "phost0"):
|
||||
return ReqMeta(
|
||||
transfer_id="t",
|
||||
local_block_ids=[1],
|
||||
remote_block_ids=[2],
|
||||
remote_host=host,
|
||||
remote_port=1234,
|
||||
remote_handshake_port=6301,
|
||||
remote_notify_port=61005,
|
||||
remote_engine_id=f"{host}:6301",
|
||||
tp_size=p_tp,
|
||||
remote_dp_size=p_dp,
|
||||
remote_dp_rank=remote_dp_rank,
|
||||
)
|
||||
|
||||
|
||||
def build_decode_workers(cfg: PDConfig) -> list:
|
||||
"""Decode workers that issue reads for one prefill instance. A TP decode
|
||||
instance reads from every tp rank; a DP decode instance from each dp-rank
|
||||
worker. Reused across requests so per-worker round-robin state advances."""
|
||||
if cfg.d_tp > 1:
|
||||
return [
|
||||
make_decode_worker(world_size=cfg.d_tp, tp_rank=r, dp_rank=0)
|
||||
for r in range(cfg.d_tp)
|
||||
]
|
||||
return [
|
||||
make_decode_worker(world_size=1, tp_rank=0, dp_rank=d) for d in range(cfg.d_dp)
|
||||
]
|
||||
|
||||
|
||||
def prefill_target_multiset(cfg: PDConfig, rounds: int) -> Counter:
|
||||
"""Drive the REAL _resolve_read_source over a fair input stream; tally which
|
||||
prefill GPU each read targets. The only modelled assumption is the proxy
|
||||
contract -- each owner dp-rank delivered equally (``for owner_dp in
|
||||
range(p_dp)``); no proxy algorithm is reproduced."""
|
||||
workers = build_decode_workers(cfg)
|
||||
hits: Counter = Counter()
|
||||
for _ in range(rounds):
|
||||
for owner_dp in range(cfg.p_dp):
|
||||
for worker in workers:
|
||||
meta = make_meta(p_tp=cfg.p_tp, p_dp=cfg.p_dp, remote_dp_rank=owner_dp)
|
||||
chosen_tp, _flexible = worker._resolve_read_source(meta)
|
||||
target = get_port_offset(owner_dp, chosen_tp, cfg.p_tp)
|
||||
assert 0 <= target < cfg.n_prefill_gpus
|
||||
hits[target] += 1
|
||||
return hits
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cfg", CONFIGS, ids=CONFIG_IDS)
|
||||
def test_kv_read_load_is_hardware_fair(cfg: PDConfig) -> None:
|
||||
# rounds a multiple of p_tp so the round-robin closes on an exact cycle.
|
||||
hits = prefill_target_multiset(cfg, rounds=16)
|
||||
gpus = set(range(cfg.n_prefill_gpus))
|
||||
assert set(hits) == gpus, f"unused prefill GPUs: {sorted(gpus - set(hits))}"
|
||||
assert max(hits.values()) == min(hits.values()), dict(sorted(hits.items()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cfg", CONFIGS, ids=CONFIG_IDS)
|
||||
def test_flexible_gate_fires_only_for_tp_prefill_dp_decode(cfg: PDConfig) -> None:
|
||||
flags = set()
|
||||
for worker in build_decode_workers(cfg):
|
||||
meta = make_meta(p_tp=cfg.p_tp, p_dp=cfg.p_dp, remote_dp_rank=0)
|
||||
_chosen, flexible = worker._resolve_read_source(meta)
|
||||
flags.add(flexible)
|
||||
is_mirror = cfg.d_tp == 1 and cfg.p_dp == 1 and cfg.p_tp > 1
|
||||
assert flags == {is_mirror}
|
||||
|
||||
|
||||
def test_symmetric_tp_is_a_bijection() -> None:
|
||||
targets = []
|
||||
for tp_rank in range(8):
|
||||
worker = make_decode_worker(world_size=8, tp_rank=tp_rank, dp_rank=0)
|
||||
chosen_tp, flexible = worker._resolve_read_source(
|
||||
make_meta(p_tp=8, p_dp=1, remote_dp_rank=0)
|
||||
)
|
||||
assert not flexible
|
||||
targets.append(chosen_tp)
|
||||
assert sorted(targets) == list(range(8))
|
||||
|
||||
|
||||
def test_owner_dp_read_is_faithful_and_covers_every_rank() -> None:
|
||||
worker = make_decode_worker(world_size=8, tp_rank=3, dp_rank=0)
|
||||
targets = []
|
||||
for owner_dp in range(8):
|
||||
chosen_tp, flexible = worker._resolve_read_source(
|
||||
make_meta(p_tp=1, p_dp=8, remote_dp_rank=owner_dp)
|
||||
)
|
||||
assert not flexible
|
||||
assert chosen_tp == 0 # p_tp == 1, only tp0 exists
|
||||
targets.append(get_port_offset(owner_dp, chosen_tp, 1))
|
||||
assert sorted(targets) == list(range(8))
|
||||
|
||||
|
||||
def test_flexible_round_robin_is_deterministic_uniform_and_staggered() -> None:
|
||||
w0 = make_decode_worker(world_size=1, tp_rank=0, dp_rank=0)
|
||||
seq = [w0._next_flex_tp_rank(8) for _ in range(64)]
|
||||
assert seq[:8] == list(range(8))
|
||||
assert Counter(seq) == Counter({t: 8 for t in range(8)})
|
||||
|
||||
first_pick = [
|
||||
make_decode_worker(world_size=1, tp_rank=0, dp_rank=d)._next_flex_tp_rank(8)
|
||||
for d in range(8)
|
||||
]
|
||||
assert sorted(first_pick) == list(range(8))
|
||||
|
||||
|
||||
def test_read_blocks_for_req_threads_chosen_tp() -> None:
|
||||
# The resolved (chosen_tp, flexible) must reach _read_blocks, which keys the
|
||||
# session AND the notify port off that single value -- so a read and its
|
||||
# completion notify address the same prefill rank.
|
||||
worker = make_decode_worker(world_size=1, tp_rank=0, dp_rank=3)
|
||||
worker._read_blocks = MagicMock()
|
||||
worker._read_blocks_for_req("r", make_meta(p_tp=8, p_dp=1, remote_dp_rank=0))
|
||||
kw = worker._read_blocks.call_args.kwargs
|
||||
assert kw["flexible"] is True
|
||||
assert kw["chosen_tp"] == 3 # first flexible pick = dp_rank seed
|
||||
assert get_port_offset(0, kw["chosen_tp"], 8) == 3
|
||||
@@ -1,494 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOMode,
|
||||
MoRIIOTransferAck,
|
||||
TransferError,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnector,
|
||||
MoRIIOConnectorWorker,
|
||||
get_moriio_expected_ack_count,
|
||||
get_moriio_remote_tp_rank,
|
||||
resolve_moriio_transfer_ack,
|
||||
validate_moriio_heterogeneous_tp_kv_heads,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWrapper,
|
||||
)
|
||||
|
||||
|
||||
def test_remote_tp_rank_same_tp_maps_to_self():
|
||||
assert [get_moriio_remote_tp_rank(rank, 4, 4) for rank in range(4)] == [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
]
|
||||
|
||||
|
||||
def test_remote_tp_rank_p4_d8_floor_maps_decode_to_prefill():
|
||||
assert [get_moriio_remote_tp_rank(rank, 8, 4) for rank in range(8)] == [
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
]
|
||||
|
||||
|
||||
def test_remote_tp_rank_p8_d4_maps_to_first_prefill_rank_per_pair():
|
||||
assert [get_moriio_remote_tp_rank(rank, 4, 8) for rank in range(4)] == [
|
||||
0,
|
||||
2,
|
||||
4,
|
||||
6,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_rank", "local_tp_size", "remote_tp_size"),
|
||||
[
|
||||
(0, 6, 4),
|
||||
(0, 4, 6),
|
||||
],
|
||||
)
|
||||
def test_remote_tp_rank_invalid_non_multiple_tp_raises(
|
||||
local_tp_rank: int, local_tp_size: int, remote_tp_size: int
|
||||
):
|
||||
with pytest.raises(ValueError, match="multiple"):
|
||||
get_moriio_remote_tp_rank(local_tp_rank, local_tp_size, remote_tp_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
|
||||
[
|
||||
(4, 4, 8),
|
||||
(8, 4, 4),
|
||||
(4, 8, 4),
|
||||
],
|
||||
)
|
||||
def test_heterogeneous_tp_head_guard_allows_supported_layouts(
|
||||
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
|
||||
):
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size,
|
||||
remote_tp_size,
|
||||
total_num_kv_heads,
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
def test_heterogeneous_tp_head_guard_allows_mla_layouts():
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size=2,
|
||||
remote_tp_size=4,
|
||||
total_num_kv_heads=4,
|
||||
is_mla=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_tp_size", "remote_tp_size", "total_num_kv_heads"),
|
||||
[
|
||||
(4, 2, 4),
|
||||
(2, 4, 4),
|
||||
],
|
||||
)
|
||||
def test_heterogeneous_tp_head_guard_rejects_split_kv_heads(
|
||||
local_tp_size: int, remote_tp_size: int, total_num_kv_heads: int
|
||||
):
|
||||
with pytest.raises(NotImplementedError, match="replicated KV heads"):
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size,
|
||||
remote_tp_size,
|
||||
total_num_kv_heads,
|
||||
is_mla=False,
|
||||
)
|
||||
|
||||
|
||||
def test_expected_ack_count_for_homogeneous_or_smaller_consumer_tp_is_one():
|
||||
assert get_moriio_expected_ack_count(4, 4) == 1
|
||||
assert get_moriio_expected_ack_count(8, 4) == 1
|
||||
|
||||
|
||||
def test_expected_ack_count_for_decode_fan_in():
|
||||
assert get_moriio_expected_ack_count(4, 8) == 2
|
||||
|
||||
|
||||
def test_expected_ack_count_rejects_non_multiple_fan_in():
|
||||
with pytest.raises(ValueError, match="multiple"):
|
||||
get_moriio_expected_ack_count(4, 6)
|
||||
|
||||
|
||||
def test_plain_string_ack_is_backward_compatible_single_ack():
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
"tx-plain",
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-plain"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-plain"
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-plain"}
|
||||
|
||||
|
||||
def test_structured_release_ack_waits_for_all_expected_acks():
|
||||
ack = MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-fanin"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {"tx-fanin": 1}
|
||||
assert completed_transfer_ids == set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-fanin"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-fanin"
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-fanin"}
|
||||
|
||||
|
||||
def test_duplicate_ack_after_completion_does_not_resolve_twice():
|
||||
ack = MoRIIOTransferAck("tx-dup", consumer_tp_size=8)
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
== "tx-dup"
|
||||
)
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
ack,
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-dup"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == {"tx-dup"}
|
||||
|
||||
|
||||
def test_ack_for_non_live_transfer_is_ignored():
|
||||
notification_counts: dict[str, int] = {}
|
||||
completed_transfer_ids: set[str] = set()
|
||||
|
||||
assert (
|
||||
resolve_moriio_transfer_ack(
|
||||
MoRIIOTransferAck("tx-stale", consumer_tp_size=8),
|
||||
producer_tp_size=4,
|
||||
live_transfer_ids={"tx-live"},
|
||||
notification_counts=notification_counts,
|
||||
completed_transfer_ids=completed_transfer_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert notification_counts == {}
|
||||
assert completed_transfer_ids == set()
|
||||
|
||||
|
||||
def test_worker_get_finished_counts_structured_release_fan_in():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.batches = [
|
||||
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
|
||||
[MoRIIOTransferAck("tx-fanin", consumer_tp_size=8)],
|
||||
]
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
return self.batches.pop(0)
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.is_producer = True
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.world_size = 4
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
|
||||
worker._consumer_notification_counts = {}
|
||||
worker._completed_consumer_notifications = set()
|
||||
worker._pending_unmapped_acks = []
|
||||
|
||||
assert worker.get_finished() == (set(), set())
|
||||
assert worker._consumer_notification_counts == {"tx-fanin": 1}
|
||||
|
||||
assert worker.get_finished() == ({"req-fanin"}, set())
|
||||
assert worker._consumer_notification_counts == {}
|
||||
assert worker._completed_consumer_notifications == {"tx-fanin"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exit_kind", ["killed", "eof"])
|
||||
def test_worker_reports_unexpected_heartbeat_exit(exit_kind):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
|
||||
MoRIIOHeartbeat,
|
||||
)
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.is_producer = True
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.moriio_wrapper = SimpleNamespace(
|
||||
pop_finished_req_ids=lambda: [], shutdown=lambda: None
|
||||
)
|
||||
worker._pending_unmapped_acks = []
|
||||
worker._heartbeat = MoRIIOHeartbeat("tcp://127.0.0.1:1", {"type": "P"}, 0.02)
|
||||
process = worker._heartbeat._process
|
||||
try:
|
||||
assert worker.get_finished() == (set(), set())
|
||||
if exit_kind == "killed":
|
||||
process.kill()
|
||||
else:
|
||||
process.stdin.close()
|
||||
returncode = process.wait(timeout=5)
|
||||
with pytest.raises(RuntimeError, match=f"heartbeat.*{returncode}"):
|
||||
worker.get_finished()
|
||||
finally:
|
||||
worker.shutdown()
|
||||
|
||||
|
||||
def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
class DoneStatus:
|
||||
def Succeeded(self):
|
||||
return True
|
||||
|
||||
def Failed(self):
|
||||
return False
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.sent = []
|
||||
|
||||
# Exercise the real batch verdict rather than restating it here.
|
||||
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
|
||||
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.sent.append((transfer_id, host, port, message_type, message_fields))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.world_size = 8
|
||||
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
# A layer maps to the list of reads posted for it (a KDA layer posts two).
|
||||
worker._recving_transfers = {"req": {"layer0": [DoneStatus(), DoneStatus()]}}
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-release")
|
||||
}
|
||||
# Transfer-timeout reaping state consulted by _pop_done_transfers.
|
||||
worker._recving_transfers_start = {}
|
||||
# Load-error bookkeeping cleared alongside a completed transfer.
|
||||
worker._recving_local_blocks = {}
|
||||
worker._invalid_block_ids = set()
|
||||
|
||||
assert worker._pop_done_transfers() == {"tx-release"}
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
(
|
||||
"tx-release",
|
||||
"127.0.0.1",
|
||||
"7000",
|
||||
"release",
|
||||
{"consumer_tp_size": 8},
|
||||
)
|
||||
]
|
||||
assert worker._recving_transfers == {}
|
||||
assert worker._recving_transfers_callback_addr == {}
|
||||
|
||||
|
||||
def test_aborted_read_without_local_blocks_only_releases_remote_blocks():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.sent = []
|
||||
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.sent.append((transfer_id, host, port, message_type, message_fields))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.world_size = 8
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
|
||||
worker._read_blocks(
|
||||
local_block_ids=[],
|
||||
remote_block_ids=[[10, 11], [90]],
|
||||
dst_engine_id="prefill",
|
||||
request_id="req-aborted",
|
||||
transfer_id="tx-aborted",
|
||||
remote_host="127.0.0.1",
|
||||
remote_notify_port=7000,
|
||||
remote_tp_size=8,
|
||||
remote_dp_rank=0,
|
||||
chosen_tp=2,
|
||||
)
|
||||
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
(
|
||||
"tx-aborted",
|
||||
"127.0.0.1",
|
||||
"7002",
|
||||
"release",
|
||||
{"consumer_tp_size": 8},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_read_completion_waits_for_every_posted_read():
|
||||
class Status:
|
||||
def __init__(self, done: bool):
|
||||
self.done = done
|
||||
|
||||
def Succeeded(self):
|
||||
return self.done
|
||||
|
||||
def Failed(self):
|
||||
return False
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.sent = []
|
||||
|
||||
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
|
||||
|
||||
def send_notify(self, *args, **kwargs):
|
||||
self.sent.append((args, kwargs))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.world_size = 8
|
||||
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker._recving_transfers = {
|
||||
"req": {"kda_layer": [Status(done=True), Status(done=False)]}
|
||||
}
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-pending")
|
||||
}
|
||||
worker._recving_transfers_start = {"req": float("inf")}
|
||||
worker._recving_local_blocks = {"req": [1, 2]}
|
||||
worker._invalid_block_ids = set()
|
||||
|
||||
assert worker._pop_done_transfers() == set()
|
||||
assert "req" in worker._recving_transfers
|
||||
assert worker.moriio_wrapper.sent == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("has_mamba", "expected_invalid"),
|
||||
[(False, {1, 2}), (True, set())],
|
||||
)
|
||||
def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid):
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = has_mamba
|
||||
worker._recving_local_blocks = {"req": [1, 2]}
|
||||
worker._invalid_block_ids = set()
|
||||
|
||||
worker._record_failed_recv("req")
|
||||
|
||||
assert worker.get_block_ids_with_load_errors() == expected_invalid
|
||||
|
||||
|
||||
def test_hybrid_step_barrier_fails_closed(monkeypatch):
|
||||
class FailingWrapper:
|
||||
def waiting_for_transfer_complete(self, _statuses):
|
||||
raise TransferError("failed")
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = True
|
||||
worker._reads_issued_this_step = [object()]
|
||||
worker._mamba_reads_this_step = [object()]
|
||||
worker.moriio_wrapper = FailingWrapper()
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.get_forward_context",
|
||||
lambda: SimpleNamespace(cudagraph_runtime_mode=None),
|
||||
)
|
||||
|
||||
with pytest.raises(TransferError, match="failed"):
|
||||
worker._await_reads_issued_this_step()
|
||||
|
||||
|
||||
def test_requested_cudagraph_mode_is_never_overridden():
|
||||
# The configured cudagraph mode is always honored: the barrier fires when
|
||||
# the operator sets cudagraph_mode=PIECEWISE, and READ mode with full
|
||||
# graphs only warns instead of silently forcing PIECEWISE.
|
||||
assert (
|
||||
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": True}) is False
|
||||
)
|
||||
assert (
|
||||
MoRIIOConnector.requires_piecewise_for_cudagraph({"read_mode": False}) is False
|
||||
)
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MoRIIOConnectorScheduler.{map,unmap}_request_id.
|
||||
|
||||
vLLM's input_processor mutates a request_id after the connector has stored it
|
||||
(e.g. ``cmpl-abc-0`` -> ``cmpl-abc-0-956053a4``), so an exact-match unmap can
|
||||
miss. These tests cover the stable-``transfer_id`` fallback that handles that
|
||||
case (and the clean exact-match and miss paths).
|
||||
|
||||
The map/unmap methods touch only the two id<->id dicts, so we bind them to a
|
||||
lightweight stand-in rather than constructing a full scheduler.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
)
|
||||
|
||||
_map = MoRIIOConnectorScheduler.map_request_id
|
||||
_unmap = MoRIIOConnectorScheduler.unmap_request_id
|
||||
|
||||
|
||||
def _sched():
|
||||
return SimpleNamespace(
|
||||
transfer_id_to_request_id={},
|
||||
request_id_to_transfer_id={},
|
||||
)
|
||||
|
||||
|
||||
def test_exact_match_unmap_clears_both_tables():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
_unmap(s, "rid-1")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
|
||||
|
||||
def test_unmap_via_transfer_id_after_request_id_mutation():
|
||||
s = _sched()
|
||||
# Connector stores the ORIGINAL request_id at map time.
|
||||
_map(s, "cmpl-abc-0", "tid-42")
|
||||
# input_processor later mutates the rid; exact match would miss, so the
|
||||
# stable transfer_id is used to find and clear the original entry.
|
||||
_unmap(s, "cmpl-abc-0-956053a4", transfer_id="tid-42")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
|
||||
|
||||
def test_unmap_miss_is_noop_and_does_not_raise():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
# Unknown rid and unknown/None transfer_id: must not raise, must not touch
|
||||
# the existing (unrelated) mapping.
|
||||
_unmap(s, "rid-unknown", transfer_id=None)
|
||||
_unmap(s, "rid-unknown", transfer_id="tid-unknown")
|
||||
assert s.request_id_to_transfer_id == {"rid-1": "tid-1"}
|
||||
assert s.transfer_id_to_request_id == {"tid-1": "rid-1"}
|
||||
|
||||
|
||||
def test_exact_match_preferred_over_transfer_id_fallback():
|
||||
s = _sched()
|
||||
_map(s, "rid-1", "tid-1")
|
||||
# Exact rid is present, so it is removed directly even if a transfer_id is
|
||||
# also supplied; no stale entries left behind.
|
||||
_unmap(s, "rid-1", transfer_id="tid-1")
|
||||
assert s.request_id_to_transfer_id == {}
|
||||
assert s.transfer_id_to_request_id == {}
|
||||
@@ -1,449 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for mori-io write_done routing (#51681)."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
ROLE,
|
||||
LayerTransferPlan,
|
||||
MoRIIOError,
|
||||
MoRIIOMode,
|
||||
RemoteAllocInfo,
|
||||
ReqMeta,
|
||||
WriteTask,
|
||||
get_port_offset,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnectorScheduler,
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWrapper,
|
||||
)
|
||||
|
||||
from .utils import make_moriio_writer
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _fake_worker(
|
||||
alloc: dict[str, Any],
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
set_remote_dp_size_local: int | None = None,
|
||||
) -> Any:
|
||||
wrapper = SimpleNamespace(
|
||||
done_remote_allocate_req_dict=alloc,
|
||||
lock=threading.Lock(),
|
||||
done_req_ids=[],
|
||||
sent_notifies=[],
|
||||
)
|
||||
wrapper.send_notify = lambda transfer_id, remote_ip, remote_port, message_type: (
|
||||
wrapper.sent_notifies.append(
|
||||
(transfer_id, remote_ip, remote_port, message_type)
|
||||
)
|
||||
)
|
||||
wrapper.waiting_for_transfer_complete = lambda statuses: None
|
||||
wrapper._mark_transfer_terminal_locked = lambda transfer_id: None
|
||||
wrapper._is_transfer_terminal_locked = lambda transfer_id: False
|
||||
worker = SimpleNamespace(moriio_wrapper=wrapper, tp_rank=tp_rank)
|
||||
if set_remote_dp_size_local is not None:
|
||||
worker.remote_dp_size_local = set_remote_dp_size_local
|
||||
return worker
|
||||
|
||||
|
||||
def _req_meta(
|
||||
*,
|
||||
transfer_id: str = "tA",
|
||||
remote_host: str = "10.0.0.1",
|
||||
multi_pod_hosts: list[str] | None = None,
|
||||
remote_dp_size: int = 16,
|
||||
remote_dp_size_local: int = 8,
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
transfer_id=transfer_id,
|
||||
local_block_ids=[[1]],
|
||||
remote_block_ids=[[2]],
|
||||
remote_host=remote_host,
|
||||
remote_port=1,
|
||||
remote_handshake_port=2,
|
||||
remote_notify_port=20000,
|
||||
remote_engine_id="remote-engine",
|
||||
tp_size=1,
|
||||
remote_dp_size=remote_dp_size,
|
||||
multi_pod_hosts=list(multi_pod_hosts or []),
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
)
|
||||
|
||||
|
||||
def _write_task(
|
||||
transfer_id: str,
|
||||
remote_ip: str,
|
||||
*,
|
||||
multi_pod_hosts: list[str] | None = None,
|
||||
remote_dp_size_local: int = 0,
|
||||
remote_dp_size: int = 1,
|
||||
request_id: str = "req",
|
||||
) -> WriteTask:
|
||||
return WriteTask(
|
||||
request_id=request_id,
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[[1, 3]],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name="layer0",
|
||||
event=None,
|
||||
remote_notify_port=20000,
|
||||
remote_ip=remote_ip,
|
||||
multi_pod_hosts=multi_pod_hosts or [],
|
||||
remote_dp_size_local=remote_dp_size_local,
|
||||
remote_dp_size=remote_dp_size,
|
||||
)
|
||||
|
||||
|
||||
def _real_wrapper() -> MoRIIOWrapper:
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
wrapper.done_req_ids = []
|
||||
wrapper.done_remote_allocate_req_dict = {}
|
||||
wrapper.done_write_cache_req_ids = []
|
||||
wrapper.failed_write_cache_req_ids = []
|
||||
wrapper._terminal_transfer_ids = OrderedDict()
|
||||
return wrapper
|
||||
|
||||
|
||||
def test_execute_write_task_ip_is_per_task_under_overwrite():
|
||||
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
|
||||
worker = _fake_worker(alloc)
|
||||
worker.multi_pod_hosts = ["10.0.0.2"]
|
||||
worker.remote_dp_size_local = 1
|
||||
|
||||
writer = make_moriio_writer(worker)
|
||||
task_a = _write_task(
|
||||
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
|
||||
)
|
||||
writer._execute_write_task(task_a)
|
||||
|
||||
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
|
||||
assert alloc["tA"].completion_remote_ip != "10.0.0.2"
|
||||
|
||||
|
||||
def test_deferred_task_ip_is_per_task_under_overwrite():
|
||||
alloc = {"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0)}
|
||||
worker = _fake_worker(alloc)
|
||||
worker.multi_pod_hosts = ["10.0.0.2"]
|
||||
worker.remote_dp_size_local = 1
|
||||
|
||||
writer = make_moriio_writer(worker)
|
||||
task_a = _write_task(
|
||||
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
|
||||
)
|
||||
writer._deferred_tasks.append(task_a)
|
||||
writer._process_deferred_tasks()
|
||||
|
||||
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
|
||||
assert writer._deferred_tasks == []
|
||||
|
||||
|
||||
def test_two_requests_resolve_distinct_hosts():
|
||||
alloc = {
|
||||
"tA": RemoteAllocInfo(block_ids=None, decode_dp_rank=0),
|
||||
"tB": RemoteAllocInfo(block_ids=None, decode_dp_rank=3),
|
||||
}
|
||||
worker = _fake_worker(alloc)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task_a = _write_task(
|
||||
"tA", "10.0.0.1", multi_pod_hosts=["10.0.0.1"], remote_dp_size_local=1
|
||||
)
|
||||
task_b = _write_task(
|
||||
"tB", "10.0.0.2", multi_pod_hosts=["10.0.0.2"], remote_dp_size_local=1
|
||||
)
|
||||
worker.multi_pod_hosts = ["10.0.0.2"]
|
||||
writer._execute_write_task(task_a)
|
||||
writer._execute_write_task(task_b)
|
||||
|
||||
assert alloc["tA"].completion_remote_ip == "10.0.0.1"
|
||||
assert alloc["tB"].completion_remote_ip == "10.0.0.2"
|
||||
|
||||
|
||||
def test_finalize_notify_port_uses_pod_local_rank():
|
||||
base = 20000
|
||||
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
|
||||
alloc = {"tA": info}
|
||||
worker = _fake_worker(alloc, tp_rank=0)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task = _write_task(
|
||||
"tA",
|
||||
"10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=8,
|
||||
)
|
||||
writer._execute_write_task(task)
|
||||
assert info.completion_remote_ip == "10.0.0.2"
|
||||
|
||||
info.writes_expected = 1
|
||||
info.writes_done = 1
|
||||
writer._finalize_if_complete("tA", info)
|
||||
|
||||
assert worker.moriio_wrapper.sent_notifies
|
||||
_tid, ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
|
||||
assert mtype == "write_done"
|
||||
assert ip == "10.0.0.2"
|
||||
assert port == base + get_port_offset(9 % 8, 0)
|
||||
|
||||
|
||||
def test_finalize_port_ignores_stale_worker_dp_local():
|
||||
base = 20000
|
||||
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=9)
|
||||
alloc = {"tA": info}
|
||||
worker = _fake_worker(alloc, tp_rank=0, set_remote_dp_size_local=1)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task = _write_task(
|
||||
"tA",
|
||||
"10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=8,
|
||||
)
|
||||
writer._execute_write_task(task)
|
||||
assert info.completion_notify_port == base + get_port_offset(9 % 8, 0)
|
||||
|
||||
info.writes_expected = 1
|
||||
info.writes_done = 1
|
||||
writer._finalize_if_complete("tA", info)
|
||||
|
||||
_tid, _ip, port, mtype = worker.moriio_wrapper.sent_notifies[-1]
|
||||
assert mtype == "write_done"
|
||||
assert port == base + get_port_offset(9 % 8, 0)
|
||||
assert port != base + get_port_offset(9 % 1, 0)
|
||||
|
||||
|
||||
def test_finalize_single_pod_port_uses_global_rank():
|
||||
base = 20000
|
||||
info = RemoteAllocInfo(block_ids=None, decode_dp_rank=3)
|
||||
alloc = {"tA": info}
|
||||
worker = _fake_worker(alloc, tp_rank=2)
|
||||
writer = make_moriio_writer(worker)
|
||||
|
||||
task = _write_task("tA", "10.0.0.1", multi_pod_hosts=[], remote_dp_size_local=0)
|
||||
writer._execute_write_task(task)
|
||||
assert info.completion_remote_ip == "10.0.0.1"
|
||||
assert info.completion_notify_port == base + get_port_offset(3, 2)
|
||||
|
||||
info.writes_expected = 1
|
||||
info.writes_done = 1
|
||||
writer._finalize_if_complete("tA", info)
|
||||
|
||||
_tid, _ip, port, _mtype = worker.moriio_wrapper.sent_notifies[-1]
|
||||
assert port == base + get_port_offset(3, 2)
|
||||
|
||||
|
||||
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = False
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
|
||||
|
||||
meta = _req_meta(
|
||||
remote_host="10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=8,
|
||||
)
|
||||
worker._write_blocks_for_req("req-a", meta, "layer0", kv_layer=None)
|
||||
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
|
||||
assert captured["remote_dp_size_local"] == 8
|
||||
assert captured["remote_ip"] == "10.0.0.1"
|
||||
assert captured["local_block_ids"] == [[1]]
|
||||
assert captured["remote_block_ids"] == [[2]]
|
||||
assert not hasattr(worker, "multi_pod_hosts")
|
||||
|
||||
|
||||
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = False
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
worker.schedule_write_blocks = _capture_schedule_write_blocks # type: ignore[method-assign]
|
||||
|
||||
meta = _req_meta(
|
||||
remote_host="10.0.0.9",
|
||||
multi_pod_hosts=[],
|
||||
remote_dp_size=4,
|
||||
remote_dp_size_local=0,
|
||||
)
|
||||
worker._write_blocks_for_req("req-b", meta, "layer0", kv_layer=None)
|
||||
|
||||
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
|
||||
assert captured["remote_dp_size_local"] == 4
|
||||
assert captured["local_block_ids"] == [[1]]
|
||||
assert captured["remote_block_ids"] == [[2]]
|
||||
|
||||
|
||||
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
|
||||
wrapper = _real_wrapper()
|
||||
sent: list[tuple[str, int, str]] = []
|
||||
wrapper.send_notify = lambda tid, ip, port, message_type: sent.append(
|
||||
(ip, port, message_type)
|
||||
)
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper, tp_rank=0))
|
||||
task = _write_task(
|
||||
"tA",
|
||||
"10.0.0.1",
|
||||
multi_pod_hosts=["10.0.0.1", "10.0.0.2"],
|
||||
remote_dp_size_local=2,
|
||||
remote_dp_size=4,
|
||||
)
|
||||
task.enqueue_time = time.perf_counter() - 2 * writer._defer_timeout
|
||||
writer._deferred_tasks.append(task)
|
||||
|
||||
writer._process_deferred_tasks()
|
||||
|
||||
assert writer._deferred_tasks == []
|
||||
assert [ack.transfer_id for ack in wrapper.done_req_ids] == ["tA"]
|
||||
# Global DP ranks 0-3, two per pod: the owning rank is unknown.
|
||||
assert sent == [
|
||||
(host, 20000 + get_port_offset(local_rank, 0), "write_failed")
|
||||
for host in ("10.0.0.1", "10.0.0.2")
|
||||
for local_rank in (0, 1)
|
||||
]
|
||||
|
||||
# A late decode allocation must not resurrect a write into freed blocks.
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
|
||||
wrapper._handle_remote_blocks_message(
|
||||
{"transfer_id": "tA", "block_notify_list": [7], "decode_rank": 1}
|
||||
)
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
|
||||
|
||||
def test_write_scheduler_waits_for_worker_ack_past_defer_deadline():
|
||||
scheduler = SimpleNamespace(
|
||||
is_producer=True,
|
||||
mode=MoRIIOMode.WRITE,
|
||||
_defer_timeout=60.0,
|
||||
_deferred_send_deadlines={"req-1": (time.monotonic() - 1.0, "tA")},
|
||||
_pending_sent_acks={},
|
||||
unmap_request_id=lambda *args, **kwargs: None,
|
||||
)
|
||||
output = SimpleNamespace(finished_sending=set())
|
||||
|
||||
MoRIIOConnectorScheduler.update_connector_output(scheduler, output)
|
||||
assert output.finished_sending is None
|
||||
assert "req-1" in scheduler._deferred_send_deadlines
|
||||
|
||||
output.finished_sending = {"req-1"}
|
||||
MoRIIOConnectorScheduler.update_connector_output(scheduler, output)
|
||||
assert output.finished_sending == {"req-1"}
|
||||
assert scheduler._deferred_send_deadlines == {}
|
||||
|
||||
|
||||
def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
|
||||
wrapper = _real_wrapper()
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.CONSUMER):
|
||||
for tid in ("tA", "tOtherRank"):
|
||||
wrapper._handle_structured_message(
|
||||
{"type": "write_failed", "transfer_id": tid}
|
||||
)
|
||||
worker = SimpleNamespace(
|
||||
mode=MoRIIOMode.WRITE,
|
||||
is_producer=False,
|
||||
moriio_wrapper=wrapper,
|
||||
transfer_id_to_request_id={"tA": "req-1"},
|
||||
_unmatched_write_failures=OrderedDict(),
|
||||
get_finished=lambda: (set(), set()),
|
||||
)
|
||||
|
||||
results = MoRIIOConnectorWorker.get_transfer_results(worker)
|
||||
|
||||
assert results.finished_recving == {"req-1"}
|
||||
assert results.failed_recving == {"req-1"}
|
||||
assert wrapper.pop_finished_write_req_ids() == set()
|
||||
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
|
||||
|
||||
|
||||
def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer():
|
||||
from unittest.mock import Mock
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = True
|
||||
worker.world_size = 2
|
||||
worker.schedule_write_blocks = Mock()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
|
||||
worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None)
|
||||
|
||||
worker.schedule_write_blocks.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"notify,expected",
|
||||
[
|
||||
# Plain WRITE peers still send one flat attention list.
|
||||
([7, 8], [[7, 8]]),
|
||||
# Hybrid WRITE sends one list per transfer group, kept as-is even
|
||||
# when the first group is empty (full attention hit, state only).
|
||||
([[7], [9]], [[7], [9]]),
|
||||
([[], [9]], [[], [9]]),
|
||||
],
|
||||
)
|
||||
def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected):
|
||||
wrapper = _real_wrapper()
|
||||
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
|
||||
wrapper._handle_remote_blocks_message(
|
||||
{"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2}
|
||||
)
|
||||
info = wrapper.done_remote_allocate_req_dict["tA"]
|
||||
assert info.block_ids == expected
|
||||
assert info.decode_dp_rank == 2
|
||||
|
||||
|
||||
def test_empty_layer_plan_posts_no_rdma_write():
|
||||
from unittest.mock import Mock
|
||||
|
||||
wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock())
|
||||
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
plan = LayerTransferPlan(
|
||||
request_id="req",
|
||||
transfer_id="tA",
|
||||
layer_name="layer0",
|
||||
sess_idx=0,
|
||||
transfer_local_offsets=[],
|
||||
transfer_remote_offsets=[],
|
||||
transfer_sizes=[],
|
||||
)
|
||||
|
||||
assert writer._do_layer_write(plan, ["session"]) == []
|
||||
wrapper.write_remote_data.assert_not_called()
|
||||
wrapper.write_remote_data_single.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_state", [[], [1, 2]])
|
||||
def test_hybrid_write_requires_exactly_one_running_state_slot(local_state):
|
||||
worker = SimpleNamespace(
|
||||
_mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1],
|
||||
)
|
||||
writer = make_moriio_writer(worker)
|
||||
task = _write_task("tA", "10.0.0.1")
|
||||
task.layer_name = "kda"
|
||||
task.local_block_ids = [[], local_state]
|
||||
|
||||
with pytest.raises(MoRIIOError, match="one running-state slot"):
|
||||
writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]]))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,253 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the scheduler-driven heartbeat / lease-renewal system."""
|
||||
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import create_request, make_nixl_scheduler
|
||||
|
||||
_ENGINE_A = "my-engine-id"
|
||||
|
||||
|
||||
def _sched(kv_lease_duration: int = 30):
|
||||
return make_nixl_scheduler(heartbeat=True, kv_lease_duration=kv_lease_duration)
|
||||
|
||||
|
||||
def _req(request_id: int = 1):
|
||||
return create_request(request_id=request_id, do_remote_prefill=True)
|
||||
|
||||
|
||||
def _worker_stub():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
w = object.__new__(NixlConnectorWorker)
|
||||
w._reqs_to_send = {}
|
||||
w._lease_extension = 20
|
||||
w._reqs_to_process = set()
|
||||
w.consumer_notification_counts_by_req = {}
|
||||
w.expected_consumer_notifications_by_req = {}
|
||||
w.xfer_stats = MagicMock()
|
||||
return w
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: on_new_request
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_on_new_request_tracks_and_groups():
|
||||
"""Add two reqs to same engine, one to another; verify grouping."""
|
||||
s = _sched()
|
||||
s.on_new_request(_req(1))
|
||||
s.on_new_request(_req(2))
|
||||
|
||||
assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-1", "prefill-2"}
|
||||
info = s._heartbeat_by_engine[_ENGINE_A]
|
||||
assert (info.host, info.port, info.tp_size) == ("my-host", 1234, 1)
|
||||
assert s._heartbeat_req_engine["id-1"] == (_ENGINE_A, "prefill-1")
|
||||
|
||||
# Different engine.
|
||||
r3 = _req(3)
|
||||
r3.kv_transfer_params["remote_engine_id"] = "engine-b"
|
||||
s.on_new_request(r3)
|
||||
assert len(s._heartbeat_by_engine) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"make_req",
|
||||
[
|
||||
lambda: create_request(request_id=2, do_remote_decode=True),
|
||||
lambda: create_request(request_id=3), # no kv_transfer_params
|
||||
],
|
||||
ids=["decode", "plain"],
|
||||
)
|
||||
def test_on_new_request_ignores_non_prefill(make_req):
|
||||
s = _sched()
|
||||
s.on_new_request(make_req())
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: _stop_heartbeat
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_stop_heartbeat_partial_and_full():
|
||||
"""Stop one of two reqs on same engine, then stop the other."""
|
||||
s = _sched()
|
||||
s.on_new_request(_req(1))
|
||||
s.on_new_request(_req(2))
|
||||
|
||||
s._stop_heartbeat("id-1")
|
||||
assert s._heartbeat_by_engine[_ENGINE_A].req_ids == {"prefill-2"}
|
||||
assert "id-1" not in s._heartbeat_req_engine
|
||||
|
||||
s._stop_heartbeat("id-2")
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
assert len(s._heartbeat_req_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: build_connector_meta throttling
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_build_connector_meta_heartbeat_throttling():
|
||||
# kv_lease_duration=30 => _heartbeat_interval = 30 // 6 = 5
|
||||
s = _sched(kv_lease_duration=30)
|
||||
s.on_new_request(_req(1))
|
||||
|
||||
# Ensure the first call triggers by placing last_heartbeat far in the past.
|
||||
s._last_heartbeat_time = time.perf_counter() - 10
|
||||
meta1 = s.build_connector_meta(MagicMock())
|
||||
assert _ENGINE_A in meta1.heartbeat_by_engine
|
||||
|
||||
# Immediate second call is throttled (< 5s since last).
|
||||
meta2 = s.build_connector_meta(MagicMock())
|
||||
assert len(meta2.heartbeat_by_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Scheduler: cleanup paths (update_connector_output / request_finished)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_update_connector_output_stops_heartbeat():
|
||||
s = _sched()
|
||||
s.on_new_request(_req(1))
|
||||
|
||||
s.update_connector_output(
|
||||
KVConnectorOutput(
|
||||
finished_sending=None,
|
||||
finished_recving={"id-1"},
|
||||
invalid_block_ids=set(),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
assert len(s._heartbeat_req_engine) == 0
|
||||
|
||||
|
||||
def test_request_finished_stops_heartbeat():
|
||||
s = _sched()
|
||||
r = _req(1)
|
||||
s.on_new_request(r)
|
||||
|
||||
# Simulate update_state_after_alloc having consumed do_remote_prefill.
|
||||
r.kv_transfer_params["do_remote_prefill"] = False
|
||||
s.request_finished(r, block_ids=())
|
||||
|
||||
assert len(s._heartbeat_by_engine) == 0
|
||||
assert len(s._heartbeat_req_engine) == 0
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Worker: _handle_heartbeat
|
||||
# ===================================================================
|
||||
|
||||
|
||||
def test_handle_heartbeat():
|
||||
w = _worker_stub()
|
||||
far_future = time.perf_counter() + 99999
|
||||
w._reqs_to_send = {"req-a": 100.0, "req-b": far_future}
|
||||
|
||||
before = time.perf_counter()
|
||||
w._handle_heartbeat("req-a,req-b,req-unknown")
|
||||
|
||||
# req-a: pushed forward to ~now+20.
|
||||
assert w._reqs_to_send["req-a"] >= before + 20
|
||||
# req-b: already far out, max() keeps it.
|
||||
assert w._reqs_to_send["req-b"] >= far_future
|
||||
# req-unknown: not added.
|
||||
assert "req-unknown" not in w._reqs_to_send
|
||||
|
||||
|
||||
def test_reaper_reclaims_expired_leases_behind_heartbeated_head(monkeypatch):
|
||||
"""A heartbeated head entry must not strand expired leases behind it.
|
||||
|
||||
Mirrors the stdlib repro in https://github.com/vllm-project/vllm/issues/58222:
|
||||
A is inserted first and kept alive via heartbeats; B and C expire and
|
||||
must still be reaped. The old early-exit scan returned nothing here.
|
||||
"""
|
||||
w = _worker_stub()
|
||||
w._reqs_to_process.update(("A", "B", "C"))
|
||||
# Insertion order is A, B, C. Heartbeats will keep A at the head.
|
||||
w._reqs_to_send = {"A": 30.0, "B": 32.0, "C": 34.0}
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
|
||||
|
||||
now = {"t": 0.0}
|
||||
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"])
|
||||
for t in (20.0, 40.0, 60.0, 80.0):
|
||||
now["t"] = t
|
||||
w._handle_heartbeat("A")
|
||||
|
||||
# A was renewed in place: still the dict head, expiry 100.
|
||||
assert next(iter(w._reqs_to_send)) == "A"
|
||||
assert w._reqs_to_send["A"] == pytest.approx(100.0)
|
||||
|
||||
done: set[str] = set()
|
||||
now["t"] = 50.0
|
||||
w._reap_expired_send_leases(done)
|
||||
|
||||
assert done == {"B", "C"}
|
||||
assert list(w._reqs_to_send) == ["A"]
|
||||
assert w._reqs_to_process == {"A"}
|
||||
assert w.xfer_stats.record_kv_expired_req.call_count == 2
|
||||
|
||||
# now == expires is expired (now >= expires): A is due at 100.
|
||||
done_eq: set[str] = set()
|
||||
now["t"] = 100.0
|
||||
w._reap_expired_send_leases(done_eq)
|
||||
assert done_eq == {"A"}
|
||||
assert not w._reqs_to_send
|
||||
assert not w._reqs_to_process
|
||||
assert w.xfer_stats.record_kv_expired_req.call_count == 3
|
||||
|
||||
|
||||
def test_reaper_reclaims_when_now_equals_expiry(monkeypatch):
|
||||
"""The reaper must reclaim on the now == expires boundary."""
|
||||
w = _worker_stub()
|
||||
w._reqs_to_process.add("eq")
|
||||
w._reqs_to_send = {"eq": 50.0}
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
|
||||
|
||||
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0)
|
||||
|
||||
done: set[str] = set()
|
||||
w._reap_expired_send_leases(done)
|
||||
|
||||
assert done == {"eq"}
|
||||
assert not w._reqs_to_send
|
||||
assert not w._reqs_to_process
|
||||
w.xfer_stats.record_kv_expired_req.assert_called_once()
|
||||
|
||||
|
||||
def test_reaper_reclaims_shorter_lease_behind_later_deadline(monkeypatch):
|
||||
"""A later-inserted longer TTL must not block an earlier-expiring entry.
|
||||
|
||||
Delete+reinsert on heartbeat cannot restore expiry order in this case
|
||||
(issue #58222), so the reaper must scan the whole map.
|
||||
"""
|
||||
w = _worker_stub()
|
||||
w._reqs_to_process.update(("X", "Y"))
|
||||
w._reqs_to_send = {"X": 55.0, "Y": 46.0}
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
|
||||
|
||||
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: 50.0)
|
||||
|
||||
done: set[str] = set()
|
||||
w._reap_expired_send_leases(done)
|
||||
|
||||
assert done == {"Y"}
|
||||
assert list(w._reqs_to_send) == ["X"]
|
||||
assert w._reqs_to_process == {"X"}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,977 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from dataclasses import replace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.metrics.stats import KV_FETCH_STAGES
|
||||
from vllm.v1.outputs import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.request import FinishReason, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
assert_scheduler_empty,
|
||||
create_model_runner_output,
|
||||
create_request,
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _num_waiting_requests(scheduler) -> int:
|
||||
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
|
||||
|
||||
|
||||
def _kv_fetch_stages(scheduler) -> tuple[int, ...]:
|
||||
"""(waiting_to_start, in_progress, completed_waiting) request counts."""
|
||||
return tuple(scheduler._kv_fetch_counts[s] for s in KV_FETCH_STAGES)
|
||||
|
||||
|
||||
def test_basic_lifecycle():
|
||||
"""Test lifecycle of a remote prefill."""
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
|
||||
# 2 Full Blocks and 1 Half Block.
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
NUM_EXTERNAL_FULL_BLOCKS = 2
|
||||
NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5))
|
||||
START_FREE_BLOCK_QUEUE_SIZE = (
|
||||
scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks
|
||||
)
|
||||
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
|
||||
scheduler.add_request(request)
|
||||
request_id = request.request_id
|
||||
assert _kv_fetch_stages(scheduler) == (1, 0, 0)
|
||||
|
||||
# STEP (1):
|
||||
# (1a): schedule()
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
|
||||
|
||||
# Nothing running and empty scheduler output.
|
||||
assert len(scheduler.running) == 0
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 0
|
||||
assert scheduler_output.scheduled_cached_reqs.num_reqs == 0
|
||||
assert len(scheduler_output.num_scheduled_tokens) == 0
|
||||
assert scheduler_output.total_num_scheduled_tokens == 0
|
||||
|
||||
# Req waiting for KVs with no computed/scheduled toks ...
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert request in scheduler.kv_holding_waiting
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == NUM_TOKENS
|
||||
|
||||
# ... but should have (uncached) blocks allocated to it.
|
||||
block_pool = scheduler.kv_cache_manager.block_pool
|
||||
assert block_pool.free_block_queue.num_free_blocks < START_FREE_BLOCK_QUEUE_SIZE
|
||||
assert len(block_pool.cached_block_hash_to_block) == 0
|
||||
blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].req_to_blocks[request_id]
|
||||
for block in blocks:
|
||||
assert block._block_hash is None
|
||||
|
||||
# (1b): forward()
|
||||
model_runner_output = EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
# (1c): update_from_output()
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
assert not engine_core_outputs or not engine_core_outputs[0].outputs
|
||||
|
||||
# STEP (2):
|
||||
# (2a): schedule(): nothing happens!
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert len(scheduler.running) == 0
|
||||
|
||||
# (2b): forward(): request finishes recv.
|
||||
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||
model_runner_output.kv_connector_output = KVConnectorOutput(
|
||||
finished_recving={request_id}
|
||||
)
|
||||
|
||||
# (2c): update_from_output():
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert request_id in scheduler.finished_recving_kv_req_ids
|
||||
assert _kv_fetch_stages(scheduler) == (0, 0, 1)
|
||||
|
||||
# STEP (3):
|
||||
# (3a): schedule(): this should actually schedule.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 1
|
||||
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
|
||||
|
||||
# Confirm the block are actually allocated.
|
||||
num_hashed_blocks = 0
|
||||
blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].req_to_blocks[request_id]
|
||||
for block in blocks:
|
||||
assert block.ref_cnt == 1
|
||||
num_hashed_blocks += 1 if block._block_hash is not None else 0
|
||||
assert num_hashed_blocks == NUM_EXTERNAL_FULL_BLOCKS
|
||||
|
||||
# Confirm the rest of the prompt is scheduled in this step.
|
||||
scheduled_req = scheduler_output.scheduled_new_reqs[0]
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[request_id]
|
||||
num_computed_tokens = scheduled_req.num_computed_tokens
|
||||
total_prompt_tokens = len(scheduled_req.prompt_token_ids)
|
||||
assert num_scheduled_tokens == total_prompt_tokens - num_computed_tokens
|
||||
|
||||
# (3b): execute_model()
|
||||
model_runner_output = create_model_runner_output([request])
|
||||
# (3c): update_from_output()
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# Step (4): Hit EOS.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output([request], use_eos=True)
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
scheduler.schedule()
|
||||
|
||||
outputs = engine_core_outputs[0].outputs
|
||||
assert len(outputs) == 1
|
||||
output = outputs[0]
|
||||
assert output.finish_reason == FinishReason.STOP
|
||||
assert_scheduler_empty(scheduler)
|
||||
|
||||
|
||||
def test_interleaved_lifecycle():
|
||||
"""Test Remote Prefills Work Well With Other Requests."""
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
|
||||
# 2 Full Blocks and 1 Half Block.
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
NUM_EXTERNAL_FULL_BLOCKS = 2
|
||||
NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5))
|
||||
|
||||
request_remote = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
request_local_a = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
)
|
||||
request_local_b = create_request(
|
||||
request_id=3,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
)
|
||||
|
||||
# STEP 1: Regular request is running.
|
||||
scheduler.add_request(request_local_a)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 1
|
||||
|
||||
model_runner_output = create_model_runner_output([request_local_a])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# STEP 2: Add a local and remote request.
|
||||
scheduler.add_request(request_local_b)
|
||||
scheduler.add_request(request_remote)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 2
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 1
|
||||
assert scheduler_output.scheduled_cached_reqs.num_reqs == 1
|
||||
|
||||
model_runner_output = create_model_runner_output([request_local_a, request_local_b])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# STEP 3: continue running, KVs not arrived yet.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 2
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 0
|
||||
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
|
||||
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_local_a, request_local_b]
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 2
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 0
|
||||
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
|
||||
|
||||
# STEP 4: KVs arrive.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 2
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 0
|
||||
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
|
||||
|
||||
model_runner_output = create_model_runner_output(
|
||||
[request_local_a, request_local_b], finished_recving={request_remote.request_id}
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# STEP 5: RECVed KVs are sent to ModelRunner.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 3
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
assert len(scheduler_output.scheduled_new_reqs) == 1
|
||||
assert scheduler_output.scheduled_cached_reqs.num_reqs == 2
|
||||
|
||||
model_runner_output = create_model_runner_output(
|
||||
[request_local_a, request_local_b, request_remote]
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# STEP 6: Hit EOS and free.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
[request_local_a, request_local_b, request_remote],
|
||||
use_eos=True,
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
scheduler.schedule()
|
||||
assert_scheduler_empty(scheduler)
|
||||
|
||||
|
||||
def test_no_spurious_prefix_caching():
|
||||
"""With P/D, blocks can be allocated but uncomputed for
|
||||
multiple engine steps. This test confirms that we do
|
||||
not accidentally have cache hits against uncomputed
|
||||
blocks.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
|
||||
# 2 and a half full external blocks.
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
NUM_EXTERNAL_FULL_BLOCKS = 2
|
||||
NUM_TOKENS = int(BLOCK_SIZE * (NUM_EXTERNAL_FULL_BLOCKS + 0.5))
|
||||
|
||||
# Both of these requests have prompts like [1,1,1,1,1, ...]
|
||||
request_remote = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
common_prefix_len=NUM_TOKENS,
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
|
||||
request_local = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
common_prefix_len=NUM_TOKENS,
|
||||
do_remote_prefill=False,
|
||||
)
|
||||
|
||||
# Schedule the remote prefill request. This should not
|
||||
# cause any blocks to be cached.
|
||||
scheduler.add_request(request_remote)
|
||||
scheduler_output = scheduler.schedule()
|
||||
scheduler.update_from_output(scheduler_output, EMPTY_MODEL_RUNNER_OUTPUT)
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Schedule the local prefill request. This should
|
||||
# cause blocks to be cached, but separately from
|
||||
scheduler.add_request(request_local)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
local_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].req_to_blocks[request_local.request_id]
|
||||
remote_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].req_to_blocks[request_remote.request_id]
|
||||
|
||||
# Local should have cached blocks (but not all due to preallocate).
|
||||
num_hashed_blocks = 0
|
||||
for block in local_blocks:
|
||||
assert block.ref_cnt == 1
|
||||
num_hashed_blocks += 1 if block._block_hash is not None else 0
|
||||
assert num_hashed_blocks > 0
|
||||
|
||||
# Remote blocks should not be cached.
|
||||
for block in remote_blocks:
|
||||
assert block.ref_cnt == 1
|
||||
assert block._block_hash is None
|
||||
|
||||
|
||||
def test_full_block_prompt():
|
||||
"""Test that we handle a prompt that is the full block size."""
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
|
||||
# 2 Full Blocks and 1 Half Block.
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
NUM_EXTERNAL_FULL_BLOCKS = 2
|
||||
NUM_TOKENS = int(BLOCK_SIZE * NUM_EXTERNAL_FULL_BLOCKS)
|
||||
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS,
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
|
||||
scheduler.add_request(request)
|
||||
request_id = request.request_id
|
||||
|
||||
# STEP (1): Initialize a recv.
|
||||
scheduler_output = scheduler.schedule()
|
||||
# All blocks should be allocated.
|
||||
num_blocks = len(
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
]
|
||||
)
|
||||
assert num_blocks == NUM_EXTERNAL_FULL_BLOCKS
|
||||
model_runner_output = EMPTY_MODEL_RUNNER_OUTPUT
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# # STEP (2): Recv.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||
model_runner_output.kv_connector_output = KVConnectorOutput(
|
||||
finished_recving={request_id}
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert request_id in scheduler.finished_recving_kv_req_ids
|
||||
|
||||
# # STEP (3): Run as usual.
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
# We need to recompute the final token of the prompt to generate
|
||||
# the first new token, so we should not have a new block.
|
||||
num_blocks = len(
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
]
|
||||
)
|
||||
assert num_blocks == NUM_EXTERNAL_FULL_BLOCKS
|
||||
assert scheduler_output.scheduled_new_reqs[0].num_computed_tokens == NUM_TOKENS - 1
|
||||
assert scheduler_output.num_scheduled_tokens[request_id] == 1
|
||||
|
||||
model_runner_output = create_model_runner_output([request])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
# # Step (4): Hit EOS.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output([request], use_eos=True)
|
||||
engine_core_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
scheduler.schedule()
|
||||
|
||||
outputs = engine_core_outputs[0].outputs
|
||||
assert len(outputs) == 1
|
||||
output = outputs[0]
|
||||
assert output.finish_reason == FinishReason.STOP
|
||||
assert_scheduler_empty(scheduler)
|
||||
|
||||
|
||||
def test_cannot_schedule_after_recv():
|
||||
"""Test that we can handle no schedule after recv due to not
|
||||
enough remaining KV blocks.
|
||||
"""
|
||||
# NOTE: the KVCacheManager will use 1 null block.
|
||||
# So there are 5 total working blocks.
|
||||
TOTAL_NUM_BLOCKS = 6
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=TOTAL_NUM_BLOCKS)
|
||||
|
||||
# Prime the KVCache.
|
||||
NUM_PROMPT_BLOCKS = 2
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
# Prompt will use 2 blocks + 1 block after we schedule.
|
||||
NUM_TOKENS_LOCAL = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS)
|
||||
NUM_TOKENS_REMOTE = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS)
|
||||
|
||||
request_normal = create_request(
|
||||
request_id=1, block_size=BLOCK_SIZE, num_tokens=NUM_TOKENS_LOCAL
|
||||
)
|
||||
request_remote = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS_REMOTE,
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
|
||||
# STEP 1: 3 blocks are in use (2 for prompt, 1 for decode).
|
||||
scheduler.add_request(request_normal)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_normal])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
|
||||
# Step 2: 5 blocks are in use (2 new for remote blocks).
|
||||
scheduler.add_request(request_remote)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_normal])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Step 3: finish recving (5 blocks in use)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_normal], finished_recving={request_remote.request_id}
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Step 4: try to schedule, remote request is put to running list
|
||||
# because the transfer is completed.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_normal, request_remote]
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 2
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
|
||||
# Step 5: Remote request will be put back to waiting list
|
||||
# because it needs new block to hold generated token.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_normal])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Step 6: finish the request, free it.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_normal], use_eos=True
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 0
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Step 7: now we can schedule (with 2 blocks computed),
|
||||
# request is retrieved from preempted list.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_remote])
|
||||
# V2 emits a resumed (previously preempted) request as a NewRequestData
|
||||
# rather than a cached request.
|
||||
if scheduler.use_v2_model_runner:
|
||||
num_computed = scheduler_output.scheduled_new_reqs[0].num_computed_tokens
|
||||
else:
|
||||
cached = scheduler_output.scheduled_cached_reqs
|
||||
num_computed = cached.num_computed_tokens[0]
|
||||
assert num_computed == NUM_PROMPT_BLOCKS * BLOCK_SIZE
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
|
||||
# Step 8: free everything.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_remote], use_eos=True
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
_ = scheduler.schedule()
|
||||
assert_scheduler_empty(scheduler)
|
||||
|
||||
|
||||
def test_cannot_recv():
|
||||
"""Test that we can handle no schedule KV block transfer due to not
|
||||
enough remaining KV blocks.
|
||||
"""
|
||||
# NOTE: the KVCacheManager will use 1 null block.
|
||||
# So there are 5 total working blocks.
|
||||
TOTAL_NUM_BLOCKS = 6
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=TOTAL_NUM_BLOCKS)
|
||||
|
||||
# Prime the KVCache.
|
||||
NUM_PROMPT_BLOCKS = 2
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
# Prompt will use 2 blocks + 1 block after we schedule.
|
||||
NUM_TOKENS_LOCAL = int(BLOCK_SIZE * NUM_PROMPT_BLOCKS)
|
||||
NUM_TOKENS_REMOTE = int(BLOCK_SIZE * (NUM_PROMPT_BLOCKS + 0.5))
|
||||
|
||||
request_normal = create_request(
|
||||
request_id=1, block_size=BLOCK_SIZE, num_tokens=NUM_TOKENS_LOCAL
|
||||
)
|
||||
request_remote = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=NUM_TOKENS_REMOTE,
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
|
||||
# STEP 1: 3 blocks are in use (2 for prompt, 1 for decode).
|
||||
scheduler.add_request(request_normal)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_normal])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
|
||||
# Step 2: 3 blocks are in use,
|
||||
# need 3 new for remote blocks but only 2 are available.
|
||||
scheduler.add_request(request_remote)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_normal])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
# Should not have KV transfer in progress.
|
||||
assert request_remote.status != RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
# Step 3: finish the request, free it.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_normal], use_eos=True
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 0
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Step 4: now we can initiate KV transfer (with 2 blocks computed).
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 0
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert request_remote.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
# Step 5: finish recving (5 blocks in use)
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[], finished_recving={request_remote.request_id}
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 0
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
|
||||
# Step 6: schedule remote request
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(reqs=[request_remote])
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
assert len(scheduler.running) == 1
|
||||
assert _num_waiting_requests(scheduler) == 0
|
||||
|
||||
# Step 7: free everything.
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=[request_remote], use_eos=True
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
_ = scheduler.schedule()
|
||||
assert_scheduler_empty(scheduler)
|
||||
|
||||
|
||||
def test_kv_fetch_stages_cleared_on_abort():
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=int(BLOCK_SIZE * 2.5),
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
scheduler.schedule()
|
||||
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
|
||||
|
||||
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform"
|
||||
)
|
||||
def test_p_side_chunked_prefill_mamba(mock_platform):
|
||||
"""P-side integration: Mamba N-1 truncation + chunked prefill completes.
|
||||
|
||||
A 64-token P-side request is truncated to 63 by the N-1 fix, then
|
||||
chunked into two prefill steps (32 + 31) and finishes with
|
||||
LENGTH_CAPPED because max_tokens is set to 1.
|
||||
"""
|
||||
mock_platform.device_type = "cpu"
|
||||
|
||||
BATCH_SIZE = 32
|
||||
NUM_TOKENS = 64
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
max_num_batched_tokens=BATCH_SIZE,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
|
||||
# mamba_cache_mode "none" only runs without prefix caching.
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_enabled=True,
|
||||
num_blocks=10000,
|
||||
)
|
||||
|
||||
scheduler = create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
|
||||
|
||||
request = create_request(
|
||||
num_tokens=NUM_TOKENS,
|
||||
do_remote_decode=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
request.max_tokens = 128
|
||||
scheduler.add_request(request)
|
||||
request_id = request.request_id
|
||||
|
||||
# ── Step 1: first chunk ──
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
assert len(request.prompt_token_ids) == NUM_TOKENS - 1
|
||||
assert request.max_tokens == 1
|
||||
assert scheduler_output.num_scheduled_tokens[request_id] == BATCH_SIZE
|
||||
assert request.num_computed_tokens == BATCH_SIZE
|
||||
|
||||
# Model returns no tokens for intermediate prefill chunk
|
||||
intermediate_output = ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[]],
|
||||
)
|
||||
scheduler.update_from_output(scheduler_output, intermediate_output)
|
||||
|
||||
# ── Step 2: remaining chunk ──
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
remaining = NUM_TOKENS - 1 - BATCH_SIZE # 31
|
||||
assert scheduler_output.num_scheduled_tokens[request_id] == remaining
|
||||
assert request.num_computed_tokens == NUM_TOKENS - 1
|
||||
|
||||
# Prefill complete: model generates 1 decode token
|
||||
final_output = create_model_runner_output([request])
|
||||
engine_core_outputs = scheduler.update_from_output(scheduler_output, final_output)
|
||||
|
||||
# max_tokens=1 → request finishes with LENGTH
|
||||
outputs = engine_core_outputs[0].outputs
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].finish_reason == FinishReason.LENGTH
|
||||
|
||||
|
||||
def test_async_load_reserves_blocks_for_inflight():
|
||||
"""A second async KV-connector load is not admitted if its initial
|
||||
allocation would consume blocks reserved for an already in-flight sequence.
|
||||
|
||||
req_a gets a 1-block prefix (full sequence = 4 blocks), reserving 3 more.
|
||||
req_b would need a 4-block initial allocation, but only
|
||||
(free - req_a's 3-block reservation) = 3 blocks are available to it, so it is
|
||||
held back in WAITING (holding no blocks) rather than wedging req_a.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
|
||||
|
||||
req_a = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 4,
|
||||
do_remote_prefill=True,
|
||||
num_remote_blocks=1,
|
||||
)
|
||||
req_b = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 5,
|
||||
do_remote_prefill=True,
|
||||
num_remote_blocks=1,
|
||||
)
|
||||
scheduler.add_request(req_a)
|
||||
scheduler.add_request(req_b)
|
||||
|
||||
# Partial external matches: req_a loads 1 block, req_b loads 4 blocks.
|
||||
with patch.object(
|
||||
scheduler.connector,
|
||||
"get_num_new_matched_tokens",
|
||||
side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE * 4, True)],
|
||||
):
|
||||
scheduler.schedule()
|
||||
|
||||
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert req_b.status == RequestStatus.WAITING
|
||||
|
||||
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].req_to_blocks
|
||||
assert req_a.request_id in req_to_blocks
|
||||
assert req_b.request_id not in req_to_blocks
|
||||
|
||||
|
||||
def test_async_loads_both_admitted_when_pool_fits():
|
||||
"""Sanity: with a pool large enough, the reservation gate admits both async
|
||||
loads (it is not over-conservative)."""
|
||||
vllm_config = create_vllm_config()
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=64)
|
||||
|
||||
reqs = [
|
||||
create_request(
|
||||
request_id=i,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 5,
|
||||
do_remote_prefill=True,
|
||||
num_remote_blocks=1,
|
||||
)
|
||||
for i in (1, 2)
|
||||
]
|
||||
for req in reqs:
|
||||
scheduler.add_request(req)
|
||||
|
||||
with patch.object(
|
||||
scheduler.connector,
|
||||
"get_num_new_matched_tokens",
|
||||
side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE, True)],
|
||||
):
|
||||
scheduler.schedule()
|
||||
|
||||
for req in reqs:
|
||||
assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("swa_enabled", "block_size", "num_spec_tokens", "num_blocks"),
|
||||
[(False, 16, 3, 8), (True, 16, 3, 17), (True, 64, 31, 40)],
|
||||
ids=["single-group", "multiple-groups", "different-block-sizes"],
|
||||
)
|
||||
def test_async_load_reserves_blocks_for_promotion_margin(
|
||||
swa_enabled, block_size, num_spec_tokens, num_blocks
|
||||
):
|
||||
"""Async loads must leave enough blocks for promotion in every cache group.
|
||||
|
||||
Otherwise two loads can fill the pool and prevent the head from being
|
||||
promoted, with nothing running to free blocks and break the deadlock.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
hash_block_size = 16
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
hash_block_size, swa_enabled=swa_enabled, sw_size=4096, num_blocks=num_blocks
|
||||
)
|
||||
full_group = kv_cache_config.kv_cache_groups[0]
|
||||
full_group.kv_cache_spec = replace(full_group.kv_cache_spec, block_size=block_size)
|
||||
if swa_enabled:
|
||||
sliding_group = kv_cache_config.kv_cache_groups[-1]
|
||||
sliding_group.kv_cache_spec = replace(
|
||||
sliding_group.kv_cache_spec,
|
||||
page_size_padded=full_group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
vllm_config,
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_config=kv_cache_config,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req_a = create_request(
|
||||
request_id=1,
|
||||
block_size=hash_block_size,
|
||||
num_tokens=block_size * 4,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
req_b = create_request(
|
||||
request_id=2,
|
||||
block_size=hash_block_size,
|
||||
num_tokens=block_size * 3,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
scheduler.add_request(req_a)
|
||||
scheduler.add_request(req_b)
|
||||
|
||||
# Both get a full external hit, so each would hold its whole prompt.
|
||||
with patch.object(
|
||||
scheduler.connector,
|
||||
"get_num_new_matched_tokens",
|
||||
side_effect=[(block_size * 4, True), (block_size * 3, True)],
|
||||
):
|
||||
scheduler.schedule()
|
||||
|
||||
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert req_b.status == RequestStatus.WAITING
|
||||
assert not any(scheduler.kv_cache_manager.get_block_ids(req_b.request_id))
|
||||
|
||||
# req_a's load lands: it must be promotable and actually get scheduled.
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(),
|
||||
create_model_runner_output([], finished_recving={req_a.request_id}),
|
||||
)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert req_a.status == RequestStatus.RUNNING
|
||||
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mamba_cache_mode", "mamba_block_size", "num_blocks"),
|
||||
[("none", 128, 18), ("align", 16, 18)],
|
||||
)
|
||||
def test_async_load_does_not_double_reserve_mamba_spec_blocks(
|
||||
mamba_cache_mode, mamba_block_size, num_blocks
|
||||
):
|
||||
"""Two hybrid loads fit when Mamba's speculative states are counted once."""
|
||||
block_size = 16
|
||||
num_spec_tokens = 3
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=mamba_block_size,
|
||||
max_model_len=128,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.cache_config.enable_prefix_caching = mamba_cache_mode == "align"
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size,
|
||||
mamba_enabled=True,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
mamba_group = kv_cache_config.kv_cache_groups[-1]
|
||||
mamba_group.kv_cache_spec = replace(
|
||||
mamba_group.kv_cache_spec,
|
||||
block_size=mamba_block_size,
|
||||
num_speculative_blocks=num_spec_tokens,
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
vllm_config,
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_config=kv_cache_config,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
reqs = [
|
||||
create_request(
|
||||
request_id=i,
|
||||
block_size=block_size,
|
||||
num_tokens=num_tokens,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
for i, num_tokens in enumerate((64, 48), 1)
|
||||
]
|
||||
for req in reqs:
|
||||
scheduler.add_request(req)
|
||||
|
||||
with patch.object(
|
||||
scheduler.connector,
|
||||
"get_num_new_matched_tokens",
|
||||
side_effect=[(req.num_tokens, True) for req in reqs],
|
||||
):
|
||||
scheduler.schedule()
|
||||
|
||||
assert all(req.status == RequestStatus.WAITING_FOR_REMOTE_KVS for req in reqs)
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(),
|
||||
create_model_runner_output(
|
||||
[], finished_recving={req.request_id for req in reqs}
|
||||
),
|
||||
)
|
||||
output = scheduler.schedule()
|
||||
for req in reqs:
|
||||
assert req.status == RequestStatus.RUNNING
|
||||
assert output.num_scheduled_tokens[req.request_id] > 0
|
||||
|
||||
|
||||
def test_deferred_lookup_does_not_block_parked_load():
|
||||
"""A request with a deferred connector lookup must not strand an async
|
||||
load queued behind it.
|
||||
|
||||
req_a's lookup is deferred (connector returns None), so it is parked in
|
||||
waiting holding no blocks; req_b is then admitted as an async
|
||||
load holding most of the pool. Once req_a's lookup resolves it cannot
|
||||
allocate, and the waiting scan stops at it every step. The parked load
|
||||
must still be promotable: it lives in the KV-holding queue
|
||||
that is drained first, so it runs, finishes, and frees its blocks for
|
||||
req_a.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
|
||||
|
||||
req_a = create_request(
|
||||
request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1
|
||||
)
|
||||
req_b = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 5,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
scheduler.add_request(req_a)
|
||||
scheduler.add_request(req_b)
|
||||
|
||||
deferrals = [(None, False)]
|
||||
|
||||
def lookup(request, num_computed_tokens):
|
||||
if request is req_b:
|
||||
return BLOCK_SIZE * 5, True
|
||||
return deferrals.pop() if deferrals else (0, False)
|
||||
|
||||
with patch.object(
|
||||
scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup
|
||||
):
|
||||
# Step 1: req_a's lookup is deferred; req_b is admitted as an async
|
||||
# load holding 5 of 7 usable blocks.
|
||||
scheduler.schedule()
|
||||
assert req_a.status == RequestStatus.WAITING
|
||||
assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
# req_a's lookup resolves (no hit) while req_b's transfer is in
|
||||
# flight; req_a needs 4 blocks but only 2 are free, so the waiting
|
||||
# scan stops at it. req_b's transfer then finishes.
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(),
|
||||
create_model_runner_output([], finished_recving={req_b.request_id}),
|
||||
)
|
||||
|
||||
# req_b must still be reached, promoted, and scheduled despite the
|
||||
# scan stopping at req_a.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert req_b.status == RequestStatus.RUNNING
|
||||
assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0
|
||||
|
||||
# req_b finishes and frees its blocks; now req_a fits and runs.
|
||||
scheduler.update_from_output(
|
||||
scheduler_output, create_model_runner_output([req_b])
|
||||
)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert req_a.status == RequestStatus.RUNNING
|
||||
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SimpleCPUOffloadConnector-level behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
_BYTES_PER_BLOCK,
|
||||
BLOCK_SIZE,
|
||||
DTYPE,
|
||||
HEAD_SIZE,
|
||||
NUM_KV_HEADS,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
|
||||
|
||||
def _make_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
|
||||
fa_layers = ["layer0"]
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
groups = [KVCacheGroupSpec(fa_layers, spec)]
|
||||
size = _BYTES_PER_BLOCK * num_blocks
|
||||
tensors = [
|
||||
KVCacheTensor(
|
||||
size=size,
|
||||
layers=fa_layers,
|
||||
layer_stride=size,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
)
|
||||
]
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=tensors,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
|
||||
def _make_connector(
|
||||
extra_config: dict | None = None,
|
||||
num_gpu_blocks: int = 16,
|
||||
cpu_bytes: int = _BYTES_PER_BLOCK * 8,
|
||||
) -> SimpleCPUOffloadConnector:
|
||||
kv_cache_config = _make_kv_cache_config(num_gpu_blocks)
|
||||
config_dict = {
|
||||
"cpu_bytes_to_use": cpu_bytes,
|
||||
}
|
||||
if extra_config:
|
||||
config_dict.update(extra_config)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="SimpleCPUOffloadConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=config_dict,
|
||||
)
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config=vllm_config,
|
||||
role=KVConnectorRole.SCHEDULER,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
return connector
|
||||
|
||||
|
||||
def test_disk_mode_rejects_non_positive_capacity() -> None:
|
||||
"""Disk mode with disk_capacity_bytes <= 0 raises ValueError.
|
||||
|
||||
Without this guard, the worker operates in disk mode but the scheduler
|
||||
classifies events as MEDIUM_CPU instead of MEDIUM_STORAGE, silently
|
||||
mislabeling events.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="disk_capacity_bytes > 0"):
|
||||
_make_connector(
|
||||
extra_config={
|
||||
"kv_offload_backend": "disk",
|
||||
"disk_path": "/tmp/fake",
|
||||
"disk_capacity_bytes": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _make_ring_kv_cache_config(num_blocks: int = 16) -> KVCacheConfig:
|
||||
"""A paged full-attention group beside a per-request ring group."""
|
||||
full = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
ring = CircularBufferSpec(
|
||||
block_size=6, num_kv_heads=1, head_size=32, head_size_v=0, dtype=DTYPE
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=_BYTES_PER_BLOCK * num_blocks,
|
||||
layers=["attn"],
|
||||
layer_stride=_BYTES_PER_BLOCK * num_blocks,
|
||||
block_stride=_BYTES_PER_BLOCK,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=ring.page_size_bytes * num_blocks,
|
||||
layers=["ring"],
|
||||
layer_stride=ring.page_size_bytes * num_blocks,
|
||||
block_stride=ring.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["attn"], full),
|
||||
KVCacheGroupSpec(["ring"], ring),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_ring_scratch_group_is_accepted_and_never_offloaded() -> None:
|
||||
"""The ring group's capacity is not a token granularity, it holds no
|
||||
hashed blocks, and it never participates in CPU hits."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="SimpleCPUOffloadConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"cpu_bytes_to_use": _BYTES_PER_BLOCK * 8},
|
||||
)
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config=vllm_config,
|
||||
role=KVConnectorRole.SCHEDULER,
|
||||
kv_cache_config=_make_ring_kv_cache_config(),
|
||||
)
|
||||
scheduler = connector.scheduler_manager
|
||||
assert scheduler.fa_gidx == 0
|
||||
cpu_groups = scheduler.cpu_kv_cache_config.kv_cache_groups
|
||||
assert [g.kv_cache_spec.prefix_cacheable for g in cpu_groups] == [True, False]
|
||||
# Only the paged group takes part in CPU-side hit lookup.
|
||||
attention_groups = scheduler.cpu_coordinator.attention_groups
|
||||
assert [g.group_ids for g in attention_groups] == [[0]]
|
||||
@@ -1,466 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SimpleCPUOffloadConnector metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from tests.v1.kv_connector.unit.test_simple_cpu_offload_connector import (
|
||||
_make_connector,
|
||||
)
|
||||
from tests.v1.simple_kv_offload.test_scheduler import (
|
||||
make_request,
|
||||
make_scheduler,
|
||||
make_scheduler_output,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
LoadRequestState,
|
||||
TransferMeta,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
INFO_LABELS,
|
||||
OUTCOME_TO_FIELD,
|
||||
MetricName,
|
||||
SimpleCPUOffloadStats,
|
||||
_MetricType,
|
||||
_StatsKey,
|
||||
)
|
||||
|
||||
SAVE_OUTCOMES = MetricName.SAVE_OUTCOMES
|
||||
LOAD_BLOCKS = MetricName.LOAD_BLOCKS
|
||||
USED_BLOCKS = MetricName.USED_BLOCKS
|
||||
PENDING_STORE_BLOCKS = MetricName.PENDING_STORE_BLOCKS
|
||||
INFO = MetricName.INFO
|
||||
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.children: list[_FakeMetric] = []
|
||||
self.observed: list[int | float] = []
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.labelvalues: tuple[object, ...] = ()
|
||||
|
||||
def labels(self, *labelvalues: object) -> _FakeMetric:
|
||||
child = _FakeMetric(**self.kwargs)
|
||||
child.labelvalues = labelvalues
|
||||
self.children.append(child)
|
||||
return child
|
||||
|
||||
def observe(self, value: int | float) -> None:
|
||||
self.observed.append(value)
|
||||
|
||||
def inc(self, value: int | float) -> None:
|
||||
self.increments.append(value)
|
||||
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
|
||||
def _fake_metric_types() -> dict[type, type]:
|
||||
return {Gauge: _FakeMetric, Counter: _FakeMetric, Histogram: _FakeMetric}
|
||||
|
||||
|
||||
def _make_prom_metrics(per_engine: dict[int, list[object]] | None = None) -> Any:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues=per_engine or {0: ["model", "0"]},
|
||||
)
|
||||
|
||||
|
||||
def _stats_payload(types: dict, data: dict) -> dict[str, Any]:
|
||||
return {_StatsKey.TYPES: types, _StatsKey.DATA: data}
|
||||
|
||||
|
||||
def _values(stats: SimpleCPUOffloadStats, name: str) -> dict:
|
||||
return stats.data[_StatsKey.DATA][name]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_kv_connector_stats_none_and_empty() -> None:
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data=None)
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
stats = SimpleCPUOffloadConnector.build_kv_connector_stats(data={})
|
||||
assert isinstance(stats, SimpleCPUOffloadStats)
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_stats_round_trip_through_serialized_dict() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.increase_counter(LOAD_BLOCKS, 4)
|
||||
stats.set_gauge(USED_BLOCKS, 7)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
rebuilt = SimpleCPUOffloadConnector.build_kv_connector_stats(data=stats.to_dict())
|
||||
assert isinstance(rebuilt, SimpleCPUOffloadStats)
|
||||
assert _values(rebuilt, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
assert _values(rebuilt, LOAD_BLOCKS) == {(): 4}
|
||||
assert _values(rebuilt, USED_BLOCKS) == {(): 7}
|
||||
assert _values(rebuilt, INFO) == {("cpu", "false", "false", "64"): 1}
|
||||
|
||||
|
||||
def test_stats_aggregate_sums_counters_and_keeps_latest_gauge() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats1.increase_counter(SAVE_OUTCOMES, 1, ("dropped_cpu_full",))
|
||||
stats1.set_gauge(USED_BLOCKS, 7)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(SAVE_OUTCOMES, 4, ("stored",))
|
||||
stats2.set_gauge(USED_BLOCKS, 2)
|
||||
|
||||
result = stats1.aggregate(stats2)
|
||||
assert result is stats1
|
||||
outcomes = _values(stats1, SAVE_OUTCOMES)
|
||||
assert outcomes == {("stored",): 7, ("dropped_cpu_full",): 1}
|
||||
assert _values(stats1, USED_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_aggregate_merges_types_from_other() -> None:
|
||||
stats1 = SimpleCPUOffloadStats()
|
||||
stats1.set_gauge(USED_BLOCKS, 1)
|
||||
|
||||
stats2 = SimpleCPUOffloadStats()
|
||||
stats2.increase_counter(LOAD_BLOCKS, 2)
|
||||
|
||||
stats1.aggregate(stats2)
|
||||
assert stats1.data[_StatsKey.TYPES][LOAD_BLOCKS] == _MetricType.COUNTER
|
||||
assert _values(stats1, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_stats_reduce_and_reset() -> None:
|
||||
stats = SimpleCPUOffloadStats()
|
||||
stats.increase_counter(SAVE_OUTCOMES, 3, ("stored",))
|
||||
stats.set_gauge(PENDING_STORE_BLOCKS, 4)
|
||||
stats.increase_counter(LOAD_BLOCKS, 9)
|
||||
stats.set_gauge(INFO, 1, ("cpu", "false", "false", "64"))
|
||||
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{SAVE_OUTCOMES}:{('stored',)}"] == 3
|
||||
assert reduced[PENDING_STORE_BLOCKS] == 4
|
||||
assert reduced[LOAD_BLOCKS] == 9
|
||||
assert not any(key.startswith(INFO) for key in reduced)
|
||||
|
||||
stats.reset()
|
||||
assert stats.is_empty()
|
||||
|
||||
|
||||
def test_outcome_labels_match_boundary_store_stats_fields() -> None:
|
||||
"""Every outcome label maps 1:1 to a BoundaryStoreStats counter field."""
|
||||
boundary_fields = {
|
||||
f.name for f in dataclasses.fields(BoundaryStoreStats) if f.name != "published"
|
||||
}
|
||||
assert set(OUTCOME_TO_FIELD.values()) == boundary_fields
|
||||
assert len(set(OUTCOME_TO_FIELD.values())) == len(OUTCOME_TO_FIELD)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prom metrics class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prom_metrics_registers_tier1_metrics() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
defs = prom._defs
|
||||
assert set(defs) == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
# The prom class registers literal names (required by the docs
|
||||
# generator); they must stay in sync with the emission-side constants.
|
||||
assert {m.kwargs["name"] for m in defs.values()} == {
|
||||
SAVE_OUTCOMES,
|
||||
LOAD_BLOCKS,
|
||||
USED_BLOCKS,
|
||||
PENDING_STORE_BLOCKS,
|
||||
INFO,
|
||||
}
|
||||
for metric in defs.values():
|
||||
assert metric.kwargs["documentation"]
|
||||
assert metric.kwargs["labelnames"][:2] == ["model_name", "engine"]
|
||||
|
||||
assert defs[SAVE_OUTCOMES].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
"outcome",
|
||||
]
|
||||
assert defs[LOAD_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
assert defs[INFO].kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
*INFO_LABELS,
|
||||
]
|
||||
assert defs[USED_BLOCKS].kwargs["labelnames"] == ["model_name", "engine"]
|
||||
|
||||
# mostrecent: multiprocess scrapes expose one series per label set, not
|
||||
# one pid-tagged series per writer process.
|
||||
for gauge in (USED_BLOCKS, PENDING_STORE_BLOCKS, INFO):
|
||||
assert defs[gauge].kwargs["multiprocess_mode"] == "mostrecent"
|
||||
|
||||
|
||||
def test_prom_metrics_observe_routes_each_series() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={
|
||||
SAVE_OUTCOMES: _MetricType.COUNTER,
|
||||
LOAD_BLOCKS: _MetricType.COUNTER,
|
||||
USED_BLOCKS: _MetricType.GAUGE,
|
||||
PENDING_STORE_BLOCKS: _MetricType.GAUGE,
|
||||
INFO: _MetricType.GAUGE,
|
||||
},
|
||||
data={
|
||||
SAVE_OUTCOMES: {("stored",): 3},
|
||||
LOAD_BLOCKS: {(): 4},
|
||||
USED_BLOCKS: {(): 5},
|
||||
PENDING_STORE_BLOCKS: {(): 1},
|
||||
INFO: {("cpu", "false", "false", "64"): 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].increments == [3]
|
||||
assert prom._metrics[(0, LOAD_BLOCKS, ())].increments == [4]
|
||||
assert prom._metrics[(0, USED_BLOCKS, ())].set_values == [5]
|
||||
assert prom._metrics[(0, PENDING_STORE_BLOCKS, ())].set_values == [1]
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].set_values == [1]
|
||||
assert prom._metrics[(0, SAVE_OUTCOMES, ("stored",))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"stored",
|
||||
)
|
||||
assert prom._metrics[(0, INFO, ("cpu", "false", "false", "64"))].labelvalues == (
|
||||
"model",
|
||||
"0",
|
||||
"cpu",
|
||||
"false",
|
||||
"false",
|
||||
"64",
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_reuses_bound_children() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
payload = _stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored",): 1}},
|
||||
)
|
||||
prom.observe(payload)
|
||||
prom.observe(payload)
|
||||
|
||||
child = prom._metrics[(0, SAVE_OUTCOMES, ("stored",))]
|
||||
assert child.increments == [1, 1]
|
||||
assert len(prom._defs[SAVE_OUTCOMES].children) == 1
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_unknown_metric() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="Unknown"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={"vllm:not_a_metric": _MetricType.COUNTER},
|
||||
data={"vllm:not_a_metric": {(): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_rejects_wrong_label_count() -> None:
|
||||
prom = _make_prom_metrics()
|
||||
with pytest.raises(AssertionError, match="labels"):
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={SAVE_OUTCOMES: _MetricType.COUNTER},
|
||||
data={SAVE_OUTCOMES: {("stored", "extra"): 1}},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_routes_per_engine() -> None:
|
||||
prom = _make_prom_metrics(per_engine={0: ["model", "0"], 1: ["model", "1"]})
|
||||
prom.observe(
|
||||
_stats_payload(
|
||||
types={USED_BLOCKS: _MetricType.GAUGE}, data={USED_BLOCKS: {(): 3}}
|
||||
),
|
||||
engine_idx=1,
|
||||
)
|
||||
assert (0, USED_BLOCKS, ()) not in prom._metrics
|
||||
assert prom._metrics[(1, USED_BLOCKS, ())].set_values == [3]
|
||||
|
||||
|
||||
def test_connector_build_prom_metrics() -> None:
|
||||
from vllm.v1.simple_kv_offload.metrics import SimpleCPUOffloadPromMetrics
|
||||
|
||||
prom = SimpleCPUOffloadConnector.build_prom_metrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
metric_types=_fake_metric_types(),
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"]},
|
||||
)
|
||||
assert isinstance(prom, SimpleCPUOffloadPromMetrics)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manager: interval stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_manager_get_stats_reports_outcome_deltas_per_interval() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.boundary_store_stats.dropped_cpu_full += 2
|
||||
sched.boundary_store_stats.skipped_in_flight += 1
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {
|
||||
("stored",): 3,
|
||||
("dropped_cpu_full",): 2,
|
||||
("skipped_in_flight",): 1,
|
||||
}
|
||||
|
||||
# Deltas are per-interval: a second drain reports no new outcomes.
|
||||
stats2 = sched.get_stats()
|
||||
assert SAVE_OUTCOMES not in stats2.data[_StatsKey.DATA]
|
||||
|
||||
|
||||
def test_manager_reset_preserves_undrained_outcome_deltas() -> None:
|
||||
fixture = make_scheduler()
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched.boundary_store_stats.stored += 3
|
||||
sched.reset()
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, SAVE_OUTCOMES) == {("stored",): 3}
|
||||
|
||||
|
||||
def test_manager_get_stats_gauges_and_info_labels() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
sched._store_event_to_blocks[0] = TransferMeta([10], [11])
|
||||
sched._pending_finished_stores.append(TransferMeta([12], [13]))
|
||||
sched._abandoned_store_event_to_blocks[1] = TransferMeta([14], [15])
|
||||
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
used = _values(stats, USED_BLOCKS)[()]
|
||||
assert used == sched.num_cpu_blocks - sched.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
info = _values(stats, INFO)
|
||||
assert info == {("cpu", "false", "false", str(sched.num_cpu_blocks)): 1}
|
||||
|
||||
# Gauges are re-reported every drain.
|
||||
stats2 = sched.get_stats()
|
||||
assert _values(stats2, PENDING_STORE_BLOCKS) == {(): 3}
|
||||
|
||||
|
||||
def test_manager_info_labels_reflect_disk_mode() -> None:
|
||||
connector = _make_connector(
|
||||
extra_config={
|
||||
"kv_offload_backend": "disk",
|
||||
"disk_path": "/tmp/fake-disk-metrics",
|
||||
"disk_capacity_bytes": 1024**3,
|
||||
"use_page_cache": True,
|
||||
}
|
||||
)
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[0] == "disk"
|
||||
assert sched._info_labelvalues[1] == "true"
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_info_labels_page_cache_false_for_cpu_backend() -> None:
|
||||
"""CPU backend ignores use_page_cache; the info gauge must not claim it."""
|
||||
connector = _make_connector(extra_config={"use_page_cache": True})
|
||||
sched = connector.scheduler_manager
|
||||
assert sched is not None
|
||||
assert sched._info_labelvalues[:2] == ("cpu", "false")
|
||||
stats = sched.get_stats()
|
||||
assert set(_values(stats, INFO)) == {sched._info_labelvalues}
|
||||
|
||||
|
||||
def test_manager_counts_load_blocks_completed() -> None:
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
|
||||
|
||||
def test_manager_abandoned_load_counts_as_completed_after_reset() -> None:
|
||||
"""Loads abandoned by reset() that still finish on the worker are
|
||||
counted as loaded blocks.
|
||||
"""
|
||||
fixture = make_scheduler(num_cpu_blocks=8)
|
||||
sched = fixture.scheduler
|
||||
|
||||
request = make_request(num_blocks=2)
|
||||
gpu_blocks = fixture.gpu_block_pool.get_new_blocks(2)
|
||||
cpu_blocks = sched.cpu_block_pool.get_new_blocks(2)
|
||||
sched._reqs_to_load[request.request_id] = LoadRequestState(
|
||||
request=request,
|
||||
transfer_meta=TransferMeta(
|
||||
[b.block_id for b in gpu_blocks], [b.block_id for b in cpu_blocks]
|
||||
),
|
||||
)
|
||||
|
||||
sched.build_connector_meta(make_scheduler_output({}))
|
||||
stats = sched.get_stats()
|
||||
assert LOAD_BLOCKS not in stats.data[_StatsKey.DATA]
|
||||
|
||||
assert sched.reset() is False
|
||||
assert set(sched._abandoned_reqs_to_load) == {request.request_id}
|
||||
|
||||
sched.update_connector_output(
|
||||
KVConnectorOutput(finished_recving={request.request_id})
|
||||
)
|
||||
assert sched._abandoned_reqs_to_load == {}
|
||||
stats = sched.get_stats()
|
||||
assert _values(stats, LOAD_BLOCKS) == {(): 2}
|
||||
@@ -1,316 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for TP mapping and transfer plan utilities.
|
||||
|
||||
These tests verify that TP mapping produces correct outputs
|
||||
(source ranks, split handles, desc IDs).
|
||||
No GPU or NIXL required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
compute_tp_mapping,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||
|
||||
# ======================================================================
|
||||
# Test fixtures / helpers
|
||||
# ======================================================================
|
||||
|
||||
|
||||
def _compute_mapping(
|
||||
tp_rank: int = 0,
|
||||
tp_size: int = 1,
|
||||
remote_tp_size: int = 1,
|
||||
is_mla: bool = False,
|
||||
num_kv_heads: int = 8,
|
||||
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
|
||||
dcp_size: int = 1,
|
||||
remote_dcp_size: int = 1,
|
||||
) -> TPMapping:
|
||||
transfer_topology = object.__new__(TransferTopology)
|
||||
transfer_topology.tp_rank = tp_rank
|
||||
transfer_topology.tp_size = tp_size
|
||||
transfer_topology.is_mla = is_mla
|
||||
transfer_topology.total_num_kv_heads = num_kv_heads
|
||||
transfer_topology.dcp_size = dcp_size
|
||||
return compute_tp_mapping(
|
||||
transfer_topology=transfer_topology,
|
||||
remote_tp_size=remote_tp_size,
|
||||
group_spec_types=group_spec_types,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# TP mapping structure tests
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestTPMappingStructure:
|
||||
def test_source_ranks_homogeneous(self):
|
||||
m = _compute_mapping(tp_size=2, tp_rank=1, remote_tp_size=2)
|
||||
assert m.all_source_ranks == (1,)
|
||||
|
||||
def test_source_ranks_d_gt_p(self):
|
||||
m = _compute_mapping(tp_size=4, tp_rank=2, remote_tp_size=2)
|
||||
assert m.all_source_ranks == (1,)
|
||||
|
||||
def test_source_ranks_p_gt_d(self):
|
||||
m = _compute_mapping(tp_size=1, tp_rank=0, remote_tp_size=2)
|
||||
assert m.all_source_ranks == (0, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks",
|
||||
[
|
||||
(0, 1, 8, 1, 8, tuple(range(8))),
|
||||
(0, 4, 4, 1, 4, (0, 1, 2, 3)),
|
||||
(2, 4, 4, 4, 4, (2,)),
|
||||
(0, 2, 4, 2, 4, (0, 2)),
|
||||
(3, 4, 2, 4, 2, (1,)),
|
||||
],
|
||||
)
|
||||
def test_mla_dcp_source_ranks(
|
||||
tp_rank,
|
||||
tp_size,
|
||||
remote_tp_size,
|
||||
dcp_size,
|
||||
remote_dcp_size,
|
||||
expected_ranks,
|
||||
):
|
||||
mapping = _compute_mapping(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
remote_tp_size=remote_tp_size,
|
||||
is_mla=True,
|
||||
num_kv_heads=1,
|
||||
dcp_size=dcp_size,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
)
|
||||
|
||||
assert mapping.all_source_ranks == expected_ranks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,remote_tp_size,dcp_size,remote_dcp_size",
|
||||
[
|
||||
(4, 2, 4, 2),
|
||||
(4, 4, 4, 4),
|
||||
(4, 1, 4, 1),
|
||||
(4, 4, 1, 4),
|
||||
(2, 4, 2, 4),
|
||||
],
|
||||
)
|
||||
def test_dcp_consumer_count_matches_readers(
|
||||
tp_size, remote_tp_size, dcp_size, remote_dcp_size
|
||||
):
|
||||
"""Each producer waits for exactly the local ranks that read from it."""
|
||||
mappings = [
|
||||
_compute_mapping(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
remote_tp_size=remote_tp_size,
|
||||
is_mla=True,
|
||||
num_kv_heads=1,
|
||||
dcp_size=dcp_size,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
)
|
||||
for tp_rank in range(tp_size)
|
||||
]
|
||||
|
||||
for remote_rank in range(remote_tp_size):
|
||||
readers = [m for m in mappings if remote_rank in m.all_source_ranks]
|
||||
for mapping in readers:
|
||||
assert mapping.local_consumers == len(readers)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Split handle tests
|
||||
# ======================================================================
|
||||
|
||||
|
||||
def _make_mock_worker_for_splits(group_spec_types):
|
||||
"""Build a mock NixlConnectorWorker with _group_spec_types for split tests.
|
||||
|
||||
No per-region replicate flags are configured (``block_len_per_layer`` empty
|
||||
and ``num_regions == 0``), so ``_build_local_splits_from_plan`` treats every
|
||||
FA descriptor as SPLIT, matching the legacy behavior these tests assert.
|
||||
"""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.transfer_topo = SimpleNamespace(virtually_split_kv_in_blocks=False)
|
||||
worker.block_len_per_layer = []
|
||||
worker.num_regions = 0
|
||||
worker._region_is_mla = []
|
||||
worker._conv_decomp = SimpleNamespace(local_conv_offsets=())
|
||||
worker._ssm_region_indices = [0] if MambaSpec in group_spec_types else []
|
||||
worker._ple_region_index = None
|
||||
return worker
|
||||
|
||||
|
||||
class TestBuildSrcSplitHandles:
|
||||
@pytest.mark.parametrize("remote_tp_size", [2, 4])
|
||||
def test_build_src_split_handles(self, remote_tp_size):
|
||||
tp_rank = 0
|
||||
tp_size = 1
|
||||
|
||||
plan = _compute_mapping(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
remote_tp_size=remote_tp_size,
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
|
||||
# One region of 8 dense 1024-byte blocks as a single strided run.
|
||||
src_blocks_data = np.array([(0x2000, 1024, 0, 1024, 8)], dtype=np.uint64)
|
||||
num_descs = 8
|
||||
splits = list(
|
||||
worker._build_local_splits_from_plan(
|
||||
plan,
|
||||
src_blocks_data,
|
||||
num_descs,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(splits) == remote_tp_size
|
||||
for handle in splits:
|
||||
assert handle.shape == src_blocks_data.shape
|
||||
# The head slice shrinks the length; block stride and count are kept.
|
||||
assert handle[:, 1].tolist() == [1024 // remote_tp_size]
|
||||
assert handle[:, 3:].tolist() == [[1024, 8]]
|
||||
|
||||
|
||||
class TestMambaPlanSplitHandles:
|
||||
"""Verify split handles for Mamba with FA/SSM distinction."""
|
||||
|
||||
def test_fa_and_ssm_different_split_factors(self):
|
||||
"""Section 0 split by num_attn_reads, section 1 by abs_tp."""
|
||||
fa_readers = (0,)
|
||||
ssm_readers = (0, 1)
|
||||
plan = TPMapping(
|
||||
source_ranks_per_group=(fa_readers, ssm_readers),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
# 2 FA runs + 1 SSM run of one block each: (addr, len, dev, stride, count)
|
||||
src_blocks_data = np.array(
|
||||
[
|
||||
(1000, 200, 0, 200, 1), # FA desc 0
|
||||
(2000, 200, 0, 200, 1), # FA desc 1
|
||||
(3000, 400, 0, 400, 1), # SSM desc 0
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2))
|
||||
|
||||
assert len(splits) == 2 # 2 source ranks
|
||||
|
||||
# Rank 0 (FA source, p_idx=0):
|
||||
# FA: chunk=200//1=200, slot=0 → (1000, 200), (2000, 200)
|
||||
# SSM: chunk=400//2=200, idx=0 → (3000, 200); block stride stays 400
|
||||
assert splits[0].tolist() == [
|
||||
[1000, 200, 0, 200, 1],
|
||||
[2000, 200, 0, 200, 1],
|
||||
[3000, 200, 0, 400, 1],
|
||||
]
|
||||
|
||||
# Rank 1 (not FA source, p_idx=1):
|
||||
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200), (2000, 200)
|
||||
# SSM: chunk=400//2=200, idx=1 → (3200, 200); block stride stays 400
|
||||
assert splits[1].tolist() == [
|
||||
[1000, 200, 0, 200, 1],
|
||||
[2000, 200, 0, 200, 1],
|
||||
[3200, 200, 0, 400, 1],
|
||||
]
|
||||
|
||||
def test_hetero_block_size_splits(self):
|
||||
"""With a block-size ratio, single-source FA sub-block descs pass
|
||||
through whole; SSM descs are unexpanded and split per source."""
|
||||
plan = TPMapping(
|
||||
source_ranks_per_group=((0,), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
# 2 FA blocks x ratio 2 sub-blocks (one run of 2 per block) + 1 SSM desc
|
||||
# (never expanded).
|
||||
src_blocks_data = np.array(
|
||||
[
|
||||
(1000, 100, 0, 100, 2),
|
||||
(2000, 100, 0, 100, 2),
|
||||
(3000, 400, 0, 400, 1),
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 4, 2))
|
||||
|
||||
assert len(splits) == 2
|
||||
fa_passthrough = [[1000, 100, 0, 100, 2], [2000, 100, 0, 100, 2]]
|
||||
assert splits[0].tolist() == fa_passthrough + [[3000, 200, 0, 400, 1]]
|
||||
assert splits[1].tolist() == fa_passthrough + [[3200, 200, 0, 400, 1]]
|
||||
|
||||
def test_hetero_block_size_head_sharded_asserts(self):
|
||||
"""Head-sharded FA reads (multiple FA sources) are incompatible with
|
||||
a block-size mismatch and must fail loudly."""
|
||||
plan = TPMapping(
|
||||
source_ranks_per_group=((0, 1), (0, 1)),
|
||||
all_source_ranks=(0, 1),
|
||||
rank_to_attention_slot={0: 0, 1: 1},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
|
||||
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||
src_blocks_data = np.array(
|
||||
[(1000, 100, 0, 100, 2), (3000, 400, 0, 400, 1)],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError, match="Head-sharded"):
|
||||
list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2, 2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_kv_heads", "local_tp", "remote_tp", "compatible"),
|
||||
[
|
||||
# 2 heads: TP1 packs both heads into a page, TP >= 2 packs one.
|
||||
(2, 1, 2, False),
|
||||
(2, 2, 1, False),
|
||||
(2, 1, 1, True),
|
||||
(2, 2, 4, True),
|
||||
(2, 4, 2, True),
|
||||
# 4 heads: the boundary moves with the head count.
|
||||
(4, 2, 4, False),
|
||||
(4, 2, 2, True),
|
||||
(4, 4, 8, True),
|
||||
],
|
||||
)
|
||||
def test_csa_linear_tp_layout_boundary(total_kv_heads, local_tp, remote_tp, compatible):
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.world_size = local_tp
|
||||
worker._is_csa_linear = True
|
||||
worker.transfer_topo = SimpleNamespace(total_num_kv_heads=total_kv_heads)
|
||||
|
||||
if compatible:
|
||||
worker._validate_csa_linear_tp_layout(remote_tp)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="KV-head sharding boundary"):
|
||||
worker._validate_csa_linear_tp_layout(remote_tp)
|
||||
@@ -537,154 +537,6 @@ def make_kv_cache_config(
|
||||
)
|
||||
|
||||
|
||||
def make_nixl_scheduler(
|
||||
has_mamba: bool = False,
|
||||
is_hma_required: bool = False,
|
||||
bounded_replay: bool = False,
|
||||
heartbeat: bool = False,
|
||||
kv_lease_duration: int = 30,
|
||||
):
|
||||
"""Create a NixlConnectorScheduler via __new__ (skipping __init__).
|
||||
|
||||
Only sets the flags needed by the tests. When *heartbeat=True* the
|
||||
scheduler-side heartbeat bookkeeping fields are also initialised.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
|
||||
NixlConnectorScheduler,
|
||||
)
|
||||
|
||||
sched = object.__new__(NixlConnectorScheduler)
|
||||
sched._has_mamba = has_mamba
|
||||
sched._bounded_replay = bounded_replay
|
||||
sched._is_hma_required = is_hma_required
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0)
|
||||
|
||||
if heartbeat:
|
||||
sched._heartbeat_by_engine = {}
|
||||
sched._heartbeat_req_engine = {}
|
||||
sched._last_heartbeat_time = 0.0
|
||||
sched._kv_lease_duration = kv_lease_duration
|
||||
sched._heartbeat_interval = kv_lease_duration // 6
|
||||
# Fields touched by build_connector_meta / request_finished:
|
||||
sched._reqs_need_recv = {}
|
||||
sched._hisparse_host_blocks_to_recv = {}
|
||||
sched._reqs_need_send = {}
|
||||
sched._reqs_in_batch = set()
|
||||
sched._reqs_not_processed = set()
|
||||
sched._reqs_need_save = {}
|
||||
sched.use_host_buffer = False
|
||||
sched.engine_id = "test-engine"
|
||||
sched.transfer_tp_size = 1
|
||||
sched.side_channel_host = "localhost"
|
||||
sched.side_channel_port = 5555
|
||||
sched.blocks_per_sw = []
|
||||
sched.is_bidirectional_kv_xfer_enabled = False
|
||||
return sched
|
||||
|
||||
|
||||
def make_nixl_push_scheduler(
|
||||
*,
|
||||
decoder_kv_blocks_ttl: float = 30.0,
|
||||
push_registration_timeout: float | None = None,
|
||||
is_bidirectional_kv_xfer_enabled: bool = False,
|
||||
has_mamba: bool = False,
|
||||
):
|
||||
"""Create a NixlPushConnectorScheduler via __new__ (skipping __init__).
|
||||
|
||||
The push scheduler can't reuse :func:`make_nixl_scheduler` because it
|
||||
is a different class (``NixlPushConnectorScheduler`` vs
|
||||
``NixlConnectorScheduler``) and carries push-specific state. Only the
|
||||
fields touched by the unit tests are populated.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
|
||||
NixlPushConnectorScheduler,
|
||||
)
|
||||
|
||||
sched = object.__new__(NixlPushConnectorScheduler)
|
||||
|
||||
# Base scheduler fields (shared with pull / heartbeat path).
|
||||
sched._reqs_need_recv = {}
|
||||
sched._reqs_need_send = {}
|
||||
sched._reqs_in_batch = set()
|
||||
sched._reqs_not_processed = set()
|
||||
sched._reqs_need_save = {}
|
||||
sched._kv_lease_duration = 30
|
||||
sched.decoder_kv_blocks_ttl = decoder_kv_blocks_ttl
|
||||
sched.use_host_buffer = False
|
||||
sched.engine_id = "decode-engine"
|
||||
sched.transfer_tp_size = 1
|
||||
sched.side_channel_host = "127.0.0.1"
|
||||
sched.side_channel_port = 5600
|
||||
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
||||
sched._has_mamba = has_mamba
|
||||
sched._bounded_replay = False
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
# vllm_config is consulted for parallel_config.tensor_parallel_size, and by
|
||||
# `_prefill_backoff` on both the P and D prefill paths.
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.parallel_config.tensor_parallel_size = 1
|
||||
vllm_config.num_prefill_lookahead_tokens = 0
|
||||
sched.vllm_config = vllm_config
|
||||
|
||||
# Push-specific state.
|
||||
sched._push_pending_registrations = {}
|
||||
sched._push_registration_deadlines = {}
|
||||
sched._finished_request_blocks = {}
|
||||
sched._newly_finished_push_blocks = {}
|
||||
sched._push_registration_timeout = (
|
||||
push_registration_timeout
|
||||
if push_registration_timeout is not None
|
||||
else decoder_kv_blocks_ttl
|
||||
)
|
||||
|
||||
# Heartbeat fields touched by base request_finished /
|
||||
# update_connector_output.
|
||||
sched._heartbeat_by_engine = {}
|
||||
sched._heartbeat_req_engine = {}
|
||||
sched._last_heartbeat_time = 0.0
|
||||
sched.blocks_per_sw = []
|
||||
|
||||
return sched
|
||||
|
||||
|
||||
def make_moriio_writer(fake_worker: Any) -> Any:
|
||||
"""Build a MoRIIOWriter with internals stubbed for unit tests.
|
||||
|
||||
Bypasses ``__init__`` and wires only the write/finalize state the tests
|
||||
touch, including the deferred-task fields used by the routing suite.
|
||||
"""
|
||||
import threading
|
||||
from queue import Queue
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWriter,
|
||||
)
|
||||
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
writer._write_task_q = Queue()
|
||||
writer._write_state_lock = threading.Lock()
|
||||
writer._scheduled_writes = defaultdict(int)
|
||||
writer._scheduled_layers = defaultdict(set)
|
||||
writer._sealed_writes = {}
|
||||
writer._deferred_tasks = []
|
||||
writer._defer_timeout = 60.0
|
||||
writer.ensure_worker_started = lambda: None
|
||||
return writer
|
||||
|
||||
|
||||
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
|
||||
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user