Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 22:06:10 +07:00
327 changed files with 13704 additions and 5433 deletions
@@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early):
allocator.release_pools()
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_runtime_state_survives_sleep(level):
"""Runtime state is offloaded at both levels and restored by the first
selective wake; a nested kv_cache pool keeps its own tag."""
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("runtime"):
state = torch.arange(1 << 20, device=DEVICE_TYPE)
with allocator.use_memory_pool("kv_cache"):
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
assert {d.tag for d in allocator.pointer_to_data.values()} == {
"runtime",
"kv_cache",
}
backend = CuMemBackend()
backend.suspend(level=level)
assert mapped_usage(allocator) == 0
backend.resume(tags=["weights"])
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
backend.resume(tags=["kv_cache"])
kv.fill_(1)
assert int(kv.sum()) == kv.numel()
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
+42 -27
View File
@@ -6,14 +6,17 @@ from collections import defaultdict
import pytest
import regex as re
from vllm import LLM, SamplingParams
from vllm import SamplingParams
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
from .common import FUSION_LOG_PATTERNS, AttentionBackendCase, Matches
def run_model(
compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs
vllm_runner,
compile_config: CompilationMode | CompilationConfig,
model: str,
**model_kwargs,
):
"""Run a model with the given compilation config for E2E fusion tests."""
compilation_config = (
@@ -36,38 +39,48 @@ def run_model(
# No cudagraphs by default
if compilation_config.cudagraph_mode is None:
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
llm = LLM(
model=model,
# trust_remote_code/enable_chunked_prefill/block_size/kernel_config are
# pinned to LLM()'s own defaults since VllmRunner overrides them for
# other test needs (e.g. it forces block_size=16, which some attention
# backends like FLASHINFER_MLA reject, and disables JIT warmup).
model_kwargs.setdefault("kernel_config", {"enable_jit_warmup": True})
with vllm_runner(
model,
compilation_config=compilation_config,
trust_remote_code=False,
enable_chunked_prefill=None,
block_size=None,
**model_kwargs,
)
outputs = llm.generate(prompts, sampling_params)
) as vllm_model:
llm = vllm_model.llm
outputs = llm.generate(prompts, sampling_params)
# Print the outputs.
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
# Print the outputs.
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
# Get the compile ranges endpoints after vllm config post init
# in order to compute compile ranges correctly
compilation_config.compile_ranges_endpoints = (
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
)
# Get the compile ranges endpoints after vllm config post init
# in order to compute compile ranges correctly
compilation_config.compile_ranges_endpoints = (
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
)
# Fetch match table from each worker via RPC and sum across workers.
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
"get_compilation_match_table"
)
combined: defaultdict[str, int] = defaultdict(int)
for table in worker_tables:
for k, v in table.items():
combined[k] += v
return dict(combined)
# Fetch match table from each worker via RPC and sum across workers.
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
"get_compilation_match_table"
)
combined: defaultdict[str, int] = defaultdict(int)
for table in worker_tables:
for k, v in table.items():
combined[k] += v
return dict(combined)
@pytest.fixture
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
def run(
model_name: str,
matches: Matches,
@@ -137,7 +150,9 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
)
with caplog_mp_spawn(logging.DEBUG) as log_holder:
match_table = run_model(full_compilation_config, model_name, **model_kwargs)
match_table = run_model(
vllm_runner, full_compilation_config, model_name, **model_kwargs
)
num_compile_ranges = len(full_compilation_config.get_compile_ranges())
assert num_compile_ranges in [1, 2, 3]
+14
View File
@@ -101,6 +101,20 @@ llama3_8b_fp4 = ModelFusionInfo(
),
)
# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
# every MLP boundary is left to ActivationQuantFusionPass.
qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
model_name="nvidia/Qwen3.8-27B-NVFP4",
hf_overrides=lambda n_layers: {
"text_config": {
"num_hidden_layers": n_layers,
"layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
}
},
matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
)
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
@@ -30,6 +30,7 @@ from .models import (
llama3_8b_fp8,
llama4_scout_fp4,
llama4_scout_fp8,
qwen3_8_27b_mixed_fp4,
qwen3_a3b_fp8,
)
@@ -202,3 +203,46 @@ def test_tp1_fp4_fusions(
compilation_config,
matches_check,
)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
)
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
def test_tp1_fp4_mixed_precision_act_fusion(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
"""Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
pass_config=PassConfig(
fuse_norm_quant=False,
fuse_act_quant=True,
fuse_attn_quant=False,
enable_qk_norm_rope_fusion=False,
),
)
run_e2e_fusion_test(
model_name,
matches_fn(n_layers),
model_kwargs,
attn_backend,
compilation_config,
["act_quant_fusion"],
)
+43
View File
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NVFP4 detection for ModelOpt MIXED_PRECISION checkpoints.
A checkpoint that quantizes some layers to NVFP4 and others to FP8 declares
``quant_algo: MIXED_PRECISION`` and resolves to ``modelopt_mixed``, so NVFP4
has to be detected from the per-layer algorithms.
"""
import pytest
from vllm.config.model import _modelopt_mixed_has_nvfp4
def _quantized_layers(*algos: str) -> dict:
return {
"quantized_layers": {
f"model.layers.{i}.proj": {"quant_algo": algo}
for i, algo in enumerate(algos)
}
}
def test_detects_nvfp4_alongside_another_algorithm():
"""The shipped shape: FP8 attention projections, NVFP4 MLP."""
assert _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "NVFP4", "FP8"))
def test_ignores_a_checkpoint_with_no_nvfp4_layer():
assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "MXFP8"))
def test_weight_only_nvfp4_does_not_count():
"""W4A16_NVFP4 leaves activations in high precision, so nothing to fuse."""
assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "W4A16_NVFP4"))
@pytest.mark.parametrize(
"quant_config",
[None, {}, {"quantized_layers": None}, {"quantized_layers": ["NVFP4"]}],
)
def test_absent_or_malformed_config_is_not_nvfp4(quant_config):
assert not _modelopt_mixed_has_nvfp4(quant_config)
+1 -1
View File
@@ -511,7 +511,7 @@ def _check_dummy_hash_model_forward(
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.use_mega_moe = False
model.use_native_mega_moe = False
model.use_sequence_parallel = False
model.fuse_mhc_all_reduce = False
model.engram_hash = state
@@ -0,0 +1,278 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from vllm.config.parallel import EPLBConfig
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
from vllm.distributed.eplb.migration_scheduler import (
MigrationFlow,
schedule_migration_batches,
schedule_migration_batches_for_layers,
)
from vllm.distributed.eplb.rebalance_execute import move_from_buffer, move_to_buffer
class _MockEplbCommunicator(EplbCommunicator):
def __init__(self) -> None:
self.send_calls: list[tuple[list[torch.Tensor], int, int]] = []
self.recv_calls: list[tuple[list[torch.Tensor], int, int]] = []
self.context_calls = 0
self.execute_count = 0
def add_send(
self, tensors: list[torch.Tensor], dst_rank: int, expert_id: int
) -> None:
self.send_calls.append((tensors, dst_rank, expert_id))
def add_recv(
self, tensors: list[torch.Tensor], src_rank: int, expert_id: int
) -> None:
self.recv_calls.append((tensors, src_rank, expert_id))
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
self.context_calls += 1
def execute(self) -> None:
self.execute_count += 1
def test_migration_batching_is_disabled_by_default_and_can_be_enabled() -> None:
assert not EPLBConfig().enable_migration_batching
assert EPLBConfig(enable_migration_batching=True).enable_migration_batching
def test_migration_batching_only_applies_to_async_eplb() -> None:
assert not EPLBConfig(use_async=True).migration_batching_enabled
assert EPLBConfig(
use_async=True, enable_migration_batching=True
).migration_batching_enabled
assert not EPLBConfig(
use_async=True, enable_migration_batching=False
).migration_batching_enabled
assert not EPLBConfig(
use_async=False, enable_migration_batching=True
).migration_batching_enabled
def test_schedule_migration_batches_is_deterministic() -> None:
transfers = [
(1, 3, 0),
(2, 4, 1),
(0, 1, 2),
(0, 2, 3),
(0, 3, 4),
(0, 4, 5),
]
num_local_experts, old_indices, new_indices = _placements_for_transfers(
5, transfers
)
batches = schedule_migration_batches(num_local_experts, old_indices, new_indices)
assert [_endpoints(batch) for batch in batches] == [
[(1, 3), (2, 4)],
[(0, 1)],
[(0, 2)],
[(0, 3)],
[(0, 4)],
]
for batch in batches:
_assert_no_endpoint_conflict(batch)
def test_schedule_migration_batches_coalesces_rank_pair() -> None:
transfers = [
(0, 1, 0),
(0, 1, 1),
(2, 3, 2),
]
num_local_experts, old_indices, new_indices = _placements_for_transfers(
4, transfers
)
assert schedule_migration_batches(num_local_experts, old_indices, new_indices) == [
[
MigrationFlow(0, 1, expert_ids=(0, 1)),
MigrationFlow(2, 3, expert_ids=(2,)),
]
]
def test_schedule_migration_batches_covers_random_instructions() -> None:
rng = np.random.default_rng(42)
transfers = []
for expert_id in range(80):
src_rank, dst_rank = rng.integers(0, 8, size=2).tolist()
if src_rank != dst_rank:
transfers.append((src_rank, dst_rank, expert_id))
num_local_experts, old_indices, new_indices = _placements_for_transfers(
8, transfers
)
batches = schedule_migration_batches(num_local_experts, old_indices, new_indices)
scheduled = _flatten_transfers(batches)
assert len(scheduled) == len(transfers)
assert set(scheduled) == set(transfers)
for batch in batches:
_assert_no_endpoint_conflict(batch)
def test_schedule_migration_batches_excludes_local_copies() -> None:
# Old: rank 0 [0, 1], rank 1 [1, 2]. New: rank 1 needs expert 0.
old_indices = np.array([0, 1, 1, 2], dtype=np.int64)
new_indices = np.array([0, 1, 0, 2], dtype=np.int64)
assert schedule_migration_batches(2, old_indices, new_indices) == [
[MigrationFlow(0, 1, expert_ids=(0,))]
]
def test_schedule_migration_batches_requires_matching_shapes() -> None:
old_indices = np.array([0, 1], dtype=np.int64)
new_indices = np.array([0], dtype=np.int64)
with pytest.raises(AssertionError):
schedule_migration_batches(1, old_indices, new_indices)
def test_schedule_migration_batches_balances_replicas() -> None:
# Rank 0 and rank 1 hold expert 0; ranks 2, 3, and 4 need it.
old_indices = np.array([0, 0, 1, 1, 1], dtype=np.int64)
new_indices = np.array([0, 0, 0, 0, 0], dtype=np.int64)
batches = schedule_migration_batches(1, old_indices, new_indices)
assert set(_flatten_transfers(batches)) == {
(0, 2, 0),
(0, 4, 0),
(1, 3, 0),
}
def test_schedule_migration_batches_for_layers_matches_individual_calls() -> None:
old_indices = np.array([[0, 1, 2, 3], [3, 2, 1, 0]], dtype=np.int64)
new_indices = np.array([[1, 2, 3, 0], [0, 3, 2, 1]], dtype=np.int64)
assert schedule_migration_batches_for_layers(1, old_indices, new_indices) == [
schedule_migration_batches(1, old_indices[0], new_indices[0]),
schedule_migration_batches(1, old_indices[1], new_indices[1]),
]
@pytest.mark.parametrize("use_precomputed", [False, True])
def test_move_to_buffer_uses_multiple_batches(use_precomputed: bool) -> None:
old_indices = np.array([0, 1, 2, 3], dtype=np.int64)
new_indices = np.array([1, 2, 3, 0], dtype=np.int64)
communicator = _MockEplbCommunicator()
migration_batches = (
schedule_migration_batches(1, old_indices, new_indices)
if use_precomputed
else None
)
move_to_buffer(
num_local_experts=1,
old_indices=old_indices,
new_indices=new_indices,
expert_weights=[torch.zeros(1, 1)],
expert_weights_buffers=[torch.zeros(1, 1)],
stream=None,
ep_rank=0,
communicator=communicator,
layer_idx=7,
enable_migration_batching=True,
migration_batches=migration_batches,
)
assert communicator.context_calls == 2
assert communicator.execute_count == 2
assert [(dst, expert) for _, dst, expert in communicator.send_calls] == [(3, 0)]
assert [(src, expert) for _, src, expert in communicator.recv_calls] == [(1, 1)]
def test_move_to_buffer_uses_primary_duplicate_destination() -> None:
old_indices = np.array([0, 1, 2, 3], dtype=np.int64)
new_indices = np.array([2, 2, 0, 3], dtype=np.int64)
expert_weights = [torch.zeros(2, 1)]
recv_buffer = torch.zeros(2, 1)
communicator = _MockEplbCommunicator()
transfer_metadata = move_to_buffer(
num_local_experts=2,
old_indices=old_indices,
new_indices=new_indices,
expert_weights=expert_weights,
expert_weights_buffers=[recv_buffer],
stream=None,
ep_rank=0,
communicator=communicator,
enable_migration_batching=True,
)
assert len(communicator.recv_calls) == 1
recv_tensors, src_rank, expert_id = communicator.recv_calls[0]
assert (src_rank, expert_id) == (1, 2)
assert recv_tensors[0].data_ptr() == recv_buffer[0].data_ptr()
recv_buffer[0].fill_(42)
move_from_buffer(
expert_weights=expert_weights,
expert_weights_buffers=[recv_buffer],
transfer_metadata=transfer_metadata,
new_indices=new_indices,
ep_rank=0,
)
torch.testing.assert_close(expert_weights[0], torch.full((2, 1), 42.0))
def _endpoints(
flows: list[MigrationFlow],
) -> list[tuple[int, int]]:
return [(item.src_rank, item.dst_rank) for item in flows]
def _assert_no_endpoint_conflict(batch: list[MigrationFlow]) -> None:
endpoints: set[int] = set()
for flow in batch:
assert flow.src_rank not in endpoints
assert flow.dst_rank not in endpoints
endpoints.update((flow.src_rank, flow.dst_rank))
def _flatten_transfers(
batches: list[list[MigrationFlow]],
) -> list[tuple[int, int, int]]:
return [
(flow.src_rank, flow.dst_rank, expert_id)
for batch in batches
for flow in batch
for expert_id in flow.expert_ids
]
def _placements_for_transfers(
num_ranks: int,
transfers: list[tuple[int, int, int]],
) -> tuple[int, np.ndarray, np.ndarray]:
old_counts = [0] * num_ranks
new_counts = [0] * num_ranks
for src_rank, dst_rank, _ in transfers:
old_counts[src_rank] += 1
new_counts[dst_rank] += 1
num_local_experts = max(1, *old_counts, *new_counts)
old_indices = np.full(num_ranks * num_local_experts, -1, dtype=np.int64)
new_indices = np.full_like(old_indices, -1)
old_counts = [0] * num_ranks
new_counts = [0] * num_ranks
for src_rank, dst_rank, expert_id in transfers:
old_offset = src_rank * num_local_experts + old_counts[src_rank]
new_offset = dst_rank * num_local_experts + new_counts[dst_rank]
old_indices[old_offset] = expert_id
new_indices[new_offset] = expert_id
old_counts[src_rank] += 1
new_counts[dst_rank] += 1
return num_local_experts, old_indices, new_indices
+129
View File
@@ -231,6 +231,8 @@ def test_one_sided_combine_into_compatibility(supports_output):
payload,
runtime_max_tokens_per_rank,
output=None,
*,
use_low_precision=False,
):
result = payload + runtime_max_tokens_per_rank
if output is None:
@@ -249,6 +251,33 @@ def test_one_sided_combine_into_compatibility(supports_output):
torch.testing.assert_close(output, payload + 2)
@pytest.mark.parametrize("low_precision_combine", [False, True])
def test_one_sided_combine_into_low_precision(low_precision_combine):
"""The fp8 combine setting is passed through to the kernel."""
from vllm.distributed.device_communicators.all2all import (
FlashInferNVLinkOneSidedManager,
)
seen_kwargs = {}
class FakeMoeAlltoAll:
def combine(self, payload, runtime_max_tokens_per_rank, output, **kwargs):
seen_kwargs.update(kwargs)
output.copy_(payload + runtime_max_tokens_per_rank)
manager = FlashInferNVLinkOneSidedManager.__new__(FlashInferNVLinkOneSidedManager)
manager.moe_alltoall = FakeMoeAlltoAll()
manager._combine_supports_output = True
manager.low_precision_combine = low_precision_combine
payload = torch.arange(4, dtype=torch.float32)
output = torch.empty_like(payload)
manager.combine_into(payload, runtime_max_tokens_per_rank=2, output=output)
assert seen_kwargs == {"use_low_precision": low_precision_combine}
torch.testing.assert_close(output, payload + 2)
# ---------------------------------------------------------------------------
# Test 1: Two-sided manager lifecycle (init, cleanup, reinit, ensure_init)
# ---------------------------------------------------------------------------
@@ -915,6 +944,106 @@ def test_one_sided_dispatch_combine(world_size):
_spawn_workers(_one_sided_data_worker, world_size, dp_size=world_size)
def _one_sided_low_precision_worker(rank, world_size):
from vllm.distributed.device_communicators.all2all import (
FlashInferNVLinkOneSidedManager,
)
from vllm.distributed.parallel_state import get_dp_group
device = torch.device(f"cuda:{rank}")
hidden_size = 256
tokens_per_rank = 32
experts_per_token = 2
num_experts = world_size * 8
manager = FlashInferNVLinkOneSidedManager(get_dp_group().cpu_group)
manager.initialize(
max_num_tokens=tokens_per_rank,
top_k=experts_per_token,
num_experts=num_experts,
hidden_size=hidden_size,
x_bytes_per_token=hidden_size // 2,
x_sf_bytes_per_token=hidden_size // 16,
)
assert manager.low_precision_combine
torch.manual_seed(rank + 42)
x = torch.randint(
0, 256, (tokens_per_rank, hidden_size // 2), device=device, dtype=torch.uint8
)
x_sf = torch.randint(
0, 256, (tokens_per_rank, hidden_size // 16), device=device, dtype=torch.uint8
)
topk_ids = torch.randint(
0,
num_experts,
(tokens_per_rank, experts_per_token),
device=device,
dtype=torch.int32,
)
topk_weights = torch.rand(
tokens_per_rank, experts_per_token, device=device, dtype=torch.float32
)
manager.moe_alltoall.dispatch(
token_selected_experts=topk_ids,
input_payloads=[x, x_sf, topk_ids, topk_weights],
runtime_max_tokens_per_rank=tokens_per_rank,
)
# Go through combine_into rather than moe_alltoall.combine: the workspace
# was sized for fp8, so the kernel must be told to send fp8. Calling the
# raw kernel here would send bf16 into fp8-sized slots and corrupt it.
expert_output = torch.ones(
world_size, tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
)
output = torch.empty(
tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
)
manager.combine_into(
payload=expert_output,
runtime_max_tokens_per_rank=tokens_per_rank,
output=output,
)
# Integers 1..top_k are exact in fp8_e4m3, so the fp8 transport must match
# bf16 exactly: one contribution per distinct expert-owning rank.
experts_per_rank = num_experts // world_size
expert_ranks = topk_ids // experts_per_rank
num_distinct = torch.tensor(
[len(set(row.tolist())) for row in expert_ranks],
device=device,
dtype=torch.bfloat16,
).unsqueeze(1)
torch.testing.assert_close(output, num_distinct.expand_as(output))
torch.distributed.barrier()
manager.cleanup()
@requires_multi_gpu
@requires_one_sided
@requires_ptrace
@pytest.mark.parametrize("world_size", [2])
def test_one_sided_combine_low_precision(world_size, monkeypatch):
"""Exercise the real fp8 combine transport, not a fake kernel.
A workspace/dtype mismatch corrupts the combine buffer silently rather
than raising, so this asserts the combined values are exactly right.
"""
from flashinfer.comm.trtllm_moe_alltoall import MoeAlltoAll
from vllm.utils.func_utils import supports_kw
if not supports_kw(
MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False
):
pytest.skip("installed FlashInfer has no use_low_precision support")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "1")
_spawn_workers(_one_sided_low_precision_worker, world_size, dp_size=world_size)
# ---------------------------------------------------------------------------
# Test 6: DeepEP v2 (ElasticBuffer) manager lifecycle
# ---------------------------------------------------------------------------
+87
View File
@@ -291,6 +291,93 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
reader.touch("key", address, monotonic_id, signature)
class TestShmObjectStorageReferenceCounting(unittest.TestCase):
"""One writer and four readers, as with tensor-parallel workers."""
def setUp(self):
ring_buffer = SingleWriterShmRingBuffer(
data_buffer_size=1024 * 100, create=True
)
self.writer = SingleWriterShmObjectStorage(
max_object_size=1024 * 10,
n_readers=4,
ring_buffer=ring_buffer,
serde_class=MsgpackSerde,
)
self.addCleanup(self.writer.close)
handle = self.writer.handle()
handle.reader_lock = Lock()
self.readers = [
SingleWriterShmObjectStorage.create_from_handle(handle) for _ in range(4)
]
for reader in self.readers:
self.addCleanup(reader.close)
def _reference(self, key: str) -> tuple[int, int, list[int], str]:
"""Take one writer reference to the key, as one request item does."""
if self.writer.is_cached(key):
address, monotonic_id = self.writer.get_cached(key)
else:
address, monotonic_id = self.writer.put(key, "value")
return address, monotonic_id, self.writer.get_signature(key), key
@staticmethod
def _touch(reader: SingleWriterShmObjectStorage, reference) -> None:
address, monotonic_id, signature, key = reference
reader.touch(key, address, monotonic_id, signature)
def _is_evictable(self, key: str) -> bool:
self.writer.free_unused()
return not self.writer.is_cached(key)
def test_item_referenced_again_before_every_reader_fetched_it(self):
"""A second request must not leave the item unevictable forever."""
first = self._reference("image")
self.writer.touch("image")
second = self._reference("image")
self.writer.release_touches()
# Every reader starts on the first request; the ones that finish it
# go on to the second while the others are still fetching.
for reader in self.readers:
self._touch(reader, first)
for reader in self.readers:
reader.get(*first)
self._touch(reader, second)
reader.get(*second)
self.assertTrue(self._is_evictable("image"))
def test_new_item_twice_in_one_request_is_not_evicted_early(self):
"""The item must stay until the last reader has fetched it."""
self.writer.touch("image")
references = [self._reference("image"), self._reference("image")]
self.writer.release_touches()
for reader in self.readers[:-1]:
for reference in references:
self._touch(reader, reference)
for reference in references:
reader.get(*reference)
for reference in references:
self._touch(self.readers[-1], reference)
self.assertFalse(self._is_evictable("image"))
for reference in references:
self.readers[-1].get(*reference)
self.assertTrue(self._is_evictable("image"))
def test_touch_protects_item_until_released(self):
reference = self._reference("image")
for reader in self.readers:
reader.get(*reference)
self.writer.touch("image")
self.assertFalse(self._is_evictable("image"))
self.writer.release_touches()
self.assertTrue(self._is_evictable("image"))
# Reader process function
def reader_process(process_id, storage_handle, items_to_read):
"""Reader process that connects to existing shared memory and reads data."""
+11 -9
View File
@@ -7,12 +7,12 @@ Integration tests for NCCL and IPC weight transfer between processes using Ray.
"""
import builtins
import importlib.util
import pickle
import runpy
import threading
import time
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -1497,10 +1497,12 @@ class TestDeferredProcessingContract:
that caller ends up reaching through a getattr."""
def _engines(self):
return {
name: loader()
for name, loader in WeightTransferEngineFactory._registry.items()
}
registry = dict(WeightTransferEngineFactory._registry)
# ModelExpress is an optional, separately installed package, but its
# backend is always registered. Skip it when the package is missing.
if importlib.util.find_spec("modelexpress") is None:
registry.pop("modelexpress")
return {name: loader() for name, loader in registry.items()}
def test_every_engine_declares_whether_it_defers(self):
for name, cls in self._engines().items():
@@ -2054,11 +2056,11 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
def _import_modelexpress_shim():
path = (
Path(__file__).parents[2]
/ "vllm/distributed/weight_transfer/modelexpress_engine.py"
spec = importlib.util.find_spec(
"vllm.distributed.weight_transfer.modelexpress_engine"
)
runpy.run_path(str(path), run_name="_test_modelexpress_shim")
assert spec is not None and spec.origin is not None
runpy.run_path(spec.origin, run_name="_test_modelexpress_shim")
@pytest.mark.parametrize(
@@ -237,6 +237,46 @@ def test_snapshot_environment_contract(
)
@pytest.mark.parametrize(
("configured", "expected"),
[(None, "1"), ("0", "0"), ("1", "1")],
)
def test_snapshot_child_defaults_nccl_ib_without_mutating_parent(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
configured: str | None,
expected: str,
):
if configured is None:
monkeypatch.delenv("NCCL_IB_DISABLE", raising=False)
else:
monkeypatch.setenv("NCCL_IB_DISABLE", configured)
parent_environment = os.environ.copy()
process = create_autospec(subprocess.Popen, instance=True)
process.pid = 100
popen = create_autospec(subprocess.Popen, return_value=process)
monkeypatch.setattr(snapshot_runtime.subprocess, "Popen", popen)
LocalSnapshotTools().launch_child(tmp_path, ("model",))
assert popen.call_args.kwargs["env"]["NCCL_IB_DISABLE"] == expected
assert os.environ == parent_environment
def test_snapshot_environment_identity_applies_effective_nccl_ib_default(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.delenv("NCCL_IB_DISABLE", raising=False)
implicit = LocalSnapshotTools()._environment_identity()
monkeypatch.setenv("NCCL_IB_DISABLE", "1")
explicit_disabled = LocalSnapshotTools()._environment_identity()
monkeypatch.setenv("NCCL_IB_DISABLE", "0")
explicit_enabled = LocalSnapshotTools()._environment_identity()
assert implicit == explicit_disabled
assert explicit_enabled != implicit
@pytest.mark.parametrize(
"case",
[
@@ -369,6 +409,33 @@ ss() { :; }
assert result.returncode == (0 if probe is None else 1), result.stderr
@pytest.mark.parametrize("container_attempted", [False, True])
def test_snapshot_e2e_cleanup_checks_state_only_after_container_attempt(
container_attempted: bool,
):
script = (
Path(__file__).resolve().parents[3]
/ ".buildkite/scripts/initialized-snapshot-e2e.sh"
)
if not script.exists():
pytest.skip("snapshot E2E script is not packaged in this test image")
source = script.read_text()
cleanup = source[source.index("cleanup() {") : source.index("on_exit() {")]
shell = (
"set -Eeuo pipefail\n"
"CONTAINER_NAME='' PORT_TWO=18002\n"
f"CHECK_CLEANUP_STATE={int(container_attempted)}\n"
"wait_clean() { echo checked-state; return 1; }\n"
+ cleanup
+ "\nif cleanup; then exit 0; else exit 1; fi\n"
)
result = subprocess.run(["bash", "-c", shell], capture_output=True, timeout=10)
assert result.returncode == int(container_attempted), result.stderr
assert (b"checked-state" in result.stdout) == container_attempted
def test_snapshot_create_cli_accepts_only_pinned_compact_mode(
monkeypatch: pytest.MonkeyPatch,
):
@@ -652,6 +719,111 @@ def test_snapshot_create_rolls_back_failed_dump(tmp_path: Path):
assert not target.exists()
@pytest.mark.parametrize(
"failure_kind", ["called-process", "timeout", "missing", "unreadable"]
)
def test_snapshot_create_preserves_bounded_criu_diagnostics_after_cleanup(
tmp_path: Path, failure_kind: str, caplog: pytest.LogCaptureFixture
):
target = tmp_path / "snapshot"
tools = _fake_snapshot_tools()
criu_tools = LocalSnapshotTools()
criu_tools.plugin_dir = tmp_path
original_run = criu_tools._run
criu_tools._privileged = lambda: [] # type: ignore[method-assign]
captured = "captured stderr " + "x" * 9000
if failure_kind == "timeout":
primary: subprocess.SubprocessError = subprocess.TimeoutExpired(
["criu", "dump"], 1, output=b"captured stdout\xff", stderr=captured.encode()
)
else:
primary = subprocess.CalledProcessError(
1, ["criu", "dump"], output="captured stdout", stderr=captured
)
def run(command: list[str], **_kwargs: object):
if "tail" not in command:
raise primary
if failure_kind == "unreadable":
raise PermissionError("log is root-owned")
if failure_kind == "timeout":
return subprocess.CompletedProcess(command, 0, "timeout log tail", "")
return original_run(command, **_kwargs)
criu_tools._run = run # type: ignore[method-assign]
def fail_dump(workdir: Path, _inventory: ProcessInventory) -> None:
images = workdir / "images"
images.mkdir()
if failure_kind != "missing":
(images / "dump.log").write_text(
"omitted dump prefix" + "d" * 9000 + "distinctive dump diagnostic"
)
criu_tools._criu("dump", workdir, [])
tools.dump.side_effect = fail_dump
with pytest.raises(type(primary)) as excinfo:
create_snapshot(
argparse.Namespace(snapshot_dir=str(target), model_tag="Qwen/Qwen3-0.6B"),
tools=tools,
)
assert excinfo.value is primary
assert not target.exists()
diagnostics = "\n".join(caplog.messages)
assert "captured stdout" in diagnostics
assert "captured stderr" not in diagnostics
assert "x" * 100 in diagnostics
assert len(diagnostics) < 17000
if failure_kind in {"missing", "unreadable"}:
assert "dump.log unavailable" in diagnostics
expected_error = (
"CalledProcessError" if failure_kind == "missing" else "PermissionError"
)
assert expected_error in diagnostics
elif failure_kind == "called-process":
assert "distinctive dump diagnostic" in diagnostics
assert "omitted dump prefix" not in diagnostics
else:
assert "timeout log tail" in diagnostics
if failure_kind == "timeout":
assert "captured stdout�" in diagnostics
def test_snapshot_criu_failure_selects_restore_log(
tmp_path: Path, caplog: pytest.LogCaptureFixture
):
tools = LocalSnapshotTools()
tools.plugin_dir = tmp_path
tools._privileged = lambda: [] # type: ignore[method-assign]
artifact = tmp_path / "snapshot"
images = artifact / "images"
images.mkdir(parents=True)
(images / "restore.log").write_text(
"omitted restore prefix" + "r" * 9000 + "distinctive restore diagnostic"
)
primary = subprocess.CalledProcessError(1, ["criu", "restore"])
original_run = tools._run
tailed: list[str] = []
def run(command: list[str], **_kwargs: object):
if "tail" not in command:
raise primary
tailed.append(command[-1])
return original_run(command, **_kwargs)
tools._run = run # type: ignore[method-assign]
with pytest.raises(subprocess.CalledProcessError) as excinfo:
tools._criu("restore", artifact, [])
assert excinfo.value is primary
assert tailed == [str(images / "restore.log")]
diagnostics = "\n".join(caplog.messages)
assert "distinctive restore diagnostic" in diagnostics
assert "omitted restore prefix" not in diagnostics
def _restore_fixture(tmp_path: Path) -> argparse.Namespace:
artifact = tmp_path / "snapshot"
artifact.mkdir(mode=0o700)
@@ -868,6 +1040,96 @@ def test_snapshot_private_path_and_link_remap_security(
tools.restore(artifact, _manifest())
@pytest.mark.parametrize("inventory_error", [None, PermissionError("denied")])
def test_snapshot_dump_rejects_unavailable_link_remap_inventory_before_criu(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
inventory_error: PermissionError | None,
):
tools = LocalSnapshotTools()
workdir = tmp_path / "snapshot"
workdir.mkdir()
tools.shm_dir = tmp_path / "missing-shm"
if inventory_error is not None:
tools.shm_dir.mkdir()
def fail_inventory(_path: Path):
raise inventory_error
monkeypatch.setattr(Path, "iterdir", fail_inventory)
monkeypatch.setattr(tools, "_criu", lambda *_args: pytest.fail("CRIU called"))
with pytest.raises(SnapshotCreateError, match="inventory CRIU link remaps"):
tools.dump(workdir, ProcessInventory(100, (100,), (100,), "GPU"))
def test_snapshot_restore_rejects_saved_remap_without_prefix(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
_mark_snapshot_pids_free(monkeypatch)
artifact = _local_restore_artifact(tmp_path)
saved = artifact / "link-remaps"
saved.mkdir()
(saved / "270").write_bytes(b"invalid remap name")
tools = LocalSnapshotTools()
tools.shm_dir = tmp_path / "shm"
tools.shm_dir.mkdir()
monkeypatch.setattr(tools, "_criu", lambda *_args: pytest.fail("CRIU called"))
with pytest.raises(SnapshotRestoreError, match="invalid saved link remap"):
tools.restore(artifact, _manifest())
assert not tuple(tools.shm_dir.iterdir())
def test_snapshot_link_remap_inventory_matches_only_numeric_suffixes(tmp_path: Path):
for name in (
"link_remap.0",
"link_remap.270",
"link_remap.",
"link_remap.12a",
"link-remap.12",
"prefix.link_remap.12",
"270",
):
(tmp_path / name).touch()
tools = LocalSnapshotTools()
tools.shm_dir = tmp_path
assert tools._link_remap_names() == {"link_remap.0", "link_remap.270"}
def test_snapshot_dump_preserves_primary_failure_when_cleanup_inventory_fails(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
tools = LocalSnapshotTools()
workdir = tmp_path / "snapshot"
workdir.mkdir()
inventory_calls = 0
def inventory_names() -> set[str]:
nonlocal inventory_calls
inventory_calls += 1
if inventory_calls > 1:
raise SnapshotCreateError(
"could not inventory CRIU link remaps: PermissionError: denied"
)
return set()
monkeypatch.setattr(tools, "_link_remap_names", inventory_names)
def fail_dump(*_args):
raise RuntimeError("dump failed")
monkeypatch.setattr(tools, "_criu", fail_dump)
with pytest.raises(SnapshotCreateError) as failure:
tools.dump(workdir, ProcessInventory(100, (100,), (100,), "GPU"))
assert "RuntimeError: dump failed" in str(failure.value)
assert "cleanup inventory failed" in str(failure.value)
@pytest.mark.parametrize(
("blocked_state", "message"),
[("tcp", "external established TCP"), ("io_uring", "kernel.io_uring_disabled=1")],
@@ -900,6 +1162,28 @@ def test_snapshot_rejects_unsafe_process_state_before_criu(
assert "1.1.1.1:443" in str(excinfo.value)
def test_snapshot_rejects_open_infiniband_device_before_criu(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
tools = LocalSnapshotTools()
monkeypatch.setattr(tools, "_tree_pids", lambda _root_pid: (100, 101))
monkeypatch.setattr(tools, "_cuda_process_rows", lambda: ("101, GPU-abc",))
monkeypatch.setattr(
tools,
"_descriptor_targets",
lambda pid: ("/dev/infiniband/uverbs7",) if pid == 101 else (),
)
monkeypatch.setattr(tools, "_criu", lambda *_args: pytest.fail("CRIU called"))
with pytest.raises(SnapshotCreateError) as failure:
tools.inventory(100, tmp_path)
message = str(failure.value)
assert "pid 101" in message
assert "/dev/infiniband/uverbs7" in message
assert "NCCL_IB_DISABLE=1" in message
def test_decode_endpoint_families():
assert _decode_endpoint("AF_INET", "3C00000A:D6F6") == "10.0.0.60:55030"
assert (
@@ -40,13 +40,11 @@ def server():
"--trust-request-mm-kwargs",
]
# ROCm: Increase timeouts to handle potential network delays and slower
# video processing when downloading multiple videos from external sources
# ROCm: Allow more time to download videos from external sources.
env_overrides = {}
if current_platform.is_rocm():
env_overrides = {
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "300",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
@@ -88,13 +88,11 @@ def server():
*ROCM_EXTRA_ARGS,
]
# ROCm: Increase timeouts to handle potential network delays and slower
# video processing when downloading multiple videos from external sources
# ROCm: Allow more time to download videos from external sources.
env_overrides = {
**(
{
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "300",
}
if current_platform.is_rocm()
else {}
@@ -1,20 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ScoringIOProcessor post-tokenization helpers."""
"""Unit tests for ScoringIOProcessor helpers."""
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
import torch
from tokenizers import Tokenizer, models, pre_tokenizers, processors
from transformers import TokenizersBackend
from vllm import TokensPrompt
from vllm.entrypoints.pooling.scoring.io_processor import (
BiEncoderIOProcessor,
CrossEncoderIOProcessor,
JinaRankingIOProcessor,
LateInteractionIOProcessor,
_apply_post_tokenization_to_token_type_ids,
)
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
from vllm.renderers import TokenizeParams
pytestmark = pytest.mark.skip_global_cleanup
@@ -113,3 +118,59 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
assert first_doc == 10
assert prompt_token_ids[:first_doc] == list(range(10, num_query))
assert prompt_token_ids[first_doc:40] == list(range(num_query, 50))
def _pooling_output(
request_id: str,
data: torch.Tensor,
error: RequestError | None = None,
) -> PoolingRequestOutput:
return PoolingRequestOutput(
request_id=request_id,
outputs=PoolingOutput(data),
prompt_token_ids=[1],
num_cached_tokens=0,
finished=True,
error=error,
)
@pytest.mark.parametrize(
"processor_type", [BiEncoderIOProcessor, LateInteractionIOProcessor]
)
@pytest.mark.parametrize("failed_index", [0, 1])
def test_paired_scoring_preserves_request_error(processor_type, failed_index):
processor = processor_type.__new__(processor_type)
processor.pad_token_id = None
error = RequestError(
code="multimodal_cache_miss",
message="Multi-modal processor cache miss.",
retryable=True,
)
outputs = [
_pooling_output("query", torch.tensor([1.0, 2.0])),
_pooling_output("document", torch.tensor([3.0, 4.0])),
]
outputs[failed_index] = _pooling_output(
outputs[failed_index].request_id,
torch.empty(0),
error,
)
(result,) = processor._post_process(outputs, n_queries=1)
assert result.request_id == "query_document"
assert result.error is error
assert result.outputs is outputs[failed_index].outputs
def test_jina_scoring_preserves_request_error_before_indexing_output():
processor = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
error = RequestError(
code="multimodal_cache_miss",
message="Multi-modal processor cache miss.",
retryable=True,
)
failed = _pooling_output("request", torch.empty(0), error)
assert processor._post_process([failed], n_queries=1) == [failed]
@@ -49,10 +49,60 @@ def _query_key(context: PoolingServeContext) -> str:
@pytest.mark.asyncio
async def test_colliding_request_ids_use_distinct_query_cache_keys(
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("failed_stage", ["query", "doc"])
async def test_flash_late_interaction_cleans_up_queries_on_failure(
failed_stage: str,
monkeypatch: pytest.MonkeyPatch,
):
serving = object.__new__(ServingScores)
serving.io_processor = Mock()
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
context = _make_context()
monkeypatch.setattr(serving, "_init_ctx", AsyncMock(return_value=context))
monkeypatch.setattr(serving, "_preprocessing", AsyncMock())
async def encode_queries(ctx):
ctx.late_interaction_query_keys = ["query-key"]
if failed_stage == "query":
raise RuntimeError("query failed")
async def encode_docs(ctx):
ctx.late_interaction_doc_keys = ["doc-key"]
if failed_stage == "doc":
raise RuntimeError("doc failed")
monkeypatch.setattr(
serving,
"_flash_late_interaction_encode_queries",
AsyncMock(side_effect=encode_queries),
)
monkeypatch.setattr(
serving,
"_flash_late_interaction_encode_docs",
AsyncMock(side_effect=encode_docs),
)
with pytest.raises(RuntimeError, match=f"{failed_stage} failed"):
await serving.flash_late_interaction()
expected_doc_keys = [] if failed_stage == "query" else ["doc-key"]
serving.engine_client.abort.assert_awaited_once_with(
["query-key", *expected_doc_keys]
)
serving.engine_client.collective_rpc.assert_awaited_once_with(
"release_late_interaction_query_cache",
args=(["query-key"],),
)
@pytest.mark.asyncio
async def test_colliding_request_ids_use_distinct_late_interaction_keys(
monkeypatch: pytest.MonkeyPatch,
):
serving = object.__new__(ServingScores)
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
prepare_generators = AsyncMock()
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
@@ -68,9 +118,24 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys(
first_doc_key = _query_key(prepared_contexts[1])
second_query_key = _query_key(prepared_contexts[2])
second_doc_key = _query_key(prepared_contexts[3])
first_doc_request_ids = prepared_contexts[1].prompt_request_ids
second_doc_request_ids = prepared_contexts[3].prompt_request_ids
assert first_query_key == first_doc_key
assert second_query_key == second_doc_key
assert first_query_key != second_query_key
assert "caller-controlled-id" not in first_query_key
assert "caller-controlled-id" not in second_query_key
assert first_doc_request_ids is not None
assert second_doc_request_ids is not None
assert first.late_interaction_doc_keys == first_doc_request_ids
assert second.late_interaction_doc_keys == second_doc_request_ids
assert first_doc_request_ids != second_doc_request_ids
assert all("caller-controlled-id" not in key for key in first_doc_request_ids)
assert all("caller-controlled-id" not in key for key in second_doc_request_ids)
assert first.late_interaction_query_keys is not None
await serving._cleanup_flash_late_interaction(first)
serving.engine_client.abort.assert_awaited_once_with(
[*first.late_interaction_query_keys, *first_doc_request_ids]
)
@@ -22,6 +22,10 @@ The streaming cases do the same for `stream=true`. Each coupled chunk's
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
through the chunked derender path with `stream_state` threaded across calls
the way a client would, so both parsers see the same chunk boundaries.
The text level cases pin `/inference/v1/generate` with `output_mode="text"`
against `/v1/completions/derender` on the same token IDs, batch and streaming,
plus logprob resolution and stop string handling.
"""
import json
@@ -440,3 +444,241 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
if not (coupled["reasoning"] and coupled["tool_calls"]):
pytest.skip("Model did not emit both a <think> block and a tool call")
# ---------------------------------------------------------------------------
# Inline text level: generate(output_mode="text") == derender(generate(...))
# ---------------------------------------------------------------------------
TEXT_MESSAGES = [
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
]
async def _render_token_ids(
client: httpx.AsyncClient, messages: list[dict]
) -> list[int]:
resp = await client.post(
"/v1/chat/completions/render", json={"model": MODEL, "messages": messages}
)
assert resp.status_code == 200, resp.text
return resp.json()["token_ids"]
def _generate_payload(
token_ids: list[int], output_mode: str, stream: bool = False, **sampling
) -> dict:
return {
"model": MODEL,
"token_ids": token_ids,
"sampling_params": {"temperature": 0, "max_tokens": 128, **sampling},
"output_mode": output_mode,
"stream": stream,
}
async def _generate(
client: httpx.AsyncClient, token_ids: list[int], output_mode: str, **sampling
) -> dict:
resp = await client.post(
"/inference/v1/generate",
json=_generate_payload(token_ids, output_mode, **sampling),
)
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["output_mode"] == output_mode
return data
async def _generate_text_stream(
client: httpx.AsyncClient, token_ids: list[int], **sampling
) -> list[dict]:
"""Stream generate at the text level and return the choice of every chunk
that has one."""
choices: list[dict] = []
async with client.stream(
"POST",
"/inference/v1/generate",
json=_generate_payload(token_ids, "text", stream=True, **sampling),
) as resp:
assert resp.status_code == 200, await resp.aread()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
assert "error" not in chunk, chunk
assert chunk["output_mode"] == "text"
choices.extend(chunk["choices"])
return choices
async def _derender_completion_text(
client: httpx.AsyncClient, output_ids: list[int], finish_reason: str
) -> str:
resp = await client.post(
"/v1/completions/derender",
json={
"model": MODEL,
"generate_responses": [
{
"request_id": "text-parity",
"choices": [
{
"index": 0,
"token_ids": output_ids,
"finish_reason": finish_reason,
}
],
}
],
},
)
assert resp.status_code == 200, resp.text
return resp.json()["choices"][0]["text"]
async def _derender_completion_stream_text(
client: httpx.AsyncClient, choices: list[dict]
) -> str:
"""Replay each generate chunk's token IDs through the streaming
completions derender endpoint with the same chunk boundaries."""
state = None
text = ""
for choice in choices:
resp = await client.post(
"/v1/completions/derender",
json={
"stream": True,
"model": MODEL,
"generate_chunk": {
"request_id": "text-parity-stream",
"choices": [
{
"index": 0,
"token_ids": choice.get("token_ids") or [],
"finish_reason": choice.get("finish_reason"),
}
],
},
"stream_state": state,
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
state = data["stream_state"]
text += "".join(ch["text"] for ch in data["chunk"]["choices"])
return text
@pytest.mark.asyncio
async def test_text_parity_batch(client):
"""Inline text matches /v1/completions/derender on the same token IDs."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
inline = (await _generate(client, token_ids, "text"))["choices"][0]
derendered = await _derender_completion_text(
client, inline["token_ids"], inline["finish_reason"]
)
assert inline["text"]
assert inline["text"] == derendered
@pytest.mark.asyncio
async def test_text_parity_stream(client):
"""Streamed inline text matches both the streaming and the non-streaming
derender of the same token IDs."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
choices = await _generate_text_stream(client, token_ids)
inline = "".join(ch["text"] for ch in choices)
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
finish_reason = choices[-1]["finish_reason"]
assert finish_reason is not None
assert inline == await _derender_completion_stream_text(client, choices)
assert inline == await _derender_completion_text(client, output_ids, finish_reason)
@pytest.mark.asyncio
async def test_text_parity_logprobs(client):
"""Inline logprob tokens and bytes match /derender resolving the
token_id:N placeholders of the tokens level."""
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
sampling = {"logprobs": 3}
tokens_choice = (await _generate(client, token_ids, "tokens", **sampling))[
"choices"
][0]
text_choice = (await _generate(client, token_ids, "text", **sampling))["choices"][0]
assert text_choice["token_ids"] == tokens_choice["token_ids"], (
"greedy (temperature=0) generation was expected to be deterministic "
"across the two generate calls"
)
disagg = await _disagg(
client,
tokens_choice["token_ids"],
len(token_ids),
tokens_choice["finish_reason"],
{"model": MODEL, "messages": TEXT_MESSAGES},
logprobs=tokens_choice["logprobs"],
)
inline_content = text_choice["logprobs"]["content"]
derender_content = disagg["choices"][0]["logprobs"]["content"]
assert len(inline_content) == len(derender_content)
for inline_entry, derender_entry in zip(inline_content, derender_content):
assert inline_entry["token"] == derender_entry["token"]
assert inline_entry["bytes"] == derender_entry["bytes"]
assert [
(top["token"], top["bytes"]) for top in inline_entry["top_logprobs"]
] == [(top["token"], top["bytes"]) for top in derender_entry["top_logprobs"]]
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_text_parity_stop_string(client, stream):
"""Inline text follows the engine's stop handling, like /v1/completions.
token_ids keep every generated token, so /derender brings the matched
stop string back. Its text must start with the inline text and only
differ after it.
"""
stop = "\n"
token_ids = await _render_token_ids(
client,
[{"role": "user", "content": "Write two short lines about the sea."}],
)
resp = await client.post(
"/v1/completions",
json={
"model": MODEL,
"prompt": token_ids,
"temperature": 0,
"max_tokens": 128,
"stop": [stop],
},
)
assert resp.status_code == 200, resp.text
coupled = resp.json()["choices"][0]
if stream:
choices = await _generate_text_stream(client, token_ids, stop=[stop])
inline_text = "".join(ch["text"] for ch in choices)
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
finish_reason = choices[-1]["finish_reason"]
else:
inline = (await _generate(client, token_ids, "text", stop=[stop]))["choices"][0]
inline_text = inline["text"]
output_ids = inline["token_ids"]
finish_reason = inline["finish_reason"]
assert inline_text == coupled["text"]
assert finish_reason == coupled["finish_reason"]
assert stop not in inline_text
derendered = await _derender_completion_text(client, output_ids, finish_reason)
assert derendered.startswith(inline_text)
if coupled["stop_reason"] != stop:
pytest.skip("Model did not emit the stop string")
assert derendered[len(inline_text) :].startswith(stop)
@@ -50,10 +50,11 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
DerenderStreamState,
GenerateResponse,
GenerateResponseChoice,
GenerateResponseStreamChoice,
GenerateStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamChoice,
GenerateTokensStreamResponse,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.parser import Parser
@@ -195,10 +196,10 @@ def _make_stream_chunk(
prompt_token_ids: list[int] | None = None,
) -> GenerateStreamResponse:
"""Build a GenerateStreamResponse SSE chunk."""
return GenerateStreamResponse(
return GenerateTokensStreamResponse(
request_id=request_id,
choices=[
GenerateResponseStreamChoice(
GenerateTokensStreamChoice(
index=index,
token_ids=token_ids,
finish_reason=finish_reason,
@@ -233,7 +234,7 @@ def _make_usage_chunk(
request_id: str = "test-req",
) -> GenerateStreamResponse:
"""Build a usage only final SSE chunk (empty choices)."""
return GenerateStreamResponse(
return GenerateTokensStreamResponse(
request_id=request_id,
choices=[],
usage=UsageInfo(
@@ -326,11 +327,11 @@ def test_non_streaming_derender_initializes_parser_prefix(
)
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
response = GenerateResponse(
response = GenerateTokensResponse(
request_id="test",
prompt_token_ids=[11, 12],
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0,
token_ids=generated_ids,
finish_reason="length",
@@ -583,10 +584,10 @@ class TestPromptSeededLeadingSpace:
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -601,10 +602,10 @@ class TestPromptSeededLeadingSpace:
):
prompt_ids, output_ids, expected = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -620,10 +621,10 @@ class TestPromptSeededLeadingSpace:
prompt_ids, output_ids, expected = leading_space_ids
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -639,10 +640,10 @@ class TestPromptSeededLeadingSpace:
):
_, output_ids, _ = leading_space_ids
choices = await derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -655,10 +656,10 @@ class TestPromptSeededLeadingSpace:
prompt_ids, output_ids, expected = leading_space_ids
choices, _, _ = await derenderer.derender_completion(
[
GenerateResponse(
GenerateTokensResponse(
request_id="t",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -1494,10 +1495,10 @@ class TestDerenderChatStreamHarmony:
content = "".join(d.content or "" for d in deltas)
batch_choices = await harmony_derenderer.derender_chat(
GenerateResponse(
GenerateTokensResponse(
request_id="test-harmony-batch",
choices=[
GenerateResponseChoice(
GenerateTokensChoice(
index=0, token_ids=output_ids, finish_reason="stop"
)
],
@@ -1743,10 +1744,10 @@ class TestServingDerenderStreamValidation:
could never fire since derender_chat_stream itself rejects anything
above 1 first."""
serving = self._make_serving(parser_configured=False)
two_choices = GenerateStreamResponse(
two_choices = GenerateTokensStreamResponse(
request_id="t",
choices=[
GenerateResponseStreamChoice(index=i, token_ids=[1]) for i in range(2)
GenerateTokensStreamChoice(index=i, token_ids=[1]) for i in range(2)
],
)
request = DerenderChatStreamRequest(
+33 -2
View File
@@ -6,6 +6,8 @@ from argparse import Namespace
from fastapi import FastAPI
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX
from vllm.tasks import SupportedTask
RENDER_PATHS = {
@@ -17,7 +19,9 @@ RENDER_PATHS = {
"/v1/responses/render",
}
GENERATE_PATH = "/inference/v1/generate"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
ABORT_PATH = "/inference/v1/abort_requests"
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH, ABORT_PATH}
def registered_paths(
@@ -66,4 +70,31 @@ def test_render_routes_remain_enabled_for_render_server():
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, "/abort_requests"}
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
def test_abort_route_without_tokens_only_requires_api_key():
"""A decode pool that loads a tokenizer serves generate without
--tokens-only and still needs to abort, but only behind --api-key."""
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
assert ABORT_PATH in paths
assert ABORT_PATH.startswith(GUARDED_PREFIX)
assert UNAUTHENTICATED_ABORT_PATH not in paths
def test_unauthenticated_abort_route_is_not_registered_twice():
"""The RLHF dev router registers /abort_requests before the scale-out
routers, so attaching the scale-out one too would add a shadowed duplicate."""
app = FastAPI()
app.state.args = Namespace(tokens_only=True)
@app.post(UNAUTHENTICATED_ABORT_PATH)
async def existing_abort_requests():
return None
attach_router(app)
paths = [route.path for route in app.routes]
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
assert ABORT_PATH in paths
@@ -20,7 +20,8 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
GenerateResponse,
GenerateTextResponse,
GenerateTokensResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
@@ -144,6 +145,7 @@ def _make_request_output(
num_cached_tokens: int | None = None,
index: int = 0,
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
text: str = "",
) -> RequestOutput:
return RequestOutput(
request_id=request_id,
@@ -153,7 +155,7 @@ def _make_request_output(
outputs=[
CompletionOutput(
index=index,
text="",
text=text,
token_ids=token_ids,
cumulative_logprob=None,
logprobs=logprobs,
@@ -239,7 +241,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert (
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
@@ -411,7 +413,7 @@ async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
if isinstance(chunk, dict) and chunk.get("metrics")
)["metrics"]["speculative_decoding"]
else:
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.metrics is not None
assert response.metrics.speculative_decoding is not None
payload = response.metrics.speculative_decoding.model_dump()
@@ -511,7 +513,7 @@ async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
if isinstance(chunk, dict)
)
else:
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.metrics is None
@@ -540,7 +542,7 @@ async def test_non_stream_returns_final_prompt_token_ids_when_requested():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.prompt_token_ids == final_prompt_token_ids
@@ -622,7 +624,7 @@ async def test_non_stream_prompt_metadata_requires_return_token_ids(
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
if return_token_ids:
assert response.prompt_token_ids == [1, 2, 3]
assert response.model_dump()["mm_placeholders"] == MM_PLACEHOLDERS
@@ -652,7 +654,7 @@ async def test_non_stream_text_only_prompt_has_no_mm_placeholders():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.prompt_token_ids == [1, 2, 3]
assert response.mm_placeholders is None
@@ -678,7 +680,7 @@ async def test_non_stream_encoder_decoder_omits_mm_placeholders():
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert isinstance(response, GenerateTokensResponse)
assert response.prompt_token_ids == [1, 2, 3]
assert response.mm_placeholders is None
@@ -1195,3 +1197,251 @@ async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
)
assert choice["sampling_mask"] == expected
async def _collect_sse(response) -> list[Any]:
return _parse_sse_chunks([chunk async for chunk in response])
def _text_request(**kwargs) -> GenerateRequest:
kwargs.setdefault("sampling_params", SamplingParams(max_tokens=10))
return GenerateRequest(
token_ids=[1, 2, 3], model=MODEL_NAME, output_mode="text", **kwargs
)
@pytest.mark.asyncio
async def test_non_stream_tokens_mode_omits_text_and_echoes_output_mode():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], text="ignored", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
)
)
assert isinstance(response, GenerateTokensResponse)
dumped = response.model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
@pytest.mark.asyncio
async def test_non_stream_text_mode_returns_text_and_token_ids():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10, 20],
text="Hello",
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request())
assert isinstance(response, GenerateTextResponse)
assert response.output_mode == "text"
assert response.choices[0].text == "Hello"
assert response.choices[0].token_ids == [10, 20]
assert response.choices[0].finish_reason == "stop"
assert response.usage is not None
assert response.usage.completion_tokens == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("no_tokenizer", ["tokens_only", "skip_tokenizer_init"])
async def test_text_mode_rejected_without_tokenizer(no_tokenizer: str):
"""The no-op detokenizer returns "", so text mode must fail instead
of answering 200 with empty text."""
engine = _mock_engine()
kwargs: dict[str, Any] = {}
if no_tokenizer == "tokens_only":
kwargs["force_no_detokenize"] = True
else:
engine.model_config.skip_tokenizer_init = True
engine.renderer = _build_renderer(engine.model_config)
engine.generate = MagicMock()
serving = _build_serving_tokens(engine, **kwargs)
response = await serving.serve_tokens(_text_request())
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert "requires a tokenizer" in response.error.message
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_text_mode_logprobs_have_decoded_tokens_and_bytes():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
text="é",
logprobs=[
{
10: Logprob(logprob=-0.5, decoded_token="é"),
11: Logprob(logprob=-1.5, decoded_token="e"),
}
],
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=2))
)
assert isinstance(response, GenerateTextResponse)
logprobs = response.choices[0].logprobs
assert logprobs is not None and logprobs.content is not None
entry = logprobs.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
@pytest.mark.asyncio
async def test_text_mode_logprobs_keep_placeholders_with_return_tokens_as_ids():
"""--return-tokens-as-token-ids keeps token_id:N placeholders, as it does
on /v1/completions, while text is still decoded."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
text="é",
logprobs=[{10: Logprob(logprob=-0.5, decoded_token="é")}],
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine, return_tokens_as_token_ids=True)
response = await serving.serve_tokens(
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=1))
)
assert isinstance(response, GenerateTextResponse)
assert response.choices[0].text == "é"
logprobs = response.choices[0].logprobs
assert logprobs is not None and logprobs.content is not None
entry = logprobs.content[0]
assert (entry.token, entry.bytes) == ("token_id:10", None)
assert [t.token for t in entry.top_logprobs] == ["token_id:10"]
@pytest.mark.asyncio
async def test_stream_text_mode_emits_text_deltas():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], text="Hel")
yield _make_request_output("req-1", token_ids=[20], text="lo")
yield _make_request_output(
"req-1", token_ids=[30], text=" world", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request(stream=True))
parsed = await _collect_sse(response)
assert parsed[-1] == "[DONE]"
data_chunks = parsed[:-1]
assert {c["output_mode"] for c in data_chunks} == {"text"}
assert [c["choices"][0]["text"] for c in data_chunks] == ["Hel", "lo", " world"]
assert [c["choices"][0]["token_ids"] for c in data_chunks] == [[10], [20], [30]]
assert data_chunks[-1]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
async def test_stream_text_mode_emits_chunks_without_new_token_ids():
"""Text flushed from the stop string buffer and the final abort output
carry no token IDs but must still reach the client."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], text="a")
yield _make_request_output("req-1", token_ids=[], text="b")
yield _make_request_output(
"req-1", token_ids=[], text="", finish_reason="abort", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request(stream=True))
data_chunks = (await _collect_sse(response))[:-1]
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b", ""]
assert data_chunks[1]["choices"][0]["token_ids"] == []
assert data_chunks[2]["choices"][0]["finish_reason"] == "abort"
@pytest.mark.asyncio
async def test_stream_text_mode_skips_outputs_without_text_or_finish():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10], text="a")
yield _make_request_output("req-1", token_ids=[], text="")
yield _make_request_output(
"req-1", token_ids=[20], text="b", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(_text_request(stream=True))
data_chunks = (await _collect_sse(response))[:-1]
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b"]
@pytest.mark.asyncio
async def test_stream_text_mode_usage_chunk_echoes_output_mode():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], text="a", finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
_text_request(stream=True, stream_options=StreamOptions(include_usage=True))
)
parsed = await _collect_sse(response)
usage_chunk = parsed[-2]
assert usage_chunk["output_mode"] == "text"
assert usage_chunk["choices"] == []
assert usage_chunk["usage"]["completion_tokens"] == 1
@@ -10,9 +10,21 @@ fail loudly if the validator semantics ever drift.
import json
import pytest
from pydantic import TypeAdapter, ValidationError
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
DerenderChatRequest,
DerenderChatStreamRequest,
DerenderCompletionRequest,
DerenderCompletionStreamRequest,
GenerateRequest,
GenerateResponse,
GenerateStreamResponse,
GenerateTextResponse,
GenerateTextStreamResponse,
GenerateTokensChoice,
GenerateTokensResponse,
GenerateTokensStreamResponse,
MultiModalFeatures,
PlaceholderRangeInfo,
)
@@ -110,3 +122,120 @@ def test_generate_request_rejects_placeholder_outside_prompt():
kwargs_data={"image": ["encoded"]},
),
)
def test_output_mode_defaults_to_tokens():
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
def test_output_mode_text_is_accepted():
payload = {**_base_payload(), "output_mode": "text"}
assert GenerateRequest.model_validate(payload).output_mode == "text"
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
def test_output_mode_rejects_unsupported_levels(output_mode):
"""Levels the server does not implement fail instead of falling back to
tokens which would answer 200 with the wrong response shape."""
payload = {**_base_payload(), "output_mode": output_mode}
with pytest.raises(ValidationError, match="output_mode"):
GenerateRequest.model_validate(payload)
def test_text_mode_rejects_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "text",
}
with pytest.raises(ValidationError, match="detokenize"):
GenerateRequest.model_validate(payload)
def test_tokens_mode_allows_detokenize_false():
payload = {
"token_ids": [1, 2, 3],
"sampling_params": {"detokenize": False},
"output_mode": "tokens",
}
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
def test_response_without_output_mode_parses_as_tokens():
"""Existing clients and older servers never send output_mode."""
parsed = TypeAdapter(GenerateResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensResponse)
assert parsed.output_mode == "tokens"
def test_stream_chunk_without_output_mode_parses_as_tokens():
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
{"choices": [{"index": 0, "token_ids": [1]}]}
)
assert isinstance(parsed, GenerateTokensStreamResponse)
def test_text_response_and_chunk_dispatch_on_output_mode():
response = TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
)
assert isinstance(response, GenerateTextResponse)
assert isinstance(chunk, GenerateTextStreamResponse)
def test_text_response_requires_text_on_every_choice():
with pytest.raises(ValidationError, match="text"):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": "text", "choices": [{"index": 0}]}
)
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
def test_response_rejects_unknown_output_mode(output_mode):
with pytest.raises(ValidationError):
TypeAdapter(GenerateResponse).validate_python(
{"output_mode": output_mode, "choices": []}
)
def test_derender_requests_accept_responses_without_output_mode():
DerenderChatRequest.model_validate(
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
)
DerenderChatStreamRequest.model_validate(
{
"stream": True,
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
}
)
def test_derender_requests_reject_text_responses():
"""Text responses are already detokenized with stop strings applied, so
re-decoding their token_ids would silently undo the truncation."""
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatRequest.model_validate({"generate_response": text})
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
with pytest.raises(ValidationError, match="output_mode"):
DerenderChatStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
with pytest.raises(ValidationError, match="output_mode"):
DerenderCompletionStreamRequest.model_validate(
{"stream": True, "generate_chunk": text}
)
def test_tokens_response_echoes_output_mode_without_text():
dumped = GenerateTokensResponse(
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
).model_dump()
assert dumped["output_mode"] == "tokens"
assert "text" not in dumped["choices"][0]
@@ -19,6 +19,7 @@ from vllm.v1.engine.detokenizer import check_stop_strings
MODEL_NAME = "Qwen/Qwen3-0.6B"
GEN_ENDPOINT = "/inference/v1/generate"
ABORT_ENDPOINT = "/inference/v1/abort_requests"
def get_vocab_size(model_name):
@@ -317,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value):
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
candidates = [22, 33, 44, 55]
rows = [[33], [44, 100], [55]]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": candidates,
"prompt_logprob_token_ids": rows,
"prompt_logprob_start": 1,
}
payload = {
@@ -334,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client):
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (len(token_ids) - 2, len(candidates))
assert scores.shape == (3, 2)
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, candidates.index(target)] == pytest.approx(
expected, abs=1e-3
)
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
assert np.isneginf(scores[[0, 2], 1]).all()
stream_only_ids = {"prompt_logprob_token_ids": candidates}
stream_only_ids = {"prompt_logprob_token_ids": rows}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
@@ -349,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client):
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
)
resp.raise_for_status()
empty = np.load(
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
)
assert empty.shape == (0, len(candidates))
assert resp.status_code == 400
assert "scored rows" in resp.text
@pytest.mark.asyncio
@@ -590,3 +587,187 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
return tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
def _text_mode_payload(token_ids, **sampling_params) -> dict:
return {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
"output_mode": "text",
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
resp.raise_for_status()
data = resp.json()
assert data["output_mode"] == "text"
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": {"enable_thinking": False},
}
chat_resp = await client.post("/v1/chat/completions", json=payload)
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
assert data["choices"][0]["text"] == chat_content
assert data["choices"][0]["token_ids"]
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
token_ids = _chat_prompt_token_ids(tokenizer, messages)
payload = _text_mode_payload(token_ids)
non_stream = await client.post(GEN_ENDPOINT, json=payload)
non_stream.raise_for_status()
chunks = []
async with client.stream(
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json.loads(line[len("data: ") :]))
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
assert chunks[-1]["choices"][0]["finish_reason"] is not None
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
assert streamed_text == non_stream.json()["choices"][0]["text"]
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
"overrides, expected",
[
pytest.param(
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
),
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
],
)
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
payload = {
**_text_mode_payload([1, 2, 3], max_tokens=5),
**overrides,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
assert resp.status_code == 400
assert expected in resp.json()["error"]["message"]
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
async def test_text_mode_rejected_when_tokens_only(client):
"""Without a tokenizer the detokenizer returns "", so the request must
fail instead of answering 200 with empty text."""
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
assert resp.status_code == 400
assert "requires a tokenizer" in resp.json()["error"]["message"]
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
payload["sampling_params"]["detokenize"] = False
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_delivers_abort_finish_chunk(
client, tokenizer, messages
):
"""The final output after an abort has no new token IDs, so it
reaches a text stream only because text mode emits finish-only chunks."""
payload = {
**_text_mode_payload(
_chat_prompt_token_ids(tokenizer, messages),
max_tokens=900,
ignore_eos=True,
),
"stream": True,
"request_id": "text-abort-e2e",
}
chunks = []
aborted = False
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
chunk = json.loads(line[len("data: ") :])
chunks.append(chunk)
if not aborted and chunk["choices"] and chunk["choices"][0]["text"]:
abort = await client.post(
ABORT_ENDPOINT,
json={"request_ids": [chunk["request_id"]]},
)
assert abort.status_code == 200
aborted = True
assert aborted
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
final_choice = chunks[-1]["choices"][0]
assert final_choice["finish_reason"] == "abort"
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
assert generated < 900
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_abort_requests_is_served_without_tokens_only(client):
resp = await client.post(
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 200
resp = await client.post(ABORT_ENDPOINT, json={})
assert resp.status_code == 400
# The unauthenticated path is only served with --tokens-only.
resp = await client.post(
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
)
assert resp.status_code == 404
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
@@ -36,3 +38,50 @@ def test_logprobs_minus_one_emits_all_tokens():
num_output_top_logprobs=-1,
)
assert len(result.content[0].top_logprobs) == 2
def test_tokens_level_logprobs_have_no_bytes():
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=2,
)
entry = result.content[0]
assert entry.bytes is None
assert all(t.bytes is None for t in entry.top_logprobs)
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
tokenizer = MagicMock()
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
],
num_output_top_logprobs=2,
tokenizer=tokenizer,
)
entry = result.content[0]
assert (entry.token, entry.bytes) == ("é", [195, 169])
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
("é", [195, 169]),
("e", [101]),
]
tokenizer.decode.assert_not_called()
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
tokenizer = MagicMock()
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7, 9],
top_logprobs=[{7: Logprob(-0.9)}, None],
num_output_top_logprobs=1,
tokenizer=tokenizer,
)
assert [e.token for e in result.content] == ["<7>", "<9>"]
assert result.content[0].top_logprobs[0].token == "<7>"
assert result.content[1].bytes == list(b"<9>")
@@ -14,12 +14,6 @@ from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.assets.audio import AudioAsset
from vllm.multimodal.media.audio import load_audio
# Increase engine iteration timeout for ROCm where first-use JIT compilation
# can exceed the default 60s, causing a silent deadlock in feed_tokens.
REALTIME_ENV_OVERRIDES = {
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",
}
MISTRAL_FORMAT_ARGS = [
"--tokenizer_mode",
"mistral",
@@ -77,9 +71,7 @@ async def test_multi_chunk_streaming(model_name, mary_had_lamb_audio_chunks):
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
with RemoteOpenAIServer(model_name, server_args) as remote_server:
ws_url = _get_websocket_url(remote_server)
async with websockets.connect(ws_url) as ws:
# Receive session.created
@@ -183,9 +175,7 @@ async def test_empty_commit_does_not_crash_engine(
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
with RemoteOpenAIServer(model_name, server_args) as remote_server:
ws_url = _get_websocket_url(remote_server)
# --- First connection: empty commit (no audio appended) ----------
@@ -272,9 +262,7 @@ async def test_session_update_invalid_model_returns_error(model_name):
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
with RemoteOpenAIServer(model_name, server_args) as remote_server:
ws_url = _get_websocket_url(remote_server)
async with websockets.connect(ws_url) as ws:
event = await receive_event(ws, timeout=30.0)
@@ -301,9 +289,7 @@ async def test_commit_without_session_update_returns_error(model_name):
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
with RemoteOpenAIServer(model_name, server_args) as remote_server:
ws_url = _get_websocket_url(remote_server)
async with websockets.connect(ws_url) as ws:
event = await receive_event(ws, timeout=30.0)
@@ -107,3 +107,37 @@ def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
Production allocates index_q with the index-K cache dtype, so the scorers
normally see a matched pair and keep the native FP8 MMA. This pins the
other branch: a bf16 query against an e4m3 cache still compiles and still
ranks the planted blocks.
"""
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
idx = minimax_m3_index_decode(
q, # bf16 query, deliberately not cast to the cache dtype
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@@ -596,6 +596,114 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
device = torch.device("cuda")
set_random_seed(7)
head_dim = nope_dim + rope_dim
return dict(
q=torch.randn(3, 3, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
kv=torch.randn(5, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
indices=torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device),
indptr=torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device),
scale=head_dim**-0.5,
attn_sink=torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device),
nope_head_dim=nope_dim,
rope_head_dim=rope_dim,
)
# 448+64 is DeepSeek V4/V4.1, 512+64 is V3.2, and 256+0 stands in for the
# NoPE-only layouts where the destination spans the whole head dim.
SPARSE_PREFILL_DIMS = [(NOPE_HEAD_DIM, ROPE_HEAD_DIM), (512, 64), (256, 0)]
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_matches_allocated(
nope_dim: int, rope_dim: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
allocated = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)
q = kwargs["q"]
dest = torch.empty(q.shape[0], q.shape[1], nope_dim, dtype=q.dtype, device=q.device)
returned = _rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
assert returned is dest
torch.testing.assert_close(dest, allocated[..., :nope_dim], atol=0.0, rtol=0.0)
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_view_leaves_neighbors_untouched(
nope_dim: int, rope_dim: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
q = kwargs["q"]
num_queries, num_heads = q.shape[0], q.shape[1]
buffer = torch.full(
(num_queries + 4, num_heads + 2, nope_dim),
-7.0,
dtype=q.dtype,
device=q.device,
)
dest = buffer[2 : 2 + num_queries, :num_heads]
assert not dest.is_contiguous()
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :nope_dim]
torch.testing.assert_close(dest, expected, atol=0.0, rtol=0.0)
untouched = torch.ones_like(buffer, dtype=torch.bool)
untouched[2 : 2 + num_queries, :num_heads] = False
assert torch.all(buffer[untouched] == -7.0)
@pytest.mark.parametrize("out_dtype", [torch.float32, torch.float16])
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_casts_to_dest_dtype(
out_dtype: torch.dtype,
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(512, 64)
q = kwargs["q"]
dest = torch.empty(q.shape[0], q.shape[1], 512, dtype=out_dtype, device=q.device)
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :512]
assert dest.dtype == out_dtype
torch.testing.assert_close(dest.float(), expected.float(), atol=2e-2, rtol=2e-2)
@torch.inference_mode()
def test_sparse_attn_prefill_head_dim_wide_out_keeps_rope_rows() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(NOPE_HEAD_DIM, ROPE_HEAD_DIM)
q = kwargs["q"]
dest = torch.empty_like(q)
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _ref_sparse_prefill_ragged(
q, kwargs["kv"], [[0, 2], [1, 3, 4], []], kwargs["scale"], kwargs["attn_sink"]
)
torch.testing.assert_close(dest, expected, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("num_queries", "on_gfx950", "expected"),
[(1023, True, False), (1024, True, True), (1024, False, False)],
@@ -718,10 +826,11 @@ def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> Non
output = torch.empty_like(q)
dense_fallback_calls = 0
def fake_dense_fallback(*args, **kwargs):
def fake_dense_fallback(*args, out, **kwargs):
nonlocal dense_fallback_calls
dense_fallback_calls += 1
return torch.zeros_like(q)
out.zero_()
return out
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)
+1 -1
View File
@@ -194,7 +194,7 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
layer = types.SimpleNamespace(
prefix=PREFIX,
enable_fused_gdn_decode=True,
enable_fused_gdn_spec_decode=True,
norm=types.SimpleNamespace(
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
),
+1 -1
View File
@@ -167,7 +167,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
draft = SimpleNamespace(
use_mega_moe=False,
use_native_mega_moe=False,
use_sequence_parallel=False,
hc_mult=hc_mult,
layers=[make_layer(mix) for mix in mixes],
@@ -88,6 +88,15 @@ class Config:
torch_trace_dir_path: str | None = None
# Force AiterExperts's hidden_pad/intermediate_pad computation
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
# None (default) preserves today's behavior: FusedMoEConfig defaults both
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
# intermediate_pad come out to 0.
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
hidden_dim_unpadded: int | None = None
intermediate_size_per_partition_unpadded: int | None = None
def __post_init__(self):
if self.quant_config is None:
self.quant_config = TestMoEQuantConfig(None, False, False, None)
@@ -664,6 +673,10 @@ def make_modular_kernel(
activation=config.activation,
device=vllm_config.device_config.device,
routing_method=RoutingMethodType.DeepSeekV3,
hidden_dim_unpadded=config.hidden_dim_unpadded,
intermediate_size_per_partition_unpadded=(
config.intermediate_size_per_partition_unpadded
),
)
prepare_finalize = maybe_make_prepare_finalize(
+2
View File
@@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
expert_map=None,
apply_router_weight_on_input=True,
routed_scaling_factor=1.0,
routing_sink=None,
)
check_accuracy(
@@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
router_logits=logits,
e_score_correction_bias=bias,
routed_scaling_factor=scale,
routing_sink=None,
**common,
)
topk_weights, topk_ids = fused_topk_bias(
+307
View File
@@ -0,0 +1,307 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
MoE geometry of a real model, warms it up and runs a forward whose inputs are
chosen so that every quantization step in the megakernel is exact, for the
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
32). The expected output is then known in closed form and compared bit for
bit.
"""
from dataclasses import dataclass
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_gemm
@dataclass(frozen=True)
class Geometry:
name: str
hidden_size: int
intermediate_size: int
num_experts: int
top_k: int
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
GEOMETRIES = (
Geometry("dsv4-pro", 7168, 3072, 384, 6),
Geometry("dsv4-flash", 4096, 2048, 256, 6),
Geometry("mistral-small-4", 4096, 2048, 128, 4),
Geometry("mistral-large-3", 7168, 4096, 128, 4),
)
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
NUM_TOKENS_PER_RANK = 4
MAX_TOKENS_PER_RANK = 128
# Every block scale below is either 1 or a power of two, so the FP4 codes
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
#
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
# it; every weight column the sentinel meets is zero.
X_SIGNAL = 2.0
X_SENTINEL = 6.0
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
W13_DIAG = 6.0
H_SIGNAL = 144.0
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
# sentinel sits next to a zero intermediate channel and pins the block scale.
W2_SENTINEL = 6.0
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
def _expert_out(expert: int) -> float:
return E2M1_CODES[expert % len(E2M1_CODES)]
def _topk_weights(top_k: int) -> tuple[float, ...]:
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
# weight * 144 must requantize exactly too: powers of two do.
return tuple(2.0 ** -(k + 1) for k in range(top_k))
def _routing_method(backend: str) -> RoutingMethodType:
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
return RoutingMethodType.DeepseekV4
return RoutingMethodType.Default
def _routing(
geometry: Geometry, rank: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor]:
"""top_k distinct experts per token, strided across ranks so dispatch goes
remote."""
base = (
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
) % geometry.num_experts
stride = geometry.num_experts // geometry.top_k
offsets = torch.arange(geometry.top_k, device=device) * stride
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
torch.int32
)
topk_weights = torch.tensor(
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
dtype=torch.float32,
device=device,
)
return topk_ids, topk_weights
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
x = torch.zeros(
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
)
for t in range(NUM_TOKENS_PER_RANK):
x[t, 16 * t] = X_SIGNAL
x[t, 16 * t + 1] = X_SENTINEL
return x
def _local_expert_weights(
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
) -> FlashInferMoeEpWeights:
w13 = torch.zeros(
num_local_experts,
2 * geometry.intermediate_size,
geometry.hidden_size,
dtype=torch.bfloat16,
device=device,
)
w2 = torch.zeros(
num_local_experts,
geometry.hidden_size,
geometry.intermediate_size,
dtype=torch.bfloat16,
device=device,
)
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
for local in range(num_local_experts):
expert = rank * num_local_experts + local
w13[local, 0, token_columns] = W13_DIAG
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
w2[local, expert, 0] = _expert_out(expert)
w2[local, expert, 2] = W2_SENTINEL
return FlashInferMoeEpWeights(w13=w13, w2=w2)
def _expected_output(
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
) -> torch.Tensor:
expected = torch.zeros(
NUM_TOKENS_PER_RANK,
geometry.hidden_size,
dtype=torch.float32,
device=topk_ids.device,
)
for t in range(NUM_TOKENS_PER_RANK):
for k in range(geometry.top_k):
expert = int(topk_ids[t, k])
weight = float(topk_weights[t, k])
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
return expected.to(torch.bfloat16)
def _moe_config(
vllm_config: VllmConfig,
geometry: Geometry,
world_size: int,
device: torch.device,
backend: str,
) -> FusedMoEConfig:
parallel = FusedMoEParallelConfig.make(
tp_size_=1,
pcp_size_=1,
dp_size_=world_size,
sp_size_=1,
vllm_parallel_config=vllm_config.parallel_config,
)
return FusedMoEConfig(
num_experts=geometry.num_experts,
experts_per_token=geometry.top_k,
hidden_dim=geometry.hidden_size,
intermediate_size=geometry.intermediate_size,
num_local_experts=geometry.num_experts // world_size,
num_logical_experts=geometry.num_experts,
activation=MoEActivation.SILU,
device=device,
routing_method=_routing_method(backend),
moe_parallel_config=parallel,
in_dtype=torch.bfloat16,
moe_backend=backend,
max_num_tokens=MAX_TOKENS_PER_RANK,
)
def _check_expert_views_alias_kernel_memory(
adapter: FlashInferMoeEp,
geometry: Geometry,
rank: int,
num_local_experts: int,
device: torch.device,
) -> None:
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
Swap local experts 0 and 1 in place on every rank, then route through the
swapped physical slots: the output must be the one the unswapped logical
experts would have produced.
"""
for view in adapter.expert_weight_views():
first = view[0].clone()
view[0].copy_(view[1])
view[1].copy_(first)
torch.distributed.barrier()
topk_ids, topk_weights = _routing(geometry, rank, device)
slot = topk_ids % num_local_experts
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
torch.accelerator.synchronize()
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
def _init_warmup_forward(
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = backend
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
)
rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{rank}")
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
adapter = FlashInferMoeEp(
moe,
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
apply_topk_in_fc1=apply_topk_in_fc1(moe),
)
try:
adapter.warmup()
topk_ids, topk_weights = _routing(geometry, rank, device)
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
torch.accelerator.synchronize()
expected = _expected_output(geometry, topk_ids, topk_weights)
assert output.dtype == expected.dtype and output.shape == expected.shape
assert torch.equal(output, expected), (
f"rank {rank}: max abs diff "
f"{(output.float() - expected.float()).abs().max().item()}, "
f"nonzero output channels per token "
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
)
if backend == FLASHINFER_MOE_EP_CUTEDSL:
_check_expert_views_alias_kernel_memory(
adapter, geometry, rank, moe.num_local_experts, device
)
finally:
adapter.destroy()
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="The FlashInfer MoE-EP megakernels require SM100",
)
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
@pytest.mark.parametrize("world_size", [2, 4])
def test_init_warmup_forward_is_exact_on_grid_inputs(
world_size: int, backend: str, geometry: Geometry
):
pytest.importorskip("flashinfer.moe_ep")
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
pytest.skip("DeepGEMM is not available")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs")
distributed_run(_init_warmup_forward, world_size, geometry, backend)
@@ -42,7 +42,7 @@ from .modular_kernel_tools.parallel_utils import (
ProcessGroupInfo,
parallel_launch_with_config,
)
from .utils import check_accuracy
from .utils import check_accuracy, make_test_weights
has_any_multi_gpu_package = (
has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe()
@@ -691,6 +691,303 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
)
# --- 4b: hidden_dim_unpadded/intermediate_size_per_partition_unpadded matrix -
# K/N intentionally unaligned to AITER's 64/128 granularity (unlike the
# shared mk_objects.py defaults) so hidden_pad/intermediate_pad are forced.
_PADDING_E = 8
_PADDING_M = 16
_PADDING_TOPK = 2
_PADDING_K_UNPADDED = 896 # 7 * 128
_PADDING_K_PADDED = 1024 # +128
_PADDING_N_UNPADDED = 384 # 3 * 128
_PADDING_N_PADDED = 512 # +128
# hidden_pad = (K_padded - hidden_dim_unpadded) // 128 * 128 (rocm_aiter_moe.py)
_PADDING_HIDDEN_PAD_EXPECTED = 128
# intermediate_pad = (N_padded - intermediate_size_per_partition_unpadded)
# // 64 * 64 * (2 if tp_size == 1 else 1) (rocm_aiter_moe.py); tp_size==1 here.
_PADDING_INTERMEDIATE_PAD_EXPECTED = 256
# {no padding, hidden-only, intermediate-only, both} -> (pad_hidden, pad_intermediate)
_PADDING_MODES: dict[str, tuple[bool, bool]] = {
"none": (False, False),
"hidden": (True, False),
"intermediate": (False, True),
"both": (True, True),
}
# unquantized + the 4 fp8 configs AiterExperts supports (excludes MXFP4,
# stubbed below pending MI350/gfx950, and MK_QUANT_CONFIGS[1], unsupported).
_PADDING_QUANT_CONFIGS = [
MK_QUANT_CONFIGS[0], # unquantized
MK_QUANT_CONFIGS[2], # fp8 channel weights / per-token activations
MK_QUANT_CONFIGS[3], # fp8 per-tensor weights / per-tensor activations
MK_QUANT_CONFIGS[4], # fp8 per-tensor weights / per-token activations
MK_QUANT_CONFIGS[5], # fp8 128x128-block weights / 128-block activations
]
_PADDING_QUANT_IDS = [
"unquantized",
"fp8_channel_token",
"fp8_tensor_tensor",
"fp8_tensor_token",
"fp8_block_token",
]
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
gate/up halves of `real_per_half` size each."""
padded_per_half = t.shape[1] // 2
return torch.cat(
[
t[:, :real_per_half],
t[:, padded_per_half : padded_per_half + real_per_half],
],
dim=1,
)
def _slice_unpadded_weights(
weights: WeightTensors,
quant_config: TestMoEQuantConfig | None,
k_unpadded: int,
n_unpadded: int,
) -> WeightTensors:
"""Derive the unpadded sub-block of a padded WeightTensors: same values,
restricted to the region hidden_pad/intermediate_pad keeps.
Scale-shape choice (per_out_ch) must match _make_padding_matrix_weights()."""
block_shape = quant_config.block_shape if quant_config is not None else None
per_out_ch = quant_config is not None and quant_config.per_out_ch_quant
w1 = _slice_gate_up_rows(weights.w1[:, :, :k_unpadded], n_unpadded)
w2 = weights.w2[:, :k_unpadded, :n_unpadded]
if weights.w1_scale is None:
w1_scale = w2_scale = None
elif weights.w2_scale is None:
raise AssertionError("w1_scale and w2_scale must both be set or both None")
elif block_shape is not None:
block_n, block_k = block_shape
assert n_unpadded % block_n == 0 and k_unpadded % block_k == 0
n_scale, k_scale = n_unpadded // block_n, k_unpadded // block_k
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_scale)[..., :k_scale]
w2_scale = weights.w2_scale[:, :k_scale, :n_scale]
elif per_out_ch:
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_unpadded)
w2_scale = weights.w2_scale[:, :k_unpadded, :]
else:
# Per-tensor scale: shape (E, 1, 1), independent of K/N -- reuse as-is.
w1_scale = weights.w1_scale
w2_scale = weights.w2_scale
return WeightTensors(w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale)
def _make_padding_matrix_weights(config: Config) -> WeightTensors:
"""Like WeightTensors.make(), but scales weights by
config.is_per_out_ch_quant instead of is_per_act_token_quant.
WeightTensors.make() ties weight-scale shape to activation quant, so
fp8_tensor_token would get per-channel scales despite being a per-tensor
weight scheme. Scoped here rather than fixing WeightTensors.make(),
which other tests depend on."""
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
e=config.E,
n=config.N,
k=config.K,
in_dtype=config.dtype,
quant_dtype=config.quant_dtype,
block_shape=config.quant_block_shape,
per_out_ch_quant=config.is_per_out_ch_quant,
)
return WeightTensors(
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
)
def _slice_unpadded_rank_tensors(
rank_tensors: RankTensors, k_unpadded: int
) -> RankTensors:
return RankTensors(
hidden_states=rank_tensors.hidden_states[:, :k_unpadded].contiguous(),
hidden_states_scale=rank_tensors.hidden_states_scale,
topk_weights=rank_tensors.topk_weights,
topk_ids=rank_tensors.topk_ids,
expert_map=rank_tensors.expert_map,
)
def _aiter_padding_matrix_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
padded_config: Config,
padded_weights: WeightTensors,
verbose: bool,
hidden_pad_expected: int,
intermediate_pad_expected: int,
):
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
set_random_seed(pgi.rank)
weights = copy.deepcopy(padded_weights)
weights.to_current_device()
rank_tensors = RankTensors.make(padded_config, pgi)
with mock.patch.object(
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
) as fused_moe_mock:
mk_out = run_modular_kernel(
pgi, vllm_config, padded_config, weights, rank_tensors
)
assert fused_moe_mock.call_count > 0, (
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
)
for call in fused_moe_mock.call_args_list:
assert call.kwargs["hidden_pad"] == hidden_pad_expected, (
f"AiterExperts.apply() forwarded hidden_pad="
f"{call.kwargs['hidden_pad']}, expected {hidden_pad_expected}."
)
assert call.kwargs["intermediate_pad"] == intermediate_pad_expected, (
f"AiterExperts.apply() forwarded intermediate_pad="
f"{call.kwargs['intermediate_pad']}, expected "
f"{intermediate_pad_expected}."
)
# AiterExperts.apply() never slices its own output -- the buffer stays
# raw (padded) width; only the caller-sliced unpadded prefix is checked.
assert mk_out.shape[-1] == padded_config.K, (
f"AiterExperts output width {mk_out.shape[-1]} != raw hidden_dim "
f"{padded_config.K}."
)
mk_out = mk_out[..., :_PADDING_K_UNPADDED]
# Only the real slice is checked: beyond it is unspecified/reused memory.
assert torch.isfinite(mk_out).all(), (
"AiterExperts output must not contain NaN/Inf in the real "
"hidden_dim_unpadded-wide output slice."
)
unpadded_weights = _slice_unpadded_weights(
weights,
padded_config.quant_config,
_PADDING_K_UNPADDED,
_PADDING_N_UNPADDED,
)
unpadded_rank_tensors = _slice_unpadded_rank_tensors(
rank_tensors, _PADDING_K_UNPADDED
)
with set_current_vllm_config(vllm_config):
ref_out = reference_moe_impl(
padded_config, unpadded_weights, unpadded_rank_tensors
)
# ref_out's magnitude here (~1e-3-1e-2) is below atol=3e-2, so a zeroed-out
# mk_out would still pass check_accuracy below -- guard against that.
ref_scale = ref_out.abs().mean()
mk_scale = mk_out.abs().mean()
assert mk_scale > 0.5 * ref_scale, (
f"AiterExperts output magnitude (mean |mk_out|={mk_scale:.6f}) looks "
f"degenerate/zeroed vs. reference (mean |ref_out|={ref_scale:.6f})."
)
# Lenient check for every scheme, including unquantized: a strict
# assert_close(atol=rtol=3e-2) spuriously fails ~0.1-0.2% of elements
# on MI300 even for unquantized AiterExperts (confirmed on hardware).
check_accuracy(ref_out, mk_out, atol=3e-2, rtol=3e-2, percent=0.9)
@require_aiter_moe
@pytest.mark.parametrize("quant_config", _PADDING_QUANT_CONFIGS, ids=_PADDING_QUANT_IDS)
@pytest.mark.parametrize("mode", list(_PADDING_MODES))
def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig | None):
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
Exercises AiterExperts's hidden_pad/intermediate_pad (rocm_aiter_moe.py)
across {no padding, hidden-only, intermediate-only, both} x the
unquantized + 4 fp8 quant schemes it supports. Compares against an
unpadded reference and verifies the hidden_pad/intermediate_pad
AiterExperts.apply() forwards to rocm_aiter_ops.fused_moe.
"""
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
pad_hidden, pad_intermediate = _PADDING_MODES[mode]
if (
quant_config is not None
and quant_config.block_shape is not None
and not pad_hidden
):
pytest.skip(
"AITER's block-quantized (128x128) CK GEMM kernel does not "
f"support the raw (unpadded) hidden_dim={_PADDING_K_UNPADDED} "
"shape on this hardware -- it raises 'wrong! device_gemm with "
"the specified compilation parameters does not support this "
"GEMM problem'. Only hidden_pad-forcing modes (raw "
f"hidden_dim={_PADDING_K_PADDED}) are exercised for this quant "
"scheme."
)
k = _PADDING_K_PADDED if pad_hidden else _PADDING_K_UNPADDED
n = _PADDING_N_PADDED if pad_intermediate else _PADDING_N_UNPADDED
config = Config(
Ms=_PADDING_M,
K=k,
N=n,
E=_PADDING_E,
topks=_PADDING_TOPK,
dtype=torch.bfloat16,
quant_config=quant_config,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
world_size=1,
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
intermediate_size_per_partition_unpadded=(
_PADDING_N_UNPADDED if pad_intermediate else None
),
)
assert config.is_valid()[0]
assert config.fe_supports_quant_scheme(), (
f"AiterExperts does not support quant scheme {quant_config}."
)
weights = _make_padding_matrix_weights(config)
vllm_config, env_dict = config.make_env_data()
hidden_pad_expected = _PADDING_HIDDEN_PAD_EXPECTED if pad_hidden else 0
intermediate_pad_expected = (
_PADDING_INTERMEDIATE_PAD_EXPECTED if pad_intermediate else 0
)
parallel_launch_with_config(
config.world_size,
_aiter_padding_matrix_worker,
vllm_config,
env_dict,
config,
weights,
False,
hidden_pad_expected,
intermediate_pad_expected,
)
@pytest.mark.skip(
reason="MXFP4 AiterExperts padding requires MI350/gfx950 hardware. "
'See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").'
)
def test_aiter_moe_padding_matrix_mxfp4():
"""Stub for the {no padding, hidden-only, intermediate-only, both} MXFP4
padding matrix once MI350/gfx950 hardware is available in CI."""
if __name__ == "__main__":
# Ability to test individual PrepareAndFinalize and FusedExperts combination
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
+9 -7
View File
@@ -16,10 +16,8 @@ from vllm.utils.torch_utils import set_random_seed
NUM_TOKENS = [1, 17, 256]
TOP_KS = [2, 8]
HIDDEN_SIZE = 512
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
HIDDEN_SIZES = [512, 1000, 3072, 7168]
# Include partial tiles and hidden sizes above the CUDA 4096-element tile cap.
HIDDEN_SIZES = [512, 1000, 3072, 4097, 7168]
DTYPES = [torch.bfloat16, torch.float32]
NUM_EXPERTS = 16
@@ -233,7 +231,7 @@ def test_nonlocal_and_padding_adjacent():
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("dtype", [*DTYPES, torch.float16])
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_weighted_sum_matches_reference(
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
@@ -268,7 +266,11 @@ def test_weighted_sum_matches_reference(
assert not out.isnan().any(), "invalid slots leaked into the reduction"
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
atol, rtol = {
torch.bfloat16: (1e-2, 1e-2),
torch.float16: (1e-3, 1e-3),
torch.float32: (1e-4, 1e-4),
}[dtype]
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
@@ -360,7 +362,7 @@ def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
"""
set_random_seed(0)
device = "cuda"
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 7168
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
+2
View File
@@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize(
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=None,
),
)
return
@@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle(
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=None,
)
else:
# Modular impl uses topk_weights and topk_ids
File diff suppressed because it is too large Load Diff
+30
View File
@@ -91,6 +91,12 @@ QWEN4_EXP_SM100_CASES = [
for num_tokens, config in plans.items()
]
QWEN4_EXP_SM121_CASES = [
(n, k, num_tokens, config)
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS.items()
for num_tokens, config in plans.items()
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
@@ -725,6 +731,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS),
((8, 0), {}),
],
)
@@ -939,6 +946,29 @@ def test_qwen4_exp_sm100_selected_shapes(
assert cosine > 0.999
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM121_CASES)
def test_qwen4_exp_sm121_selected_shapes(
n: int,
k: int,
num_tokens: int,
config: SkinnyGemmConfig,
) -> None:
_require_capability_and_cute((12, 1))
torch.manual_seed(42 + num_tokens)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
selected = qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS[(n, k)][num_tokens]
assert selected == config
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
_require_sm103_and_cute()
spec = glm52_gemm.GLM52_Q_B_PROJECTION
+225 -118
View File
@@ -636,8 +636,8 @@ def test_top_k_per_row_decode_gfx950_k512_masked_graph_replay() -> None:
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_get_aiter_top_k_kernel,
_launch_aiter_top_k_per_row_prefill,
)
@@ -647,17 +647,14 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
top_k = 4
indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda")
aiter_topk_kernel = _get_aiter_top_k_kernel(
if not rocm_aiter_ops.is_indexer_top_k_supported(
is_prefill=True,
compress_ratio=4,
num_rows=logits.shape[0],
on_gfx950=True,
)
if aiter_topk_kernel is None:
):
pytest.skip("AITER top-k is unavailable")
assert (
_launch_aiter_top_k_per_row_prefill(
aiter_topk_kernel,
logits,
row_starts,
row_ends,
@@ -683,10 +680,7 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
num_speculative_tokens: int,
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_get_aiter_top_k_kernel,
_launch_aiter_top_k_per_row_decode,
)
from vllm._aiter_ops import rocm_aiter_ops
query_tokens = num_speculative_tokens + 1
offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
@@ -701,20 +695,18 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
top_k = 4
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
aiter_topk_kernel = _get_aiter_top_k_kernel(
if not rocm_aiter_ops.is_indexer_top_k_supported(
is_prefill=False,
compress_ratio=4,
num_rows=num_rows,
max_valid_seq_len=seq_lens.max().item(),
on_gfx950=True,
)
if aiter_topk_kernel is None:
):
pytest.skip("AITER top-k is unavailable")
assert (
_launch_aiter_top_k_per_row_decode(
aiter_topk_kernel,
rocm_aiter_ops.indexer_top_k_decode(
logits,
seq_lens,
1,
seq_lens.reshape(-1),
indices,
top_k,
)
@@ -729,130 +721,107 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
assert torch.all(indices[row_idx, num_valid:] == -1)
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse
@requires_gfx950
@torch.inference_mode()
def test_aiter_decode_topk_matches_per_row_under_mtp() -> None:
"""The aiter backend expands (B, 1) seq_lens into per-row ends before
launching, so next_n > 1 must agree with the in-tree kernel, including
the rows that clamp to a zero-length prefix.
"""
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
def prefill_kernel(*args, **kwargs) -> None:
pass
next_n = 4
top_k = 512
num_columns = 1024
seq_lens = torch.tensor([[2], [513], [700]], dtype=torch.int32, device="cuda")
num_rows = seq_lens.shape[0] * next_n
def decode_kernel(*args, **kwargs) -> None:
pass
torch.manual_seed(0)
logits = torch.randn(num_rows, num_columns, dtype=torch.float32, device="cuda")
outputs = {}
for backend in ("aiter", "per_row"):
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
get_indexer_topk(backend)(logits, seq_lens, next_n, indices, top_k, num_columns)
outputs[backend] = indices
row_ends = (
(seq_lens - next_n + 1 + torch.arange(next_n, device="cuda"))
.clamp_(min=0)
.reshape(-1)
.tolist()
)
assert row_ends[:next_n] == [0, 0, 1, 2]
for row_idx, row_end in enumerate(row_ends):
num_valid = min(top_k, row_end)
aiter_row = outputs["aiter"][row_idx]
assert set(aiter_row[:num_valid].tolist()) == set(
outputs["per_row"][row_idx][:num_valid].tolist()
)
assert torch.all(aiter_row[num_valid:] == -1)
def _enable_aiter_topk(monkeypatch, enabled: bool = True) -> None:
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setattr(
rocm_aiter_mla_sparse,
"_get_aiter_topk_ops",
lambda: (prefill_kernel, decode_kernel),
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: enabled, raising=True
)
get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
from vllm._aiter_ops import rocm_aiter_ops
_enable_aiter_topk(monkeypatch)
is_supported = rocm_aiter_ops.is_indexer_top_k_supported
eligible_cases = [
(
dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True),
prefill_kernel,
dict(is_prefill=True, compress_ratio=4, num_rows=1),
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_536),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=257,
max_valid_seq_len=250_000,
),
(
dict(
is_prefill=False,
compress_ratio=4,
num_rows=1,
max_valid_seq_len=65_536,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=4,
num_rows=257,
max_valid_seq_len=250_000,
num_columns=524_288,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=2,
num_rows=385,
max_valid_seq_len=250_000,
num_columns=524_288,
topk_tokens=512,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=2,
num_rows=384,
max_valid_seq_len=50_000,
num_columns=1_048_577,
topk_tokens=512,
on_gfx950=True,
),
decode_kernel,
),
]
for kwargs, expected_kernel in eligible_cases:
assert get_kernel(**kwargs) is expected_kernel
def fail_if_imported() -> None:
pytest.fail("ineligible shapes must not import AITER top-k")
monkeypatch.setattr(
rocm_aiter_mla_sparse,
"_get_aiter_topk_ops",
fail_if_imported,
)
ineligible_cases = [
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
dict(
is_prefill=False,
compress_ratio=2,
num_rows=384,
max_valid_seq_len=1_048_576,
num_columns=1_048_576,
topk_tokens=512,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=1,
max_valid_seq_len=65_537,
on_gfx950=True,
),
]
for kwargs in eligible_cases:
assert is_supported(**kwargs) is True
ineligible_cases = [
dict(is_prefill=True, compress_ratio=1, num_rows=1),
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_537),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=256,
max_valid_seq_len=125_000,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=320,
max_valid_seq_len=2_500,
on_gfx950=False,
),
]
for kwargs in ineligible_cases:
assert get_kernel(**kwargs) is None
assert is_supported(**kwargs) is False
monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None)
assert (
get_kernel(
is_prefill=True,
compress_ratio=4,
num_rows=1,
on_gfx950=True,
)
is None
)
_enable_aiter_topk(monkeypatch, enabled=False)
assert is_supported(is_prefill=True, compress_ratio=4, num_rows=1) is False
def test_dsv4_native_top_k_window() -> None:
"""The in-tree decode kernel's measured advantage band over AITER, which
applies to the DSV4 indexer only."""
from vllm._aiter_ops import rocm_aiter_ops
prefers_native = rocm_aiter_ops.dsv4_indexer_prefers_native_top_k
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=512) is True
assert prefers_native(num_rows=385, num_columns=524_288, topk_tokens=512) is False
assert prefers_native(num_rows=128, num_columns=1_048_577, topk_tokens=512) is False
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=1024) is False
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@@ -1927,3 +1896,141 @@ def test_sparse_indexer_topk_backend_resolution() -> None:
if not is_sm100:
with pytest.raises(RuntimeError, match="SM100a/SM103a"):
resolve("deep_select")
def _patch_aiter_topk(monkeypatch, decode_kernel, enabled: bool = True) -> None:
from vllm._aiter_ops import rocm_aiter_ops
_enable_aiter_topk(monkeypatch, enabled=enabled)
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: enabled, raising=True)
monkeypatch.setattr(rocm_aiter_ops, "indexer_top_k_decode", decode_kernel)
def _make_topk(backend: str):
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
return SparseIndexerTopk()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
def test_sparse_indexer_topk_aiter_requires_rocm() -> None:
"""Explicitly requesting the ROCm-only backend must fail fast elsewhere."""
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(RuntimeError, match="ROCm"):
_make_topk("aiter").resolve_backend(logits, 2048, 8)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
def test_sparse_indexer_topk_backend_resolution_aiter(monkeypatch) -> None:
"""Aiter is opt-in only: "auto" never selects it (the shape-dependent perf
gate lives at the call site), and an explicit request validates hard
capability only."""
def decode_kernel(*args, **kwargs) -> None:
pass
_patch_aiter_topk(monkeypatch, decode_kernel)
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
assert _make_topk("aiter").resolve_backend(logits, 2048, 8) == "aiter"
# Shapes the call-site perf gate would decline still resolve.
assert _make_topk("aiter").resolve_backend(logits, 512, 8) == "aiter"
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
_make_topk("aiter").resolve_backend(logits, 3000, 8)
_enable_aiter_topk(monkeypatch, enabled=False)
with pytest.raises(RuntimeError, match="no tuned indexer top-k kernels"):
_make_topk("aiter").resolve_backend(logits, 2048, 8)
_patch_aiter_topk(monkeypatch, decode_kernel, enabled=False)
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
with pytest.raises(RuntimeError, match="VLLM_ROCM_USE_AITER"):
_make_topk("aiter").resolve_backend(logits, 2048, 8)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
@torch.inference_mode()
def test_sparse_indexer_topk_aiter_applies_no_shape_gate(monkeypatch) -> None:
"""Shape-dependent perf gating belongs to the call site: once "aiter" is
requested the dispatcher must run it for every supported k, including the
k=512 band the in-tree kernel wins on DSV4's compressed-KV logits."""
from vllm.model_executor.layers import indexer_topk
calls: list[tuple] = []
_patch_aiter_topk(monkeypatch, lambda *args: calls.append(args))
monkeypatch.setattr(
indexer_topk.ops,
"top_k_per_row_decode",
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
)
num_rows = 8
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
indices = torch.empty((num_rows, 512), dtype=torch.int32, device="cuda")
op = _make_topk("aiter")
op(logits, seq_lens, 1, indices, 512, 4096)
# Past the compressed-context boundary the call-site gate declines on.
op(logits, seq_lens, 1, indices, 512, 4 * (64 * 1024 + 1))
assert len(calls) == 2
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
@torch.inference_mode()
def test_sparse_indexer_topk_auto_stays_on_per_row(monkeypatch) -> None:
"""The "auto" chain must not reach AITER on its own: the decode top-k runs
through top_k_per_row_decode unless the caller asks for the aiter backend."""
from vllm.model_executor.layers import indexer_topk
def decode_kernel(*args, **kwargs) -> None:
pytest.fail("AITER is opt-in only")
_patch_aiter_topk(monkeypatch, decode_kernel)
calls: list[tuple] = []
monkeypatch.setattr(
indexer_topk.ops, "top_k_per_row_decode", lambda *args: calls.append(args)
)
num_rows = 8
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
indices = torch.empty((num_rows, 1024), dtype=torch.int32, device="cuda")
_make_topk("auto")(logits, seq_lens, 1, indices, 1024, 4096)
assert len(calls) == 1
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
@torch.inference_mode()
def test_sparse_indexer_topk_aiter_expands_mtp_row_ends(monkeypatch) -> None:
"""AITER's decode kernel only implements the next_n == 1 form, so spec
decode batches must be handed the equivalent expanded per-row ends."""
from vllm.model_executor.layers import indexer_topk
recorded: dict = {}
def decode_kernel(logits, next_n, seq_lens, *args, **kwargs) -> None:
recorded["next_n"] = next_n
recorded["row_ends"] = seq_lens.tolist()
_patch_aiter_topk(monkeypatch, decode_kernel)
monkeypatch.setattr(
indexer_topk.ops,
"top_k_per_row_decode",
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
)
next_n = 2
logits = torch.randn(4, 4096, dtype=torch.float32, device="cuda")
seq_lens = torch.tensor([[100], [200]], dtype=torch.int32, device="cuda")
indices = torch.empty((4, 1024), dtype=torch.int32, device="cuda")
_make_topk("aiter")(logits, seq_lens, next_n, indices, 1024, 800)
assert recorded["next_n"] == 1
assert recorded["row_ends"] == [99, 100, 199, 200]
+77
View File
@@ -362,6 +362,83 @@ def test_fused_moe_lora_kernel(
torch.testing.assert_close(output, output2, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("max_lora_rank", [128, 256])
@pytest.mark.parametrize("add_inputs", [True, False])
@pytest.mark.parametrize("num_slices", [1, 2])
@pytest.mark.parametrize("device", DEVICES)
def test_fused_moe_lora_kernel_large_rank_fallback(
max_lora_rank,
add_inputs,
num_slices,
device,
):
"""Ranks above the one-shot limit fall back to shrink/expand kernels."""
torch.set_default_device(device)
set_random_seed(42)
num_tokens = 8
num_sequences = 4
top_k_num = 2
num_experts = 4
max_loras = 2
N = 256
K = 256
block_size = 16
dtype = torch.bfloat16
topk_ids, topk_weights, token_lora_mapping, lora_ids = sample_data(
num_tokens, num_sequences, max_loras, num_experts, top_k_num
)
lora_a_stacked = [
torch.rand(
(max_loras, num_experts, max_lora_rank, K),
dtype=dtype,
)
for _ in range(num_slices)
]
lora_b_stacked = [
torch.rand(
(max_loras, num_experts, N // num_slices, max_lora_rank),
dtype=dtype,
)
for _ in range(num_slices)
]
hidden_states = torch.rand((num_tokens, K), dtype=dtype)
residual = torch.randn((num_tokens, top_k_num, N), dtype=dtype)
output = residual.clone()
use_fused_moe_lora_kernel(
topk_ids,
topk_weights,
token_lora_mapping,
max_lora_rank,
top_k_num,
lora_ids,
lora_a_stacked,
lora_b_stacked,
hidden_states,
output,
max_loras,
num_experts,
block_size,
add_inputs=add_inputs,
)
expected = use_torch(
hidden_states,
token_lora_mapping,
topk_ids,
lora_a_stacked,
lora_b_stacked,
top_k_num,
num_slices,
)
if add_inputs:
expected += residual
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
def use_fused_moe_lora_kernel_naive(
topk_ids,
topk_weights,
@@ -0,0 +1,198 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from dataclasses import dataclass
import pytest
import torch
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerForConditionalGeneration,
)
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
PlaceholderRange,
)
@pytest.fixture(autouse=True, scope="module")
def _force_cpu_default_device():
# get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
# Ensure the default device is CPU so the rest of the test tensors match.
original = torch.get_default_device()
torch.set_default_device("cpu")
yield
torch.set_default_device(original)
TEXT_TOKEN_ID = 1
VISION_START_TOKEN_ID = 100
VISION_END_TOKEN_ID = 101
IMAGE_TOKEN_ID = 200
VIDEO_TOKEN_ID = 201
AUDIO_TOKEN_ID = 202
AUDIO_START_TOKEN_ID = 300
AUDIO_END_TOKEN_ID = 301
@dataclass
class DummyVisionConfig:
spatial_merge_size: int = 1
@dataclass
class DummyConfig:
position_id_per_seconds: int = 25
vision_config: DummyVisionConfig = dataclasses.field(
default_factory=DummyVisionConfig
)
def make_model() -> Qwen3OmniMoeThinkerForConditionalGeneration:
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
Qwen3OmniMoeThinkerForConditionalGeneration
)
model.config = DummyConfig()
return model
def _elem(data):
return MultiModalFieldElem(
data=torch.tensor(data),
field=None, # HACK.
)
def make_image_feature(offset: int, grid_thw: tuple[int, int, int]):
return MultiModalFeatureSpec(
data=MultiModalKwargsItem({"image_grid_thw": _elem(grid_thw)}),
modality="image",
identifier="DUMMY",
mm_position=PlaceholderRange(
offset=offset, length=int(torch.tensor(grid_thw).prod())
),
)
def make_video_feature(
offset: int, grid_thw: tuple[int, int, int], use_audio_in_video: bool = False
):
data = {"video_grid_thw": _elem(grid_thw)}
if use_audio_in_video:
data["use_audio_in_video"] = _elem(True)
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(data),
modality="video",
identifier="DUMMY",
mm_position=PlaceholderRange(
offset=offset, length=int(torch.tensor(grid_thw).prod())
),
)
def make_audio_feature(offset: int, audio_feature_length: int):
return MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{"audio_feature_lengths": _elem(audio_feature_length)}
),
modality="audio",
identifier="DUMMY",
mm_position=PlaceholderRange(offset=offset, length=0),
)
def test_mrope_positions_image_at_prompt_end():
"""A prompt ending with <|vision_end|> must still produce seq_len positions."""
model = make_model()
# [<|vision_start|>, <|image_pad|> x 4, <|vision_end|>]
input_tokens = (
[VISION_START_TOKEN_ID] + [IMAGE_TOKEN_ID] * 4 + [VISION_END_TOKEN_ID]
)
mm_features = [make_image_feature(offset=1, grid_thw=(1, 2, 2))]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_audio_at_prompt_end():
model = make_model()
# audio_feature_length=16 -> 2 audio tokens.
# [<|audio_start|>, <|audio_pad|> x 2, <|audio_end|>]
input_tokens = [AUDIO_START_TOKEN_ID] + [AUDIO_TOKEN_ID] * 2 + [AUDIO_END_TOKEN_ID]
mm_features = [make_audio_feature(offset=1, audio_feature_length=16)]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_video_at_prompt_end():
model = make_model()
# [T, <|vision_start|>, <|video_pad|> x 8, <|vision_end|>]
input_tokens = (
[TEXT_TOKEN_ID]
+ [VISION_START_TOKEN_ID]
+ [VIDEO_TOKEN_ID] * 8
+ [VISION_END_TOKEN_ID]
)
mm_features = [make_video_feature(offset=2, grid_thw=(2, 2, 2))]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_audio_in_video_at_prompt_end():
model = make_model()
# video grid (1, 2, 2) -> 4 video tokens; audio_feature_length=16 -> 2
# audio tokens, interleaved after the video tokens.
# [T, T, <|vision_start|>, <|audio_start|>, V x 4, A x 2,
# <|audio_end|>, <|vision_end|>]
input_tokens = (
[TEXT_TOKEN_ID] * 2
+ [VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
+ [VIDEO_TOKEN_ID] * 4
+ [AUDIO_TOKEN_ID] * 2
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
)
mm_features = [
make_video_feature(offset=3, grid_thw=(1, 2, 2), use_audio_in_video=True),
make_audio_feature(offset=3, audio_feature_length=16),
]
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
assert positions.shape == (3, len(input_tokens))
def test_mrope_positions_image_then_text():
"""Media followed by text: positions must match the expected layout exactly.
Before the fix this case passed the length guard but every position from
the first <|image_pad|> onward was silently shifted by one slot.
"""
model = make_model()
# [T, T, <|vision_start|>, <|image_pad|> x 4, <|vision_end|>, T, T, T]
input_tokens = (
[TEXT_TOKEN_ID] * 2
+ [VISION_START_TOKEN_ID]
+ [IMAGE_TOKEN_ID] * 4
+ [VISION_END_TOKEN_ID]
+ [TEXT_TOKEN_ID] * 3
)
mm_features = [make_image_feature(offset=3, grid_thw=(1, 2, 2))]
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
expected = torch.tensor(
[
[0, 1, 2, 3, 3, 3, 3, 5, 6, 7, 8],
[0, 1, 2, 3, 3, 4, 4, 5, 6, 7, 8],
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8],
]
)
torch.testing.assert_close(positions, expected)
assert delta == 9 - len(input_tokens)
@@ -11,8 +11,16 @@ from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEKernelMonolithicImpl,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
MoEPrepareAndFinalizeNoDPEPMonolithic,
)
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
RoutedExpertsSink,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
@@ -174,45 +182,139 @@ def test_public_binding_supports_direct_capture_source():
assert calls == [(3, topk_ids)]
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
class DummyFusedMoE:
def __init__(self):
self.layer_id = 3
self.router = _make_router()
# Use a concrete monolithic expert and override its capability
# instead of instantiating the abstract base class directly.
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsFp8,
)
_SINK_CONFIG = SimpleNamespace(
use_ep=False,
dp_size=1,
ep_size=1,
max_num_tokens=4,
experts_per_token=2,
device="cpu",
)
fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8)
self.routed_experts = types.SimpleNamespace(
quant_method=types.SimpleNamespace(
is_monolithic=True,
moe_kernel=types.SimpleNamespace(
impl=types.SimpleNamespace(fused_experts=fused_experts)
),
)
)
self._quant_method = self.routed_experts.quant_method
self._quant_method.moe_kernel.impl.fused_experts = fused_experts
fused_experts.supports_routing_replay_capture = lambda: False
class DummyCapturer:
def capture(self, layer_id, topk_ids):
pass
class _ToyMonolithicExperts(CPUExpertsInt4):
"""A monolithic kernel instance, as a weight reload rebuilds it: routes
token t to experts (t + offset, t + offset + 1)."""
dummy_module = DummyFusedMoE()
moe_config = _SINK_CONFIG
quant_config = None
def __init__(self, offset: int = 0, supports_capture: bool = True):
self.offset = offset
self.supports_capture = supports_capture
@property
def expects_unquantized_inputs(self) -> bool:
return True
def supports_routing_replay_capture(self) -> bool:
return self.supports_capture
def apply(self, hidden_states, *args, routing_replay_out=None, **kwargs):
if routing_replay_out is not None:
t = torch.arange(hidden_states.shape[0])
routing_replay_out[: len(t)] = torch.stack([t, t + 1], 1) + self.offset
return hidden_states
class _MonolithicMoE:
"""The parts of a MoE layer that capture binding reads."""
layer_id = 5
def __init__(self, experts: _ToyMonolithicExperts):
self.router = _make_router()
kernel = SimpleNamespace(impl=SimpleNamespace(fused_experts=experts))
self._quant_method = SimpleNamespace(is_monolithic=True, moe_kernel=kernel)
self.routed_experts = SimpleNamespace(
quant_method=self._quant_method, routing_sink=None
)
def _bind_monolithic(
monkeypatch, experts: _ToyMonolithicExperts, capture
) -> _MonolithicMoE:
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
monkeypatch.setattr(fused_moe_layer, "MoERunner", _MonolithicMoE)
layer = _MonolithicMoE(experts)
bind_routed_experts_capturer(
SimpleNamespace(modules=lambda: [layer]), SimpleNamespace(capture=capture)
)
return layer
def test_public_binding_gives_monolithic_layers_a_sink(monkeypatch):
"""A monolithic kernel routes internally, so binding gives the layer, which
outlives kernel rebuilds, a sink that captures under its layer id."""
calls = []
layer = _bind_monolithic(
monkeypatch, _ToyMonolithicExperts(), lambda *args: calls.append(args)
)
ids = torch.tensor([[1, 2]])
layer.routed_experts.routing_sink.capture_fn(ids)
assert calls == [(_MonolithicMoE.layer_id, ids)]
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
with pytest.raises(ValueError, match="monolithic MoE kernel"):
bind_routed_experts_capturer(
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
_bind_monolithic(
monkeypatch, _ToyMonolithicExperts(supports_capture=False), Mock()
)
@pytest.mark.parametrize(
("use_ep", "dp_size", "ep_size", "rows"),
[(False, 1, 1, 4), (False, 2, 4, 8), (True, 2, 4, 16)],
ids=["single-rank", "dp", "ep"],
)
def test_routed_experts_sink_holds_a_dispatched_batch(use_ep, dp_size, ep_size, rows):
"""One int16 buffer, allocated once, large enough for the batch the kernel
sees after a DP or EP gather."""
config = SimpleNamespace(
**{
**vars(_SINK_CONFIG),
"use_ep": use_ep,
"dp_size": dp_size,
"ep_size": ep_size,
}
)
sink = RoutedExpertsSink(config, Mock())
assert sink.buffer.shape == (rows, 2) and sink.buffer.dtype == torch.int16
def test_monolithic_capture_survives_kernel_rebuilds():
"""The bug in #59449: a weight reload rebuilds the kernel. Through the
production kernel path, the layer's sink captures what the current kernel
routed, before and after a rebuild, and a rebuilt kernel that cannot
capture is refused instead of leaving stale rows."""
captured = []
sink = RoutedExpertsSink(_SINK_CONFIG, lambda ids: captured.append(ids.tolist()))
def forward(experts: _ToyMonolithicExperts) -> None:
kernel = FusedMoEKernelMonolithicImpl(
MoEPrepareAndFinalizeNoDPEPMonolithic(), experts
)
kernel.apply(
torch.zeros(3, 8),
w1=None,
w2=None,
router_logits=torch.zeros(3, 16),
activation=None,
global_num_experts=16,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=sink,
)
for offset in (0, 7): # the kernel before and after a reload
forward(_ToyMonolithicExperts(offset))
assert captured[-1] == [[t + offset, t + offset + 1] for t in range(3)]
with pytest.raises(ValueError, match="not supported"):
forward(_ToyMonolithicExperts(supports_capture=False))
def test_routed_experts_capturer_single_dp_no_metadata():
"""dp_metadata is None: capture writes the full topk_ids rows."""
capturer = _capturer_with_buffer(dp_rank=0)
@@ -8,10 +8,6 @@ import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA-only dispatch"
)
def _require_deep_gemm() -> None:
from vllm.utils.deep_gemm import has_deep_gemm
@@ -20,17 +16,10 @@ def _require_deep_gemm() -> None:
pytest.skip("kpool indexer requires DeepGEMM")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
def _build(indexer_cls, backend: str):
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
op = SparseAttnIndexerKpool(
return indexer_cls(
k_cache=None,
quant_block_size=128,
scale_fmt="ue8m0",
@@ -40,4 +29,73 @@ def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> N
max_total_seq_len=8192,
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
)
assert op.topk_backend == backend
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
backend: str,
) -> None:
"""The AMD kpool indexer must go through the same dispatcher, so the
AITER decode top-k is reachable from kernel_config."""
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize(
"configured,full_cudagraph,expected",
[
("auto", False, "aiter"),
("auto", True, "auto"),
("aiter", True, "aiter"),
("per_row", False, "per_row"),
],
)
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
monkeypatch: pytest.MonkeyPatch,
configured: str,
full_cudagraph: bool,
expected: str,
) -> None:
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
use the context length to pick AITER and must stay on the in-tree kernel.
Explicit backends keep working, and outside FULL the long-context heuristic
still narrows to AITER."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.models.glm5next.amd import sparse_indexer
monkeypatch.setattr(
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
)
monkeypatch.setattr(
rocm_aiter_ops,
"is_indexer_top_k_supported",
lambda **kwargs: True,
raising=False,
)
assert (
sparse_indexer._kpool_decode_topk_backend(
configured,
num_rows=8,
max_valid_seq_len=64 * 1024,
select_k=512,
index_kpool=4,
full_cudagraph=full_cudagraph,
)
== expected
)
+108
View File
@@ -0,0 +1,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from collections.abc import Callable
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock, patch
import pytest
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.mxfp4 import (
_moe_weight_override_is_int4,
_use_k3_situ_int4_gfx942,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
pytestmark = pytest.mark.cpu_test
class _FakeRocmModule(ModuleType):
on_gfx942: Callable[[], bool]
class _FakeAiterOpsModule(ModuleType):
rocm_aiter_ops: SimpleNamespace
def _vllm_config(moe_weight: str | None):
quantization_config = (
None
if moe_weight is None
else QuantizationConfigArgs(moe={"weight": moe_weight})
)
return SimpleNamespace(
model_config=SimpleNamespace(quantization_config=quantization_config)
)
@pytest.mark.parametrize(
("moe_weight", "expected"),
[
(None, False),
("mxfp4", False),
("int4_per_group_32", True),
],
)
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
monkeypatch.setattr(
"vllm.config.get_current_vllm_config",
lambda: _vllm_config(moe_weight),
)
assert _moe_weight_override_is_int4() is expected
@pytest.mark.parametrize("int4_requested", [False, True])
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
moe = SimpleNamespace(
activation=MoEActivation.SITU,
activation_situ_linear_beta=25.0,
)
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
fake_rocm.on_gfx942 = lambda: True
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
with (
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
patch.dict(
sys.modules,
{
"vllm.platforms.rocm": fake_rocm,
"vllm._aiter_ops": fake_aiter_ops,
},
),
patch(
"vllm.model_executor.layers.quantization.mxfp4."
"_moe_weight_override_is_int4",
return_value=int4_requested,
),
):
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
def test_int4_per_group_32_is_not_an_online_moe_method():
# Load-time gfx942 requant. Must not match the online MoE table.
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
)
assert (
config.resolve_quant_method_cls(
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
)
is None
)
def test_unknown_online_moe_weight_still_raises():
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
)
with pytest.raises(ValueError, match="online quantization"):
config.resolve_quant_method_cls(
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
)
+375
View File
@@ -0,0 +1,375 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kimi-K3 multi-turn prefix cache reuse with KV offload, P/D and DCP."""
import contextlib
import json
import random
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
import pytest
import requests
from prometheus_client.parser import text_string_to_metric_families
from tests.models.utils import check_logprobs_close
from tests.utils import RemoteOpenAIServer, multi_gpu_marks
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
# Official Kimi-K3 config with 8 layers and no weights.
MODEL = "riverclouds/Kimi-K3-8L-dummy"
# The recipe's DSpark draft, reading target layers that exist in MODEL.
DRAFT = "riverclouds/Kimi-K3-DSpark-for-8L-dummy"
# Conversation: the first prompt spans FIRST_PROMPT_BLOCKS blocks plus an
# unaligned tail, and each later turn appends TURN_TOKENS random tokens.
NUM_TURNS = 3
FIRST_PROMPT_BLOCKS = 3
UNALIGNED_TAIL = 37
TURN_TOKENS = 777
MAX_TOKENS = 8
NUM_LOGPROBS = 5
STARTUP_TIMEOUT = 1200
BASE_ARGS = [
"--trust-remote-code",
"--language-model-only",
"--load-format",
"dummy",
"--no-enable-flashinfer-autotune",
"--enable-prompt-tokens-details",
# Default max_num_seqs OOMs one GPU during warmup.
"--max-num-seqs",
"8",
]
pytestmark = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Kimi-K3 NVIDIA kernels require the SM100 family",
)
SPEC_CONFIG = {
"method": "dspark",
"model": DRAFT,
"num_speculative_tokens": 7,
"draft_sample_method": "probabilistic",
"rejection_sample_method": "block",
}
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
OFFLOAD = {
"kv_connector": "SimpleCPUOffloadConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"cpu_bytes_to_use_per_rank": 32 << 30,
"lazy_offload": False,
},
}
NIXL_OFFLOAD = {
"kv_connector": "MultiConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"connectors": [NIXL, OFFLOAD]},
}
@dataclass(frozen=True)
class Mode:
# A finer prefix_match_unit enables partial hits inside a Mamba block.
prefix_match_unit: int | None
spec: bool
@property
def name(self) -> str:
name = "block" if self.prefix_match_unit is None else "partial"
return f"{name}-dspark" if self.spec else name
MODES = [Mode(unit, spec) for spec in (False, True) for unit in (None, 128)]
@dataclass(frozen=True)
class Instance:
tp: int = 1
dp: int = 1
dcp: int = 1
kv_config: dict | None = None
prefix_caching: bool = True
@property
def num_gpus(self) -> int:
return self.tp * self.dp
def args(self, mode: Mode) -> list[str]:
args = BASE_ARGS + ["--tensor-parallel-size", str(self.tp)]
if mode.spec:
args += ["--speculative-config", json.dumps(SPEC_CONFIG)]
if self.dp > 1:
args += ["--data-parallel-size", str(self.dp), "--enable-expert-parallel"]
if self.dcp > 1:
args += ["--decode-context-parallel-size", str(self.dcp)]
if self.kv_config is not None:
args += ["--kv-transfer-config", json.dumps(self.kv_config)]
if not self.prefix_caching:
return args + ["--no-enable-prefix-caching"]
args.append("--enable-prefix-caching")
if mode.prefix_match_unit is not None:
args += ["--prefix-match-unit", str(mode.prefix_match_unit)]
return args
@dataclass(frozen=True)
class Deployment:
decode: Instance
prefill: Instance | None = None
# Reset GPU prefix caches each turn so hits must come from CPU.
offload: bool = False
@property
def instances(self) -> tuple[Instance, ...]:
return (self.prefill, self.decode) if self.prefill else (self.decode,)
@property
def num_gpus(self) -> int:
return sum(i.num_gpus for i in self.instances)
DEPLOYMENTS = {
"plain": Deployment(Instance()),
"offload": Deployment(Instance(kv_config=OFFLOAD), offload=True),
"dcp2": Deployment(Instance(tp=2, dcp=2)),
"dcp2-offload": Deployment(Instance(tp=2, dcp=2, kv_config=OFFLOAD), offload=True),
"pd": Deployment(Instance(kv_config=NIXL), prefill=Instance(kv_config=NIXL)),
# NIXL rejects prefix caching on a hybrid decoder with a different TP.
"pd-tp2-dep2": Deployment(
Instance(dp=2, kv_config=NIXL, prefix_caching=False),
prefill=Instance(tp=2, kv_config=NIXL),
),
"pd-offload": Deployment(
Instance(kv_config=NIXL_OFFLOAD),
prefill=Instance(kv_config=NIXL_OFFLOAD),
offload=True,
),
}
def _known_failure(name: str, mode: Mode) -> str | None:
deployment = DEPLOYMENTS[name]
if mode.spec and deployment.decode.dcp > 1:
return "FlashInfer MLA DCP decode breaks on ragged spec batches (#59392)"
return None
def _case(name: str, mode: Mode):
marks = []
if (num_gpus := DEPLOYMENTS[name].num_gpus) > 1:
marks += multi_gpu_marks(num_gpus=num_gpus)
if reason := _known_failure(name, mode):
marks.append(pytest.mark.xfail(reason=reason, strict=True))
return pytest.param(name, mode, marks=marks, id=f"{name}-{mode.name}")
def _block_size(url: str) -> int:
response = requests.get(f"{url}/metrics", timeout=30)
response.raise_for_status()
for family in text_string_to_metric_families(response.text):
for sample in family.samples:
if sample.name == "vllm:cache_config_info":
return lcm(
*(
int(value)
for key in ("block_size", "mamba_block_size")
if (value := sample.labels.get(key, "None")) != "None"
)
)
raise AssertionError("missing vllm:cache_config_info metric")
def _complete(url: str, prompt: list[int], salt: str, **extra) -> dict:
body = {
"model": MODEL,
"prompt": prompt,
"max_tokens": MAX_TOKENS,
"temperature": 0,
"logprobs": NUM_LOGPROBS,
"return_tokens_as_token_ids": True,
"cache_salt": salt,
**extra,
}
response = requests.post(f"{url}/v1/completions", json=body, timeout=600)
response.raise_for_status()
return response.json()
def _pd_complete(
prefill_url: str, decode_url: str, prompt: list[int], salt: str
) -> tuple[dict, dict]:
remote_decode = {
"do_remote_decode": True,
"do_remote_prefill": False,
"remote_engine_id": None,
"remote_block_ids": None,
"remote_host": None,
"remote_port": None,
}
prefill = _complete(
prefill_url, prompt, salt, max_tokens=1, kv_transfer_params=remote_decode
)
decode = _complete(
decode_url, prompt, salt, kv_transfer_params=prefill["kv_transfer_params"]
)
return prefill, decode
def _cached(response: dict) -> int:
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
def _token_id(token: str) -> int:
return int(token.removeprefix("token_id:"))
def _tokens_text_logprobs(response: dict):
choice = response["choices"][0]
ids = [_token_id(t) for t in choice["logprobs"]["tokens"]]
top = [
{_token_id(t): lp for t, lp in d.items()}
for d in choice["logprobs"]["top_logprobs"]
]
return ids, choice["text"], top
def _reset_gpu_prefix_cache(url: str) -> None:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
@dataclass(frozen=True)
class Turn:
prompt_len: int
cached: int
expected_cached: int
recompute_cached: int
output: tuple
recompute: tuple
@property
def matches_recompute(self) -> bool:
return self.output[0] == self.recompute[0]
def _expected_cached(prev_prompt_len: int, hit_unit: int, spec: bool) -> int:
"""A turn must reuse the previous prompt up to its last checkpoint.
EAGLE-style drafts drop the trailing unit, so the checkpoint is one earlier.
"""
if not prev_prompt_len:
return 0
checkpoint = (prev_prompt_len - 1) // hit_unit * hit_unit
return max(checkpoint - hit_unit, 0) if spec else checkpoint
@contextlib.contextmanager
def _serve(deployment: Deployment, mode: Mode) -> Iterator[list[RemoteOpenAIServer]]:
with contextlib.ExitStack() as stack:
# Sequential shutdown waits on memory still held by sibling servers.
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
next_gpu = 0
for instance in deployment.instances:
env = {}
if deployment.offload:
# Enables /reset_prefix_cache.
env["VLLM_SERVER_DEV_MODE"] = "1"
if deployment.prefill is not None:
gpus = range(next_gpu, next_gpu + instance.num_gpus)
next_gpu += instance.num_gpus
env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus))
env["VLLM_SSM_CONV_STATE_LAYOUT"] = "DS"
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(get_open_port())
servers.append(
RemoteOpenAIServer(
MODEL,
instance.args(mode),
env_dict=env,
max_wait_seconds=STARTUP_TIMEOUT,
)
)
yield servers
def _run_conversation(
deployment: Deployment,
servers: list[RemoteOpenAIServer],
mode: Mode,
) -> list[Turn]:
# Under P/D the prefiller computes the prompt, so hits are read from it.
compute_url, decode_url = servers[0].url_root, servers[-1].url_root
block_size = _block_size(compute_url)
hit_unit = mode.prefix_match_unit or block_size
rng = random.Random(0)
def new_tokens(n: int) -> list[int]:
return [rng.randint(1000, 150000) for _ in range(n)]
prompt = new_tokens(FIRST_PROMPT_BLOCKS * block_size + UNALIGNED_TAIL)
prev_len = 0
turns = []
for turn in range(NUM_TURNS):
if deployment.offload:
for server in servers:
_reset_gpu_prefix_cache(server.url_root)
if deployment.prefill is not None:
computed, output = _pd_complete(compute_url, decode_url, prompt, "conv")
else:
computed = output = _complete(decode_url, prompt, "conv")
recompute = _complete(decode_url, prompt, f"recompute-{turn}")
turns.append(
Turn(
prompt_len=len(prompt),
cached=_cached(computed),
expected_cached=_expected_cached(prev_len, hit_unit, mode.spec),
recompute_cached=_cached(recompute),
output=_tokens_text_logprobs(output),
recompute=_tokens_text_logprobs(recompute),
)
)
prev_len = len(prompt)
prompt = prompt + new_tokens(TURN_TOKENS)
return turns
def _format(turns: list[Turn]) -> str:
rows = ["turn prompt cached expected matches_recompute"]
rows += [
f"{i:>4} {t.prompt_len:>6} {t.cached:>6} {t.expected_cached:>8} "
f"{t.matches_recompute}"
for i, t in enumerate(turns)
]
return "\n".join(rows)
@pytest.mark.parametrize(
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
)
def test_turns_reuse_prefix_and_match_recompute(name: str, mode: Mode) -> None:
deployment = DEPLOYMENTS[name]
with _serve(deployment, mode) as servers:
turns = _run_conversation(deployment, servers, mode)
table = _format(turns)
print(f"\n{name}-{mode.name}\n{table}")
assert all(t.recompute_cached == 0 for t in turns), table
assert all(t.cached >= t.expected_cached for t in turns), table
# Dummy weights differ across TP sizes.
if len({i.tp for i in deployment.instances}) == 1:
check_logprobs_close(
outputs_0_lst=[t.recompute for t in turns],
outputs_1_lst=[t.output for t in turns],
name_0="recompute",
name_1=name,
)
@@ -5,16 +5,15 @@ from transformers import GraniteConfig
from vllm.model_executor.models.granite import granite_layer_attn_params
from ...utils import check_logprobs_close, check_transformers_version
from ...utils import check_logprobs_close
# model -> minimum transformers version, or None if unconstrained
MODELS = {
MODELS = [
# TODO(sang): Sliding window should be tested separately.
"ibm/PowerLM-3b": None,
"ibm/PowerMoE-3b": None,
"ibm-granite/granite-swash-2b": "5.15.1",
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
}
"ibm/PowerLM-3b",
"ibm/PowerMoE-3b",
"ibm-granite/granite-swash-2b",
"ibm-granite/granite-swash-3b-a600m",
]
@pytest.mark.parametrize("model", MODELS)
@@ -31,8 +30,6 @@ def test_models(
max_tokens: int,
num_logprobs: int,
) -> None:
check_transformers_version(model, min_transformers_version=MODELS[model])
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
@@ -799,7 +799,6 @@ VLM_TEST_SETTINGS = {
dtype="half",
num_logprobs=10,
patch_hf_runner=model_utils.ovis2_5_patch_hf_runner,
hf_model_kwargs={"revision": "refs/pr/5"},
),
"paddleocr_vl": VLMTestInfo(
models=["PaddlePaddle/PaddleOCR-VL"],
@@ -104,16 +104,7 @@ def run_test(
)
@pytest.mark.parametrize(
"model",
[
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
# instead of shipping a separate lm_head.weight tensor; this
# exercises that path (see tie_weights() in nemotron_parse.py).
"nvidia/NVIDIA-Nemotron-Parse-2.0",
],
)
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"])
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
@@ -12,12 +12,22 @@ Run with:
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
"""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from transformers import AutoTokenizer
from vllm.model_executor.models.deepseek_ocr import DeepseekOCRImagePixelInputs
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
from vllm.model_executor.models.deepseek_ocr import (
DeepseekOCRForCausalLM,
DeepseekOCRImagePixelInputs,
)
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
from vllm.transformers_utils.processors.deepseek_ocr import (
DeepseekOCRProcessor,
ImageTransform,
)
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
@@ -155,3 +165,80 @@ class TestDeepseekOCRInputValidation:
def test_missing_images_raises_value_error(self, processor):
with pytest.raises(ValueError, match="prompt and images"):
processor(prompt="<image>\nDescribe this image.", images=None)
def _balanced_normalized_pixels(
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
) -> torch.Tensor:
"""Build a pixel tensor whose values cancel to an exact sum of 0.
Matches the normalized half-black / half-white case from the DeepSeek-OCR
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
"""
assert width % 2 == 0
pixel_values = torch.ones(batch, channels, height, width)
pixel_values[..., : width // 2] = -1.0
assert pixel_values.sum().item() == 0.0
return pixel_values
class TestDeepseekOCRZeroSumPixelsAccepted:
"""Zero-sum normalized pixels must still produce validated image inputs.
Returning None from ``_parse_and_validate_image_input`` makes
``embed_multimodal`` return None and trips the worker encoder-output
sanity check, killing EngineCore. The zero-sum shortcut is therefore
removed; balanced images must parse like any other valid tensor.
"""
def test_ocr_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_ocr2_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
model,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_image_transform_half_black_half_white_sums_to_zero(self):
"""A 1024×1024 half-black/half-white PNG through ImageTransform
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
must still accept it.
"""
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
pixel_values = ImageTransform()(image).unsqueeze(0)
assert pixel_values.sum().item() == 0.0
images_crop = torch.zeros(0, 3, 640, 640)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert torch.equal(image_input.data, pixel_values)
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)
def _image_info(**kwargs):
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 1},
**kwargs,
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
).info
def test_image_encoder_cache_covers_full_token_budget():
"""The most-features probe must reach the processor's token ceiling.
The inherited square probe refits to 89x89 = 7921 tokens under the
max_image_tokens=8000 budget, so the encoder cache came up short and
ordinary non-square images in 7922-8000 tokens were refused with
HTTP 400 (#59539).
"""
info = _image_info()
assert info.get_max_image_tokens() == 8000
# The shapes from the issue, with the token counts the processor
# actually produces; every one must fit the cache.
for width, height, expected_tokens in [
(4032, 3024, 7931), # phone photo, 4:3
(3840, 2160, 7973), # 4K frame, 16:9
(3508, 2480, 7950), # A4 at 300 dpi
(2600, 2400, 7998), # 13:12
(3000, 3000, 7921), # square worst case before the fix
]:
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
assert num_tokens == expected_tokens
assert num_tokens <= info.get_max_image_tokens()
# The profiling dummy covers the real worst case, not the square one.
size = info.get_image_size_with_most_features()
assert size.width * size.height == 2240 * 2800
def test_image_encoder_cache_follows_max_pixels_override():
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
assert info.get_max_image_tokens() == 100
size = info.get_image_size_with_most_features()
assert (
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
== 100
)
@@ -4,11 +4,8 @@ import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
from vllm.multimodal import MULTIMODAL_REGISTRY
from .transformers_backend import PROCESSOR_CLASSES, create_processor
AUDIO_MODEL_SETTINGS = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
@@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = {
}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize(
"model_id",
[
@@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = {
"zai-org/GLM-ASR-Nano-2512",
],
)
def test_audio_multimodal_processor(model_id, processor_cls):
def test_audio_multimodal_processor(model_id):
settings = AUDIO_MODEL_SETTINGS[model_id]
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
audio = np.zeros(16000, dtype=np.float32)
mm_data = {"audio": (audio, 16000)}
@@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("separator", [" and ", ""])
def test_audio_multiple_inputs(separator, processor_cls):
def test_audio_multiple_inputs(separator):
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
model_id = "ibm-granite/granite-speech-3.3-2b"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
audio_token = mm_processor.info.get_hf_processor().audio_token
# One token per audio; the processor expands each to its placeholder run.
@@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls):
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
def process():
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
# The legacy path reads placeholders off contiguous runs of the audio token, so
# it cannot tell adjacent ones apart and says so instead of merging them
if processor_cls is LegacyMultiModalProcessor and not separator:
with pytest.raises(ValueError, match="Separate them in the prompt"):
process()
return
result = process()
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
@@ -5,23 +5,16 @@ from unittest.mock import patch
import pytest
from vllm.assets.image import ImageAsset
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
OffsetsMultiModalProcessor,
)
from .transformers_backend import (
PROCESSOR_CLASSES,
create_cached_processor,
create_processor,
offsets_only,
)
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_multimodal_processor(model_id, processor_cls):
mm_processor = create_processor(model_id, processor_cls)
def test_multimodal_processor(model_id):
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image_pil = ImageAsset("cherry_blossom").pil_image
mm_data = {"image": image_pil}
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
)
def _process_two_images(processor_cls, separator: str):
def _process_two_images(separator: str):
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_multiple_inputs(processor_cls):
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_two_images(processor_cls, separator="\n and ")
result = _process_two_images(separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_adjacent_inputs(processor_cls):
def test_image_adjacent_inputs():
"""Adjacent images stay separate placeholders rather than merging into one."""
result = _process_two_images(processor_cls, separator="")
result = _process_two_images(separator="")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_batch_padding_removed_from_image_items(processor_cls):
def test_batch_padding_removed_from_image_items():
"""Emu3 pads every image up to the largest in the batch, which would leave an
item's data dependent on what it was processed with and so uncacheable."""
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
)
image_token = mm_processor.info.get_hf_processor().image_token
images = [
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
assert len(shapes) == 2
def _process_one_gemma3_image(processor_cls):
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
hf_processor = mm_processor.info.get_hf_processor()
result = mm_processor(
prompt=f"{hf_processor.boi_token} What is this?",
def _process_one_gemma3_image():
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
)
boi_token = mm_processor.info.get_hf_processor().boi_token
return mm_processor(
prompt=f"{boi_token} What is this?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
return hf_processor, result
@offsets_only
def test_non_embedding_tokens_excluded_from_placeholders():
"""Gemma3 wraps each image in text that carries no embeddings, which must be
inside the placeholder range but masked out of it."""
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
result = _process_one_gemma3_image()
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.is_embed is not None
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
def test_legacy_placeholders_hold_only_image_tokens():
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
marker in the unexpanded prompt, not the token the expansion repeats."""
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.length == hf_processor.image_seq_length
prompt_ids = result["prompt_token_ids"]
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
assert set(covered) == {hf_processor.tokenizer.image_token_id}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
def test_tokens_structuring_an_image_are_masked_not_dropped():
"""SmolVLM splits each image into tiles introduced by tokens carrying no
embeddings. Those belong inside the placeholder and masked out, because the token
count the processor reports is over the whole span. Idefics3 also refuses a prompt
holding `<image>` when no images are passed, which is how the offsets path has to
tokenize it before splicing in the expansion."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
holding `<image>` when no images are passed, which is how the prompt has to be
tokenized before splicing in the expansion."""
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<image>What is this?",
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
@offsets_only
def test_missing_replacement_offsets_names_the_processor():
"""A processor that reports no replacement offsets cannot be served, which must
be said plainly rather than surfacing later as a field config mismatch."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
hf_processor_cls = type(mm_processor.info.get_hf_processor())
hf_call = hf_processor_cls.__call__
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_text_only_prompt(processor_cls):
def test_text_only_prompt():
"""An image model still accepts a prompt with no images."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
assert not result["mm_placeholders"]
@offsets_only
def test_repeated_image_hits_the_processor_cache():
"""Check that mm caching is actually working."""
mm_processor, cache = create_cached_processor(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
)
model_config.multimodal_config.mm_processor_cache_gb = 4
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
cache = MultiModalProcessorOnlyCache(model_config)
image = ImageAsset("cherry_blossom").pil_image
def process():
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
assert first["mm_hashes"] == second["mm_hashes"]
@offsets_only
@pytest.mark.parametrize(
("model_id", "prompt"),
[
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
"<image> and more text",
marks=pytest.mark.xfail(
reason="Emu3Processor prepends its BOS token only when images are "
"passed, so the unexpanded prompt the offsets path tokenizes never "
"passed, so the unexpanded prompt vLLM tokenizes never "
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
strict=False,
),
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
],
)
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
"""The offsets path splices the expansion into a prompt tokenized without any
multi-modal data, so its token ids have to come out the same as the ones the HF
processor produces itself, which is what the legacy path returns."""
prompt_ids = []
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
mm_processor = create_processor(model_id, processor_cls)
prompt_ids.append(
mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)["prompt_token_ids"]
)
"""The prompt is tokenized without any multi-modal data and the expansion spliced
in, so its token ids have to come out the same as the ones the HF processor
produces itself."""
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
info = mm_processor.info
image = ImageAsset("cherry_blossom").pil_image
legacy_ids, offsets_ids = prompt_ids
assert legacy_ids == offsets_ids
hf_processor = info.get_hf_processor()
prompt_ids = info.get_tokenizer().encode(
prompt, **info.default_tok_params.get_encode_kwargs()
)
hf_ids = info.ctx.call_hf_processor(
hf_processor,
dict(text=hf_processor.decode(prompt_ids), images=[image]),
dict(truncation=False, add_special_tokens=False),
)["input_ids"][0].tolist()
result = mm_processor(
prompt=prompt,
mm_items=info.parse_mm_data({"image": image}),
hf_processor_mm_kwargs={},
)
assert result["prompt_token_ids"] == hf_ids
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_nested_image_fields_split_per_image(processor_cls):
def test_nested_image_fields_split_per_image():
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
belonging to each image one dimension further in. Slicing the batch dimension
instead handed the first image every row and the second an empty tensor."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
result = mm_processor(
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers for testing both processors of the Transformers modelling backend."""
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
MultiModalDummyInputsBuilder,
MultiModalProcessingInfo,
MultiModalProcessor,
OffsetsMultiModalProcessor,
)
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers.registry import cached_tokenizer_from_config
offsets_only = pytest.mark.skipif(
MultiModalProcessor is not OffsetsMultiModalProcessor,
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
)
PROCESSOR_CLASSES = [
pytest.param(LegacyMultiModalProcessor, id="legacy"),
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
]
def create_processor(model_id: str, processor_cls):
"""Build a processor directly, because the registry only ever builds the one the
installed transformers version selects, leaving the other path untested."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
return processor_cls(info, MultiModalDummyInputsBuilder(info))
def create_cached_processor(model_id: str, processor_cls):
"""Build a processor backed by a real multi-modal processor cache, and hand the
cache back so a test can check it was actually used."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
model_config.multimodal_config.mm_processor_cache_gb = 4
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
cache = MultiModalProcessorOnlyCache(model_config)
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
+15 -81
View File
@@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"CohereLabs/North-Mini-Code",
trust_remote_code=True,
is_available_online=False,
min_transformers_version="5.9.0",
),
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm"),
# FIXME: databricks/dbrx-instruct has been deleted
"DbrxForCausalLM": _HfExamplesInfo(
"databricks/dbrx-instruct", is_available_online=False
@@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
),
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
"ExaoneMoeForCausalLM": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
),
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
@@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
"Gemma4ForCausalLM": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.0.0",
),
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
@@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
"zai-org/GLM-4.7-Flash",
min_transformers_version="5.0.0",
),
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
),
"GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False),
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
"GPTBigCodeForCausalLM": _HfExamplesInfo(
@@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
"ibm-research/moe-7b-1b-active-shared-experts"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
),
"GraniteSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"),
"GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"),
"HrmTextForCausalLM": _HfExamplesInfo(
"sapientinc/HRM-Text-1B",
min_transformers_version="5.9.0",
),
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
@@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
min_transformers_version="5.9.0",
),
"InklingForCausalLM": _HfExamplesInfo(
"thinkingmachines/Inkling-NVFP4",
@@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"IQuestLoopCoderForCausalLM": _HfExamplesInfo(
"IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
),
"Jais2ForCausalLM": _HfExamplesInfo(
"inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
),
"Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"),
"JambaForCausalLM": _HfExamplesInfo(
"ai21labs/AI21-Jamba-1.5-Mini",
extras={
@@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
"Lfm2MoeForCausalLM": _HfExamplesInfo(
"LiquidAI/LFM2-8B-A1B",
min_transformers_version="5.0.0",
use_original_num_layers=True,
# Initialize at least one MoE layer
hf_overrides={"num_hidden_layers": 4},
@@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
),
"MiniCPM4ForCausalLM": _HfExamplesInfo(
"openbmb/MiniCPM4.1-8B",
min_transformers_version="4.56",
max_transformers_version="4.57",
transformers_version_reason={
"hf": "HF remote code imports removed `is_torch_fx_available`; "
@@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Qwen3NextForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
extras={"tiny-random": "tiny-random/qwen3-next-moe"},
min_transformers_version="4.56.3",
),
"Rnj1ForCausalLM": _HfExamplesInfo(
"EssentialAI/rnj-1-instruct",
@@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
),
"OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo(
"openai/privacy-filter",
min_transformers_version="5.6.0.dev0",
),
"RobertaForTokenClassification": _HfExamplesInfo(
"Jean-Baptiste/roberta-large-ner-english"
@@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
"AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/audio-flamingo-3-hf",
min_transformers_version="5.3.0",
transformers_version_reason={
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
},
@@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025",
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
min_transformers_version="5.9.0",
),
"CohereCompassForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/North-Micro-Vision-Instruct",
min_transformers_version="5.16.0",
),
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Nano",
extras={"super": "nvidia/Cosmos3-Super"},
max_model_len=4096,
min_transformers_version="4.57",
),
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Edge",
max_model_len=4096,
min_transformers_version="5.15",
use_original_num_layers=True,
hf_overrides={
"text_config": {
@@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
"LGAI-EXAONE/EXAONE-4.5-33B",
min_transformers_version="5.6.0",
),
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
"allendou/FireRedASR2-LLM-vllm",
@@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.5.0",
),
# TODO: update min_transformers_version when Gemma4 Unified lands in
# a stable transformers release.
"Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-12B-it",
min_transformers_version="5.8.0",
is_available_online=False,
),
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-ASR-Nano-2512",
min_transformers_version="5.0.0",
),
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-vision-4.1-4b",
@@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-speech-4.1-2b-plus",
min_transformers_version="5.8.0",
),
"GLM4VForCausalLM": _HfExamplesInfo(
"zai-org/glm-4v-9b",
@@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"H2OVLChatModel": _HfExamplesInfo(
@@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
"tencent/HunyuanOCR",
min_transformers_version="5.13",
),
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
"HuggingFaceM4/Idefics3-8B-Llama3",
@@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Lfm2VlForConditionalGeneration": _HfExamplesInfo(
"LiquidAI/LFM2-VL-450M",
min_transformers_version="5.0.0",
),
"Llama4ForConditionalGeneration": _HfExamplesInfo(
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
@@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
min_transformers_version="5.7.0",
),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
@@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
"allenai/Molmo2-8B",
extras={"olmo": "allenai/Molmo2-O-7B"},
min_transformers_version="4.51",
trust_remote_code=True,
# required by current PrefixLM implementation
max_num_batched_tokens=31872,
@@ -1296,19 +1260,15 @@ _MULTIMODAL_EXAMPLE_MODELS = {
trust_remote_code=True,
max_transformers_version="4.57",
transformers_version_reason={
"vllm": "Custom processor code is not compatible with Transformers v5."
"hf": "HF remote code reads `is_parallelizable`, which was removed "
"in Transformers v5."
},
),
"Ovis2_6ForCausalLM": _HfExamplesInfo(
"AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
),
"Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
"AIDC-AI/Ovis2.6-30B-A3B",
trust_remote_code=True,
max_transformers_version="4.57",
transformers_version_reason={
"vllm": "Custom processor code is not compatible with Transformers v5."
},
"AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
),
"PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
"PaddlePaddle/PaddleOCR-VL",
@@ -1353,7 +1313,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
"baidu/Qianfan-OCR",
min_transformers_version="5.6.0",
),
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen2-Audio-7B-Instruct"
@@ -1368,12 +1327,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen3VLForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-VL-4B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-VL-30B-A3B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3.5-0.8B",
@@ -1391,23 +1348,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
),
"Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
"Qwen/Qwen3-ForcedAligner-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
),
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
@@ -1456,7 +1409,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
"microsoft/VibeVoice-ASR-HF",
min_transformers_version="5.13.0",
),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
@@ -1530,7 +1482,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
use_original_num_layers=True, # DFlash requires all layers
max_model_len=8192, # Reduce for CI
max_num_seqs=32,
min_transformers_version="4.56.3", # Required for Qwen3Next
),
"MuseGlimmerAssistantModel": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
@@ -1576,7 +1527,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
is_available_online=False,
use_original_num_layers=True,
min_transformers_version="4.57",
),
"Gemma4DSparkModel": _HfExamplesInfo(
"google/gemma-4-12B-it",
@@ -1737,7 +1687,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"google/gemma-4-E4B-it",
speculative_model="google/gemma-4-E4B-it-assistant",
trust_remote_code=True,
min_transformers_version="5.8.0",
),
"ErnieMTPModel": _HfExamplesInfo(
"baidu/ERNIE-4.5-21B-A3B-PT",
@@ -1747,13 +1696,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"ExaoneMoeMTP": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B",
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
min_transformers_version="5.1.0",
enable_prefix_caching=False,
),
"Exaone4_5_MTP": _HfExamplesInfo(
"LGAI-EXAONE/EXAONE-4.5-33B",
speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
min_transformers_version="5.6.0",
),
"ExtractHiddenStatesModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
@@ -1766,12 +1713,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
"zai-org/GLM-4.7-Flash",
speculative_model="zai-org/GLM-4.7-Flash",
min_transformers_version="5.0.0",
),
"GlmOcrMTPModel": _HfExamplesInfo(
"zai-org/GLM-OCR",
speculative_model="zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextMTPModel": _HfExamplesInfo(
"zai-org/GLM-5.3-Flash",
@@ -1842,9 +1787,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
is_available_online=False,
),
"Qwen3NextMTP": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
),
"Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"),
"Qwen3_5MTP": _HfExamplesInfo(
"Qwen/Qwen3.5-0.8B",
speculative_model="Qwen/Qwen3.5-0.8B",
@@ -1865,29 +1808,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
}
_TRANSFORMERS_BACKEND_MODELS = {
"TransformersEmbeddingModel": _HfExamplesInfo(
"BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
),
"TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
"TransformersForSequenceClassification": _HfExamplesInfo(
"papluca/xlm-roberta-base-language-detection",
min_transformers_version="5.0.0",
),
"TransformersForCausalLM": _HfExamplesInfo(
"hmellor/Ilama-3.2-1B", trust_remote_code=True
),
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
"TransformersMoEForCausalLM": _HfExamplesInfo(
"allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
),
"TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"),
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
),
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
),
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
"Qwen/Qwen3-VL-30B-A3B-Instruct"
),
"TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
"TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
"google/gemma-3-4b-it"
+313 -195
View File
@@ -1,116 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the flashinfer moe_ep backend plumbing.
Everything here runs without a GPU or a flashinfer install: the flashinfer
modules the helpers import lazily are replaced with capture fakes.
"""
import sys
from dataclasses import dataclass, field
from types import ModuleType, SimpleNamespace
from typing import Any
from dataclasses import replace
from types import SimpleNamespace
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.config.kernel import (
FLASHINFER_MOE_EP_BACKENDS,
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
MEGA_MOE_BACKENDS,
NATIVE_MEGA_MOE_BACKENDS,
PASSTHROUGH_ALL2ALL_BACKEND,
bind_passthrough_all2all_backend,
validate_flashinfer_moe_ep_model,
)
from vllm.utils.flashinfer_moe_ep import (
_E2M1_LUT,
FI_MOE_EP_BACKEND_SPECS,
_dequant_fp4_ue8m0_gran32,
build_fi_mega_config,
fi_moe_ep_backend_spec,
make_fi_moe_ep_bootstrap,
megakernel_runtime_requirements,
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
epilogue_from_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
make_mxfp4_moe_quant_config,
map_mxfp4_backend,
mxfp4_round_up_hidden_size_and_intermediate_size,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
backend_to_kernel_cls as mxfp4_backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
NvFp4MoeBackend,
convert_to_nvfp4_moe_kernel_format,
map_nvfp4_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
backend_to_kernel_cls as nvfp4_backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
PassThroughPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Static,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
@dataclass
class _FakeBootstrapConfig:
world_size: int
rank: int
process_group: Any = None
auto_bootstrap: bool = True
device: int | None = field(default=None, kw_only=True)
def test_flashinfer_backends_are_megakernels_outside_the_native_model_path():
assert FLASHINFER_MOE_EP_BACKENDS <= MEGA_MOE_BACKENDS
assert FLASHINFER_MOE_EP_BACKENDS.isdisjoint(NATIVE_MEGA_MOE_BACKENDS)
@dataclass
class _FakeDeepGemmMegaMoeConfig:
intermediate_size: int
top_k: int
activation_clamp: float | None
fast_math: bool
@dataclass
class _FakeNvfp4CutedslMegaMoeConfig:
intermediate_size: int
top_k: int
activation_clamp: float | None
fast_math: bool
@dataclass
class _FakeMegaConfig:
megakernel: Any
preprocess_weights: bool
quantize_input: bool
@pytest.fixture
def fake_flashinfer(monkeypatch):
"""Install a minimal fake flashinfer.moe_ep for the lazy imports."""
moe_ep = ModuleType("flashinfer.moe_ep")
core = ModuleType("flashinfer.moe_ep.core")
runtime = ModuleType("flashinfer.moe_ep.core.runtime")
flashinfer = ModuleType("flashinfer")
fake_attrs: dict[ModuleType, dict[str, Any]] = {
moe_ep: {
"BootstrapConfig": _FakeBootstrapConfig,
"DeepGemmMegaMoeConfig": _FakeDeepGemmMegaMoeConfig,
"Nvfp4CutedslMegaMoeConfig": _FakeNvfp4CutedslMegaMoeConfig,
"MegaConfig": _FakeMegaConfig,
"core": core,
},
runtime: {"TORCH_DIST": "torch_dist", "NVSHMEM": "nvshmem"},
flashinfer: {"moe_ep": moe_ep},
core: {"runtime": runtime},
}
for mod, attrs in fake_attrs.items():
for attr, value in attrs.items():
setattr(mod, attr, value)
for name, mod in {
"flashinfer": flashinfer,
"flashinfer.moe_ep": moe_ep,
"flashinfer.moe_ep.core": core,
"flashinfer.moe_ep.core.runtime": runtime,
}.items():
monkeypatch.setitem(sys.modules, name, mod)
return moe_ep
def test_fi_backend_strings_are_registered_mega_moe_backends():
assert set(FI_MOE_EP_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
assert FLASHINFER_MOE_EP_BACKENDS < MEGA_MOE_BACKENDS
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
instead of silently falling through to the generic FusedMoE path."""
def test_only_deep_gemm_backend_is_dsv4_specific():
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_CUTEDSL,
["MixtralForCausalLM"],
)
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
validate_flashinfer_moe_ep_model(moe_backend, ["DeepseekV4ForCausalLM"])
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_DEEP_GEMM,
["MixtralForCausalLM"],
)
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_DEEP_GEMM,
["DeepseekV4ForCausalLM"],
)
@pytest.mark.parametrize(
@@ -118,8 +92,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
)
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
models validate their own constraints at construction time."""
"""Models validate native deep_gemm mega constraints at construction time."""
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
@@ -128,11 +101,7 @@ def test_non_fi_backend_ignores_architectures():
@pytest.mark.parametrize("moe_backend", sorted(MEGA_MOE_BACKENDS))
def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
"""Every mega backend must qualify for sequence-parallel MoE at
TP>1/EP: the predicate once matched only the native backend string,
which silently ran the fi backends full-batch with an all-reduce on
every rank — 0.42-0.65x native e2e at TP8."""
def test_token_sharding_backends_enable_dsv4_sequence_parallel(moe_backend: str):
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
vllm_config = SimpleNamespace(
@@ -147,113 +116,262 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
assert _use_sequence_parallel(vllm_config)
def test_fi_moe_ep_backend_spec_kernel_and_nvshmem_contract():
dg = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
assert dg.megakernel == "deep_gemm_mega"
assert not dg.needs_nvshmem
cd = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
assert cd.megakernel == "nvfp4_cutedsl"
assert cd.needs_nvshmem
with pytest.raises(ValueError, match="not a flashinfer moe_ep backend"):
fi_moe_ep_backend_spec("deep_gemm_mega_moe")
def test_dsv4_requests_pre_fc2_router_weight_placement():
dsv4 = SimpleNamespace(routing_method=RoutingMethodType.DeepseekV4)
default = SimpleNamespace(routing_method=RoutingMethodType.Default)
assert fi_ep.apply_topk_in_fc1(dsv4)
assert not fi_ep.apply_topk_in_fc1(default)
def test_megakernel_runtime_requirements(fake_flashinfer):
dg = megakernel_runtime_requirements(
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
def test_backend_specs_preserve_weight_format_contracts():
cutedsl = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_CUTEDSL)
assert cutedsl.kernel == "cutedsl"
assert cutedsl.weight_formats == frozenset({"nvfp4", "mxfp4"})
deep_gemm = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_DEEP_GEMM)
assert deep_gemm.kernel == "deep_gemm"
assert deep_gemm.weight_formats == frozenset({"mxfp4"})
def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
fields = dict(
moe_backend=moe_backend,
moe_parallel_config=SimpleNamespace(
use_ep=True, use_batched_activation_format=False
),
is_act_and_mul=True,
activation=MoEActivation.SILU,
has_hash_routing=False,
routing_method=RoutingMethodType.DeepseekV4,
router_logits_dtype=torch.float32,
hidden_dim=256,
is_lora_enabled=False,
skip_final_all_reduce=False,
in_dtype=torch.bfloat16,
has_bias=False,
swiglu_alpha=None,
swiglu_beta=None,
)
assert dg == frozenset({"torch_dist"})
fields.update(overrides)
return SimpleNamespace(**fields)
cd = megakernel_runtime_requirements(
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
def _is_supported(moe, weight_key, activation_key) -> tuple[bool, str]:
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
weight_key,
activation_key,
mk.FusedMoEActivationFormat.Standard,
)
assert cd == frozenset({"torch_dist", "nvshmem"})
return supported, reason or ""
def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
device it already bound via BootstrapConfig.device."""
import vllm.utils.flashinfer_moe_ep as mod
pg = object()
@pytest.fixture
def megakernel_host(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
"""A vLLM config without DBO/EPLB/weight transfer, on a supported device."""
config = SimpleNamespace(
weight_transfer_config=None,
parallel_config=SimpleNamespace(enable_dbo=False, enable_eplb=False),
)
monkeypatch.setattr(fi_ep, "get_current_vllm_config", lambda: config)
monkeypatch.setattr(
mod,
"get_ep_group",
lambda: SimpleNamespace(world_size=4, rank_in_group=2, device_group=pg),
FlashInferMoeEpExperts, "_supports_current_device", staticmethod(lambda: True)
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 3)
bootstrap = make_fi_moe_ep_bootstrap()
assert bootstrap.world_size == 4
assert bootstrap.rank == 2
assert bootstrap.process_group is pg
assert bootstrap.auto_bootstrap is False
assert bootstrap.device == 3
return config
def test_build_fi_mega_config_selects_kernel_config(fake_flashinfer):
dg = build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=7.0,
megakernel="deep_gemm_mega",
def test_megakernel_requires_expert_parallel(megakernel_host):
"""The oracle rejects the megakernel through the experts' support check."""
moe = _megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL)
assert _is_supported(moe, kNvfp4Static, kNvfp4Dynamic) == (True, "")
moe.moe_parallel_config.use_ep = False
supported, reason = _is_supported(moe, kNvfp4Static, kNvfp4Dynamic)
assert not supported and "parallel config" in reason
def test_megakernel_reports_its_own_constraints(megakernel_host):
"""Constraints the framework does not know about surface as reasons."""
_, reason = _is_supported(
_megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL), kNvfp4Static, None
)
assert isinstance(dg.megakernel, _FakeDeepGemmMegaMoeConfig)
assert dg.megakernel.intermediate_size == 2048
assert dg.megakernel.top_k == 8
assert dg.megakernel.activation_clamp == 7.0
assert dg.preprocess_weights and dg.quantize_input
assert "A16 activations" in reason
cd = build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=None,
megakernel="nvfp4_cutedsl",
deep_gemm = _megakernel_moe(
FLASHINFER_MOE_EP_DEEP_GEMM, routing_method=RoutingMethodType.Renormalize
)
assert isinstance(cd.megakernel, _FakeNvfp4CutedslMegaMoeConfig)
_, reason = _is_supported(deep_gemm, kMxfp4Static, None)
assert "routing method" in reason
with pytest.raises(ValueError, match="Unsupported fi_moe_ep megakernel"):
build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=None,
megakernel="deep_gemm",
megakernel_host.parallel_config.enable_eplb = True
_, reason = _is_supported(
_megakernel_moe(FLASHINFER_MOE_EP_DEEP_GEMM), kMxfp4Static, None
)
assert reason.endswith("EPLB")
def test_megakernels_are_oracle_backends():
"""Quant methods reach the megakernel through the NVFP4/MXFP4 oracles."""
nvfp4 = map_nvfp4_backend(FLASHINFER_MOE_EP_CUTEDSL)
assert nvfp4 is NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
assert nvfp4_backend_to_kernel_cls(nvfp4) == [FlashInferMoeEpExperts]
assert map_mxfp4_backend(FLASHINFER_MOE_EP_DEEP_GEMM) == [
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
]
for backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
assert mxfp4_backend_to_kernel_cls(backend) == [FlashInferMoeEpExperts]
# The megakernel takes the model's own dimensions.
assert mxfp4_round_up_hidden_size_and_intermediate_size(
backend, 2880, 2880
) == (2880, 2880)
quant_config = make_mxfp4_moe_quant_config(
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL, w1_scale=None, w2_scale=None
)
assert quant_config is not None and quant_config.use_nvfp4_w4a4
def test_megakernel_is_a_modular_kernel_with_pass_through_stages():
"""The megakernel plugs into ``FusedMoEKernel`` like any other experts impl.
Routing stays outside (top-k ids in), dispatch and combine happen inside,
so prepare passes tokens through untouched and finalize has nothing to do.
"""
moe = make_dummy_moe_config(
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
)
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
kernel = mk.FusedMoEKernel(
PassThroughPrepareAndFinalize(),
FlashInferMoeEpExperts(moe, quant_config),
)
assert not kernel.is_monolithic
assert kernel.prepare_finalize.output_is_reduced()
# Generic stages: the int32 top-k ids the megakernel wants are its concern.
assert kernel.prepare_finalize.topk_indices_dtype() is None
assert kernel.fused_experts.expects_unquantized_inputs
assert isinstance(
kernel.fused_experts.finalize_weight_and_reduce_impl(), TopKWeightAndReduceNoOP
)
assert kernel.fused_experts.workspace_shapes(
16, 256, 256, 2, 8, 8, None, MoEActivation.SILU
) == ((0,), (0,), (16, 256))
tokens = torch.zeros(4, 256, dtype=torch.bfloat16)
prepared = kernel.prepare_finalize.prepare(
tokens,
torch.ones(4, 2),
torch.zeros(4, 2, dtype=torch.int32),
8,
None,
False,
quant_config,
)
assert prepared[0] is tokens and all(item is None for item in prepared[1:])
def test_epilogue_takes_the_weight_global_scales_from_the_quant_config():
"""The canonical ``g1/g2_alphas`` become the per-expert fc1/fc2 alphas and
keep aliasing the layer's tensors, so EPLB permutations reach the kernel."""
g1_alphas = torch.tensor([1.0, 2.0])
g2_alphas = torch.tensor([3.0, 4.0])
quant_config = nvfp4_moe_quant_config(
g1_alphas=g1_alphas,
g2_alphas=g2_alphas,
a1_gscale=torch.ones(2),
a2_gscale=torch.ones(2),
w1_scale=torch.ones(2, 4, 1),
w2_scale=torch.ones(2, 2, 1),
)
epilogue = epilogue_from_quant_config(quant_config)
assert epilogue.fc1_alpha is not None and epilogue.fc2_alpha is not None
assert epilogue.fc1_alpha.data_ptr() == g1_alphas.data_ptr()
assert epilogue.fc2_alpha.data_ptr() == g2_alphas.data_ptr()
# Activations are quantized dynamically: no norm constants.
assert epilogue.input_norm_const == 1.0 and epilogue.fc1_norm_const is None
def test_epilogue_rejects_unloaded_weight_global_scales():
quant_config = nvfp4_moe_quant_config(
g1_alphas=torch.tensor([1.0, float("nan")]),
g2_alphas=torch.ones(2),
a1_gscale=torch.ones(2),
a2_gscale=torch.ones(2),
w1_scale=torch.ones(2, 4, 1),
w2_scale=torch.ones(2, 2, 1),
)
with pytest.raises(ValueError, match="g1_alphas"):
epilogue_from_quant_config(quant_config)
def test_mxfp4_checkpoints_keep_the_default_epilogue():
"""MXFP4 has no per-tensor global scales; the megakernel needs no alphas."""
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
assert epilogue_from_quant_config(quant_config) == fi_ep.FlashInferMoeEpEpilogue()
def test_nvfp4_kernel_format_passes_megakernel_weights_through():
"""The megakernel takes the canonical tensors as they are; the oracle must
not reject or reshape them."""
tensors = tuple(torch.arange(4.0).reshape(2, 2) + i for i in range(8))
converted = convert_to_nvfp4_moe_kernel_format(
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
SimpleNamespace(),
*tensors,
is_act_and_mul=True,
)
assert all(out is src for out, src in zip(converted, tensors))
def test_passthrough_all2all_backend_selects_the_pass_through_stages():
"""``all2all_backend="passthrough"`` is how the oracles get the
pass-through prepare/finalize."""
moe = make_dummy_moe_config(
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
)
moe.moe_parallel_config = replace(
moe.moe_parallel_config, all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND
)
assert not moe.moe_parallel_config.use_all2all_kernels
prepare_finalize = maybe_make_prepare_finalize(
moe, FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
)
assert isinstance(prepare_finalize, PassThroughPrepareAndFinalize)
def test_passthrough_all2all_backend_is_not_a_cli_choice():
"""Users cannot pick the passthrough all2all backend explicitly."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
with pytest.raises(SystemExit):
parser.parse_args(["--all2all-backend", PASSTHROUGH_ALL2ALL_BACKEND])
parser.parse_args(["--all2all-backend", "deepep_low_latency"])
def test_megakernel_moe_backend_binds_the_all2all_backend():
"""A megakernel deployment gets its all2all backend by default."""
parallel = SimpleNamespace(all2all_backend="allgather_reducescatter")
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
assert parallel.all2all_backend == PASSTHROUGH_ALL2ALL_BACKEND
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
with pytest.raises(ValueError, match="cannot be paired"):
bind_passthrough_all2all_backend(
FLASHINFER_MOE_EP_DEEP_GEMM,
SimpleNamespace(all2all_backend="deepep_low_latency"),
)
with pytest.raises(ValueError, match="only valid with"):
bind_passthrough_all2all_backend(
"flashinfer_trtllm",
SimpleNamespace(all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND),
)
def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
from vllm.models.deepseek_v4.nvidia.fi_moe import ckpt_uses_nvfp4_experts
nvfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo="NVFP4"))
assert ckpt_uses_nvfp4_experts(nvfp4)
mxfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo=None))
assert not ckpt_uses_nvfp4_experts(mxfp4)
no_algo = SimpleNamespace(quant_config=SimpleNamespace())
assert not ckpt_uses_nvfp4_experts(no_algo)
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
"""One 32-element scale group per row: low nibble is the even element,
high nibble the odd one, ue8m0 scale applies to the whole group."""
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
out = _dequant_fp4_ue8m0_gran32(packed, sf)
assert out.shape == (2, 32)
assert out.dtype == torch.bfloat16
expected = torch.empty(2, 32)
for row in range(2):
for col in range(16):
byte = int(packed[row, col])
expected[row, 2 * col] = _E2M1_LUT[byte & 0x0F]
expected[row, 2 * col + 1] = _E2M1_LUT[byte >> 4]
expected[row] *= 2.0**row
assert torch.equal(out, expected.to(torch.bfloat16))
untouched = SimpleNamespace(all2all_backend="deepep_low_latency")
bind_passthrough_all2all_backend("flashinfer_trtllm", untouched)
assert untouched.all2all_backend == "deepep_low_latency"
+1 -1
View File
@@ -657,7 +657,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.use_mega_moe = True
moe.use_native_mega_moe = True
moe.gate = FakeGate()
moe.experts = FakeExperts()
moe.shared_experts = FakeSharedExperts()
+1 -1
View File
@@ -241,7 +241,7 @@ def test_v41_dspark_loads_linear_scales(
linear_scale_name=scale_name,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
confidence_head=None,
),
named_parameters=lambda: [(runtime_name, param)],
-11
View File
@@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
import transformers
from packaging.version import Version
installed = Version(transformers.__version__)
required = Version("5.15.0.dev0")
if installed < required:
pytest.skip(
"MLA models with the Transformers modeling backend require "
f"transformers>={required}, but got {installed}"
)
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
args = (example_prompts, 32, 5)
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
-14
View File
@@ -609,17 +609,3 @@ def dummy_hf_overrides(
)
return hf_config
def check_transformers_version(
model: str,
min_transformers_version: str | None = None,
max_transformers_version: str | None = None,
):
from .registry import _HfExamplesInfo
return _HfExamplesInfo(
model,
min_transformers_version=min_transformers_version,
max_transformers_version=max_transformers_version,
).check_transformers_version(on_fail="skip")
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt4Static32,
kInt8StaticChannelSym,
kMxfp8Dynamic,
)
@@ -46,6 +47,12 @@ def test_quant_spec_string_representation_uses_quantization_name():
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
def test_quant_spec_resolves_groupwise_int4_weight():
spec = QuantSpec(weight="int4_per_group_32")
assert spec.weight == kInt4Static32
assert spec.activation is None
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
quant_spec(weight="not_a_real_format")
+426
View File
@@ -0,0 +1,426 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the `VLLM_ROCM_MOE_PADDING` weight memory-layout padding trick.
`VLLM_ROCM_MOE_PADDING` (see `UnquantizedFusedMoEMethod._maybe_pad_weight`)
is a memory-stride trick, not a logical-shape one: when eligible, it enlarges
a weight tensor's storage so per-expert allocations land farther apart in
HBM. Shape and values are unaffected, but `stride(-2)` increases by
`num_pad` elements -- that's the whole mechanism. For the **AITER** path
tested here, the padded storage never persists on `w13_weight`/`w2_weight`:
`convert_to_unquantized_kernel_format` reallocates via
`rocm_aiter_ops.shuffle_weights`. This is AITER-specific -- on TRITON-on-ROCm,
`.contiguous()` is skipped specifically to keep this same padding, so its
persisted parameters *do* stay padded.
This file verifies:
1. `_maybe_pad_weight` in isolation (any ROCm backend, AITER not required):
grows storage/`stride(-2)` only when eligible (512-byte-aligned) and the
flag is on -- covering no padding, hidden-only, intermediate-only, and
both-dimensions-padded, since w13 and w2 key their alignment gate off
different dims; a true no-op (identical object) otherwise.
2. End to end, AITER only (per #54966's "Test padding" ask):
`AiterExperts.apply()` matches an independent reference regardless of
the flag, and persisted parameter storage size is unaffected by it.
Logical `hidden_dim_unpadded`/`intermediate_size_per_partition_unpadded`
padding and HIP-graph token-padding are separate mechanisms, covered in
#59333 and #59334.
See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
"""
import pytest
import torch
import torch.nn.functional as F
import vllm.envs as envs
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
ExpertMapManager,
)
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
aiter_available = is_aiter_found_and_supported()
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="VLLM_ROCM_MOE_PADDING only takes effect on ROCm",
)
requires_aiter = pytest.mark.skipif(
not aiter_available,
reason="requires AITER to exercise AiterExperts.apply()",
)
DEVICE = current_platform.device_type
DTYPE = torch.bfloat16
NUM_EXPERTS = 4
TOPK = 2
NUM_TOKENS = 16
# Mirrors `_maybe_pad_weight`'s `num_pad` constant in unquantized_fused_moe_method.py.
PAD_BYTES = 256
# 256 bf16 elems * 2 bytes = 512 bytes: eligible for `_maybe_pad_weight` on
# both w13 and w2.
HIDDEN_SIZE = 256
INTERMEDIATE_SIZE = 256
def _set_padding_env(monkeypatch: pytest.MonkeyPatch, padding: bool) -> None:
"""Sets the flag only; `_maybe_pad_weight` reads it lazily, no AITER needed."""
monkeypatch.setenv("VLLM_ROCM_MOE_PADDING", "1" if padding else "0")
def _set_padding_env_with_aiter(monkeypatch: pytest.MonkeyPatch, padding: bool) -> None:
"""Also enables AITER, for tests routing through RoutedExperts's backend
selection."""
_set_padding_env(monkeypatch, padding)
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
def _make_moe_config(
hidden_size: int,
intermediate_size: int,
dtype: torch.dtype = DTYPE,
) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=NUM_EXPERTS,
experts_per_token=TOPK,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=NUM_EXPERTS,
num_logical_experts=NUM_EXPERTS,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=dtype,
device=DEVICE,
routing_method=RoutingMethodType.TopK,
max_num_tokens=NUM_TOKENS,
)
def _make_routed_experts(hidden_size: int, intermediate_size: int) -> RoutedExperts:
moe_config = _make_moe_config(hidden_size, intermediate_size)
expert_map_manager = ExpertMapManager(
max_num_batched_tokens=NUM_TOKENS,
top_k=TOPK,
global_num_experts=NUM_EXPERTS,
num_redundant_experts=0,
num_expert_group=None,
moe_parallel_config=moe_config.moe_parallel_config,
placement_strategy="linear",
enable_eplb=False,
)
return RoutedExperts(
"experts",
DTYPE,
moe_config,
quant_config=None,
expert_map_manager=expert_map_manager,
)
def _expert_weight_iterator(seed: int, hidden_size: int, intermediate_size: int):
set_random_seed(seed)
for expert_id in range(NUM_EXPERTS):
yield (
f"{expert_id}.gate_proj.weight",
torch.randn(
intermediate_size,
hidden_size,
device=DEVICE,
dtype=DTYPE,
),
)
yield (
f"{expert_id}.up_proj.weight",
torch.randn(
intermediate_size,
hidden_size,
device=DEVICE,
dtype=DTYPE,
),
)
yield (
f"{expert_id}.down_proj.weight",
torch.randn(
hidden_size,
intermediate_size,
device=DEVICE,
dtype=DTYPE,
),
)
def _load_and_process_weights(
layer: RoutedExperts, seed: int, hidden_size: int, intermediate_size: int
) -> None:
loaded = set(
layer.load_weights(
_expert_weight_iterator(seed, hidden_size, intermediate_size)
)
)
assert {"w13_weight", "w2_weight"} <= loaded
layer.quant_method.process_weights_after_loading(layer)
torch.accelerator.synchronize()
def _make_static_inputs(
hidden_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
set_random_seed(2026)
x = torch.randn(
NUM_TOKENS,
hidden_size,
device=DEVICE,
dtype=DTYPE,
)
topk_ids = torch.arange(NUM_TOKENS, device=DEVICE, dtype=torch.int64).unsqueeze(1)
topk_ids = torch.cat(
[topk_ids % NUM_EXPERTS, (topk_ids + 1) % NUM_EXPERTS],
dim=1,
)
topk_weights = torch.full(
(NUM_TOKENS, TOPK),
1.0 / TOPK,
device=DEVICE,
dtype=DTYPE,
)
return x, topk_weights, topk_ids
def _forward(
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
return layer.forward_modular(
x,
topk_weights,
topk_ids,
shared_experts=None,
shared_experts_input=None,
)
def _assert_backend_is_aiter(layer: RoutedExperts) -> None:
selected = layer.quant_method.unquantized_backend
assert selected is UnquantizedMoeBackend.AITER, (
f"expected AiterExperts to be selected, got backend={selected!r}"
)
# --- 1. `_maybe_pad_weight` in isolation -----------------------------------
@pytest.mark.parametrize(
"padding,hidden_size,intermediate_size,dtype,expect_padded",
[
# eligible shape (512 bytes): padding fires iff the env var is on.
(True, HIDDEN_SIZE, INTERMEDIATE_SIZE, DTYPE, True),
(False, HIDDEN_SIZE, INTERMEDIATE_SIZE, DTYPE, False),
# ineligible shape (400 bytes, not 512-aligned): never fires.
(True, 200, 200, DTYPE, False),
# fp32: alignment threshold and num_pad both scale with
# element_size() (128 fp32 elems * 4 bytes == 512).
(True, 128, 128, torch.float32, True),
],
ids=["eligible-on", "eligible-off", "ineligible-on", "eligible-on-fp32"],
)
def test_maybe_pad_weight_transparent(
monkeypatch: pytest.MonkeyPatch,
default_vllm_config,
padding: bool,
hidden_size: int,
intermediate_size: int,
dtype: torch.dtype,
expect_padded: bool,
) -> None:
"""Shape/values never change; storage only grows (`stride(-2) += num_pad`)
when eligible and the flag is on. Otherwise it's a true no-op (identical
object)."""
assert default_vllm_config is not None
_set_padding_env(monkeypatch, padding)
assert envs.VLLM_ROCM_MOE_PADDING is padding
moe_config = _make_moe_config(hidden_size, intermediate_size, dtype=dtype)
method = UnquantizedFusedMoEMethod(moe_config)
original = torch.randn(
NUM_EXPERTS,
2 * intermediate_size,
hidden_size,
device=DEVICE,
dtype=dtype,
)
result = method._maybe_pad_weight(original)
assert result.shape == original.shape
assert torch.equal(result, original), "padding must not alter weight values"
if expect_padded:
num_pad = PAD_BYTES // original.element_size()
assert result.stride(-1) == 1
assert result.stride(-2) == original.stride(-2) + num_pad
assert result.untyped_storage().nbytes() > original.untyped_storage().nbytes()
assert result.data_ptr() != original.data_ptr()
else:
assert result is original
@pytest.mark.parametrize(
"hidden_size,intermediate_size,expect_w13_padded,expect_w2_padded",
[
# only hidden_size aligned: only w13 (last dim hidden_size) eligible.
(HIDDEN_SIZE, 200, True, False),
# only intermediate_size aligned: only w2 (last dim
# intermediate_size) eligible.
(200, HIDDEN_SIZE, False, True),
],
ids=["only-w13-eligible", "only-w2-eligible"],
)
def test_maybe_pad_weight_asymmetric_w13_w2_eligibility(
monkeypatch: pytest.MonkeyPatch,
default_vllm_config,
hidden_size: int,
intermediate_size: int,
expect_w13_padded: bool,
expect_w2_padded: bool,
) -> None:
"""w13 and w2 key their alignment gate off different dims (hidden_size
vs. intermediate_size), so they're independently eligible."""
assert default_vllm_config is not None
_set_padding_env(monkeypatch, True)
moe_config = _make_moe_config(hidden_size, intermediate_size)
method = UnquantizedFusedMoEMethod(moe_config)
w13 = torch.randn(
NUM_EXPERTS, 2 * intermediate_size, hidden_size, device=DEVICE, dtype=DTYPE
)
w2 = torch.randn(
NUM_EXPERTS, hidden_size, intermediate_size, device=DEVICE, dtype=DTYPE
)
w13_result = method._maybe_pad_weight(w13)
w2_result = method._maybe_pad_weight(w2)
assert (w13_result is not w13) == expect_w13_padded
assert (w2_result is not w2) == expect_w2_padded
def _reference_moe_forward(
seed: int,
hidden_size: int,
intermediate_size: int,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""A minimal, independent (no AITER) reference MoE forward, so the E2E
test below can't pass merely because both flag settings are wrong the
same way."""
weights: dict[int, dict[str, torch.Tensor]] = {}
for name, tensor in _expert_weight_iterator(seed, hidden_size, intermediate_size):
expert_id_str, proj_name = name.split(".", 1)
weights.setdefault(int(expert_id_str), {})[proj_name] = tensor
output = torch.zeros_like(x)
for expert_id, proj in weights.items():
gate = F.silu(x @ proj["gate_proj.weight"].T)
up = x @ proj["up_proj.weight"].T
expert_out = (gate * up) @ proj["down_proj.weight"].T
mask = topk_ids == expert_id
weight = (topk_weights * mask).sum(dim=1, keepdim=True)
output = output + weight * expert_out
return output
# --- 2. End-to-end numerical transparency through AiterExperts -------------
@requires_aiter
@torch.inference_mode()
def test_aiter_moe_padding_numerically_transparent(
monkeypatch: pytest.MonkeyPatch,
default_vllm_config,
workspace_init,
) -> None:
"""`AiterExperts.apply()` output must match an independent reference
(not just itself across flag settings) for an eligible weight shape,
regardless of `VLLM_ROCM_MOE_PADDING`. Persisted parameter storage size
must also be unaffected by the flag (see module docstring)."""
assert default_vllm_config is not None
assert workspace_init is None
outputs: dict[bool, torch.Tensor] = {}
storage_nbytes: dict[bool, tuple[int, int]] = {}
for padding in (True, False):
_set_padding_env_with_aiter(monkeypatch, padding)
with torch.device(DEVICE):
layer = _make_routed_experts(HIDDEN_SIZE, INTERMEDIATE_SIZE)
_load_and_process_weights(
layer,
seed=1,
hidden_size=HIDDEN_SIZE,
intermediate_size=INTERMEDIATE_SIZE,
)
_assert_backend_is_aiter(layer)
storage_nbytes[padding] = (
layer.w13_weight.data.untyped_storage().nbytes(),
layer.w2_weight.data.untyped_storage().nbytes(),
)
x, topk_weights, topk_ids = _make_static_inputs(HIDDEN_SIZE)
outputs[padding] = _forward(layer, x, topk_weights, topk_ids).clone()
assert storage_nbytes[True] == storage_nbytes[False], (
"VLLM_ROCM_MOE_PADDING must not change the persisted parameters' "
"storage size for the AITER path"
)
reference = _reference_moe_forward(
seed=1,
hidden_size=HIDDEN_SIZE,
intermediate_size=INTERMEDIATE_SIZE,
x=x,
topk_weights=topk_weights,
topk_ids=topk_ids,
)
# Same AITER kernel, byte-identical weights and inputs either way (only
# a transient weight-loading buffer differs): bitwise-identical results.
torch.testing.assert_close(outputs[True], outputs[False], atol=0, rtol=0)
# Cosine similarity, not elementwise assert_close, for the reference
# comparison: two independently computed bf16 GEMM chains can have
# near-zero elements where rounding noise blows up elementwise relative
# error. Same pattern as test_b12x.py / test_mxfp4_moe.py.
for padded, output in outputs.items():
cos_sim = F.cosine_similarity(
output.flatten().float(), reference.flatten().float(), dim=0
)
assert cos_sim > 0.99, (
f"padding={padded} output diverged from reference "
f"(cosine similarity {cos_sim:.4f})"
)
+43 -1
View File
@@ -2,8 +2,17 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.outputs import RequestOutput
from vllm.outputs import (
ClassificationRequestOutput,
EmbeddingRequestOutput,
PoolingOutput,
PoolingRequestOutput,
RequestError,
RequestOutput,
ScoringRequestOutput,
)
pytestmark = pytest.mark.cpu_test
@@ -19,3 +28,36 @@ def test_request_output_forward_compatible():
example_arg_added_in_new_version="some_value",
)
assert output is not None
@pytest.mark.parametrize(
("output_type", "empty_value"),
[
(EmbeddingRequestOutput, []),
(ClassificationRequestOutput, []),
(ScoringRequestOutput, 0.0),
],
)
@pytest.mark.skip_global_cleanup
def test_specialized_pooling_outputs_preserve_request_error(output_type, empty_value):
error = RequestError(
code="multimodal_cache_miss",
message="Multi-modal processor cache miss.",
retryable=True,
)
base_output = PoolingRequestOutput(
request_id="request",
outputs=PoolingOutput(torch.empty(0)),
prompt_token_ids=[1, 2],
num_cached_tokens=0,
finished=True,
error=error,
)
output = output_type.from_base(base_output)
assert output.error is error
value = getattr(output.outputs, "embedding", None)
value = getattr(output.outputs, "probs", value)
value = getattr(output.outputs, "score", value)
assert value == empty_value
-49
View File
@@ -10,7 +10,6 @@ from transformers import AutoConfig
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.hf import CachedHfTokenizer
from vllm.tokenizers.registry import (
TokenizerRegistry,
cached_get_tokenizer,
@@ -93,54 +92,6 @@ def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
)
@pytest.mark.parametrize(
("transformers_version", "model_type", "native_filename", "expect_error"),
[
pytest.param("5.14.0", "mistral", "tekken.json", True),
pytest.param("5.15.0", "mistral", "tekken.json", False),
pytest.param("5.14.0", "mistral3", "tekken.json", True),
pytest.param("5.15.0", "mistral3", "tekken.json", False),
pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
pytest.param("5.14.0", "mistral", None, False),
pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
],
)
def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
tmp_path: Path,
transformers_version: str,
model_type: str,
native_filename: str | None,
expect_error: bool,
):
if native_filename is not None:
(tmp_path / native_filename).touch()
with (
patch(
"vllm.tokenizers.registry.get_config",
return_value=SimpleNamespace(model_type=model_type),
),
patch(
"vllm.tokenizers.registry.transformers.__version__",
transformers_version,
),
patch.object(
CachedHfTokenizer,
"from_pretrained",
return_value=SimpleNamespace(is_fast=True),
) as from_pretrained,
):
if expect_error:
with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
get_tokenizer(str(tmp_path), tokenizer_mode="hf")
else:
tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
assert tokenizer.is_fast is True
assert from_pretrained.called == (not expect_error)
def test_customized_tokenizer():
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
@@ -3,11 +3,14 @@
import json
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
from openai.types.responses import ToolChoiceFunction
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
from xgrammar import Grammar, StructuralTag
from xgrammar import get_model_structural_tag as get_xgrammar_builtin_structural_tag
from xgrammar.testing import _is_grammar_accept_string
import vllm.envs as envs
@@ -1311,6 +1314,536 @@ def test_tool_strict_level_from_name():
ToolStrictLevel.from_name("off")
# ---------------------------------------------------------------------------
# GLM-4.7 structural tag: builtin content for strict tools, shallow otherwise
# ---------------------------------------------------------------------------
def _glm47_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"days": {"type": "integer"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["city"],
},
},
),
ChatCompletionToolsParam(
type="function",
function={
"name": "run_command",
"parameters": {
"type": "object",
"properties": {"command": {"type": "string"}},
},
},
),
]
def _glm47_arg(key: str, value: str) -> str:
return f"<arg_key>{key}</arg_key><arg_value>{value}</arg_value>"
def _glm47_call(name: str, *args: str) -> str:
return f"<tool_call>{name}{''.join(args)}</tool_call>"
def _glm47_shallow_grammar(
tool_choice, tools=None, reasoning: bool = False, *, auto: bool = False
):
# auto needs the FUNCTION level to lift the non-strict auto gate.
tag = get_model_structural_tag(
model="glm_4_7",
tools=tools if tools is not None else _glm47_tools(),
tool_choice=tool_choice,
reasoning=reasoning,
strict_level=ToolStrictLevel.FUNCTION if auto else ToolStrictLevel.AUTO,
)
assert isinstance(tag, StructuralTag)
return Grammar.from_structural_tag(tag)
@pytest.mark.parametrize(
"body",
[
# plain text, no tool call
"Hello there",
# text followed by a call
"Sure." + _glm47_call("get_weather", _glm47_arg("city", "Paris")),
# call only, required arg present
_glm47_call("get_weather", _glm47_arg("city", "Paris")),
# all arguments omitted
_glm47_call("get_weather"),
# repeated argument
_glm47_call(
"get_weather", _glm47_arg("city", "Paris"), _glm47_arg("city", "Rome")
),
# arguments in reverse declaration order
_glm47_call(
"get_weather", _glm47_arg("days", "3"), _glm47_arg("city", "Paris")
),
# enum-constrained value
_glm47_call("get_weather", _glm47_arg("unit", "celsius")),
# string values with spaces and a bare '<'
_glm47_call("run_command", _glm47_arg("command", "grep 'a<b' x.py")),
# second tool, two calls
_glm47_call("run_command", _glm47_arg("command", "ls -la"))
+ _glm47_call("get_weather", _glm47_arg("city", "Paris")),
],
)
def test_glm47_non_strict_auto_accepts_valid_tool_calls(body: str):
grammar = _glm47_shallow_grammar("auto", auto=True)
assert _is_grammar_accept_string(grammar, body)
@pytest.mark.parametrize(
"body",
[
# unknown tool name
_glm47_call("unknown_tool", _glm47_arg("city", "Paris")),
# undeclared argument key
_glm47_call("get_weather", _glm47_arg("zzz", "x")),
# structural marker inside a string value
_glm47_call("get_weather", _glm47_arg("city", "a<arg_key>b")),
_glm47_call("get_weather", _glm47_arg("city", "a</tool_call>b")),
# integer-typed value given non-numeric text
_glm47_call("get_weather", _glm47_arg("days", "abc")),
# enum-typed value outside the enum
_glm47_call("get_weather", _glm47_arg("unit", "kelvin")),
# unterminated tool call
"<tool_call>get_weather" + _glm47_arg("city", "Paris"),
],
)
def test_glm47_non_strict_auto_rejects_invalid(body: str):
grammar = _glm47_shallow_grammar("auto", auto=True)
assert not _is_grammar_accept_string(grammar, body)
def test_glm47_non_strict_required_requires_at_least_one_call():
grammar = _glm47_shallow_grammar("required")
assert not _is_grammar_accept_string(grammar, "Just answering.")
assert _is_grammar_accept_string(
grammar, _glm47_call("get_weather", _glm47_arg("city", "Paris"))
)
assert _is_grammar_accept_string(
grammar,
_glm47_call("run_command", _glm47_arg("command", "ls"))
+ _glm47_call("get_weather", _glm47_arg("city", "Paris")),
)
def test_glm47_non_strict_forced_emits_single_named_call():
grammar = _glm47_shallow_grammar(
ChatCompletionNamedToolChoiceParam(
type="function",
function=ChatCompletionNamedFunction(name="get_weather"),
)
)
assert _is_grammar_accept_string(
grammar, _glm47_call("get_weather", _glm47_arg("city", "Paris"))
)
assert not _is_grammar_accept_string(grammar, "No call here")
assert not _is_grammar_accept_string(grammar, _glm47_call("run_command"))
assert not _is_grammar_accept_string(
grammar,
_glm47_call("get_weather", _glm47_arg("city", "Paris"))
+ _glm47_call("run_command"),
)
def test_glm47_non_strict_reasoning_gates_on_think_close():
grammar = _glm47_shallow_grammar("required", reasoning=True)
call = _glm47_call("get_weather", _glm47_arg("city", "Paris"))
assert _is_grammar_accept_string(grammar, "thinking...</think>" + call)
assert not _is_grammar_accept_string(grammar, call)
@pytest.mark.parametrize(
("parameters", "key"),
[
(
{
"$ref": "#/$defs/args",
"$defs": {
"args": {
"type": "object",
"properties": {"city": {"type": "string"}},
}
},
},
"city",
),
(
{
"anyOf": [
{"type": "object", "properties": {"city": {"type": "string"}}},
{"type": "null"},
]
},
"city",
),
(
{"allOf": [{"type": "object", "properties": {"city": {"type": "string"}}}]},
"city",
),
({"type": "object", "patternProperties": {"^arg": {"type": "string"}}}, "city"),
({"type": "object"}, "city"),
({}, "city"),
(
{
"type": "object",
"properties": {"city": {"type": "string"}},
"additionalProperties": True,
},
"undeclared_key",
),
],
)
def test_glm47_non_plain_object_schemas_keep_free_arguments(parameters, key):
# Schemas that are not a plain object with declared properties fall back
# to the builtin's non-strict envelope content, which accepts any key.
tools = [
ChatCompletionToolsParam(
type="function", function={"name": "f", "parameters": parameters}
)
]
tag = get_model_structural_tag(
model="glm_4_7",
tools=tools,
tool_choice="required",
reasoning=False,
)
assert isinstance(tag, StructuralTag)
grammar = Grammar.from_structural_tag(tag)
call = _glm47_call("f", _glm47_arg(key, "Paris"))
assert _is_grammar_accept_string(grammar, call)
def _glm47_strict_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"strict": True,
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["city"],
"additionalProperties": False,
},
},
),
ChatCompletionToolsParam(
type="function",
function={
"name": "run_command",
"strict": True,
"parameters": {
"type": "object",
"properties": {"command": {"type": "string"}},
},
},
),
ChatCompletionToolsParam(
type="function",
function={"name": "no_params", "strict": True, "parameters": None},
),
]
@pytest.mark.parametrize(
("tool_choice", "reasoning"),
[
("auto", False),
("required", False),
("named", False),
("required", True),
("allowed", False),
],
)
def test_glm47_strict_only_matches_xgrammar_builtin(tool_choice: str, reasoning: bool):
# Strict-only requests stay byte-identical to the xgrammar builtin.
strict_tools = _glm47_strict_tools()
dumped = [
{"type": "function", "function": t.function.model_dump(exclude_none=True)}
for t in strict_tools
]
if tool_choice == "named":
choice = ChatCompletionNamedToolChoiceParam(
type="function",
function=ChatCompletionNamedFunction(name="run_command"),
)
builtin_choice: Any = {"type": "function", "function": {"name": "run_command"}}
elif tool_choice == "allowed":
choice = ToolChoiceAllowed(
type="allowed_tools",
mode="auto",
tools=[{"type": "function", "function": {"name": "get_weather"}}],
)
builtin_choice = {
"type": "allowed_tools",
"allowed_tools": {
"mode": "auto",
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
},
}
else:
choice = tool_choice
builtin_choice = tool_choice
ours = get_model_structural_tag(
model="glm_4_7",
tools=strict_tools,
tool_choice=choice,
reasoning=reasoning,
)
expected = get_xgrammar_builtin_structural_tag(
model="glm_4_7",
tools=dumped,
tool_choice=builtin_choice,
reasoning=reasoning,
)
assert isinstance(ours, StructuralTag)
assert ours.model_dump() == expected.model_dump()
def test_glm47_parameter_level_pins_schemas_like_the_builtin(
sample_tools: list[ChatCompletionToolsParam],
):
# The operator floor overrides per-tool strictness: every tool is pinned.
ours = get_model_structural_tag(
model="glm_4_7",
tools=sample_tools,
tool_choice="required",
reasoning=False,
strict_level=ToolStrictLevel.PARAMETER,
)
expected = get_xgrammar_builtin_structural_tag(
model="glm_4_7",
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"strict": True,
"parameters": sample_tools[0].function.parameters,
},
}
],
tool_choice="required",
reasoning=False,
)
assert isinstance(ours, StructuralTag)
assert ours.model_dump() == expected.model_dump()
def test_glm47_mixed_tools_dispatch_per_tool(
sample_tools_strict: list[ChatCompletionToolsParam],
sample_tools: list[ChatCompletionToolsParam],
):
mixed = [sample_tools_strict[0], sample_tools[0]]
mixed[1] = mixed[1].model_copy(deep=True)
mixed[1].function.name = "run_command"
tag = get_model_structural_tag(
model="glm_4_7",
tools=mixed,
tool_choice="required",
reasoning=False,
)
assert isinstance(tag, StructuralTag)
tags = tag.model_dump()["format"]["tags"]
assert tags[0]["content"]["json_schema"] == (
sample_tools_strict[0].function.parameters
)
assert tags[1]["content"]["type"] != "json_schema"
def _glm47_parser(
tools, monkeypatch: pytest.MonkeyPatch, *, strict_calling: bool = True
) -> Glm47MoeModelToolParser:
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", strict_calling)
return Glm47MoeModelToolParser(MagicMock(), tools=tools)
@pytest.mark.parametrize(
("tool_choice", "tools"),
[("auto", "plain"), ("required", "plain"), ("required", "strict")],
)
def test_glm47_get_structural_tag_disabled_when_flag_off(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
sample_tools_strict: list[ChatCompletionToolsParam],
tool_choice: str,
tools: str,
):
parser = _glm47_parser(sample_tools, monkeypatch, strict_calling=False)
request = ChatCompletionRequest(
messages=[],
model="m",
tools=sample_tools_strict if tools == "strict" else sample_tools,
tool_choice=tool_choice,
)
assert parser.get_structural_tag(request) is None
def test_glm47_default_strict_level_constrains_auto(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
):
# --tool-strict-level auto: the parser's FUNCTION default lifts the
# non-strict auto gate.
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
class TestParser(DelegatingParser):
tool_parser_cls = Glm47MoeModelToolParser
request = ChatCompletionRequest(
messages=[],
model="m",
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
tool_choice="auto",
)
parser = TestParser(MagicMock(), tools=sample_tools)
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
assert out.structured_outputs.json is None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
call = (
"<tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Paris</arg_value></tool_call>"
)
assert _is_grammar_accept_string(grammar, "Sure. " + call)
assert _is_grammar_accept_string(grammar, "Plain answer.")
assert not _is_grammar_accept_string(
grammar, "Sure. " + call.replace("city", "ctiy")
)
def test_glm47_responses_request_attaches_shallow_tag(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
class TestParser(DelegatingParser):
tool_parser_cls = Glm47MoeModelToolParser
request = ResponsesRequest.model_validate(
{
"input": "hi",
"tools": [
{
"type": "function",
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
}
],
"tool_choice": "required",
}
)
parser = TestParser(MagicMock(), tools=None)
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
assert out.structured_outputs.structural_tag is not None
assert out.text is None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
assert _is_grammar_accept_string(
grammar,
"<tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Paris</arg_value></tool_call>",
)
def test_glm47_operator_level_overrides_parser_default(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
):
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
class TestParser(DelegatingParser):
tool_parser_cls = Glm47MoeModelToolParser
tool_strict_level = ToolStrictLevel.PARAMETER
request = ChatCompletionRequest(
messages=[],
model="m",
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
tool_choice="required",
)
parser = TestParser(MagicMock(), tools=sample_tools)
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
content = json.loads(out.structured_outputs.structural_tag)["format"]["tags"][0][
"content"
]
assert content["json_schema"] == sample_tools[0].function.parameters
def test_glm47_required_with_non_strict_tools_attaches_shallow_tag(
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
):
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
class TestParser(DelegatingParser):
tool_parser_cls = Glm47MoeModelToolParser
request = ChatCompletionRequest(
messages=[],
model="m",
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
tool_choice="required",
)
parser = TestParser(MagicMock(), tools=sample_tools)
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
out = parser.adjust_request(request)
assert out.structured_outputs is not None
assert out.structured_outputs.json is None
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
call = (
"<tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Paris</arg_value></tool_call>"
)
assert _is_grammar_accept_string(grammar, call)
assert not _is_grammar_accept_string(grammar, "It is sunny in Paris.")
assert not _is_grammar_accept_string(grammar, call.replace("city", "ctiy"))
@pytest.mark.parametrize("policy", ["auto", "required", "named"])
def test_mimo_strict_compact_xml(policy):
tools = [
+129
View File
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the agent-file pre-commit check.
The hook passes silently on a healthy tree, so CI would not notice a rule that
stopped firing. Each case breaks one convention in a minimal repo and asserts
the check reports it.
"""
import os
from pathlib import Path
import pytest
from tools.pre_commit.check_agent_files import check
SKILL_MD = ".agents/skills/demo/SKILL.md"
SKILL = "---\nname: demo\ndescription: Demo things. Use when testing.\n---\n"
def write(path: str, text: str) -> None:
Path(path).parent.mkdir(parents=True, exist_ok=True)
Path(path).write_text(text)
def tracked() -> dict[str, str]:
"""Mimic `git ls-files -s` for every file and symlink under the cwd."""
files = {}
for root, dirs, names in os.walk("."):
for name in dirs + names:
path = os.path.normpath(os.path.join(root, name))
if os.path.islink(path):
files[path] = "120000"
elif os.path.isfile(path):
files[path] = "100644"
return files
@pytest.fixture(autouse=True)
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
write("AGENTS.md", "See the [guide](docs/guide.md#some-heading).\n")
write("docs/guide.md", "## Some Heading\n")
write(SKILL_MD, SKILL)
os.makedirs(".claude/skills")
os.symlink("../../.agents/skills/demo", ".claude/skills/demo")
def test_valid_repo_passes():
assert check(tracked()) == ([], [])
@pytest.mark.parametrize(
("break_repo", "expected"),
[
pytest.param(
lambda: os.unlink(".claude/skills/demo"),
"must be a symlink",
id="skill-not-linked",
),
pytest.param(
lambda: write(".claude/skills/other/SKILL.md", SKILL),
"only symlinks",
id="real-file-in-claude-skills",
),
pytest.param(
lambda: os.symlink("../../.agents/skills/gone", ".claude/skills/gone"),
"only symlinks",
id="dangling-link",
),
pytest.param(
lambda: write(SKILL_MD, SKILL.replace("demo", "other")),
"must be 'demo'",
id="name-differs-from-dir",
),
pytest.param(
lambda: write(SKILL_MD, SKILL.replace("Demo things", "Demo: things")),
"not valid YAML",
id="unquoted-colon-in-description",
),
pytest.param(
lambda: write(SKILL_MD, "# Demo\n"),
"missing YAML frontmatter",
id="no-frontmatter",
),
pytest.param(
lambda: write("vllm/CLAUDE.md", "@AGENTS.md\n"),
"remove it",
id="claude-md",
),
pytest.param(
lambda: write("AGENTS.md", "line\n" * 201),
"200-line budget",
id="root-guide-over-budget",
),
pytest.param(
lambda: write("vllm/AGENTS.md", "line\n" * 301),
"300-line budget",
id="domain-guide-over-budget",
),
pytest.param(
lambda: write(SKILL_MD, SKILL + "line\n" * 500),
"500-line budget",
id="skill-over-budget",
),
pytest.param(
lambda: write("AGENTS.md", "[gone](docs/gone.md)\n"),
"does not exist",
id="dead-link",
),
pytest.param(
lambda: write(SKILL_MD, SKILL + "[guide](../../../docs/guide.md#gone)\n"),
"no heading",
id="dead-anchor",
),
],
)
def test_broken_convention_is_reported(break_repo, expected):
break_repo()
errors, _ = check(tracked())
assert len(errors) == 1
assert expected in errors[0]
def test_description_without_trigger_is_advisory():
write(SKILL_MD, SKILL.replace(" Use when testing.", ""))
errors, notes = check(tracked())
assert not errors
assert "when to use" in notes[0]
@@ -23,6 +23,8 @@ from vllm.v1.attention.backends.mla.compressor_utils import (
from vllm.v1.attention.backends.mla.indexer import (
BuildPrefillChunkMetadataKernel,
DeepseekV4IndexerBackend,
DeepSeekV32IndexerDecodeMetadata,
DeepseekV32IndexerMetadata,
DeepseekV32IndexerMetadataBuilder,
DeepseekV41IndexerBackend,
get_max_prefill_buffer_size,
@@ -99,6 +101,72 @@ def test_fused_indexer_decode_metadata(query_lens, padding):
torch.testing.assert_close(per_req[:reqs], lengths)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_indexer_draft_decode_metadata_update_is_capture_safe():
if not indexer_module.has_deep_gemm():
pytest.skip("requires DeepGEMM")
device = torch.device("cuda")
num_reqs = 3
seq_lens = torch.tensor([17, 33, 65], dtype=torch.int32, device=device)
decode_seq_lens = torch.zeros((num_reqs, 1), dtype=torch.int32, device=device)
block_table = torch.arange(num_reqs * 4, dtype=torch.int32, device=device).view(
num_reqs, 4
)
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
schedule_metadata = indexer_module.get_paged_mqa_logits_metadata(
torch.div(seq_lens, 16, rounding_mode="floor").unsqueeze(1),
64,
num_sms,
).clone()
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = 16
builder.num_sms = num_sms
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(num_states=64)
metadata = DeepseekV32IndexerMetadata(
seq_lens=seq_lens,
max_seq_len=128,
slot_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
num_decodes=num_reqs,
num_decode_tokens=num_reqs,
num_prefills=0,
num_prefill_tokens=0,
decode=DeepSeekV32IndexerDecodeMetadata(
block_table=block_table,
seq_lens=decode_seq_lens,
decode_lens=torch.zeros(num_reqs, dtype=torch.int32, device=device),
requires_padding=False,
schedule_metadata=schedule_metadata,
),
)
builder.update_draft_decode_metadata(metadata)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
builder.update_draft_decode_metadata(metadata)
seq_lens.copy_(torch.tensor([32, 48, 80], dtype=torch.int32, device=device))
graph.replay()
torch.accelerator.synchronize()
expected_decode_seq_lens = torch.div(seq_lens, 16, rounding_mode="floor")
torch.testing.assert_close(decode_seq_lens.flatten(), expected_decode_seq_lens)
torch.testing.assert_close(
schedule_metadata,
indexer_module.get_paged_mqa_logits_metadata(
expected_decode_seq_lens.unsqueeze(1), 64, num_sms
),
)
assert metadata.decode is not None
assert torch.all(metadata.decode.decode_lens == 1)
expected_slot_mapping = block_table[:, 0].to(torch.int64) * 64 + torch.tensor(
[1, 2, 4], dtype=torch.int64, device=device
)
torch.testing.assert_close(metadata.slot_mapping, expected_slot_mapping)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0], [0, 0, 0]]
@@ -300,6 +300,16 @@ def test_builder_build_falls_back_without_positions():
assert meta.slot_mapping is cam.slot_mapping
def test_builder_updates_draft_mapping():
builder = make_tail_builder()
cam = make_common_metadata([[15], [16]], [5, 9])
meta = KpoolTailMetadataBuilder.build(builder, 0, cam)
assert cam.positions is not None
cam.positions.add_(1)
builder.update_draft_decode_metadata(meta)
assert meta.slot_mapping[:2].tolist() == [5 * KPOOL, 9 * KPOOL + 1]
def test_builder_reuses_slot_mapping_storage():
builder = make_tail_builder()
first = make_common_metadata([list(range(10))], [5])
@@ -65,6 +65,9 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
)
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backends.mla import (
flashattn_mla_sparse as flashattn_sparse_module,
)
from vllm.v1.attention.backends.mla import index_group as index_group_module
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
FlashAttnMLASparseImpl,
@@ -3405,6 +3408,129 @@ def test_flashinfer_hisparse_decode_runs_batched_attention():
assert lse is None
@pytest.mark.skipif(
not current_platform.is_device_capability_family(90),
reason="FlashAttention MLA requires Hopper",
)
@pytest.mark.parametrize("use_cuda_graph", [False, True])
@pytest.mark.parametrize("block_stride_rows", [64, 128])
@pytest.mark.parametrize(
"query_len,num_decode_tokens,max_decode_rows,num_layers",
[
pytest.param(1, 2, 2, 4, id="decode"),
pytest.param(4, 8, 8, 4, id="mtp4"),
pytest.param(6, 12, 12, 4, id="mtp6"),
pytest.param(4, 8, 2, 4, id="scratch_overflow"),
pytest.param(4, 4, 8, 4, id="mixed"),
pytest.param(4, 0, 8, 4, id="prefill"),
pytest.param(4, 8, 8, 1, id="single_layer"),
],
)
def test_flashattn_shared_indices_match_independent_conversion(
monkeypatch,
query_len,
num_decode_tokens,
max_decode_rows,
num_layers,
block_stride_rows,
use_cuda_graph,
):
"""Reuse must preserve attention outputs as requests and pages change."""
device = torch.device(DEVICE_TYPE)
num_tokens, block_size, topk, seq_len = 2 * query_len, 64, 2048, 8192
blocks_per_req = seq_len // block_size
logical = torch.arange(
0, seq_len, seq_len // topk, dtype=torch.int32, device=device
).repeat(num_tokens, 1)
logical[:, ::7] = -1
builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=max_decode_rows)
metadata = SimpleNamespace(
block_table=torch.randperm(
2 * blocks_per_req, dtype=torch.int32, device=device
).view(2, blocks_per_req),
block_size=block_size,
req_id_per_token=torch.arange(
2, dtype=torch.int32, device=device
).repeat_interleave(query_len),
num_decode_tokens=num_decode_tokens,
)
q_nope = torch.randn(num_tokens, 8, 512, dtype=torch.bfloat16, device=device)
q_rope = torch.randn(num_tokens, 8, 64, dtype=torch.bfloat16, device=device)
caches = torch.randn(
num_layers,
2 * blocks_per_req,
block_stride_rows,
576,
dtype=torch.bfloat16,
device=device,
)[:, :, :block_size]
impls = []
for i in range(num_layers):
impl = object.__new__(FlashAttnMLASparseImpl)
impl.topk_indices_buffer = logical
impl.index_group, impl.index_group_index = builder.register_layer(i == 0)
impl.dcp_world_size = 1
impl.kv_lora_rank, impl.qk_rope_head_dim = 512, 64
impl.scale = 256**-0.5
impl.cu_seqlens_q_buffer = torch.arange(
num_tokens + 1, dtype=torch.int32, device=device
)
impls.append(impl)
convert = triton_convert_req_index_to_global_index
convert_spy = MagicMock(wraps=convert)
for module in (index_group_module, flashattn_sparse_module):
monkeypatch.setattr(
module, "triton_convert_req_index_to_global_index", convert_spy
)
def run():
impls[0].record_logical_topk_ready()
return [
impl.forward_mqa((q_nope, q_rope), cache, metadata, None)[0]
for impl, cache in zip(impls, caches)
]
stream = torch.cuda.Stream(device=device)
stream.wait_stream(current_stream())
with torch.cuda.stream(stream):
run() # Warm up the conversion and FA3 kernels before capture.
stream.synchronize()
convert_spy.reset_mock()
if use_cuda_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
outputs = run()
else:
outputs = run()
reuse = num_decode_tokens == num_tokens and num_tokens <= max_decode_rows + 1
assert convert_spy.call_count == (1 if reuse else num_layers)
for _ in range(2):
metadata.req_id_per_token.copy_(metadata.req_id_per_token.flip(0))
metadata.block_table.copy_(metadata.block_table.roll(1, dims=1))
logical.copy_(torch.where(logical >= 0, (logical + 1) % seq_len, logical))
if use_cuda_graph:
graph.replay()
else:
outputs = run()
physical, counts = convert(
metadata.req_id_per_token,
metadata.block_table,
logical,
BLOCK_SIZE=block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk,
return_valid_counts=True,
)
for impl, cache, actual in zip(impls, caches, outputs):
expected = impl._run_mqa_kernel(
q_nope, q_rope, cache, physical, counts, block_size
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
stream.synchronize()
def test_flashattn_hisparse_decode_uses_index_group():
num_tokens = 4
q_nope = torch.empty(num_tokens, 2, 3, device=DEVICE_TYPE)
+52
View File
@@ -4,6 +4,8 @@
the hit to rebuild the non-cacheable sliding-window group, keeps the hit's
blocks, and hands the worker the replay start."""
import dataclasses
import pytest
import torch
@@ -277,3 +279,53 @@ def test_hit_no_longer_than_window_is_ignored(via_connector):
assert new_req.num_computed_tokens == 0
assert new_req.replay_start == 0
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS
@pytest.mark.parametrize("is_async", [True, False])
@pytest.mark.parametrize("loads_window", [True, False])
def test_kv_load_zeroing_respects_loaded_groups(is_async, loads_window):
"""Only groups restored by an async load skip zeroing. P/D loads include
the non-cacheable window; stores leave it to be initialized locally."""
local = 5 * BLOCK_SIZE
scheduler = _replay_scheduler(use_kv_connector=MockKVConfig())
if loads_window:
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
assert scheduler.needs_kv_cache_zeroing
first = create_requests(
num_requests=1, num_tokens=local, same_prompt=True, block_size=BLOCK_SIZE
)[0]
_prefill(scheduler, first)
scheduler.connector.config = dataclasses.replace(
scheduler.connector.config,
matched_tokens=NUM_PROMPT_TOKENS - 1 - local,
is_async=is_async,
)
request = create_requests(
num_requests=1,
num_tokens=NUM_PROMPT_TOKENS,
same_prompt=True,
block_size=BLOCK_SIZE,
req_ids=["load"],
)[0]
scheduler.add_request(request)
out = scheduler.schedule()
manager = scheduler.kv_cache_manager
hit_ids = {
b.block_id for g in manager.get_blocks(first.request_id).blocks for b in g
}
swa_blocks = manager.get_blocks(request.request_id).blocks[SWA]
assert any(
not b.is_null and b.block_id not in hit_ids
for b in swa_blocks[: local // BLOCK_SIZE]
)
zeroed = set(out.new_block_ids_to_zero or ())
for group_id, blocks in enumerate(manager.get_blocks(request.request_id).blocks):
new_ids = {
b.block_id for b in blocks if not b.is_null and b.block_id not in hit_ids
}
assert new_ids
if is_async and (group_id == FULL or loads_window):
assert not new_ids & zeroed
else:
assert new_ids <= zeroed
@@ -16,6 +16,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
KpoolTailManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
@@ -332,6 +334,37 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
assert manager.req_to_blocks[request_id] == blocks
@pytest.mark.parametrize("record_for_zeroing", [True, False])
def test_external_kpool_tail_zeroing(record_for_zeroing):
"""A transferred ring at index zero skips zeroing even with a local hit."""
spec = KpoolTailSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
sliding_window=4,
)
manager = KpoolTailManager(
spec,
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=4,
needs_kv_cache_zeroing=True,
max_admission_blocks_per_request=1,
)
manager.allocate_external_computed_blocks(
"request", 16, 4, record_for_zeroing=record_for_zeroing
)
blocks = manager.req_to_blocks["request"]
assert len(blocks) == 1
assert manager.take_new_block_ids() == (
[blocks[0].block_id] if record_for_zeroing else []
)
assert manager.allocate_new_blocks("request", 20, 20) == []
assert manager.take_new_block_ids() == []
def test_sliding_window_records_new_blocks_for_zeroing():
block_size = 2
spec = SlidingWindowSpec(
@@ -192,6 +192,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
full_sequence_must_fit: bool = False,
reserved_blocks: int = 0,
has_scheduled_reqs: bool = True,
skip_zeroing_group_ids: tuple[int, ...] = (),
):
ret = original_allocate_slots_fn(
self,
@@ -206,6 +207,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
full_sequence_must_fit,
reserved_blocks,
has_scheduled_reqs,
skip_zeroing_group_ids,
)
if cur_step_action is not None:
cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[
+2 -4
View File
@@ -491,7 +491,7 @@ async def test_streaming_input_per_chunk_sampling_params(engine: AsyncLLM):
async def test_streaming_input_empty_generator(engine: AsyncLLM):
"""Test behavior when the input generator yields nothing.
An empty generator should still produce a finished output.
An empty generator should finish without producing output.
"""
request_id = "test_empty_generator"
sampling_params = get_sampling_params(max_tokens=10)
@@ -505,9 +505,7 @@ async def test_streaming_input_empty_generator(engine: AsyncLLM):
async for output in engine.generate(empty_generator(), sampling_params, request_id):
outputs.append(output)
# Should still get a finished marker
assert len(outputs) >= 1, "Should receive at least one output"
assert outputs[-1].finished, "Should have a finished output"
assert outputs == []
@pytest.mark.asyncio(loop_scope="module")
@@ -150,6 +150,21 @@ def test_draft_model_quantization(
assert_draft_model_correctness(sd_case, vllm_runner)
@single_gpu_only
def test_draft_model_transformers_backend(vllm_runner):
"""Ensure that Attention prefixes don't collide between the target and
draft models when both models use the Transformers modelling backend."""
sd_case = ArgsTest(
target_model="allenai/OLMo-2-0425-1B-Instruct",
draft_model="allenai/OLMo-2-0425-1B-Instruct",
sampling_config=greedy_sampling(),
num_speculative_tokens=3,
expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1
expected_acceptance_rate=0.98, # slight epsilon
)
assert_draft_model_correctness(sd_case, vllm_runner)
@multi_gpu_only(num_gpus=2)
def test_draft_model_tensor_parallelism(vllm_runner):
"""Ensure spec decode works when running with TP > 1."""
@@ -48,16 +48,6 @@ def check_mtp_correctness(
elif "gemma-4" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
if draft_model is not None and "gemma-4" in draft_model:
import transformers
from packaging.version import Version
if Version(transformers.__version__) < Version("5.8.0"):
pytest.skip(
"Gemma4 MTP assistant requires transformers>=5.8.0, "
f"got {transformers.__version__}"
)
with vllm_runner(
model_name,
block_size=None,
+34
View File
@@ -2,11 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from typing import TYPE_CHECKING
from unittest.mock import MagicMock, call
import pytest
from vllm import LLM
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.engine.llm_engine import LLMEngine
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
if TYPE_CHECKING:
@@ -18,6 +20,38 @@ MODEL = "facebook/opt-125m"
DTYPE = "half"
@pytest.mark.skip_global_cleanup
def test_step_invalidates_mm_processor_cache_misses():
engine = object.__new__(LLMEngine)
engine.should_execute_dummy_batch = False
engine.log_stats = False
engine.logger_manager = None
mm_processor_cache = MagicMock()
engine.renderer = MagicMock(mm_processor_cache=mm_processor_cache)
engine_core_output = MagicMock(
mm_cache_miss_hashes=["missing-hash-1", "missing-hash-2"]
)
outputs = MagicMock(
outputs=[engine_core_output],
timestamp=0.0,
scheduler_stats=None,
)
engine.engine_core = MagicMock()
engine.engine_core.get_output.return_value = outputs
processed_outputs = MagicMock(reqs_to_abort=[], request_outputs=[])
engine.output_processor = MagicMock()
engine.output_processor.process_outputs.return_value = processed_outputs
assert engine.step() == []
assert mm_processor_cache.invalidate.call_args_list == [
call("missing-hash-1"),
call("missing-hash-2"),
]
def _vllm_model(
apc: bool,
vllm_runner: type[VllmRunner],
+96 -1
View File
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock
import numpy as np
import pytest
import torch
from tests.v1.engine.utils import (
NUM_PROMPT_LOGPROBS_UNDER_TEST,
@@ -18,7 +19,12 @@ from tests.v1.engine.utils import (
from vllm import PoolingParams
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
from vllm.outputs import (
CompletionOutput,
PoolingRequestOutput,
RequestOutput,
SamplingMask,
)
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.v1.engine import (
@@ -1630,3 +1636,92 @@ def test_request_output_add_merges_delta_sampling_masks():
assert merged.outputs[0].token_ids == [10, 20]
assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]]
@pytest.mark.skip_global_cleanup
def test_pooling_cache_miss_is_a_request_error_and_batch_continues():
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
requests = [
EngineCoreRequest(
request_id=f"request-{idx}-int",
external_req_id=f"request-{idx}",
prompt_token_ids=[1, 2],
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=None,
pooling_params=PoolingParams(task="embed"),
)
for idx in range(2)
]
for request in requests:
output_processor.add_request(request, prompt=None)
processed = output_processor.process_outputs(
[
EngineCoreOutput(
request_id=requests[0].request_id,
new_token_ids=[],
finish_reason=FinishReason.ERROR,
mm_cache_miss_hashes=["missing-hash"],
),
EngineCoreOutput(
request_id=requests[1].request_id,
new_token_ids=[],
pooling_output=torch.tensor([1.0, 2.0]),
finish_reason=FinishReason.STOP,
),
]
)
assert len(processed.request_outputs) == 2
failed, succeeded = processed.request_outputs
assert isinstance(failed, PoolingRequestOutput)
assert failed.error is not None
assert failed.error.code == "multimodal_cache_miss"
assert failed.error.retryable
assert isinstance(succeeded, PoolingRequestOutput)
assert succeeded.error is None
assert torch.equal(succeeded.outputs.data, torch.tensor([1.0, 2.0]))
@pytest.mark.skip_global_cleanup
def test_async_pooling_cache_miss_does_not_fail_output_processor():
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
request = EngineCoreRequest(
request_id="request-int",
external_req_id="request",
prompt_token_ids=[1, 2],
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=None,
pooling_params=PoolingParams(task="embed"),
)
queue = RequestOutputCollector(
output_kind=RequestOutputKind.FINAL_ONLY,
request_id=request.request_id,
)
output_processor.add_request(request, prompt=None, queue=queue)
processed = output_processor.process_outputs(
[
EngineCoreOutput(
request_id=request.request_id,
new_token_ids=[],
finish_reason=FinishReason.ERROR,
mm_cache_miss_hashes=["missing-hash"],
)
]
)
assert processed.request_outputs == []
output = queue.get_nowait()
assert isinstance(output, PoolingRequestOutput)
assert output.error is not None
assert output.error.retryable
assert not output_processor.has_unfinished_requests()
@@ -111,6 +111,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
}
fail_scheduler.connector = Mock()
fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
)
@@ -327,6 +327,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
)
recompute_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
recompute_scheduler.connector.request_finished.return_value = (False, None)
recompute_scheduler.connector.take_events.return_value = ()
@@ -42,6 +42,7 @@ def test_failed_receive_completion_honors_failure_policy(policy):
request = create_request(num_tokens=3 * scheduler.block_size)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = [
(2 * scheduler.block_size, True),
(0, False),
@@ -103,6 +104,7 @@ def test_async_load_failure(
}
scheduler.connector = Mock()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
)
@@ -332,6 +334,7 @@ def test_async_progressive_load_failure(
}
scheduler.connector = Mock()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
)
@@ -1549,6 +1549,40 @@ async def test_kv_producer(monkeypatch):
prefill_worker.shutdown()
@pytest.mark.parametrize(
("local_block_ids", "expected_finished"),
[
([], set()),
([[]], set()),
([[100]], {"d-req-1"}),
],
)
def test_pull_completion_requires_local_blocks(
local_block_ids: list[list[int]],
expected_finished: set[str],
):
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
worker.shutdown = MagicMock()
worker.finished_recving_reqs = set()
pull_meta = PullReqMeta(
d_req_id="d-req-1",
transfer_id="xfer-req-1",
local_block_ids=local_block_ids,
remote_engine_id="p-engine",
remote_bootstrap_addr="http://bootstrap:33333",
pull_tasks_count=1,
)
response = MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH,
ok_reqs=["d-req-1"],
)
worker.process_pulling_result(response, {"d-req-1": pull_meta})
assert pull_meta.pull_tasks_count == 0
assert worker.finished_recving_reqs == expected_finished
@pytest.mark.asyncio
async def test_kv_consumuer(monkeypatch):
"""Simulates a Consumer Worker (Decoder) initiating a pull from a Producer.
@@ -22,14 +22,20 @@ Role shapes below are exactly those in the RFC (#46107) deployments:
"""
import contextlib
import ctypes
import importlib.util
import os
import subprocess
import sys
import time
import types
from collections import Counter
from pathlib import Path
from typing import cast
import msgpack
import pytest
import zmq
_MISSING = object()
@@ -203,3 +209,127 @@ def test_consecutive_requests_alternate_instances(route):
# than filling one instance's ranks before moving on.
insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)]
assert insts == [0, 1, 2]
@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep")
def test_discovery_heartbeat_progresses_while_worker_holds_gil():
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
payload = {
"type": "P",
"http_address": "127.0.0.1:8000",
"zmq_address": "host:127.0.0.1,handshake:6301,notify:61005",
"dp_size": 1,
"tp_size": 8,
"transfer_mode": "READ",
}
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
port = socket.bind_to_random_port("tcp://127.0.0.1")
heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02)
try:
assert socket.poll(5000), "Discovery did not register"
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
# PyDLL retains the GIL throughout the native call.
ctypes.PyDLL(None).usleep(300_000)
received = 0
while socket.poll(0):
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
received += 1
assert received >= 3, "Discovery stalled while the worker held the GIL"
finally:
heartbeat.shutdown()
heartbeat.shutdown()
while socket.poll(0):
socket.recv_multipart()
assert not socket.poll(100), "Heartbeats continued after shutdown"
@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs")
def test_discovery_heartbeat_exits_when_worker_dies():
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat
owner_code = """
import runpy
import sys
Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"]
heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02)
print(heartbeat._process.pid, flush=True)
sys.stdin.read()
"""
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
port = socket.bind_to_random_port("tcp://127.0.0.1")
owner = subprocess.Popen(
[
sys.executable,
"-c",
owner_code,
moriio_heartbeat.__file__,
f"tcp://127.0.0.1:{port}",
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
text=True,
)
child_pid = None
child_exited = False
try:
assert socket.poll(5000), "Discovery did not register"
assert owner.stdout is not None
child_pid = int(owner.stdout.readline())
owner.kill()
owner.wait(timeout=5)
stat = Path(f"/proc/{child_pid}/stat")
deadline = time.monotonic() + 5
while stat.exists():
# A terminated child may await reaping by the container's init.
try:
if stat.read_text().split(") ")[1].startswith("Z "):
break
except FileNotFoundError:
break
assert time.monotonic() < deadline, "Orphan heartbeat survived worker"
time.sleep(0.01)
child_exited = True
finally:
if owner.poll() is None:
owner.kill()
owner.communicate(timeout=5)
if child_pid is not None and not child_exited:
with contextlib.suppress(ProcessLookupError):
os.kill(child_pid, 9)
def test_discovery_heartbeat_recovers_after_router_outage(capfd):
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
payload = {"type": "P", "http_address": "127.0.0.1:8000"}
with zmq.Context() as context:
with context.socket(zmq.ROUTER) as socket:
port = socket.bind_to_random_port("tcp://127.0.0.1")
endpoint = f"tcp://127.0.0.1:{port}"
heartbeat = MoRIIOHeartbeat(endpoint, payload, 0.001, max_retries=1)
try:
assert socket.poll(5000), "Discovery did not register"
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
except BaseException:
heartbeat.shutdown()
raise
try:
# Fill the real DEALER queue while discovery is unavailable.
deadline = time.monotonic() + 10
errors = ""
while "heartbeat send failed" not in errors:
errors += capfd.readouterr().err
assert time.monotonic() < deadline, "No send timeout during outage"
time.sleep(0.01)
with context.socket(zmq.ROUTER) as socket:
socket.bind(endpoint)
assert socket.poll(5000), "Discovery did not recover after outage"
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
assert heartbeat._process.poll() is None
finally:
heartbeat.shutdown()
@@ -269,6 +269,34 @@ def test_worker_get_finished_counts_structured_release_fan_in():
assert worker._completed_consumer_notifications == {"tx-fanin"}
@pytest.mark.parametrize("exit_kind", ["killed", "eof"])
def test_worker_reports_unexpected_heartbeat_exit(exit_kind):
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
MoRIIOHeartbeat,
)
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.is_producer = True
worker.mode = MoRIIOMode.READ
worker.moriio_wrapper = SimpleNamespace(
pop_finished_req_ids=lambda: [], shutdown=lambda: None
)
worker._pending_unmapped_acks = []
worker._heartbeat = MoRIIOHeartbeat("tcp://127.0.0.1:1", {"type": "P"}, 0.02)
process = worker._heartbeat._process
try:
assert worker.get_finished() == (set(), set())
if exit_kind == "killed":
process.kill()
else:
process.stdin.close()
returncode = process.wait(timeout=5)
with pytest.raises(RuntimeError, match=f"heartbeat.*{returncode}"):
worker.get_finished()
finally:
worker.shutdown()
def test_read_completion_sends_structured_release_with_consumer_tp_size():
class DoneStatus:
def Succeeded(self):
@@ -202,6 +202,20 @@ def mc() -> MultiConnector:
return mc
@pytest.mark.parametrize("chosen", [0, 1])
def test_loaded_groups_follow_the_connector_serving_the_request(mc, chosen):
request = MagicMock(request_id="request")
for i, connector in enumerate(mc._connectors):
connector.get_num_new_matched_tokens.return_value = (
(16, True) if i == chosen else (0, False)
)
connector.get_loaded_kv_cache_group_ids.return_value = (i,)
assert mc.get_num_new_matched_tokens(request, 0) == (16, True)
assert mc.get_loaded_kv_cache_group_ids(request) == (chosen,)
mc._connectors[1 - chosen].get_loaded_kv_cache_group_ids.assert_not_called()
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
assert mc.get_mem_pool_context() is None
@@ -1522,34 +1522,6 @@ def _make_fake_kv_cache_manager():
return manager
@pytest.mark.cpu_test
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
"""Only zero-recorded (attention) groups contribute, sliced to the
externally-loaded token range; Mamba state blocks are never zeroed."""
manager = _make_fake_kv_cache_manager()
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
assert manager.get_zeroing_block_ids_in_range("req-1", 16, 56) == [11, 12, 13]
@pytest.mark.cpu_test
def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
"""Blocks that will be loaded by the connector must not be zeroed."""
from vllm.v1.core.sched.scheduler import Scheduler
class FakeKVCacheManager:
def take_new_block_ids(self):
return [9, 10, 11, 12]
scheduler = object.__new__(Scheduler)
scheduler.needs_kv_cache_zeroing = True
scheduler.kv_cache_manager = FakeKVCacheManager()
scheduler._skip_zero_block_ids = {10, 12}
assert scheduler._get_new_block_ids_to_zero() == [9, 11]
assert not scheduler._skip_zero_block_ids
@pytest.mark.cpu_test
def test_failed_load_rezeroes_unwritten_skipped_blocks():
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
@@ -1575,7 +1547,6 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
# Attention blocks covering tokens >= 48 are re-recorded for zeroing
# and flow into the next step's zero list; Mamba blocks are not.
scheduler._skip_zero_block_ids = set()
assert scheduler._get_new_block_ids_to_zero() == [13, 14, 15]
+115 -40
View File
@@ -11,6 +11,7 @@ from unittest.mock import patch
import numpy as np
import pytest
import torch
from pydantic import TypeAdapter, ValidationError
from tests.utils import large_gpu_mark
from tests.v1.sample.utils import (
@@ -20,7 +21,7 @@ from tests.v1.sample.utils import (
compute_correct_cumulative_logprob,
get_test_batch,
)
from vllm import SamplingParams
from vllm import SamplingParams, TokensPrompt
from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
@@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
return SimpleNamespace(
max_logprobs=max_logprobs,
max_model_len=16,
logits_processors=None,
is_diffusion=False,
get_vocab_size=lambda: vocab_size,
@@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
)
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
"""The candidate count is bounded by max_logprobs."""
def test_prompt_logprob_token_ids_validation():
"""Rows may be ragged (-1 padded); the width is bounded by max_logprobs."""
model_config = _model_config(vocab_size=100, max_logprobs=4)
def verify(**kwargs):
SamplingParams(**kwargs).verify(
params = SamplingParams(**kwargs)
params.verify(
model_config,
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
)
return params.prompt_logprob_token_ids
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [
[-1] * 4,
[1, -1, -1, -1],
[4, 3, 2, 1],
]
# Any integer layout is normalized to what the worker uploads:
# C-contiguous native int32.
odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8"))
ids = verify(prompt_logprob_token_ids=odd)
assert ids.dtype == np.int32 and ids.flags.c_contiguous
assert ids.tolist() == [[1, 2], [3, -1]]
# The pydantic path (token-in/token-out API, /docs) takes strict integer lists.
adapter = TypeAdapter(SamplingParams)
for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"):
with pytest.raises(ValidationError):
adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}')
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
verify(prompt_logprob_token_ids=np.array([1, 2]))
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
verify(prompt_logprob_token_ids=[1, 2])
for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]):
with pytest.raises(VLLMValidationError, match="integer token ids"):
verify(prompt_logprob_token_ids=bad_ids)
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64))
# The error names the value to raise max_logprobs to.
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=[[1], [100]])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=[[1], [-2]])
with pytest.raises(VLLMValidationError, match="non-empty"):
verify(prompt_logprob_token_ids=[[], []])
# Rows are bounded by max_model_len before the table is padded.
with pytest.raises(VLLMValidationError, match="at most 15 scored rows"):
verify(prompt_logprob_token_ids=[[]] * 15 + [[1]])
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
verify(prompt_logprob_start=3)
@@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
params = SamplingParams(prompt_logprob_token_ids=[[1, 2]])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
@@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
"""Per-row scores stay row-aligned across chunked prefill and preemption."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = [
@@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
] + [f"Tell me about the number {i}: " for i in range(32)]
start = 2
candidate_ids = [10, 100, 1000, 10000]
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=start,
)
def candidate_ids(j):
return [10 + j, 100 + j, 1000 + j, 10000 + j]
with VllmRunner(
"Qwen/Qwen3-0.6B",
@@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
disable_log_stats=False,
gpu_memory_utilization=0.25,
) as vllm_model:
tokenizer = vllm_model.llm.get_tokenizer()
token_prompts = [
TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts
]
def make_params(row_ids):
return [
SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=[
row_ids(j)
for j in range(len(p["prompt_token_ids"]) - 1 - start)
],
prompt_logprob_start=start,
)
for p in token_prompts
]
metrics_before = vllm_model.llm.get_metrics()
outputs = vllm_model.llm.generate(prompts, sampling_params)
full_params = make_params(candidate_ids)
for params in full_params: # the array form, as a top-k tensor would be
params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids)
outputs = vllm_model.llm.generate(token_prompts, full_params)
for i, output in enumerate(outputs):
scores = output.prompt_token_id_logprobs
assert scores is not None, f"Output {i} missing fixed-ID scores"
assert scores is not None, f"Output {i} missing per-row scores"
expected_shape = (
len(output.prompt_token_ids) - 1 - start,
len(candidate_ids),
4,
)
assert scores.shape == expected_shape, (
f"Output {i} scored {scores.shape}, expected {expected_shape}"
@@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
assert math.isfinite(float(scores.min()))
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
# Ragged rows: each row keeps a prefix of the candidates and must
# reproduce that prefix of the full scores, with -inf padding.
ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4])
ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params)
for output, ragged_output in zip(outputs, ragged_outputs):
expected = np.full_like(output.prompt_token_id_logprobs, -np.inf)
for j in range(len(expected)):
n = 1 + j % 4
expected[j, :n] = output.prompt_token_id_logprobs[j, :n]
np.testing.assert_allclose(
ragged_output.prompt_token_id_logprobs, expected, atol=1e-2
)
# The row count is checked against the prompt at admission.
with pytest.raises(VLLMValidationError, match="scored rows"):
vllm_model.llm.generate(token_prompts[0], ragged_params[1])
metrics_after = vllm_model.llm.get_metrics()
preemptions_before = next(
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
@@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
# unchunked, unpreempted run of the same requests. Batch composition moves
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
max_logprobs=64,
gpu_memory_utilization=0.25,
"Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25
) as reference:
reference_outputs = reference.llm.generate(prompts, sampling_params)
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
reference_outputs = reference.llm.generate(token_prompts, full_params)
self_scored = reference.llm.generate(
prompts,
token_prompts,
[
SamplingParams(
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
max_tokens=1,
prompt_logprobs=0,
prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]],
)
for ids in own_ids
for p in token_prompts
],
)
for output, ref in zip(outputs, reference_outputs):
np.testing.assert_allclose(
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
output.prompt_token_id_logprobs,
ref.prompt_token_id_logprobs,
rtol=0.1,
atol=0.2,
)
for output, ids in zip(self_scored, own_ids):
scores = output.prompt_token_id_logprobs
for output in self_scored:
scores = output.prompt_token_id_logprobs[:, 0]
for row, target in enumerate(output.prompt_token_ids[1:]):
expected = output.prompt_logprobs[row + 1][target].logprob
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
assert scores[row] == pytest.approx(expected, abs=1e-3)
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
@@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompt = "The capital of France is Paris. " * 20
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=[10, 100, 1000],
skip_reading_prefix_cache=False,
)
text = "The capital of France is Paris. " * 20
with VllmRunner(
"Qwen/Qwen3-0.6B",
@@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
enable_prefix_caching=True,
gpu_memory_utilization=0.25,
) as vllm_model:
token_ids = vllm_model.llm.get_tokenizer()(text).input_ids
prompt = TokensPrompt(prompt_token_ids=token_ids)
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)),
skip_reading_prefix_cache=False,
)
first = vllm_model.llm.generate([prompt], sampling_params)[0]
assert first.prompt_token_id_logprobs is not None
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
+3 -1
View File
@@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list():
def test_sampling_params_trace_field_rejects_prompt_scoring():
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
params = SamplingParams(
trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]]
)
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
@@ -39,6 +39,7 @@ def _make_fake_speculator(
and a draft model.
"""
fake_input_buffers = SimpleNamespace(
positions=torch.arange(max_num_tokens, dtype=torch.int64),
query_start_loc=torch.zeros(max_num_reqs + 1, dtype=torch.int32),
seq_lens=torch.zeros(max_num_reqs, dtype=torch.int32),
dcp_local_seq_lens=torch.zeros(max_num_reqs, dtype=torch.int32),
@@ -113,6 +114,7 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound():
captured = _run_build(fake, num_reqs=3, num_reqs_padded=4, base=base, step=2)
assert torch.equal(captured["positions"], fake.input_buffers.positions[:4])
bound = captured["seq_lens_cpu_upper_bound"]
assert isinstance(bound, torch.Tensor), (
"seq_lens_cpu_upper_bound must be a tensor, not None"
@@ -8,7 +8,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.engine.protocol import StreamingInput
from vllm.outputs import RequestOutput
from vllm.outputs import STREAM_FINISHED, RequestOutput
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.output_processor import RequestOutputCollector
@@ -170,3 +170,31 @@ async def test_generate_with_async_generator():
assert outputs[2].finished is True
# Both inputs were processed
assert inputs_received == ["Hello", " world"]
@pytest.mark.asyncio
async def test_empty_input_stream_finishes_without_engine_request():
llm = AsyncLLM.__new__(AsyncLLM)
llm._validate_streaming_input_sampling_params = MagicMock()
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
final_req = MagicMock()
final_req.request_id = "request-int"
llm.input_processor = MagicMock()
llm.input_processor.process_inputs.return_value = final_req
async def empty_stream():
return
yield # pragma: no cover
queue = await llm._add_streaming_input_request(
"request", empty_stream(), SamplingParams(max_tokens=4)
)
task = queue._input_stream_task
assert task is not None
await task
assert queue.get_nowait() is STREAM_FINISHED
llm._add_request.assert_not_awaited()
@@ -85,6 +85,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
speculator.max_model_len = speculator.draft_max_seq_len = 32
speculator.draft_is_prefilling = torch.zeros(4, dtype=torch.bool)
speculator.input_buffers = SimpleNamespace(
positions=torch.arange(4, dtype=torch.int64),
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
)
@@ -137,6 +137,52 @@ def test_finished_request_releases_unscored_doc_use():
)
@pytest.mark.skip_global_cleanup
def test_release_queries_is_targeted_and_idempotent():
runner = LateInteractionRunner()
query_emb = torch.tensor([[1.0, 0.0], [0.0, 1.0]], dtype=torch.float32)
for query_key in ("query-release", "query-keep"):
query_params = _make_pooling_params(
build_late_interaction_query_params(query_key=query_key, query_uses=1)
)
runner.postprocess_pooler_output(
raw_pooler_output=[query_emb],
pooling_params=[query_params],
req_ids=[f"{query_key}-req"],
finished_mask=[True],
)
released_doc_params = _make_pooling_params(
build_late_interaction_doc_params(query_key="query-release")
)
kept_doc_params = _make_pooling_params(
build_late_interaction_doc_params(query_key="query-keep")
)
runner.register_request("released-doc", released_doc_params)
runner.register_request("kept-doc", kept_doc_params)
runner.release_queries(["query-release"])
runner.release_queries(["query-release"])
with pytest.raises(ValueError, match="query cache miss"):
runner.postprocess_pooler_output(
raw_pooler_output=[query_emb],
pooling_params=[released_doc_params],
req_ids=["released-doc"],
finished_mask=[True],
)
kept_output = runner.postprocess_pooler_output(
raw_pooler_output=[query_emb],
pooling_params=[kept_doc_params],
req_ids=["kept-doc"],
finished_mask=[True],
)
assert isinstance(kept_output, list)
assert kept_output[0] is not None
def test_invalid_query_uses_raises():
runner = LateInteractionRunner()
bad_meta = LateInteractionParams(
+48
View File
@@ -26,6 +26,7 @@ from vllm.v1.watermarking.spec_decode import (
speculative_target_watermark_key,
)
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.sample.watermark import (
draft_watermarking_mask,
@@ -1499,6 +1500,53 @@ def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
assert not draft_logits[:, :, -1].any()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
)
def test_draft_watermarker_cuda_graph_reads_current_prompt_lens():
draft_watermarker = DraftWatermarker(
GumbelWatermarker(key=42, context_width=2),
max_num_reqs=1,
device=torch.device("cuda"),
num_speculative_steps=1,
deduplicate_contexts="single_turn",
deduplicate_contexts_max_history=None,
)
prompt_lens = UvaBackedTensor(1, dtype=torch.int32)
contexts = torch.tensor([[5, 6]]).cuda()
enabled = torch.ones(1, dtype=torch.bool).cuda()
# [5, 6] repeats only while the prompt counts as history.
all_token_ids = torch.tensor([[5, 6, 9, 5, 6]]).cuda()
total_lens = torch.tensor([5]).cuda()
logits = torch.zeros(1, 8).cuda()
idx_mapping = torch.zeros(1, dtype=torch.int32).cuda()
temperature = torch.ones(1).cuda()
draft_step = torch.tensor(0).cuda()
def prepare():
draft_watermarker.prepare(
contexts, enabled, all_token_ids, prompt_lens.gpu, total_lens
)
def watermark_mask():
_, mask = draft_watermarker._sampling_state(
logits, idx_mapping, temperature, draft_step, contexts
)
return mask
prepare()
assert not watermark_mask().item()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
mask = watermark_mask()
# A step that adds requests moves prompt_lens.gpu to the next pool buffer.
prompt_lens.np[:] = 3
prompt_lens.copy_to_uva()
prepare()
graph.replay()
assert mask.item()
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
speculator = object.__new__(DSparkSpeculator)
speculator.draft_logits = torch.zeros(2, 1, 8)