Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -327,6 +327,35 @@ def test_reentered_tag(free_x_early):
|
||||
allocator.release_pools()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_runtime_state_survives_sleep(level):
|
||||
"""Runtime state is offloaded at both levels and restored by the first
|
||||
selective wake; a nested kv_cache pool keeps its own tag."""
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
with allocator.use_memory_pool("runtime"):
|
||||
state = torch.arange(1 << 20, device=DEVICE_TYPE)
|
||||
with allocator.use_memory_pool("kv_cache"):
|
||||
kv = torch.zeros(1 << 20, device=DEVICE_TYPE)
|
||||
assert {d.tag for d in allocator.pointer_to_data.values()} == {
|
||||
"runtime",
|
||||
"kv_cache",
|
||||
}
|
||||
|
||||
backend = CuMemBackend()
|
||||
backend.suspend(level=level)
|
||||
assert mapped_usage(allocator) == 0
|
||||
|
||||
backend.resume(tags=["weights"])
|
||||
assert torch.equal(state, torch.arange(1 << 20, device=DEVICE_TYPE))
|
||||
backend.resume(tags=["kv_cache"])
|
||||
kv.fill_(1)
|
||||
assert int(kv.sum()) == kv.numel()
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
|
||||
@@ -6,14 +6,17 @@ from collections import defaultdict
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
|
||||
|
||||
from .common import FUSION_LOG_PATTERNS, AttentionBackendCase, Matches
|
||||
|
||||
|
||||
def run_model(
|
||||
compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs
|
||||
vllm_runner,
|
||||
compile_config: CompilationMode | CompilationConfig,
|
||||
model: str,
|
||||
**model_kwargs,
|
||||
):
|
||||
"""Run a model with the given compilation config for E2E fusion tests."""
|
||||
compilation_config = (
|
||||
@@ -36,38 +39,48 @@ def run_model(
|
||||
# No cudagraphs by default
|
||||
if compilation_config.cudagraph_mode is None:
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
llm = LLM(
|
||||
model=model,
|
||||
|
||||
# trust_remote_code/enable_chunked_prefill/block_size/kernel_config are
|
||||
# pinned to LLM()'s own defaults since VllmRunner overrides them for
|
||||
# other test needs (e.g. it forces block_size=16, which some attention
|
||||
# backends like FLASHINFER_MLA reject, and disables JIT warmup).
|
||||
model_kwargs.setdefault("kernel_config", {"enable_jit_warmup": True})
|
||||
with vllm_runner(
|
||||
model,
|
||||
compilation_config=compilation_config,
|
||||
trust_remote_code=False,
|
||||
enable_chunked_prefill=None,
|
||||
block_size=None,
|
||||
**model_kwargs,
|
||||
)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
) as vllm_model:
|
||||
llm = vllm_model.llm
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
# Print the outputs.
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
# Print the outputs.
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
|
||||
# Get the compile ranges endpoints after vllm config post init
|
||||
# in order to compute compile ranges correctly
|
||||
compilation_config.compile_ranges_endpoints = (
|
||||
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
|
||||
)
|
||||
# Get the compile ranges endpoints after vllm config post init
|
||||
# in order to compute compile ranges correctly
|
||||
compilation_config.compile_ranges_endpoints = (
|
||||
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
|
||||
)
|
||||
|
||||
# Fetch match table from each worker via RPC and sum across workers.
|
||||
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
|
||||
"get_compilation_match_table"
|
||||
)
|
||||
combined: defaultdict[str, int] = defaultdict(int)
|
||||
for table in worker_tables:
|
||||
for k, v in table.items():
|
||||
combined[k] += v
|
||||
return dict(combined)
|
||||
# Fetch match table from each worker via RPC and sum across workers.
|
||||
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
|
||||
"get_compilation_match_table"
|
||||
)
|
||||
combined: defaultdict[str, int] = defaultdict(int)
|
||||
for table in worker_tables:
|
||||
for k, v in table.items():
|
||||
combined[k] += v
|
||||
return dict(combined)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
|
||||
def run(
|
||||
model_name: str,
|
||||
matches: Matches,
|
||||
@@ -137,7 +150,9 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
)
|
||||
|
||||
with caplog_mp_spawn(logging.DEBUG) as log_holder:
|
||||
match_table = run_model(full_compilation_config, model_name, **model_kwargs)
|
||||
match_table = run_model(
|
||||
vllm_runner, full_compilation_config, model_name, **model_kwargs
|
||||
)
|
||||
|
||||
num_compile_ranges = len(full_compilation_config.get_compile_ranges())
|
||||
assert num_compile_ranges in [1, 2, 3]
|
||||
|
||||
@@ -101,6 +101,20 @@ llama3_8b_fp4 = ModelFusionInfo(
|
||||
),
|
||||
)
|
||||
|
||||
# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
|
||||
# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
|
||||
# every MLP boundary is left to ActivationQuantFusionPass.
|
||||
qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Qwen3.8-27B-NVFP4",
|
||||
hf_overrides=lambda n_layers: {
|
||||
"text_config": {
|
||||
"num_hidden_layers": n_layers,
|
||||
"layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
|
||||
}
|
||||
},
|
||||
matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
|
||||
)
|
||||
|
||||
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
|
||||
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
|
||||
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
|
||||
|
||||
@@ -30,6 +30,7 @@ from .models import (
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp4,
|
||||
llama4_scout_fp8,
|
||||
qwen3_8_27b_mixed_fp4,
|
||||
qwen3_a3b_fp8,
|
||||
)
|
||||
|
||||
@@ -202,3 +203,46 @@ def test_tp1_fp4_fusions(
|
||||
compilation_config,
|
||||
matches_check,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
def test_tp1_fp4_mixed_precision_act_fusion(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
"""Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=True,
|
||||
fuse_attn_quant=False,
|
||||
enable_qk_norm_rope_fusion=False,
|
||||
),
|
||||
)
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches_fn(n_layers),
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
["act_quant_fusion"],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NVFP4 detection for ModelOpt MIXED_PRECISION checkpoints.
|
||||
|
||||
A checkpoint that quantizes some layers to NVFP4 and others to FP8 declares
|
||||
``quant_algo: MIXED_PRECISION`` and resolves to ``modelopt_mixed``, so NVFP4
|
||||
has to be detected from the per-layer algorithms.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.model import _modelopt_mixed_has_nvfp4
|
||||
|
||||
|
||||
def _quantized_layers(*algos: str) -> dict:
|
||||
return {
|
||||
"quantized_layers": {
|
||||
f"model.layers.{i}.proj": {"quant_algo": algo}
|
||||
for i, algo in enumerate(algos)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_detects_nvfp4_alongside_another_algorithm():
|
||||
"""The shipped shape: FP8 attention projections, NVFP4 MLP."""
|
||||
assert _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "NVFP4", "FP8"))
|
||||
|
||||
|
||||
def test_ignores_a_checkpoint_with_no_nvfp4_layer():
|
||||
assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "MXFP8"))
|
||||
|
||||
|
||||
def test_weight_only_nvfp4_does_not_count():
|
||||
"""W4A16_NVFP4 leaves activations in high precision, so nothing to fuse."""
|
||||
assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "W4A16_NVFP4"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"quant_config",
|
||||
[None, {}, {"quantized_layers": None}, {"quantized_layers": ["NVFP4"]}],
|
||||
)
|
||||
def test_absent_or_malformed_config_is_not_nvfp4(quant_config):
|
||||
assert not _modelopt_mixed_has_nvfp4(quant_config)
|
||||
@@ -511,7 +511,7 @@ def _check_dummy_hash_model_forward(
|
||||
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.use_mega_moe = False
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_hash = state
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import EPLBConfig
|
||||
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
|
||||
from vllm.distributed.eplb.migration_scheduler import (
|
||||
MigrationFlow,
|
||||
schedule_migration_batches,
|
||||
schedule_migration_batches_for_layers,
|
||||
)
|
||||
from vllm.distributed.eplb.rebalance_execute import move_from_buffer, move_to_buffer
|
||||
|
||||
|
||||
class _MockEplbCommunicator(EplbCommunicator):
|
||||
def __init__(self) -> None:
|
||||
self.send_calls: list[tuple[list[torch.Tensor], int, int]] = []
|
||||
self.recv_calls: list[tuple[list[torch.Tensor], int, int]] = []
|
||||
self.context_calls = 0
|
||||
self.execute_count = 0
|
||||
|
||||
def add_send(
|
||||
self, tensors: list[torch.Tensor], dst_rank: int, expert_id: int
|
||||
) -> None:
|
||||
self.send_calls.append((tensors, dst_rank, expert_id))
|
||||
|
||||
def add_recv(
|
||||
self, tensors: list[torch.Tensor], src_rank: int, expert_id: int
|
||||
) -> None:
|
||||
self.recv_calls.append((tensors, src_rank, expert_id))
|
||||
|
||||
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
|
||||
self.context_calls += 1
|
||||
|
||||
def execute(self) -> None:
|
||||
self.execute_count += 1
|
||||
|
||||
|
||||
def test_migration_batching_is_disabled_by_default_and_can_be_enabled() -> None:
|
||||
assert not EPLBConfig().enable_migration_batching
|
||||
assert EPLBConfig(enable_migration_batching=True).enable_migration_batching
|
||||
|
||||
|
||||
def test_migration_batching_only_applies_to_async_eplb() -> None:
|
||||
assert not EPLBConfig(use_async=True).migration_batching_enabled
|
||||
assert EPLBConfig(
|
||||
use_async=True, enable_migration_batching=True
|
||||
).migration_batching_enabled
|
||||
assert not EPLBConfig(
|
||||
use_async=True, enable_migration_batching=False
|
||||
).migration_batching_enabled
|
||||
assert not EPLBConfig(
|
||||
use_async=False, enable_migration_batching=True
|
||||
).migration_batching_enabled
|
||||
|
||||
|
||||
def test_schedule_migration_batches_is_deterministic() -> None:
|
||||
transfers = [
|
||||
(1, 3, 0),
|
||||
(2, 4, 1),
|
||||
(0, 1, 2),
|
||||
(0, 2, 3),
|
||||
(0, 3, 4),
|
||||
(0, 4, 5),
|
||||
]
|
||||
num_local_experts, old_indices, new_indices = _placements_for_transfers(
|
||||
5, transfers
|
||||
)
|
||||
|
||||
batches = schedule_migration_batches(num_local_experts, old_indices, new_indices)
|
||||
|
||||
assert [_endpoints(batch) for batch in batches] == [
|
||||
[(1, 3), (2, 4)],
|
||||
[(0, 1)],
|
||||
[(0, 2)],
|
||||
[(0, 3)],
|
||||
[(0, 4)],
|
||||
]
|
||||
for batch in batches:
|
||||
_assert_no_endpoint_conflict(batch)
|
||||
|
||||
|
||||
def test_schedule_migration_batches_coalesces_rank_pair() -> None:
|
||||
transfers = [
|
||||
(0, 1, 0),
|
||||
(0, 1, 1),
|
||||
(2, 3, 2),
|
||||
]
|
||||
num_local_experts, old_indices, new_indices = _placements_for_transfers(
|
||||
4, transfers
|
||||
)
|
||||
|
||||
assert schedule_migration_batches(num_local_experts, old_indices, new_indices) == [
|
||||
[
|
||||
MigrationFlow(0, 1, expert_ids=(0, 1)),
|
||||
MigrationFlow(2, 3, expert_ids=(2,)),
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
def test_schedule_migration_batches_covers_random_instructions() -> None:
|
||||
rng = np.random.default_rng(42)
|
||||
transfers = []
|
||||
for expert_id in range(80):
|
||||
src_rank, dst_rank = rng.integers(0, 8, size=2).tolist()
|
||||
if src_rank != dst_rank:
|
||||
transfers.append((src_rank, dst_rank, expert_id))
|
||||
num_local_experts, old_indices, new_indices = _placements_for_transfers(
|
||||
8, transfers
|
||||
)
|
||||
|
||||
batches = schedule_migration_batches(num_local_experts, old_indices, new_indices)
|
||||
|
||||
scheduled = _flatten_transfers(batches)
|
||||
assert len(scheduled) == len(transfers)
|
||||
assert set(scheduled) == set(transfers)
|
||||
for batch in batches:
|
||||
_assert_no_endpoint_conflict(batch)
|
||||
|
||||
|
||||
def test_schedule_migration_batches_excludes_local_copies() -> None:
|
||||
# Old: rank 0 [0, 1], rank 1 [1, 2]. New: rank 1 needs expert 0.
|
||||
old_indices = np.array([0, 1, 1, 2], dtype=np.int64)
|
||||
new_indices = np.array([0, 1, 0, 2], dtype=np.int64)
|
||||
|
||||
assert schedule_migration_batches(2, old_indices, new_indices) == [
|
||||
[MigrationFlow(0, 1, expert_ids=(0,))]
|
||||
]
|
||||
|
||||
|
||||
def test_schedule_migration_batches_requires_matching_shapes() -> None:
|
||||
old_indices = np.array([0, 1], dtype=np.int64)
|
||||
new_indices = np.array([0], dtype=np.int64)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
schedule_migration_batches(1, old_indices, new_indices)
|
||||
|
||||
|
||||
def test_schedule_migration_batches_balances_replicas() -> None:
|
||||
# Rank 0 and rank 1 hold expert 0; ranks 2, 3, and 4 need it.
|
||||
old_indices = np.array([0, 0, 1, 1, 1], dtype=np.int64)
|
||||
new_indices = np.array([0, 0, 0, 0, 0], dtype=np.int64)
|
||||
|
||||
batches = schedule_migration_batches(1, old_indices, new_indices)
|
||||
|
||||
assert set(_flatten_transfers(batches)) == {
|
||||
(0, 2, 0),
|
||||
(0, 4, 0),
|
||||
(1, 3, 0),
|
||||
}
|
||||
|
||||
|
||||
def test_schedule_migration_batches_for_layers_matches_individual_calls() -> None:
|
||||
old_indices = np.array([[0, 1, 2, 3], [3, 2, 1, 0]], dtype=np.int64)
|
||||
new_indices = np.array([[1, 2, 3, 0], [0, 3, 2, 1]], dtype=np.int64)
|
||||
|
||||
assert schedule_migration_batches_for_layers(1, old_indices, new_indices) == [
|
||||
schedule_migration_batches(1, old_indices[0], new_indices[0]),
|
||||
schedule_migration_batches(1, old_indices[1], new_indices[1]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_precomputed", [False, True])
|
||||
def test_move_to_buffer_uses_multiple_batches(use_precomputed: bool) -> None:
|
||||
old_indices = np.array([0, 1, 2, 3], dtype=np.int64)
|
||||
new_indices = np.array([1, 2, 3, 0], dtype=np.int64)
|
||||
communicator = _MockEplbCommunicator()
|
||||
migration_batches = (
|
||||
schedule_migration_batches(1, old_indices, new_indices)
|
||||
if use_precomputed
|
||||
else None
|
||||
)
|
||||
|
||||
move_to_buffer(
|
||||
num_local_experts=1,
|
||||
old_indices=old_indices,
|
||||
new_indices=new_indices,
|
||||
expert_weights=[torch.zeros(1, 1)],
|
||||
expert_weights_buffers=[torch.zeros(1, 1)],
|
||||
stream=None,
|
||||
ep_rank=0,
|
||||
communicator=communicator,
|
||||
layer_idx=7,
|
||||
enable_migration_batching=True,
|
||||
migration_batches=migration_batches,
|
||||
)
|
||||
|
||||
assert communicator.context_calls == 2
|
||||
assert communicator.execute_count == 2
|
||||
assert [(dst, expert) for _, dst, expert in communicator.send_calls] == [(3, 0)]
|
||||
assert [(src, expert) for _, src, expert in communicator.recv_calls] == [(1, 1)]
|
||||
|
||||
|
||||
def test_move_to_buffer_uses_primary_duplicate_destination() -> None:
|
||||
old_indices = np.array([0, 1, 2, 3], dtype=np.int64)
|
||||
new_indices = np.array([2, 2, 0, 3], dtype=np.int64)
|
||||
expert_weights = [torch.zeros(2, 1)]
|
||||
recv_buffer = torch.zeros(2, 1)
|
||||
communicator = _MockEplbCommunicator()
|
||||
|
||||
transfer_metadata = move_to_buffer(
|
||||
num_local_experts=2,
|
||||
old_indices=old_indices,
|
||||
new_indices=new_indices,
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=[recv_buffer],
|
||||
stream=None,
|
||||
ep_rank=0,
|
||||
communicator=communicator,
|
||||
enable_migration_batching=True,
|
||||
)
|
||||
|
||||
assert len(communicator.recv_calls) == 1
|
||||
recv_tensors, src_rank, expert_id = communicator.recv_calls[0]
|
||||
assert (src_rank, expert_id) == (1, 2)
|
||||
assert recv_tensors[0].data_ptr() == recv_buffer[0].data_ptr()
|
||||
|
||||
recv_buffer[0].fill_(42)
|
||||
move_from_buffer(
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=[recv_buffer],
|
||||
transfer_metadata=transfer_metadata,
|
||||
new_indices=new_indices,
|
||||
ep_rank=0,
|
||||
)
|
||||
torch.testing.assert_close(expert_weights[0], torch.full((2, 1), 42.0))
|
||||
|
||||
|
||||
def _endpoints(
|
||||
flows: list[MigrationFlow],
|
||||
) -> list[tuple[int, int]]:
|
||||
return [(item.src_rank, item.dst_rank) for item in flows]
|
||||
|
||||
|
||||
def _assert_no_endpoint_conflict(batch: list[MigrationFlow]) -> None:
|
||||
endpoints: set[int] = set()
|
||||
for flow in batch:
|
||||
assert flow.src_rank not in endpoints
|
||||
assert flow.dst_rank not in endpoints
|
||||
endpoints.update((flow.src_rank, flow.dst_rank))
|
||||
|
||||
|
||||
def _flatten_transfers(
|
||||
batches: list[list[MigrationFlow]],
|
||||
) -> list[tuple[int, int, int]]:
|
||||
return [
|
||||
(flow.src_rank, flow.dst_rank, expert_id)
|
||||
for batch in batches
|
||||
for flow in batch
|
||||
for expert_id in flow.expert_ids
|
||||
]
|
||||
|
||||
|
||||
def _placements_for_transfers(
|
||||
num_ranks: int,
|
||||
transfers: list[tuple[int, int, int]],
|
||||
) -> tuple[int, np.ndarray, np.ndarray]:
|
||||
old_counts = [0] * num_ranks
|
||||
new_counts = [0] * num_ranks
|
||||
for src_rank, dst_rank, _ in transfers:
|
||||
old_counts[src_rank] += 1
|
||||
new_counts[dst_rank] += 1
|
||||
num_local_experts = max(1, *old_counts, *new_counts)
|
||||
old_indices = np.full(num_ranks * num_local_experts, -1, dtype=np.int64)
|
||||
new_indices = np.full_like(old_indices, -1)
|
||||
old_counts = [0] * num_ranks
|
||||
new_counts = [0] * num_ranks
|
||||
for src_rank, dst_rank, expert_id in transfers:
|
||||
old_offset = src_rank * num_local_experts + old_counts[src_rank]
|
||||
new_offset = dst_rank * num_local_experts + new_counts[dst_rank]
|
||||
old_indices[old_offset] = expert_id
|
||||
new_indices[new_offset] = expert_id
|
||||
old_counts[src_rank] += 1
|
||||
new_counts[dst_rank] += 1
|
||||
return num_local_experts, old_indices, new_indices
|
||||
@@ -231,6 +231,8 @@ def test_one_sided_combine_into_compatibility(supports_output):
|
||||
payload,
|
||||
runtime_max_tokens_per_rank,
|
||||
output=None,
|
||||
*,
|
||||
use_low_precision=False,
|
||||
):
|
||||
result = payload + runtime_max_tokens_per_rank
|
||||
if output is None:
|
||||
@@ -249,6 +251,33 @@ def test_one_sided_combine_into_compatibility(supports_output):
|
||||
torch.testing.assert_close(output, payload + 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("low_precision_combine", [False, True])
|
||||
def test_one_sided_combine_into_low_precision(low_precision_combine):
|
||||
"""The fp8 combine setting is passed through to the kernel."""
|
||||
from vllm.distributed.device_communicators.all2all import (
|
||||
FlashInferNVLinkOneSidedManager,
|
||||
)
|
||||
|
||||
seen_kwargs = {}
|
||||
|
||||
class FakeMoeAlltoAll:
|
||||
def combine(self, payload, runtime_max_tokens_per_rank, output, **kwargs):
|
||||
seen_kwargs.update(kwargs)
|
||||
output.copy_(payload + runtime_max_tokens_per_rank)
|
||||
|
||||
manager = FlashInferNVLinkOneSidedManager.__new__(FlashInferNVLinkOneSidedManager)
|
||||
manager.moe_alltoall = FakeMoeAlltoAll()
|
||||
manager._combine_supports_output = True
|
||||
manager.low_precision_combine = low_precision_combine
|
||||
payload = torch.arange(4, dtype=torch.float32)
|
||||
output = torch.empty_like(payload)
|
||||
|
||||
manager.combine_into(payload, runtime_max_tokens_per_rank=2, output=output)
|
||||
|
||||
assert seen_kwargs == {"use_low_precision": low_precision_combine}
|
||||
torch.testing.assert_close(output, payload + 2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1: Two-sided manager lifecycle (init, cleanup, reinit, ensure_init)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -915,6 +944,106 @@ def test_one_sided_dispatch_combine(world_size):
|
||||
_spawn_workers(_one_sided_data_worker, world_size, dp_size=world_size)
|
||||
|
||||
|
||||
def _one_sided_low_precision_worker(rank, world_size):
|
||||
from vllm.distributed.device_communicators.all2all import (
|
||||
FlashInferNVLinkOneSidedManager,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
hidden_size = 256
|
||||
tokens_per_rank = 32
|
||||
experts_per_token = 2
|
||||
num_experts = world_size * 8
|
||||
|
||||
manager = FlashInferNVLinkOneSidedManager(get_dp_group().cpu_group)
|
||||
manager.initialize(
|
||||
max_num_tokens=tokens_per_rank,
|
||||
top_k=experts_per_token,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden_size,
|
||||
x_bytes_per_token=hidden_size // 2,
|
||||
x_sf_bytes_per_token=hidden_size // 16,
|
||||
)
|
||||
assert manager.low_precision_combine
|
||||
|
||||
torch.manual_seed(rank + 42)
|
||||
x = torch.randint(
|
||||
0, 256, (tokens_per_rank, hidden_size // 2), device=device, dtype=torch.uint8
|
||||
)
|
||||
x_sf = torch.randint(
|
||||
0, 256, (tokens_per_rank, hidden_size // 16), device=device, dtype=torch.uint8
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
num_experts,
|
||||
(tokens_per_rank, experts_per_token),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
topk_weights = torch.rand(
|
||||
tokens_per_rank, experts_per_token, device=device, dtype=torch.float32
|
||||
)
|
||||
|
||||
manager.moe_alltoall.dispatch(
|
||||
token_selected_experts=topk_ids,
|
||||
input_payloads=[x, x_sf, topk_ids, topk_weights],
|
||||
runtime_max_tokens_per_rank=tokens_per_rank,
|
||||
)
|
||||
|
||||
# Go through combine_into rather than moe_alltoall.combine: the workspace
|
||||
# was sized for fp8, so the kernel must be told to send fp8. Calling the
|
||||
# raw kernel here would send bf16 into fp8-sized slots and corrupt it.
|
||||
expert_output = torch.ones(
|
||||
world_size, tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
output = torch.empty(
|
||||
tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
manager.combine_into(
|
||||
payload=expert_output,
|
||||
runtime_max_tokens_per_rank=tokens_per_rank,
|
||||
output=output,
|
||||
)
|
||||
|
||||
# Integers 1..top_k are exact in fp8_e4m3, so the fp8 transport must match
|
||||
# bf16 exactly: one contribution per distinct expert-owning rank.
|
||||
experts_per_rank = num_experts // world_size
|
||||
expert_ranks = topk_ids // experts_per_rank
|
||||
num_distinct = torch.tensor(
|
||||
[len(set(row.tolist())) for row in expert_ranks],
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).unsqueeze(1)
|
||||
torch.testing.assert_close(output, num_distinct.expand_as(output))
|
||||
|
||||
torch.distributed.barrier()
|
||||
manager.cleanup()
|
||||
|
||||
|
||||
@requires_multi_gpu
|
||||
@requires_one_sided
|
||||
@requires_ptrace
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
def test_one_sided_combine_low_precision(world_size, monkeypatch):
|
||||
"""Exercise the real fp8 combine transport, not a fake kernel.
|
||||
|
||||
A workspace/dtype mismatch corrupts the combine buffer silently rather
|
||||
than raising, so this asserts the combined values are exactly right.
|
||||
"""
|
||||
from flashinfer.comm.trtllm_moe_alltoall import MoeAlltoAll
|
||||
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
if not supports_kw(
|
||||
MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False
|
||||
):
|
||||
pytest.skip("installed FlashInfer has no use_low_precision support")
|
||||
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "1")
|
||||
_spawn_workers(_one_sided_low_precision_worker, world_size, dp_size=world_size)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 6: DeepEP v2 (ElasticBuffer) manager lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -291,6 +291,93 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):
|
||||
reader.touch("key", address, monotonic_id, signature)
|
||||
|
||||
|
||||
class TestShmObjectStorageReferenceCounting(unittest.TestCase):
|
||||
"""One writer and four readers, as with tensor-parallel workers."""
|
||||
|
||||
def setUp(self):
|
||||
ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=1024 * 100, create=True
|
||||
)
|
||||
self.writer = SingleWriterShmObjectStorage(
|
||||
max_object_size=1024 * 10,
|
||||
n_readers=4,
|
||||
ring_buffer=ring_buffer,
|
||||
serde_class=MsgpackSerde,
|
||||
)
|
||||
self.addCleanup(self.writer.close)
|
||||
handle = self.writer.handle()
|
||||
handle.reader_lock = Lock()
|
||||
self.readers = [
|
||||
SingleWriterShmObjectStorage.create_from_handle(handle) for _ in range(4)
|
||||
]
|
||||
for reader in self.readers:
|
||||
self.addCleanup(reader.close)
|
||||
|
||||
def _reference(self, key: str) -> tuple[int, int, list[int], str]:
|
||||
"""Take one writer reference to the key, as one request item does."""
|
||||
if self.writer.is_cached(key):
|
||||
address, monotonic_id = self.writer.get_cached(key)
|
||||
else:
|
||||
address, monotonic_id = self.writer.put(key, "value")
|
||||
return address, monotonic_id, self.writer.get_signature(key), key
|
||||
|
||||
@staticmethod
|
||||
def _touch(reader: SingleWriterShmObjectStorage, reference) -> None:
|
||||
address, monotonic_id, signature, key = reference
|
||||
reader.touch(key, address, monotonic_id, signature)
|
||||
|
||||
def _is_evictable(self, key: str) -> bool:
|
||||
self.writer.free_unused()
|
||||
return not self.writer.is_cached(key)
|
||||
|
||||
def test_item_referenced_again_before_every_reader_fetched_it(self):
|
||||
"""A second request must not leave the item unevictable forever."""
|
||||
first = self._reference("image")
|
||||
self.writer.touch("image")
|
||||
second = self._reference("image")
|
||||
self.writer.release_touches()
|
||||
|
||||
# Every reader starts on the first request; the ones that finish it
|
||||
# go on to the second while the others are still fetching.
|
||||
for reader in self.readers:
|
||||
self._touch(reader, first)
|
||||
for reader in self.readers:
|
||||
reader.get(*first)
|
||||
self._touch(reader, second)
|
||||
reader.get(*second)
|
||||
|
||||
self.assertTrue(self._is_evictable("image"))
|
||||
|
||||
def test_new_item_twice_in_one_request_is_not_evicted_early(self):
|
||||
"""The item must stay until the last reader has fetched it."""
|
||||
self.writer.touch("image")
|
||||
references = [self._reference("image"), self._reference("image")]
|
||||
self.writer.release_touches()
|
||||
|
||||
for reader in self.readers[:-1]:
|
||||
for reference in references:
|
||||
self._touch(reader, reference)
|
||||
for reference in references:
|
||||
reader.get(*reference)
|
||||
for reference in references:
|
||||
self._touch(self.readers[-1], reference)
|
||||
self.assertFalse(self._is_evictable("image"))
|
||||
|
||||
for reference in references:
|
||||
self.readers[-1].get(*reference)
|
||||
self.assertTrue(self._is_evictable("image"))
|
||||
|
||||
def test_touch_protects_item_until_released(self):
|
||||
reference = self._reference("image")
|
||||
for reader in self.readers:
|
||||
reader.get(*reference)
|
||||
|
||||
self.writer.touch("image")
|
||||
self.assertFalse(self._is_evictable("image"))
|
||||
self.writer.release_touches()
|
||||
self.assertTrue(self._is_evictable("image"))
|
||||
|
||||
|
||||
# Reader process function
|
||||
def reader_process(process_id, storage_handle, items_to_read):
|
||||
"""Reader process that connects to existing shared memory and reads data."""
|
||||
|
||||
@@ -7,12 +7,12 @@ Integration tests for NCCL and IPC weight transfer between processes using Ray.
|
||||
"""
|
||||
|
||||
import builtins
|
||||
import importlib.util
|
||||
import pickle
|
||||
import runpy
|
||||
import threading
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -1497,10 +1497,12 @@ class TestDeferredProcessingContract:
|
||||
that caller ends up reaching through a getattr."""
|
||||
|
||||
def _engines(self):
|
||||
return {
|
||||
name: loader()
|
||||
for name, loader in WeightTransferEngineFactory._registry.items()
|
||||
}
|
||||
registry = dict(WeightTransferEngineFactory._registry)
|
||||
# ModelExpress is an optional, separately installed package, but its
|
||||
# backend is always registered. Skip it when the package is missing.
|
||||
if importlib.util.find_spec("modelexpress") is None:
|
||||
registry.pop("modelexpress")
|
||||
return {name: loader() for name, loader in registry.items()}
|
||||
|
||||
def test_every_engine_declares_whether_it_defers(self):
|
||||
for name, cls in self._engines().items():
|
||||
@@ -2054,11 +2056,11 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
|
||||
|
||||
|
||||
def _import_modelexpress_shim():
|
||||
path = (
|
||||
Path(__file__).parents[2]
|
||||
/ "vllm/distributed/weight_transfer/modelexpress_engine.py"
|
||||
spec = importlib.util.find_spec(
|
||||
"vllm.distributed.weight_transfer.modelexpress_engine"
|
||||
)
|
||||
runpy.run_path(str(path), run_name="_test_modelexpress_shim")
|
||||
assert spec is not None and spec.origin is not None
|
||||
runpy.run_path(spec.origin, run_name="_test_modelexpress_shim")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -237,6 +237,46 @@ def test_snapshot_environment_contract(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[(None, "1"), ("0", "0"), ("1", "1")],
|
||||
)
|
||||
def test_snapshot_child_defaults_nccl_ib_without_mutating_parent(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: str | None,
|
||||
expected: str,
|
||||
):
|
||||
if configured is None:
|
||||
monkeypatch.delenv("NCCL_IB_DISABLE", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("NCCL_IB_DISABLE", configured)
|
||||
parent_environment = os.environ.copy()
|
||||
process = create_autospec(subprocess.Popen, instance=True)
|
||||
process.pid = 100
|
||||
popen = create_autospec(subprocess.Popen, return_value=process)
|
||||
monkeypatch.setattr(snapshot_runtime.subprocess, "Popen", popen)
|
||||
|
||||
LocalSnapshotTools().launch_child(tmp_path, ("model",))
|
||||
|
||||
assert popen.call_args.kwargs["env"]["NCCL_IB_DISABLE"] == expected
|
||||
assert os.environ == parent_environment
|
||||
|
||||
|
||||
def test_snapshot_environment_identity_applies_effective_nccl_ib_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.delenv("NCCL_IB_DISABLE", raising=False)
|
||||
implicit = LocalSnapshotTools()._environment_identity()
|
||||
monkeypatch.setenv("NCCL_IB_DISABLE", "1")
|
||||
explicit_disabled = LocalSnapshotTools()._environment_identity()
|
||||
monkeypatch.setenv("NCCL_IB_DISABLE", "0")
|
||||
explicit_enabled = LocalSnapshotTools()._environment_identity()
|
||||
|
||||
assert implicit == explicit_disabled
|
||||
assert explicit_enabled != implicit
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
@@ -369,6 +409,33 @@ ss() { :; }
|
||||
assert result.returncode == (0 if probe is None else 1), result.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("container_attempted", [False, True])
|
||||
def test_snapshot_e2e_cleanup_checks_state_only_after_container_attempt(
|
||||
container_attempted: bool,
|
||||
):
|
||||
script = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ ".buildkite/scripts/initialized-snapshot-e2e.sh"
|
||||
)
|
||||
if not script.exists():
|
||||
pytest.skip("snapshot E2E script is not packaged in this test image")
|
||||
source = script.read_text()
|
||||
cleanup = source[source.index("cleanup() {") : source.index("on_exit() {")]
|
||||
shell = (
|
||||
"set -Eeuo pipefail\n"
|
||||
"CONTAINER_NAME='' PORT_TWO=18002\n"
|
||||
f"CHECK_CLEANUP_STATE={int(container_attempted)}\n"
|
||||
"wait_clean() { echo checked-state; return 1; }\n"
|
||||
+ cleanup
|
||||
+ "\nif cleanup; then exit 0; else exit 1; fi\n"
|
||||
)
|
||||
|
||||
result = subprocess.run(["bash", "-c", shell], capture_output=True, timeout=10)
|
||||
|
||||
assert result.returncode == int(container_attempted), result.stderr
|
||||
assert (b"checked-state" in result.stdout) == container_attempted
|
||||
|
||||
|
||||
def test_snapshot_create_cli_accepts_only_pinned_compact_mode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
@@ -652,6 +719,111 @@ def test_snapshot_create_rolls_back_failed_dump(tmp_path: Path):
|
||||
assert not target.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"failure_kind", ["called-process", "timeout", "missing", "unreadable"]
|
||||
)
|
||||
def test_snapshot_create_preserves_bounded_criu_diagnostics_after_cleanup(
|
||||
tmp_path: Path, failure_kind: str, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
target = tmp_path / "snapshot"
|
||||
tools = _fake_snapshot_tools()
|
||||
criu_tools = LocalSnapshotTools()
|
||||
criu_tools.plugin_dir = tmp_path
|
||||
original_run = criu_tools._run
|
||||
criu_tools._privileged = lambda: [] # type: ignore[method-assign]
|
||||
captured = "captured stderr " + "x" * 9000
|
||||
if failure_kind == "timeout":
|
||||
primary: subprocess.SubprocessError = subprocess.TimeoutExpired(
|
||||
["criu", "dump"], 1, output=b"captured stdout\xff", stderr=captured.encode()
|
||||
)
|
||||
else:
|
||||
primary = subprocess.CalledProcessError(
|
||||
1, ["criu", "dump"], output="captured stdout", stderr=captured
|
||||
)
|
||||
|
||||
def run(command: list[str], **_kwargs: object):
|
||||
if "tail" not in command:
|
||||
raise primary
|
||||
if failure_kind == "unreadable":
|
||||
raise PermissionError("log is root-owned")
|
||||
if failure_kind == "timeout":
|
||||
return subprocess.CompletedProcess(command, 0, "timeout log tail", "")
|
||||
return original_run(command, **_kwargs)
|
||||
|
||||
criu_tools._run = run # type: ignore[method-assign]
|
||||
|
||||
def fail_dump(workdir: Path, _inventory: ProcessInventory) -> None:
|
||||
images = workdir / "images"
|
||||
images.mkdir()
|
||||
if failure_kind != "missing":
|
||||
(images / "dump.log").write_text(
|
||||
"omitted dump prefix" + "d" * 9000 + "distinctive dump diagnostic"
|
||||
)
|
||||
criu_tools._criu("dump", workdir, [])
|
||||
|
||||
tools.dump.side_effect = fail_dump
|
||||
with pytest.raises(type(primary)) as excinfo:
|
||||
create_snapshot(
|
||||
argparse.Namespace(snapshot_dir=str(target), model_tag="Qwen/Qwen3-0.6B"),
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
assert excinfo.value is primary
|
||||
assert not target.exists()
|
||||
diagnostics = "\n".join(caplog.messages)
|
||||
assert "captured stdout" in diagnostics
|
||||
assert "captured stderr" not in diagnostics
|
||||
assert "x" * 100 in diagnostics
|
||||
assert len(diagnostics) < 17000
|
||||
if failure_kind in {"missing", "unreadable"}:
|
||||
assert "dump.log unavailable" in diagnostics
|
||||
expected_error = (
|
||||
"CalledProcessError" if failure_kind == "missing" else "PermissionError"
|
||||
)
|
||||
assert expected_error in diagnostics
|
||||
elif failure_kind == "called-process":
|
||||
assert "distinctive dump diagnostic" in diagnostics
|
||||
assert "omitted dump prefix" not in diagnostics
|
||||
else:
|
||||
assert "timeout log tail" in diagnostics
|
||||
if failure_kind == "timeout":
|
||||
assert "captured stdout�" in diagnostics
|
||||
|
||||
|
||||
def test_snapshot_criu_failure_selects_restore_log(
|
||||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
tools = LocalSnapshotTools()
|
||||
tools.plugin_dir = tmp_path
|
||||
tools._privileged = lambda: [] # type: ignore[method-assign]
|
||||
artifact = tmp_path / "snapshot"
|
||||
images = artifact / "images"
|
||||
images.mkdir(parents=True)
|
||||
(images / "restore.log").write_text(
|
||||
"omitted restore prefix" + "r" * 9000 + "distinctive restore diagnostic"
|
||||
)
|
||||
primary = subprocess.CalledProcessError(1, ["criu", "restore"])
|
||||
original_run = tools._run
|
||||
tailed: list[str] = []
|
||||
|
||||
def run(command: list[str], **_kwargs: object):
|
||||
if "tail" not in command:
|
||||
raise primary
|
||||
tailed.append(command[-1])
|
||||
return original_run(command, **_kwargs)
|
||||
|
||||
tools._run = run # type: ignore[method-assign]
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError) as excinfo:
|
||||
tools._criu("restore", artifact, [])
|
||||
|
||||
assert excinfo.value is primary
|
||||
assert tailed == [str(images / "restore.log")]
|
||||
diagnostics = "\n".join(caplog.messages)
|
||||
assert "distinctive restore diagnostic" in diagnostics
|
||||
assert "omitted restore prefix" not in diagnostics
|
||||
|
||||
|
||||
def _restore_fixture(tmp_path: Path) -> argparse.Namespace:
|
||||
artifact = tmp_path / "snapshot"
|
||||
artifact.mkdir(mode=0o700)
|
||||
@@ -868,6 +1040,96 @@ def test_snapshot_private_path_and_link_remap_security(
|
||||
tools.restore(artifact, _manifest())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("inventory_error", [None, PermissionError("denied")])
|
||||
def test_snapshot_dump_rejects_unavailable_link_remap_inventory_before_criu(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
inventory_error: PermissionError | None,
|
||||
):
|
||||
tools = LocalSnapshotTools()
|
||||
workdir = tmp_path / "snapshot"
|
||||
workdir.mkdir()
|
||||
tools.shm_dir = tmp_path / "missing-shm"
|
||||
if inventory_error is not None:
|
||||
tools.shm_dir.mkdir()
|
||||
|
||||
def fail_inventory(_path: Path):
|
||||
raise inventory_error
|
||||
|
||||
monkeypatch.setattr(Path, "iterdir", fail_inventory)
|
||||
monkeypatch.setattr(tools, "_criu", lambda *_args: pytest.fail("CRIU called"))
|
||||
|
||||
with pytest.raises(SnapshotCreateError, match="inventory CRIU link remaps"):
|
||||
tools.dump(workdir, ProcessInventory(100, (100,), (100,), "GPU"))
|
||||
|
||||
|
||||
def test_snapshot_restore_rejects_saved_remap_without_prefix(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
_mark_snapshot_pids_free(monkeypatch)
|
||||
artifact = _local_restore_artifact(tmp_path)
|
||||
saved = artifact / "link-remaps"
|
||||
saved.mkdir()
|
||||
(saved / "270").write_bytes(b"invalid remap name")
|
||||
tools = LocalSnapshotTools()
|
||||
tools.shm_dir = tmp_path / "shm"
|
||||
tools.shm_dir.mkdir()
|
||||
monkeypatch.setattr(tools, "_criu", lambda *_args: pytest.fail("CRIU called"))
|
||||
|
||||
with pytest.raises(SnapshotRestoreError, match="invalid saved link remap"):
|
||||
tools.restore(artifact, _manifest())
|
||||
|
||||
assert not tuple(tools.shm_dir.iterdir())
|
||||
|
||||
|
||||
def test_snapshot_link_remap_inventory_matches_only_numeric_suffixes(tmp_path: Path):
|
||||
for name in (
|
||||
"link_remap.0",
|
||||
"link_remap.270",
|
||||
"link_remap.",
|
||||
"link_remap.12a",
|
||||
"link-remap.12",
|
||||
"prefix.link_remap.12",
|
||||
"270",
|
||||
):
|
||||
(tmp_path / name).touch()
|
||||
tools = LocalSnapshotTools()
|
||||
tools.shm_dir = tmp_path
|
||||
|
||||
assert tools._link_remap_names() == {"link_remap.0", "link_remap.270"}
|
||||
|
||||
|
||||
def test_snapshot_dump_preserves_primary_failure_when_cleanup_inventory_fails(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
tools = LocalSnapshotTools()
|
||||
workdir = tmp_path / "snapshot"
|
||||
workdir.mkdir()
|
||||
inventory_calls = 0
|
||||
|
||||
def inventory_names() -> set[str]:
|
||||
nonlocal inventory_calls
|
||||
inventory_calls += 1
|
||||
if inventory_calls > 1:
|
||||
raise SnapshotCreateError(
|
||||
"could not inventory CRIU link remaps: PermissionError: denied"
|
||||
)
|
||||
return set()
|
||||
|
||||
monkeypatch.setattr(tools, "_link_remap_names", inventory_names)
|
||||
|
||||
def fail_dump(*_args):
|
||||
raise RuntimeError("dump failed")
|
||||
|
||||
monkeypatch.setattr(tools, "_criu", fail_dump)
|
||||
|
||||
with pytest.raises(SnapshotCreateError) as failure:
|
||||
tools.dump(workdir, ProcessInventory(100, (100,), (100,), "GPU"))
|
||||
|
||||
assert "RuntimeError: dump failed" in str(failure.value)
|
||||
assert "cleanup inventory failed" in str(failure.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("blocked_state", "message"),
|
||||
[("tcp", "external established TCP"), ("io_uring", "kernel.io_uring_disabled=1")],
|
||||
@@ -900,6 +1162,28 @@ def test_snapshot_rejects_unsafe_process_state_before_criu(
|
||||
assert "1.1.1.1:443" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_snapshot_rejects_open_infiniband_device_before_criu(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
tools = LocalSnapshotTools()
|
||||
monkeypatch.setattr(tools, "_tree_pids", lambda _root_pid: (100, 101))
|
||||
monkeypatch.setattr(tools, "_cuda_process_rows", lambda: ("101, GPU-abc",))
|
||||
monkeypatch.setattr(
|
||||
tools,
|
||||
"_descriptor_targets",
|
||||
lambda pid: ("/dev/infiniband/uverbs7",) if pid == 101 else (),
|
||||
)
|
||||
monkeypatch.setattr(tools, "_criu", lambda *_args: pytest.fail("CRIU called"))
|
||||
|
||||
with pytest.raises(SnapshotCreateError) as failure:
|
||||
tools.inventory(100, tmp_path)
|
||||
|
||||
message = str(failure.value)
|
||||
assert "pid 101" in message
|
||||
assert "/dev/infiniband/uverbs7" in message
|
||||
assert "NCCL_IB_DISABLE=1" in message
|
||||
|
||||
|
||||
def test_decode_endpoint_families():
|
||||
assert _decode_endpoint("AF_INET", "3C00000A:D6F6") == "10.0.0.60:55030"
|
||||
assert (
|
||||
|
||||
@@ -40,13 +40,11 @@ def server():
|
||||
"--trust-request-mm-kwargs",
|
||||
]
|
||||
|
||||
# ROCm: Increase timeouts to handle potential network delays and slower
|
||||
# video processing when downloading multiple videos from external sources
|
||||
# ROCm: Allow more time to download videos from external sources.
|
||||
env_overrides = {}
|
||||
if current_platform.is_rocm():
|
||||
env_overrides = {
|
||||
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
|
||||
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "300",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_overrides) as remote_server:
|
||||
|
||||
@@ -88,13 +88,11 @@ def server():
|
||||
*ROCM_EXTRA_ARGS,
|
||||
]
|
||||
|
||||
# ROCm: Increase timeouts to handle potential network delays and slower
|
||||
# video processing when downloading multiple videos from external sources
|
||||
# ROCm: Allow more time to download videos from external sources.
|
||||
env_overrides = {
|
||||
**(
|
||||
{
|
||||
"VLLM_VIDEO_FETCH_TIMEOUT": "120",
|
||||
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "300",
|
||||
}
|
||||
if current_platform.is_rocm()
|
||||
else {}
|
||||
|
||||
@@ -1,20 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ScoringIOProcessor post-tokenization helpers."""
|
||||
"""Unit tests for ScoringIOProcessor helpers."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers, processors
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
BiEncoderIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
JinaRankingIOProcessor,
|
||||
LateInteractionIOProcessor,
|
||||
_apply_post_tokenization_to_token_type_ids,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -113,3 +118,59 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
assert first_doc == 10
|
||||
assert prompt_token_ids[:first_doc] == list(range(10, num_query))
|
||||
assert prompt_token_ids[first_doc:40] == list(range(num_query, 50))
|
||||
|
||||
|
||||
def _pooling_output(
|
||||
request_id: str,
|
||||
data: torch.Tensor,
|
||||
error: RequestError | None = None,
|
||||
) -> PoolingRequestOutput:
|
||||
return PoolingRequestOutput(
|
||||
request_id=request_id,
|
||||
outputs=PoolingOutput(data),
|
||||
prompt_token_ids=[1],
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"processor_type", [BiEncoderIOProcessor, LateInteractionIOProcessor]
|
||||
)
|
||||
@pytest.mark.parametrize("failed_index", [0, 1])
|
||||
def test_paired_scoring_preserves_request_error(processor_type, failed_index):
|
||||
processor = processor_type.__new__(processor_type)
|
||||
processor.pad_token_id = None
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
outputs = [
|
||||
_pooling_output("query", torch.tensor([1.0, 2.0])),
|
||||
_pooling_output("document", torch.tensor([3.0, 4.0])),
|
||||
]
|
||||
outputs[failed_index] = _pooling_output(
|
||||
outputs[failed_index].request_id,
|
||||
torch.empty(0),
|
||||
error,
|
||||
)
|
||||
|
||||
(result,) = processor._post_process(outputs, n_queries=1)
|
||||
|
||||
assert result.request_id == "query_document"
|
||||
assert result.error is error
|
||||
assert result.outputs is outputs[failed_index].outputs
|
||||
|
||||
|
||||
def test_jina_scoring_preserves_request_error_before_indexing_output():
|
||||
processor = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
failed = _pooling_output("request", torch.empty(0), error)
|
||||
|
||||
assert processor._post_process([failed], n_queries=1) == [failed]
|
||||
|
||||
@@ -49,10 +49,60 @@ def _query_key(context: PoolingServeContext) -> str:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys(
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("failed_stage", ["query", "doc"])
|
||||
async def test_flash_late_interaction_cleans_up_queries_on_failure(
|
||||
failed_stage: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving.io_processor = Mock()
|
||||
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
|
||||
|
||||
context = _make_context()
|
||||
monkeypatch.setattr(serving, "_init_ctx", AsyncMock(return_value=context))
|
||||
monkeypatch.setattr(serving, "_preprocessing", AsyncMock())
|
||||
|
||||
async def encode_queries(ctx):
|
||||
ctx.late_interaction_query_keys = ["query-key"]
|
||||
if failed_stage == "query":
|
||||
raise RuntimeError("query failed")
|
||||
|
||||
async def encode_docs(ctx):
|
||||
ctx.late_interaction_doc_keys = ["doc-key"]
|
||||
if failed_stage == "doc":
|
||||
raise RuntimeError("doc failed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
serving,
|
||||
"_flash_late_interaction_encode_queries",
|
||||
AsyncMock(side_effect=encode_queries),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
serving,
|
||||
"_flash_late_interaction_encode_docs",
|
||||
AsyncMock(side_effect=encode_docs),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match=f"{failed_stage} failed"):
|
||||
await serving.flash_late_interaction()
|
||||
|
||||
expected_doc_keys = [] if failed_stage == "query" else ["doc-key"]
|
||||
serving.engine_client.abort.assert_awaited_once_with(
|
||||
["query-key", *expected_doc_keys]
|
||||
)
|
||||
serving.engine_client.collective_rpc.assert_awaited_once_with(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(["query-key"],),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_late_interaction_keys(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
|
||||
prepare_generators = AsyncMock()
|
||||
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
|
||||
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
|
||||
@@ -68,9 +118,24 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys(
|
||||
first_doc_key = _query_key(prepared_contexts[1])
|
||||
second_query_key = _query_key(prepared_contexts[2])
|
||||
second_doc_key = _query_key(prepared_contexts[3])
|
||||
first_doc_request_ids = prepared_contexts[1].prompt_request_ids
|
||||
second_doc_request_ids = prepared_contexts[3].prompt_request_ids
|
||||
|
||||
assert first_query_key == first_doc_key
|
||||
assert second_query_key == second_doc_key
|
||||
assert first_query_key != second_query_key
|
||||
assert "caller-controlled-id" not in first_query_key
|
||||
assert "caller-controlled-id" not in second_query_key
|
||||
assert first_doc_request_ids is not None
|
||||
assert second_doc_request_ids is not None
|
||||
assert first.late_interaction_doc_keys == first_doc_request_ids
|
||||
assert second.late_interaction_doc_keys == second_doc_request_ids
|
||||
assert first_doc_request_ids != second_doc_request_ids
|
||||
assert all("caller-controlled-id" not in key for key in first_doc_request_ids)
|
||||
assert all("caller-controlled-id" not in key for key in second_doc_request_ids)
|
||||
|
||||
assert first.late_interaction_query_keys is not None
|
||||
await serving._cleanup_flash_late_interaction(first)
|
||||
serving.engine_client.abort.assert_awaited_once_with(
|
||||
[*first.late_interaction_query_keys, *first_doc_request_ids]
|
||||
)
|
||||
|
||||
@@ -22,6 +22,10 @@ The streaming cases do the same for `stream=true`. Each coupled chunk's
|
||||
`token_ids` becomes one `/inference/v1/generate` stream chunk, fed
|
||||
through the chunked derender path with `stream_state` threaded across calls
|
||||
the way a client would, so both parsers see the same chunk boundaries.
|
||||
|
||||
The text level cases pin `/inference/v1/generate` with `output_mode="text"`
|
||||
against `/v1/completions/derender` on the same token IDs, batch and streaming,
|
||||
plus logprob resolution and stop string handling.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -440,3 +444,241 @@ async def test_stream_parity_reasoning_and_tool_call(client, chunking):
|
||||
assert any(len(ch["token_ids"]) > 1 for ch in coupled_choices)
|
||||
if not (coupled["reasoning"] and coupled["tool_calls"]):
|
||||
pytest.skip("Model did not emit both a <think> block and a tool call")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline text level: generate(output_mode="text") == derender(generate(...))
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TEXT_MESSAGES = [
|
||||
{"role": "user", "content": "What is 2+2? Answer in one short sentence."}
|
||||
]
|
||||
|
||||
|
||||
async def _render_token_ids(
|
||||
client: httpx.AsyncClient, messages: list[dict]
|
||||
) -> list[int]:
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/render", json={"model": MODEL, "messages": messages}
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["token_ids"]
|
||||
|
||||
|
||||
def _generate_payload(
|
||||
token_ids: list[int], output_mode: str, stream: bool = False, **sampling
|
||||
) -> dict:
|
||||
return {
|
||||
"model": MODEL,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"temperature": 0, "max_tokens": 128, **sampling},
|
||||
"output_mode": output_mode,
|
||||
"stream": stream,
|
||||
}
|
||||
|
||||
|
||||
async def _generate(
|
||||
client: httpx.AsyncClient, token_ids: list[int], output_mode: str, **sampling
|
||||
) -> dict:
|
||||
resp = await client.post(
|
||||
"/inference/v1/generate",
|
||||
json=_generate_payload(token_ids, output_mode, **sampling),
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == output_mode
|
||||
return data
|
||||
|
||||
|
||||
async def _generate_text_stream(
|
||||
client: httpx.AsyncClient, token_ids: list[int], **sampling
|
||||
) -> list[dict]:
|
||||
"""Stream generate at the text level and return the choice of every chunk
|
||||
that has one."""
|
||||
choices: list[dict] = []
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"/inference/v1/generate",
|
||||
json=_generate_payload(token_ids, "text", stream=True, **sampling),
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
assert "error" not in chunk, chunk
|
||||
assert chunk["output_mode"] == "text"
|
||||
choices.extend(chunk["choices"])
|
||||
return choices
|
||||
|
||||
|
||||
async def _derender_completion_text(
|
||||
client: httpx.AsyncClient, output_ids: list[int], finish_reason: str
|
||||
) -> str:
|
||||
resp = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"generate_responses": [
|
||||
{
|
||||
"request_id": "text-parity",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": output_ids,
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
async def _derender_completion_stream_text(
|
||||
client: httpx.AsyncClient, choices: list[dict]
|
||||
) -> str:
|
||||
"""Replay each generate chunk's token IDs through the streaming
|
||||
completions derender endpoint with the same chunk boundaries."""
|
||||
state = None
|
||||
text = ""
|
||||
for choice in choices:
|
||||
resp = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"stream": True,
|
||||
"model": MODEL,
|
||||
"generate_chunk": {
|
||||
"request_id": "text-parity-stream",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": choice.get("token_ids") or [],
|
||||
"finish_reason": choice.get("finish_reason"),
|
||||
}
|
||||
],
|
||||
},
|
||||
"stream_state": state,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
state = data["stream_state"]
|
||||
text += "".join(ch["text"] for ch in data["chunk"]["choices"])
|
||||
return text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_batch(client):
|
||||
"""Inline text matches /v1/completions/derender on the same token IDs."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
inline = (await _generate(client, token_ids, "text"))["choices"][0]
|
||||
|
||||
derendered = await _derender_completion_text(
|
||||
client, inline["token_ids"], inline["finish_reason"]
|
||||
)
|
||||
|
||||
assert inline["text"]
|
||||
assert inline["text"] == derendered
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_stream(client):
|
||||
"""Streamed inline text matches both the streaming and the non-streaming
|
||||
derender of the same token IDs."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
choices = await _generate_text_stream(client, token_ids)
|
||||
inline = "".join(ch["text"] for ch in choices)
|
||||
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
|
||||
finish_reason = choices[-1]["finish_reason"]
|
||||
|
||||
assert finish_reason is not None
|
||||
assert inline == await _derender_completion_stream_text(client, choices)
|
||||
assert inline == await _derender_completion_text(client, output_ids, finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_parity_logprobs(client):
|
||||
"""Inline logprob tokens and bytes match /derender resolving the
|
||||
token_id:N placeholders of the tokens level."""
|
||||
token_ids = await _render_token_ids(client, TEXT_MESSAGES)
|
||||
sampling = {"logprobs": 3}
|
||||
tokens_choice = (await _generate(client, token_ids, "tokens", **sampling))[
|
||||
"choices"
|
||||
][0]
|
||||
text_choice = (await _generate(client, token_ids, "text", **sampling))["choices"][0]
|
||||
assert text_choice["token_ids"] == tokens_choice["token_ids"], (
|
||||
"greedy (temperature=0) generation was expected to be deterministic "
|
||||
"across the two generate calls"
|
||||
)
|
||||
|
||||
disagg = await _disagg(
|
||||
client,
|
||||
tokens_choice["token_ids"],
|
||||
len(token_ids),
|
||||
tokens_choice["finish_reason"],
|
||||
{"model": MODEL, "messages": TEXT_MESSAGES},
|
||||
logprobs=tokens_choice["logprobs"],
|
||||
)
|
||||
|
||||
inline_content = text_choice["logprobs"]["content"]
|
||||
derender_content = disagg["choices"][0]["logprobs"]["content"]
|
||||
assert len(inline_content) == len(derender_content)
|
||||
for inline_entry, derender_entry in zip(inline_content, derender_content):
|
||||
assert inline_entry["token"] == derender_entry["token"]
|
||||
assert inline_entry["bytes"] == derender_entry["bytes"]
|
||||
assert [
|
||||
(top["token"], top["bytes"]) for top in inline_entry["top_logprobs"]
|
||||
] == [(top["token"], top["bytes"]) for top in derender_entry["top_logprobs"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_text_parity_stop_string(client, stream):
|
||||
"""Inline text follows the engine's stop handling, like /v1/completions.
|
||||
|
||||
token_ids keep every generated token, so /derender brings the matched
|
||||
stop string back. Its text must start with the inline text and only
|
||||
differ after it.
|
||||
"""
|
||||
stop = "\n"
|
||||
token_ids = await _render_token_ids(
|
||||
client,
|
||||
[{"role": "user", "content": "Write two short lines about the sea."}],
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
"/v1/completions",
|
||||
json={
|
||||
"model": MODEL,
|
||||
"prompt": token_ids,
|
||||
"temperature": 0,
|
||||
"max_tokens": 128,
|
||||
"stop": [stop],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
coupled = resp.json()["choices"][0]
|
||||
|
||||
if stream:
|
||||
choices = await _generate_text_stream(client, token_ids, stop=[stop])
|
||||
inline_text = "".join(ch["text"] for ch in choices)
|
||||
output_ids = [t for ch in choices for t in ch.get("token_ids") or []]
|
||||
finish_reason = choices[-1]["finish_reason"]
|
||||
else:
|
||||
inline = (await _generate(client, token_ids, "text", stop=[stop]))["choices"][0]
|
||||
inline_text = inline["text"]
|
||||
output_ids = inline["token_ids"]
|
||||
finish_reason = inline["finish_reason"]
|
||||
|
||||
assert inline_text == coupled["text"]
|
||||
assert finish_reason == coupled["finish_reason"]
|
||||
assert stop not in inline_text
|
||||
|
||||
derendered = await _derender_completion_text(client, output_ids, finish_reason)
|
||||
assert derendered.startswith(inline_text)
|
||||
if coupled["stop_reason"] != stop:
|
||||
pytest.skip("Model did not emit the stop string")
|
||||
assert derendered[len(inline_text) :].startswith(stop)
|
||||
|
||||
@@ -50,10 +50,11 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
DerenderStreamState,
|
||||
GenerateResponse,
|
||||
GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamChoice,
|
||||
GenerateTokensStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.parser import Parser
|
||||
@@ -195,10 +196,10 @@ def _make_stream_chunk(
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a GenerateStreamResponse SSE chunk."""
|
||||
return GenerateStreamResponse(
|
||||
return GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(
|
||||
GenerateTokensStreamChoice(
|
||||
index=index,
|
||||
token_ids=token_ids,
|
||||
finish_reason=finish_reason,
|
||||
@@ -233,7 +234,7 @@ def _make_usage_chunk(
|
||||
request_id: str = "test-req",
|
||||
) -> GenerateStreamResponse:
|
||||
"""Build a usage only final SSE chunk (empty choices)."""
|
||||
return GenerateStreamResponse(
|
||||
return GenerateTokensStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[],
|
||||
usage=UsageInfo(
|
||||
@@ -326,11 +327,11 @@ def test_non_streaming_derender_initializes_parser_prefix(
|
||||
)
|
||||
monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))
|
||||
generated_ids = tokenizer.encode("answer", add_special_tokens=False)
|
||||
response = GenerateResponse(
|
||||
response = GenerateTokensResponse(
|
||||
request_id="test",
|
||||
prompt_token_ids=[11, 12],
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0,
|
||||
token_ids=generated_ids,
|
||||
finish_reason="length",
|
||||
@@ -583,10 +584,10 @@ class TestPromptSeededLeadingSpace:
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -601,10 +602,10 @@ class TestPromptSeededLeadingSpace:
|
||||
):
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -620,10 +621,10 @@ class TestPromptSeededLeadingSpace:
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False)
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -639,10 +640,10 @@ class TestPromptSeededLeadingSpace:
|
||||
):
|
||||
_, output_ids, _ = leading_space_ids
|
||||
choices = await derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -655,10 +656,10 @@ class TestPromptSeededLeadingSpace:
|
||||
prompt_ids, output_ids, expected = leading_space_ids
|
||||
choices, _, _ = await derenderer.derender_completion(
|
||||
[
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -1494,10 +1495,10 @@ class TestDerenderChatStreamHarmony:
|
||||
content = "".join(d.content or "" for d in deltas)
|
||||
|
||||
batch_choices = await harmony_derenderer.derender_chat(
|
||||
GenerateResponse(
|
||||
GenerateTokensResponse(
|
||||
request_id="test-harmony-batch",
|
||||
choices=[
|
||||
GenerateResponseChoice(
|
||||
GenerateTokensChoice(
|
||||
index=0, token_ids=output_ids, finish_reason="stop"
|
||||
)
|
||||
],
|
||||
@@ -1743,10 +1744,10 @@ class TestServingDerenderStreamValidation:
|
||||
could never fire since derender_chat_stream itself rejects anything
|
||||
above 1 first."""
|
||||
serving = self._make_serving(parser_configured=False)
|
||||
two_choices = GenerateStreamResponse(
|
||||
two_choices = GenerateTokensStreamResponse(
|
||||
request_id="t",
|
||||
choices=[
|
||||
GenerateResponseStreamChoice(index=i, token_ids=[1]) for i in range(2)
|
||||
GenerateTokensStreamChoice(index=i, token_ids=[1]) for i in range(2)
|
||||
],
|
||||
)
|
||||
request = DerenderChatStreamRequest(
|
||||
|
||||
@@ -6,6 +6,8 @@ from argparse import Namespace
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.api_router import attach_router
|
||||
from vllm.entrypoints.serve.middleware.authenticate import GUARDED_PREFIX
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
RENDER_PATHS = {
|
||||
@@ -17,7 +19,9 @@ RENDER_PATHS = {
|
||||
"/v1/responses/render",
|
||||
}
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
ABORT_PATH = "/inference/v1/abort_requests"
|
||||
UNAUTHENTICATED_ABORT_PATH = "/abort_requests"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH, ABORT_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
@@ -66,4 +70,31 @@ def test_render_routes_remain_enabled_for_render_server():
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, "/abort_requests"}
|
||||
assert paths >= {GENERATE_PATH, ABORT_PATH, UNAUTHENTICATED_ABORT_PATH}
|
||||
|
||||
|
||||
def test_abort_route_without_tokens_only_requires_api_key():
|
||||
"""A decode pool that loads a tokenizer serves generate without
|
||||
--tokens-only and still needs to abort, but only behind --api-key."""
|
||||
paths = registered_paths(("generate",), enable_scale_out=True, tokens_only=False)
|
||||
|
||||
assert ABORT_PATH in paths
|
||||
assert ABORT_PATH.startswith(GUARDED_PREFIX)
|
||||
assert UNAUTHENTICATED_ABORT_PATH not in paths
|
||||
|
||||
|
||||
def test_unauthenticated_abort_route_is_not_registered_twice():
|
||||
"""The RLHF dev router registers /abort_requests before the scale-out
|
||||
routers, so attaching the scale-out one too would add a shadowed duplicate."""
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(tokens_only=True)
|
||||
|
||||
@app.post(UNAUTHENTICATED_ABORT_PATH)
|
||||
async def existing_abort_requests():
|
||||
return None
|
||||
|
||||
attach_router(app)
|
||||
|
||||
paths = [route.path for route in app.routes]
|
||||
assert paths.count(UNAUTHENTICATED_ABORT_PATH) == 1
|
||||
assert ABORT_PATH in paths
|
||||
|
||||
@@ -20,7 +20,8 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
|
||||
)
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateTextResponse,
|
||||
GenerateTokensResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
@@ -144,6 +145,7 @@ def _make_request_output(
|
||||
num_cached_tokens: int | None = None,
|
||||
index: int = 0,
|
||||
spec_decode_metrics: RequestSpecDecodeMetrics | None = None,
|
||||
text: str = "",
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=request_id,
|
||||
@@ -153,7 +155,7 @@ def _make_request_output(
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=index,
|
||||
text="",
|
||||
text=text,
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=None,
|
||||
logprobs=logprobs,
|
||||
@@ -239,7 +241,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert (
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
@@ -411,7 +413,7 @@ async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):
|
||||
if isinstance(chunk, dict) and chunk.get("metrics")
|
||||
)["metrics"]["speculative_decoding"]
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.metrics is not None
|
||||
assert response.metrics.speculative_decoding is not None
|
||||
payload = response.metrics.speculative_decoding.model_dump()
|
||||
@@ -511,7 +513,7 @@ async def test_serve_tokens_omits_spec_decode_metrics_for_parallel_sampling(
|
||||
if isinstance(chunk, dict)
|
||||
)
|
||||
else:
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.metrics is None
|
||||
|
||||
|
||||
@@ -540,7 +542,7 @@ async def test_non_stream_returns_final_prompt_token_ids_when_requested():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.prompt_token_ids == final_prompt_token_ids
|
||||
|
||||
|
||||
@@ -622,7 +624,7 @@ async def test_non_stream_prompt_metadata_requires_return_token_ids(
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
if return_token_ids:
|
||||
assert response.prompt_token_ids == [1, 2, 3]
|
||||
assert response.model_dump()["mm_placeholders"] == MM_PLACEHOLDERS
|
||||
@@ -652,7 +654,7 @@ async def test_non_stream_text_only_prompt_has_no_mm_placeholders():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.prompt_token_ids == [1, 2, 3]
|
||||
assert response.mm_placeholders is None
|
||||
|
||||
@@ -678,7 +680,7 @@ async def test_non_stream_encoder_decoder_omits_mm_placeholders():
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
assert response.prompt_token_ids == [1, 2, 3]
|
||||
assert response.mm_placeholders is None
|
||||
|
||||
@@ -1195,3 +1197,251 @@ async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
|
||||
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
|
||||
)
|
||||
assert choice["sampling_mask"] == expected
|
||||
|
||||
|
||||
async def _collect_sse(response) -> list[Any]:
|
||||
return _parse_sse_chunks([chunk async for chunk in response])
|
||||
|
||||
|
||||
def _text_request(**kwargs) -> GenerateRequest:
|
||||
kwargs.setdefault("sampling_params", SamplingParams(max_tokens=10))
|
||||
return GenerateRequest(
|
||||
token_ids=[1, 2, 3], model=MODEL_NAME, output_mode="text", **kwargs
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_tokens_mode_omits_text_and_echoes_output_mode():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[10], text="ignored", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(response, GenerateTokensResponse)
|
||||
dumped = response.model_dump()
|
||||
assert dumped["output_mode"] == "tokens"
|
||||
assert "text" not in dumped["choices"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_text_mode_returns_text_and_token_ids():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10, 20],
|
||||
text="Hello",
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request())
|
||||
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert response.output_mode == "text"
|
||||
assert response.choices[0].text == "Hello"
|
||||
assert response.choices[0].token_ids == [10, 20]
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert response.usage is not None
|
||||
assert response.usage.completion_tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("no_tokenizer", ["tokens_only", "skip_tokenizer_init"])
|
||||
async def test_text_mode_rejected_without_tokenizer(no_tokenizer: str):
|
||||
"""The no-op detokenizer returns "", so text mode must fail instead
|
||||
of answering 200 with empty text."""
|
||||
engine = _mock_engine()
|
||||
kwargs: dict[str, Any] = {}
|
||||
if no_tokenizer == "tokens_only":
|
||||
kwargs["force_no_detokenize"] = True
|
||||
else:
|
||||
engine.model_config.skip_tokenizer_init = True
|
||||
engine.renderer = _build_renderer(engine.model_config)
|
||||
engine.generate = MagicMock()
|
||||
serving = _build_serving_tokens(engine, **kwargs)
|
||||
|
||||
response = await serving.serve_tokens(_text_request())
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert "requires a tokenizer" in response.error.message
|
||||
engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_mode_logprobs_have_decoded_tokens_and_bytes():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
text="é",
|
||||
logprobs=[
|
||||
{
|
||||
10: Logprob(logprob=-0.5, decoded_token="é"),
|
||||
11: Logprob(logprob=-1.5, decoded_token="e"),
|
||||
}
|
||||
],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=2))
|
||||
)
|
||||
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
logprobs = response.choices[0].logprobs
|
||||
assert logprobs is not None and logprobs.content is not None
|
||||
entry = logprobs.content[0]
|
||||
assert (entry.token, entry.bytes) == ("é", [195, 169])
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("é", [195, 169]),
|
||||
("e", [101]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_mode_logprobs_keep_placeholders_with_return_tokens_as_ids():
|
||||
"""--return-tokens-as-token-ids keeps token_id:N placeholders, as it does
|
||||
on /v1/completions, while text is still decoded."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
text="é",
|
||||
logprobs=[{10: Logprob(logprob=-0.5, decoded_token="é")}],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine, return_tokens_as_token_ids=True)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
_text_request(sampling_params=SamplingParams(max_tokens=10, logprobs=1))
|
||||
)
|
||||
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert response.choices[0].text == "é"
|
||||
logprobs = response.choices[0].logprobs
|
||||
assert logprobs is not None and logprobs.content is not None
|
||||
entry = logprobs.content[0]
|
||||
assert (entry.token, entry.bytes) == ("token_id:10", None)
|
||||
assert [t.token for t in entry.top_logprobs] == ["token_id:10"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_emits_text_deltas():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], text="Hel")
|
||||
yield _make_request_output("req-1", token_ids=[20], text="lo")
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[30], text=" world", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request(stream=True))
|
||||
parsed = await _collect_sse(response)
|
||||
|
||||
assert parsed[-1] == "[DONE]"
|
||||
data_chunks = parsed[:-1]
|
||||
assert {c["output_mode"] for c in data_chunks} == {"text"}
|
||||
assert [c["choices"][0]["text"] for c in data_chunks] == ["Hel", "lo", " world"]
|
||||
assert [c["choices"][0]["token_ids"] for c in data_chunks] == [[10], [20], [30]]
|
||||
assert data_chunks[-1]["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_emits_chunks_without_new_token_ids():
|
||||
"""Text flushed from the stop string buffer and the final abort output
|
||||
carry no token IDs but must still reach the client."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], text="a")
|
||||
yield _make_request_output("req-1", token_ids=[], text="b")
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[], text="", finish_reason="abort", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request(stream=True))
|
||||
data_chunks = (await _collect_sse(response))[:-1]
|
||||
|
||||
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b", ""]
|
||||
assert data_chunks[1]["choices"][0]["token_ids"] == []
|
||||
assert data_chunks[2]["choices"][0]["finish_reason"] == "abort"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_skips_outputs_without_text_or_finish():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10], text="a")
|
||||
yield _make_request_output("req-1", token_ids=[], text="")
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[20], text="b", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(_text_request(stream=True))
|
||||
data_chunks = (await _collect_sse(response))[:-1]
|
||||
|
||||
assert [c["choices"][0]["text"] for c in data_chunks] == ["a", "b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_text_mode_usage_chunk_echoes_output_mode():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[10], text="a", finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
response = await serving.serve_tokens(
|
||||
_text_request(stream=True, stream_options=StreamOptions(include_usage=True))
|
||||
)
|
||||
parsed = await _collect_sse(response)
|
||||
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["output_mode"] == "text"
|
||||
assert usage_chunk["choices"] == []
|
||||
assert usage_chunk["usage"]["completion_tokens"] == 1
|
||||
|
||||
@@ -10,9 +10,21 @@ fail loudly if the validator semantics ever drift.
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderChatStreamRequest,
|
||||
DerenderCompletionRequest,
|
||||
DerenderCompletionStreamRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
GenerateStreamResponse,
|
||||
GenerateTextResponse,
|
||||
GenerateTextStreamResponse,
|
||||
GenerateTokensChoice,
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamResponse,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
@@ -110,3 +122,120 @@ def test_generate_request_rejects_placeholder_outside_prompt():
|
||||
kwargs_data={"image": ["encoded"]},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_output_mode_defaults_to_tokens():
|
||||
assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_output_mode_text_is_accepted():
|
||||
payload = {**_base_payload(), "output_mode": "text"}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", "bogus", None])
|
||||
def test_output_mode_rejects_unsupported_levels(output_mode):
|
||||
"""Levels the server does not implement fail instead of falling back to
|
||||
tokens which would answer 200 with the wrong response shape."""
|
||||
payload = {**_base_payload(), "output_mode": output_mode}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_text_mode_rejects_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "text",
|
||||
}
|
||||
with pytest.raises(ValidationError, match="detokenize"):
|
||||
GenerateRequest.model_validate(payload)
|
||||
|
||||
|
||||
def test_tokens_mode_allows_detokenize_false():
|
||||
payload = {
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"detokenize": False},
|
||||
"output_mode": "tokens",
|
||||
}
|
||||
assert GenerateRequest.model_validate(payload).output_mode == "tokens"
|
||||
|
||||
|
||||
def test_response_without_output_mode_parses_as_tokens():
|
||||
"""Existing clients and older servers never send output_mode."""
|
||||
parsed = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensResponse)
|
||||
assert parsed.output_mode == "tokens"
|
||||
|
||||
|
||||
def test_stream_chunk_without_output_mode_parses_as_tokens():
|
||||
parsed = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"choices": [{"index": 0, "token_ids": [1]}]}
|
||||
)
|
||||
assert isinstance(parsed, GenerateTokensStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_and_chunk_dispatch_on_output_mode():
|
||||
response = TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
chunk = TypeAdapter(GenerateStreamResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
)
|
||||
assert isinstance(response, GenerateTextResponse)
|
||||
assert isinstance(chunk, GenerateTextStreamResponse)
|
||||
|
||||
|
||||
def test_text_response_requires_text_on_every_choice():
|
||||
with pytest.raises(ValidationError, match="text"):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": "text", "choices": [{"index": 0}]}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_mode", ["derender", None, 5])
|
||||
def test_response_rejects_unknown_output_mode(output_mode):
|
||||
with pytest.raises(ValidationError):
|
||||
TypeAdapter(GenerateResponse).validate_python(
|
||||
{"output_mode": output_mode, "choices": []}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_accept_responses_without_output_mode():
|
||||
DerenderChatRequest.model_validate(
|
||||
{"generate_response": {"choices": [{"index": 0, "token_ids": [1]}]}}
|
||||
)
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{
|
||||
"stream": True,
|
||||
"generate_chunk": {"choices": [{"index": 0, "token_ids": [1]}]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_derender_requests_reject_text_responses():
|
||||
"""Text responses are already detokenized with stop strings applied, so
|
||||
re-decoding their token_ids would silently undo the truncation."""
|
||||
text = {"output_mode": "text", "choices": [{"index": 0, "text": "hi"}]}
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatRequest.model_validate({"generate_response": text})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionRequest.model_validate({"generate_responses": [text]})
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderChatStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
with pytest.raises(ValidationError, match="output_mode"):
|
||||
DerenderCompletionStreamRequest.model_validate(
|
||||
{"stream": True, "generate_chunk": text}
|
||||
)
|
||||
|
||||
|
||||
def test_tokens_response_echoes_output_mode_without_text():
|
||||
dumped = GenerateTokensResponse(
|
||||
choices=[GenerateTokensChoice(index=0, token_ids=[1])]
|
||||
).model_dump()
|
||||
assert dumped["output_mode"] == "tokens"
|
||||
assert "text" not in dumped["choices"][0]
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.v1.engine.detokenizer import check_stop_strings
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
ABORT_ENDPOINT = "/inference/v1/abort_requests"
|
||||
|
||||
|
||||
def get_vocab_size(model_name):
|
||||
@@ -317,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value):
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
candidates = [22, 33, 44, 55]
|
||||
rows = [[33], [44, 100], [55]]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": candidates,
|
||||
"prompt_logprob_token_ids": rows,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
@@ -334,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client):
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (len(token_ids) - 2, len(candidates))
|
||||
assert scores.shape == (3, 2)
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, candidates.index(target)] == pytest.approx(
|
||||
expected, abs=1e-3
|
||||
)
|
||||
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
|
||||
assert np.isneginf(scores[[0, 2], 1]).all()
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": candidates}
|
||||
stream_only_ids = {"prompt_logprob_token_ids": rows}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
@@ -349,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client):
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
|
||||
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
empty = np.load(
|
||||
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
|
||||
)
|
||||
assert empty.shape == (0, len(candidates))
|
||||
assert resp.status_code == 400
|
||||
assert "scored rows" in resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -590,3 +587,187 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
|
||||
return tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
|
||||
def _text_mode_payload(token_ids, **sampling_params) -> dict:
|
||||
return {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": 24, "temperature": 0.0, **sampling_params},
|
||||
"output_mode": "text",
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload(token_ids))
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert data["output_mode"] == "text"
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
chat_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
chat_content = chat_resp.json()["choices"][0]["message"]["content"]
|
||||
|
||||
assert data["choices"][0]["text"] == chat_content
|
||||
assert data["choices"][0]["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
|
||||
token_ids = _chat_prompt_token_ids(tokenizer, messages)
|
||||
payload = _text_mode_payload(token_ids)
|
||||
|
||||
non_stream = await client.post(GEN_ENDPOINT, json=payload)
|
||||
non_stream.raise_for_status()
|
||||
|
||||
chunks = []
|
||||
async with client.stream(
|
||||
"POST", GEN_ENDPOINT, json={**payload, "stream": True}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json.loads(line[len("data: ") :]))
|
||||
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
streamed_text = "".join(chunk["choices"][0]["text"] for chunk in chunks)
|
||||
assert streamed_text == non_stream.json()["choices"][0]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
pytest.param(
|
||||
{"sampling_params": {"detokenize": False}}, "detokenize", id="no-detok"
|
||||
),
|
||||
pytest.param({"output_mode": "derender"}, "output_mode", id="unsupported"),
|
||||
],
|
||||
)
|
||||
async def test_text_mode_invalid_requests_return_400(client, overrides, expected):
|
||||
payload = {
|
||||
**_text_mode_payload([1, 2, 3], max_tokens=5),
|
||||
**overrides,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
|
||||
assert resp.status_code == 400
|
||||
assert expected in resp.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
async def test_text_mode_rejected_when_tokens_only(client):
|
||||
"""Without a tokenizer the detokenizer returns "", so the request must
|
||||
fail instead of answering 200 with empty text."""
|
||||
resp = await client.post(GEN_ENDPOINT, json=_text_mode_payload([1, 2, 3]))
|
||||
assert resp.status_code == 400
|
||||
assert "requires a tokenizer" in resp.json()["error"]["message"]
|
||||
|
||||
payload = {**_text_mode_payload([1, 2, 3]), "output_mode": "tokens"}
|
||||
payload["sampling_params"]["detokenize"] = False
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_delivers_abort_finish_chunk(
|
||||
client, tokenizer, messages
|
||||
):
|
||||
"""The final output after an abort has no new token IDs, so it
|
||||
reaches a text stream only because text mode emits finish-only chunks."""
|
||||
payload = {
|
||||
**_text_mode_payload(
|
||||
_chat_prompt_token_ids(tokenizer, messages),
|
||||
max_tokens=900,
|
||||
ignore_eos=True,
|
||||
),
|
||||
"stream": True,
|
||||
"request_id": "text-abort-e2e",
|
||||
}
|
||||
|
||||
chunks = []
|
||||
aborted = False
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
chunk = json.loads(line[len("data: ") :])
|
||||
chunks.append(chunk)
|
||||
if not aborted and chunk["choices"] and chunk["choices"][0]["text"]:
|
||||
abort = await client.post(
|
||||
ABORT_ENDPOINT,
|
||||
json={"request_ids": [chunk["request_id"]]},
|
||||
)
|
||||
assert abort.status_code == 200
|
||||
aborted = True
|
||||
|
||||
assert aborted
|
||||
assert {chunk["output_mode"] for chunk in chunks} == {"text"}
|
||||
final_choice = chunks[-1]["choices"][0]
|
||||
assert final_choice["finish_reason"] == "abort"
|
||||
generated = sum(len(chunk["choices"][0]["token_ids"] or []) for chunk in chunks)
|
||||
assert generated < 900
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
async def test_abort_requests_is_served_without_tokens_only(client):
|
||||
resp = await client.post(
|
||||
ABORT_ENDPOINT, json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
resp = await client.post(ABORT_ENDPOINT, json={})
|
||||
assert resp.status_code == 400
|
||||
|
||||
# The unauthenticated path is only served with --tokens-only.
|
||||
resp = await client.post(
|
||||
"/abort_requests", json={"request_ids": ["generate-tokens-unknown"]}
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
@@ -36,3 +38,50 @@ def test_logprobs_minus_one_emits_all_tokens():
|
||||
num_output_top_logprobs=-1,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 2
|
||||
|
||||
|
||||
def test_tokens_level_logprobs_have_no_bytes():
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert entry.bytes is None
|
||||
assert all(t.bytes is None for t in entry.top_logprobs)
|
||||
|
||||
|
||||
def test_resolved_logprobs_use_engine_decoded_tokens_and_bytes():
|
||||
tokenizer = MagicMock()
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[
|
||||
{7: Logprob(-0.9, decoded_token="é"), 8: Logprob(-1.1, decoded_token="e")}
|
||||
],
|
||||
num_output_top_logprobs=2,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
entry = result.content[0]
|
||||
assert (entry.token, entry.bytes) == ("é", [195, 169])
|
||||
assert [(t.token, t.bytes) for t in entry.top_logprobs] == [
|
||||
("é", [195, 169]),
|
||||
("e", [101]),
|
||||
]
|
||||
tokenizer.decode.assert_not_called()
|
||||
|
||||
|
||||
def test_resolved_logprobs_decode_tokens_the_engine_did_not():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.decode.side_effect = lambda ids: f"<{ids[0]}>"
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7, 9],
|
||||
top_logprobs=[{7: Logprob(-0.9)}, None],
|
||||
num_output_top_logprobs=1,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
assert [e.token for e in result.content] == ["<7>", "<9>"]
|
||||
assert result.content[0].top_logprobs[0].token == "<7>"
|
||||
assert result.content[1].bytes == list(b"<9>")
|
||||
|
||||
@@ -14,12 +14,6 @@ from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
# Increase engine iteration timeout for ROCm where first-use JIT compilation
|
||||
# can exceed the default 60s, causing a silent deadlock in feed_tokens.
|
||||
REALTIME_ENV_OVERRIDES = {
|
||||
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",
|
||||
}
|
||||
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
"mistral",
|
||||
@@ -77,9 +71,7 @@ async def test_multi_chunk_streaming(model_name, mary_had_lamb_audio_chunks):
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
# Receive session.created
|
||||
@@ -183,9 +175,7 @@ async def test_empty_commit_does_not_crash_engine(
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
|
||||
# --- First connection: empty commit (no audio appended) ----------
|
||||
@@ -272,9 +262,7 @@ async def test_session_update_invalid_model_returns_error(model_name):
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
@@ -301,9 +289,7 @@ async def test_commit_without_session_update_returns_error(model_name):
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
ws_url = _get_websocket_url(remote_server)
|
||||
async with websockets.connect(ws_url) as ws:
|
||||
event = await receive_event(ws, timeout=30.0)
|
||||
|
||||
@@ -107,3 +107,37 @@ def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
|
||||
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
|
||||
|
||||
Production allocates index_q with the index-K cache dtype, so the scorers
|
||||
normally see a matched pair and keep the native FP8 MMA. This pins the
|
||||
other branch: a bf16 query against an e4m3 cache still compiles and still
|
||||
ranks the planted blocks.
|
||||
"""
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
idx = minimax_m3_index_decode(
|
||||
q, # bf16 query, deliberately not cast to the cache dtype
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
seq_lens,
|
||||
SEQ_LEN,
|
||||
TOPK,
|
||||
0,
|
||||
0,
|
||||
HEADS,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
@@ -596,6 +596,114 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(7)
|
||||
head_dim = nope_dim + rope_dim
|
||||
return dict(
|
||||
q=torch.randn(3, 3, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
|
||||
kv=torch.randn(5, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
|
||||
indices=torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device),
|
||||
indptr=torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device),
|
||||
scale=head_dim**-0.5,
|
||||
attn_sink=torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device),
|
||||
nope_head_dim=nope_dim,
|
||||
rope_head_dim=rope_dim,
|
||||
)
|
||||
|
||||
|
||||
# 448+64 is DeepSeek V4/V4.1, 512+64 is V3.2, and 256+0 stands in for the
|
||||
# NoPE-only layouts where the destination spans the whole head dim.
|
||||
SPARSE_PREFILL_DIMS = [(NOPE_HEAD_DIM, ROPE_HEAD_DIM), (512, 64), (256, 0)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_out_matches_allocated(
|
||||
nope_dim: int, rope_dim: int
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
|
||||
allocated = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)
|
||||
|
||||
q = kwargs["q"]
|
||||
dest = torch.empty(q.shape[0], q.shape[1], nope_dim, dtype=q.dtype, device=q.device)
|
||||
returned = _rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
assert returned is dest
|
||||
torch.testing.assert_close(dest, allocated[..., :nope_dim], atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_out_view_leaves_neighbors_untouched(
|
||||
nope_dim: int, rope_dim: int
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
|
||||
q = kwargs["q"]
|
||||
num_queries, num_heads = q.shape[0], q.shape[1]
|
||||
|
||||
buffer = torch.full(
|
||||
(num_queries + 4, num_heads + 2, nope_dim),
|
||||
-7.0,
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
)
|
||||
dest = buffer[2 : 2 + num_queries, :num_heads]
|
||||
assert not dest.is_contiguous()
|
||||
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :nope_dim]
|
||||
torch.testing.assert_close(dest, expected, atol=0.0, rtol=0.0)
|
||||
|
||||
untouched = torch.ones_like(buffer, dtype=torch.bool)
|
||||
untouched[2 : 2 + num_queries, :num_heads] = False
|
||||
assert torch.all(buffer[untouched] == -7.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("out_dtype", [torch.float32, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_out_casts_to_dest_dtype(
|
||||
out_dtype: torch.dtype,
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(512, 64)
|
||||
q = kwargs["q"]
|
||||
dest = torch.empty(q.shape[0], q.shape[1], 512, dtype=out_dtype, device=q.device)
|
||||
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :512]
|
||||
assert dest.dtype == out_dtype
|
||||
torch.testing.assert_close(dest.float(), expected.float(), atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_head_dim_wide_out_keeps_rope_rows() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(NOPE_HEAD_DIM, ROPE_HEAD_DIM)
|
||||
q = kwargs["q"]
|
||||
dest = torch.empty_like(q)
|
||||
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
expected = _ref_sparse_prefill_ragged(
|
||||
q, kwargs["kv"], [[0, 2], [1, 3, 4], []], kwargs["scale"], kwargs["attn_sink"]
|
||||
)
|
||||
torch.testing.assert_close(dest, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_queries", "on_gfx950", "expected"),
|
||||
[(1023, True, False), (1024, True, True), (1024, False, False)],
|
||||
@@ -718,10 +826,11 @@ def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> Non
|
||||
output = torch.empty_like(q)
|
||||
dense_fallback_calls = 0
|
||||
|
||||
def fake_dense_fallback(*args, **kwargs):
|
||||
def fake_dense_fallback(*args, out, **kwargs):
|
||||
nonlocal dense_fallback_calls
|
||||
dense_fallback_calls += 1
|
||||
return torch.zeros_like(q)
|
||||
out.zero_()
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
|
||||
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)
|
||||
|
||||
@@ -194,7 +194,7 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_fused_gdn_decode=True,
|
||||
enable_fused_gdn_spec_decode=True,
|
||||
norm=types.SimpleNamespace(
|
||||
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
|
||||
),
|
||||
|
||||
@@ -167,7 +167,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
|
||||
draft = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
use_native_mega_moe=False,
|
||||
use_sequence_parallel=False,
|
||||
hc_mult=hc_mult,
|
||||
layers=[make_layer(mix) for mix in mixes],
|
||||
|
||||
@@ -88,6 +88,15 @@ class Config:
|
||||
|
||||
torch_trace_dir_path: str | None = None
|
||||
|
||||
# Force AiterExperts's hidden_pad/intermediate_pad computation
|
||||
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
|
||||
# None (default) preserves today's behavior: FusedMoEConfig defaults both
|
||||
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
|
||||
# intermediate_pad come out to 0.
|
||||
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
hidden_dim_unpadded: int | None = None
|
||||
intermediate_size_per_partition_unpadded: int | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.quant_config is None:
|
||||
self.quant_config = TestMoEQuantConfig(None, False, False, None)
|
||||
@@ -664,6 +673,10 @@ def make_modular_kernel(
|
||||
activation=config.activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=RoutingMethodType.DeepSeekV3,
|
||||
hidden_dim_unpadded=config.hidden_dim_unpadded,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
config.intermediate_size_per_partition_unpadded
|
||||
),
|
||||
)
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
|
||||
@@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=True,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_sink=None,
|
||||
)
|
||||
|
||||
check_accuracy(
|
||||
@@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
|
||||
router_logits=logits,
|
||||
e_score_correction_bias=bias,
|
||||
routed_scaling_factor=scale,
|
||||
routing_sink=None,
|
||||
**common,
|
||||
)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
|
||||
|
||||
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
|
||||
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
|
||||
MoE geometry of a real model, warms it up and runs a forward whose inputs are
|
||||
chosen so that every quantization step in the megakernel is exact, for the
|
||||
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
|
||||
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
|
||||
32). The expected output is then known in closed form and compared bit for
|
||||
bit.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
FlashInferMoeEp,
|
||||
FlashInferMoeEpWeights,
|
||||
apply_topk_in_fc1,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Geometry:
|
||||
name: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_experts: int
|
||||
top_k: int
|
||||
|
||||
|
||||
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
|
||||
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
|
||||
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
|
||||
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
|
||||
GEOMETRIES = (
|
||||
Geometry("dsv4-pro", 7168, 3072, 384, 6),
|
||||
Geometry("dsv4-flash", 4096, 2048, 256, 6),
|
||||
Geometry("mistral-small-4", 4096, 2048, 128, 4),
|
||||
Geometry("mistral-large-3", 7168, 4096, 128, 4),
|
||||
)
|
||||
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
|
||||
NUM_TOKENS_PER_RANK = 4
|
||||
MAX_TOKENS_PER_RANK = 128
|
||||
|
||||
# Every block scale below is either 1 or a power of two, so the FP4 codes
|
||||
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
|
||||
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
|
||||
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
|
||||
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
|
||||
#
|
||||
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
|
||||
# it; every weight column the sentinel meets is zero.
|
||||
X_SIGNAL = 2.0
|
||||
X_SENTINEL = 6.0
|
||||
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
|
||||
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
|
||||
W13_DIAG = 6.0
|
||||
H_SIGNAL = 144.0
|
||||
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
|
||||
# sentinel sits next to a zero intermediate channel and pins the block scale.
|
||||
W2_SENTINEL = 6.0
|
||||
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
|
||||
|
||||
|
||||
def _expert_out(expert: int) -> float:
|
||||
return E2M1_CODES[expert % len(E2M1_CODES)]
|
||||
|
||||
|
||||
def _topk_weights(top_k: int) -> tuple[float, ...]:
|
||||
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
|
||||
# weight * 144 must requantize exactly too: powers of two do.
|
||||
return tuple(2.0 ** -(k + 1) for k in range(top_k))
|
||||
|
||||
|
||||
def _routing_method(backend: str) -> RoutingMethodType:
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
|
||||
return RoutingMethodType.DeepseekV4
|
||||
return RoutingMethodType.Default
|
||||
|
||||
|
||||
def _routing(
|
||||
geometry: Geometry, rank: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""top_k distinct experts per token, strided across ranks so dispatch goes
|
||||
remote."""
|
||||
base = (
|
||||
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
|
||||
) % geometry.num_experts
|
||||
stride = geometry.num_experts // geometry.top_k
|
||||
offsets = torch.arange(geometry.top_k, device=device) * stride
|
||||
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
|
||||
torch.int32
|
||||
)
|
||||
topk_weights = torch.tensor(
|
||||
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
return topk_ids, topk_weights
|
||||
|
||||
|
||||
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
|
||||
x = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
x[t, 16 * t] = X_SIGNAL
|
||||
x[t, 16 * t + 1] = X_SENTINEL
|
||||
return x
|
||||
|
||||
|
||||
def _local_expert_weights(
|
||||
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
|
||||
) -> FlashInferMoeEpWeights:
|
||||
w13 = torch.zeros(
|
||||
num_local_experts,
|
||||
2 * geometry.intermediate_size,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
w2 = torch.zeros(
|
||||
num_local_experts,
|
||||
geometry.hidden_size,
|
||||
geometry.intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
|
||||
for local in range(num_local_experts):
|
||||
expert = rank * num_local_experts + local
|
||||
w13[local, 0, token_columns] = W13_DIAG
|
||||
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
|
||||
w2[local, expert, 0] = _expert_out(expert)
|
||||
w2[local, expert, 2] = W2_SENTINEL
|
||||
return FlashInferMoeEpWeights(w13=w13, w2=w2)
|
||||
|
||||
|
||||
def _expected_output(
|
||||
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.float32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
for k in range(geometry.top_k):
|
||||
expert = int(topk_ids[t, k])
|
||||
weight = float(topk_weights[t, k])
|
||||
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
|
||||
return expected.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _moe_config(
|
||||
vllm_config: VllmConfig,
|
||||
geometry: Geometry,
|
||||
world_size: int,
|
||||
device: torch.device,
|
||||
backend: str,
|
||||
) -> FusedMoEConfig:
|
||||
parallel = FusedMoEParallelConfig.make(
|
||||
tp_size_=1,
|
||||
pcp_size_=1,
|
||||
dp_size_=world_size,
|
||||
sp_size_=1,
|
||||
vllm_parallel_config=vllm_config.parallel_config,
|
||||
)
|
||||
return FusedMoEConfig(
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
num_local_experts=geometry.num_experts // world_size,
|
||||
num_logical_experts=geometry.num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device=device,
|
||||
routing_method=_routing_method(backend),
|
||||
moe_parallel_config=parallel,
|
||||
in_dtype=torch.bfloat16,
|
||||
moe_backend=backend,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
)
|
||||
|
||||
|
||||
def _check_expert_views_alias_kernel_memory(
|
||||
adapter: FlashInferMoeEp,
|
||||
geometry: Geometry,
|
||||
rank: int,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
|
||||
|
||||
Swap local experts 0 and 1 in place on every rank, then route through the
|
||||
swapped physical slots: the output must be the one the unswapped logical
|
||||
experts would have produced.
|
||||
"""
|
||||
for view in adapter.expert_weight_views():
|
||||
first = view[0].clone()
|
||||
view[0].copy_(view[1])
|
||||
view[1].copy_(first)
|
||||
torch.distributed.barrier()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
slot = topk_ids % num_local_experts
|
||||
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
|
||||
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
|
||||
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
|
||||
|
||||
|
||||
def _init_warmup_forward(
|
||||
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
|
||||
) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.data_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.kernel_config.moe_backend = backend
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
|
||||
)
|
||||
rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
adapter = FlashInferMoeEp(
|
||||
moe,
|
||||
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
try:
|
||||
adapter.warmup()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = _expected_output(geometry, topk_ids, topk_weights)
|
||||
assert output.dtype == expected.dtype and output.shape == expected.shape
|
||||
assert torch.equal(output, expected), (
|
||||
f"rank {rank}: max abs diff "
|
||||
f"{(output.float() - expected.float()).abs().max().item()}, "
|
||||
f"nonzero output channels per token "
|
||||
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
|
||||
)
|
||||
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL:
|
||||
_check_expert_views_alias_kernel_memory(
|
||||
adapter, geometry, rank, moe.num_local_experts, device
|
||||
)
|
||||
finally:
|
||||
adapter.destroy()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="The FlashInfer MoE-EP megakernels require SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_init_warmup_forward_is_exact_on_grid_inputs(
|
||||
world_size: int, backend: str, geometry: Geometry
|
||||
):
|
||||
pytest.importorskip("flashinfer.moe_ep")
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
|
||||
pytest.skip("DeepGEMM is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs")
|
||||
distributed_run(_init_warmup_forward, world_size, geometry, backend)
|
||||
@@ -42,7 +42,7 @@ from .modular_kernel_tools.parallel_utils import (
|
||||
ProcessGroupInfo,
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from .utils import check_accuracy
|
||||
from .utils import check_accuracy, make_test_weights
|
||||
|
||||
has_any_multi_gpu_package = (
|
||||
has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe()
|
||||
@@ -691,6 +691,303 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
|
||||
)
|
||||
|
||||
|
||||
# --- 4b: hidden_dim_unpadded/intermediate_size_per_partition_unpadded matrix -
|
||||
|
||||
# K/N intentionally unaligned to AITER's 64/128 granularity (unlike the
|
||||
# shared mk_objects.py defaults) so hidden_pad/intermediate_pad are forced.
|
||||
_PADDING_E = 8
|
||||
_PADDING_M = 16
|
||||
_PADDING_TOPK = 2
|
||||
_PADDING_K_UNPADDED = 896 # 7 * 128
|
||||
_PADDING_K_PADDED = 1024 # +128
|
||||
_PADDING_N_UNPADDED = 384 # 3 * 128
|
||||
_PADDING_N_PADDED = 512 # +128
|
||||
# hidden_pad = (K_padded - hidden_dim_unpadded) // 128 * 128 (rocm_aiter_moe.py)
|
||||
_PADDING_HIDDEN_PAD_EXPECTED = 128
|
||||
# intermediate_pad = (N_padded - intermediate_size_per_partition_unpadded)
|
||||
# // 64 * 64 * (2 if tp_size == 1 else 1) (rocm_aiter_moe.py); tp_size==1 here.
|
||||
_PADDING_INTERMEDIATE_PAD_EXPECTED = 256
|
||||
|
||||
# {no padding, hidden-only, intermediate-only, both} -> (pad_hidden, pad_intermediate)
|
||||
_PADDING_MODES: dict[str, tuple[bool, bool]] = {
|
||||
"none": (False, False),
|
||||
"hidden": (True, False),
|
||||
"intermediate": (False, True),
|
||||
"both": (True, True),
|
||||
}
|
||||
|
||||
# unquantized + the 4 fp8 configs AiterExperts supports (excludes MXFP4,
|
||||
# stubbed below pending MI350/gfx950, and MK_QUANT_CONFIGS[1], unsupported).
|
||||
_PADDING_QUANT_CONFIGS = [
|
||||
MK_QUANT_CONFIGS[0], # unquantized
|
||||
MK_QUANT_CONFIGS[2], # fp8 channel weights / per-token activations
|
||||
MK_QUANT_CONFIGS[3], # fp8 per-tensor weights / per-tensor activations
|
||||
MK_QUANT_CONFIGS[4], # fp8 per-tensor weights / per-token activations
|
||||
MK_QUANT_CONFIGS[5], # fp8 128x128-block weights / 128-block activations
|
||||
]
|
||||
_PADDING_QUANT_IDS = [
|
||||
"unquantized",
|
||||
"fp8_channel_token",
|
||||
"fp8_tensor_tensor",
|
||||
"fp8_tensor_token",
|
||||
"fp8_block_token",
|
||||
]
|
||||
|
||||
|
||||
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
|
||||
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
|
||||
gate/up halves of `real_per_half` size each."""
|
||||
padded_per_half = t.shape[1] // 2
|
||||
return torch.cat(
|
||||
[
|
||||
t[:, :real_per_half],
|
||||
t[:, padded_per_half : padded_per_half + real_per_half],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
|
||||
def _slice_unpadded_weights(
|
||||
weights: WeightTensors,
|
||||
quant_config: TestMoEQuantConfig | None,
|
||||
k_unpadded: int,
|
||||
n_unpadded: int,
|
||||
) -> WeightTensors:
|
||||
"""Derive the unpadded sub-block of a padded WeightTensors: same values,
|
||||
restricted to the region hidden_pad/intermediate_pad keeps.
|
||||
|
||||
Scale-shape choice (per_out_ch) must match _make_padding_matrix_weights()."""
|
||||
block_shape = quant_config.block_shape if quant_config is not None else None
|
||||
per_out_ch = quant_config is not None and quant_config.per_out_ch_quant
|
||||
|
||||
w1 = _slice_gate_up_rows(weights.w1[:, :, :k_unpadded], n_unpadded)
|
||||
w2 = weights.w2[:, :k_unpadded, :n_unpadded]
|
||||
|
||||
if weights.w1_scale is None:
|
||||
w1_scale = w2_scale = None
|
||||
elif weights.w2_scale is None:
|
||||
raise AssertionError("w1_scale and w2_scale must both be set or both None")
|
||||
elif block_shape is not None:
|
||||
block_n, block_k = block_shape
|
||||
assert n_unpadded % block_n == 0 and k_unpadded % block_k == 0
|
||||
n_scale, k_scale = n_unpadded // block_n, k_unpadded // block_k
|
||||
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_scale)[..., :k_scale]
|
||||
w2_scale = weights.w2_scale[:, :k_scale, :n_scale]
|
||||
elif per_out_ch:
|
||||
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_unpadded)
|
||||
w2_scale = weights.w2_scale[:, :k_unpadded, :]
|
||||
else:
|
||||
# Per-tensor scale: shape (E, 1, 1), independent of K/N -- reuse as-is.
|
||||
w1_scale = weights.w1_scale
|
||||
w2_scale = weights.w2_scale
|
||||
|
||||
return WeightTensors(w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale)
|
||||
|
||||
|
||||
def _make_padding_matrix_weights(config: Config) -> WeightTensors:
|
||||
"""Like WeightTensors.make(), but scales weights by
|
||||
config.is_per_out_ch_quant instead of is_per_act_token_quant.
|
||||
|
||||
WeightTensors.make() ties weight-scale shape to activation quant, so
|
||||
fp8_tensor_token would get per-channel scales despite being a per-tensor
|
||||
weight scheme. Scoped here rather than fixing WeightTensors.make(),
|
||||
which other tests depend on."""
|
||||
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
|
||||
e=config.E,
|
||||
n=config.N,
|
||||
k=config.K,
|
||||
in_dtype=config.dtype,
|
||||
quant_dtype=config.quant_dtype,
|
||||
block_shape=config.quant_block_shape,
|
||||
per_out_ch_quant=config.is_per_out_ch_quant,
|
||||
)
|
||||
return WeightTensors(
|
||||
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
|
||||
)
|
||||
|
||||
|
||||
def _slice_unpadded_rank_tensors(
|
||||
rank_tensors: RankTensors, k_unpadded: int
|
||||
) -> RankTensors:
|
||||
return RankTensors(
|
||||
hidden_states=rank_tensors.hidden_states[:, :k_unpadded].contiguous(),
|
||||
hidden_states_scale=rank_tensors.hidden_states_scale,
|
||||
topk_weights=rank_tensors.topk_weights,
|
||||
topk_ids=rank_tensors.topk_ids,
|
||||
expert_map=rank_tensors.expert_map,
|
||||
)
|
||||
|
||||
|
||||
def _aiter_padding_matrix_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
padded_config: Config,
|
||||
padded_weights: WeightTensors,
|
||||
verbose: bool,
|
||||
hidden_pad_expected: int,
|
||||
intermediate_pad_expected: int,
|
||||
):
|
||||
device = torch.device(f"cuda:{pgi.local_rank}")
|
||||
init_workspace_manager(device)
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
weights = copy.deepcopy(padded_weights)
|
||||
weights.to_current_device()
|
||||
|
||||
rank_tensors = RankTensors.make(padded_config, pgi)
|
||||
|
||||
with mock.patch.object(
|
||||
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
|
||||
) as fused_moe_mock:
|
||||
mk_out = run_modular_kernel(
|
||||
pgi, vllm_config, padded_config, weights, rank_tensors
|
||||
)
|
||||
|
||||
assert fused_moe_mock.call_count > 0, (
|
||||
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
|
||||
)
|
||||
for call in fused_moe_mock.call_args_list:
|
||||
assert call.kwargs["hidden_pad"] == hidden_pad_expected, (
|
||||
f"AiterExperts.apply() forwarded hidden_pad="
|
||||
f"{call.kwargs['hidden_pad']}, expected {hidden_pad_expected}."
|
||||
)
|
||||
assert call.kwargs["intermediate_pad"] == intermediate_pad_expected, (
|
||||
f"AiterExperts.apply() forwarded intermediate_pad="
|
||||
f"{call.kwargs['intermediate_pad']}, expected "
|
||||
f"{intermediate_pad_expected}."
|
||||
)
|
||||
|
||||
# AiterExperts.apply() never slices its own output -- the buffer stays
|
||||
# raw (padded) width; only the caller-sliced unpadded prefix is checked.
|
||||
assert mk_out.shape[-1] == padded_config.K, (
|
||||
f"AiterExperts output width {mk_out.shape[-1]} != raw hidden_dim "
|
||||
f"{padded_config.K}."
|
||||
)
|
||||
mk_out = mk_out[..., :_PADDING_K_UNPADDED]
|
||||
# Only the real slice is checked: beyond it is unspecified/reused memory.
|
||||
assert torch.isfinite(mk_out).all(), (
|
||||
"AiterExperts output must not contain NaN/Inf in the real "
|
||||
"hidden_dim_unpadded-wide output slice."
|
||||
)
|
||||
|
||||
unpadded_weights = _slice_unpadded_weights(
|
||||
weights,
|
||||
padded_config.quant_config,
|
||||
_PADDING_K_UNPADDED,
|
||||
_PADDING_N_UNPADDED,
|
||||
)
|
||||
unpadded_rank_tensors = _slice_unpadded_rank_tensors(
|
||||
rank_tensors, _PADDING_K_UNPADDED
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = reference_moe_impl(
|
||||
padded_config, unpadded_weights, unpadded_rank_tensors
|
||||
)
|
||||
|
||||
# ref_out's magnitude here (~1e-3-1e-2) is below atol=3e-2, so a zeroed-out
|
||||
# mk_out would still pass check_accuracy below -- guard against that.
|
||||
ref_scale = ref_out.abs().mean()
|
||||
mk_scale = mk_out.abs().mean()
|
||||
assert mk_scale > 0.5 * ref_scale, (
|
||||
f"AiterExperts output magnitude (mean |mk_out|={mk_scale:.6f}) looks "
|
||||
f"degenerate/zeroed vs. reference (mean |ref_out|={ref_scale:.6f})."
|
||||
)
|
||||
|
||||
# Lenient check for every scheme, including unquantized: a strict
|
||||
# assert_close(atol=rtol=3e-2) spuriously fails ~0.1-0.2% of elements
|
||||
# on MI300 even for unquantized AiterExperts (confirmed on hardware).
|
||||
check_accuracy(ref_out, mk_out, atol=3e-2, rtol=3e-2, percent=0.9)
|
||||
|
||||
|
||||
@require_aiter_moe
|
||||
@pytest.mark.parametrize("quant_config", _PADDING_QUANT_CONFIGS, ids=_PADDING_QUANT_IDS)
|
||||
@pytest.mark.parametrize("mode", list(_PADDING_MODES))
|
||||
def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig | None):
|
||||
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
|
||||
Exercises AiterExperts's hidden_pad/intermediate_pad (rocm_aiter_moe.py)
|
||||
across {no padding, hidden-only, intermediate-only, both} x the
|
||||
unquantized + 4 fp8 quant schemes it supports. Compares against an
|
||||
unpadded reference and verifies the hidden_pad/intermediate_pad
|
||||
AiterExperts.apply() forwards to rocm_aiter_ops.fused_moe.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
pad_hidden, pad_intermediate = _PADDING_MODES[mode]
|
||||
|
||||
if (
|
||||
quant_config is not None
|
||||
and quant_config.block_shape is not None
|
||||
and not pad_hidden
|
||||
):
|
||||
pytest.skip(
|
||||
"AITER's block-quantized (128x128) CK GEMM kernel does not "
|
||||
f"support the raw (unpadded) hidden_dim={_PADDING_K_UNPADDED} "
|
||||
"shape on this hardware -- it raises 'wrong! device_gemm with "
|
||||
"the specified compilation parameters does not support this "
|
||||
"GEMM problem'. Only hidden_pad-forcing modes (raw "
|
||||
f"hidden_dim={_PADDING_K_PADDED}) are exercised for this quant "
|
||||
"scheme."
|
||||
)
|
||||
|
||||
k = _PADDING_K_PADDED if pad_hidden else _PADDING_K_UNPADDED
|
||||
n = _PADDING_N_PADDED if pad_intermediate else _PADDING_N_UNPADDED
|
||||
|
||||
config = Config(
|
||||
Ms=_PADDING_M,
|
||||
K=k,
|
||||
N=n,
|
||||
E=_PADDING_E,
|
||||
topks=_PADDING_TOPK,
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
world_size=1,
|
||||
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
_PADDING_N_UNPADDED if pad_intermediate else None
|
||||
),
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
assert config.fe_supports_quant_scheme(), (
|
||||
f"AiterExperts does not support quant scheme {quant_config}."
|
||||
)
|
||||
|
||||
weights = _make_padding_matrix_weights(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
|
||||
hidden_pad_expected = _PADDING_HIDDEN_PAD_EXPECTED if pad_hidden else 0
|
||||
intermediate_pad_expected = (
|
||||
_PADDING_INTERMEDIATE_PAD_EXPECTED if pad_intermediate else 0
|
||||
)
|
||||
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
_aiter_padding_matrix_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
False,
|
||||
hidden_pad_expected,
|
||||
intermediate_pad_expected,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="MXFP4 AiterExperts padding requires MI350/gfx950 hardware. "
|
||||
'See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").'
|
||||
)
|
||||
def test_aiter_moe_padding_matrix_mxfp4():
|
||||
"""Stub for the {no padding, hidden-only, intermediate-only, both} MXFP4
|
||||
padding matrix once MI350/gfx950 hardware is available in CI."""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Ability to test individual PrepareAndFinalize and FusedExperts combination
|
||||
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
|
||||
|
||||
@@ -16,10 +16,8 @@ from vllm.utils.torch_utils import set_random_seed
|
||||
NUM_TOKENS = [1, 17, 256]
|
||||
TOP_KS = [2, 8]
|
||||
HIDDEN_SIZE = 512
|
||||
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
|
||||
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
|
||||
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
|
||||
HIDDEN_SIZES = [512, 1000, 3072, 7168]
|
||||
# Include partial tiles and hidden sizes above the CUDA 4096-element tile cap.
|
||||
HIDDEN_SIZES = [512, 1000, 3072, 4097, 7168]
|
||||
DTYPES = [torch.bfloat16, torch.float32]
|
||||
NUM_EXPERTS = 16
|
||||
|
||||
@@ -233,7 +231,7 @@ def test_nonlocal_and_padding_adjacent():
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("dtype", [*DTYPES, torch.float16])
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_weighted_sum_matches_reference(
|
||||
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
|
||||
@@ -268,7 +266,11 @@ def test_weighted_sum_matches_reference(
|
||||
|
||||
assert not out.isnan().any(), "invalid slots leaked into the reduction"
|
||||
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
|
||||
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||
atol, rtol = {
|
||||
torch.bfloat16: (1e-2, 1e-2),
|
||||
torch.float16: (1e-3, 1e-3),
|
||||
torch.float32: (1e-4, 1e-4),
|
||||
}[dtype]
|
||||
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@@ -360,7 +362,7 @@ def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
|
||||
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 7168
|
||||
|
||||
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||
|
||||
@@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize(
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=None,
|
||||
),
|
||||
)
|
||||
return
|
||||
@@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=None,
|
||||
)
|
||||
else:
|
||||
# Modular impl uses topk_weights and topk_ids
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -91,6 +91,12 @@ QWEN4_EXP_SM100_CASES = [
|
||||
for num_tokens, config in plans.items()
|
||||
]
|
||||
|
||||
QWEN4_EXP_SM121_CASES = [
|
||||
(n, k, num_tokens, config)
|
||||
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS.items()
|
||||
for num_tokens, config in plans.items()
|
||||
]
|
||||
|
||||
EXPECTED_CUTE_CONFIGS = {
|
||||
(3072, 7168, 1): (224, 3, 4, 8),
|
||||
(3072, 7168, 2): (128, 3, 2, 8),
|
||||
@@ -725,6 +731,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
|
||||
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
|
||||
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
|
||||
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
|
||||
((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS),
|
||||
((8, 0), {}),
|
||||
],
|
||||
)
|
||||
@@ -939,6 +946,29 @@ def test_qwen4_exp_sm100_selected_shapes(
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM121_CASES)
|
||||
def test_qwen4_exp_sm121_selected_shapes(
|
||||
n: int,
|
||||
k: int,
|
||||
num_tokens: int,
|
||||
config: SkinnyGemmConfig,
|
||||
) -> None:
|
||||
_require_capability_and_cute((12, 1))
|
||||
torch.manual_seed(42 + num_tokens)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
selected = qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS[(n, k)][num_tokens]
|
||||
assert selected == config
|
||||
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
|
||||
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
|
||||
_require_sm103_and_cute()
|
||||
spec = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||
|
||||
+225
-118
@@ -636,8 +636,8 @@ def test_top_k_per_row_decode_gfx950_k512_masked_graph_replay() -> None:
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_get_aiter_top_k_kernel,
|
||||
_launch_aiter_top_k_per_row_prefill,
|
||||
)
|
||||
|
||||
@@ -647,17 +647,14 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
top_k = 4
|
||||
indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
if not rocm_aiter_ops.is_indexer_top_k_supported(
|
||||
is_prefill=True,
|
||||
compress_ratio=4,
|
||||
num_rows=logits.shape[0],
|
||||
on_gfx950=True,
|
||||
)
|
||||
if aiter_topk_kernel is None:
|
||||
):
|
||||
pytest.skip("AITER top-k is unavailable")
|
||||
assert (
|
||||
_launch_aiter_top_k_per_row_prefill(
|
||||
aiter_topk_kernel,
|
||||
logits,
|
||||
row_starts,
|
||||
row_ends,
|
||||
@@ -683,10 +680,7 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
num_speculative_tokens: int,
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_get_aiter_top_k_kernel,
|
||||
_launch_aiter_top_k_per_row_decode,
|
||||
)
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
query_tokens = num_speculative_tokens + 1
|
||||
offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
|
||||
@@ -701,20 +695,18 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
top_k = 4
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
if not rocm_aiter_ops.is_indexer_top_k_supported(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=num_rows,
|
||||
max_valid_seq_len=seq_lens.max().item(),
|
||||
on_gfx950=True,
|
||||
)
|
||||
if aiter_topk_kernel is None:
|
||||
):
|
||||
pytest.skip("AITER top-k is unavailable")
|
||||
assert (
|
||||
_launch_aiter_top_k_per_row_decode(
|
||||
aiter_topk_kernel,
|
||||
rocm_aiter_ops.indexer_top_k_decode(
|
||||
logits,
|
||||
seq_lens,
|
||||
1,
|
||||
seq_lens.reshape(-1),
|
||||
indices,
|
||||
top_k,
|
||||
)
|
||||
@@ -729,130 +721,107 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
assert torch.all(indices[row_idx, num_valid:] == -1)
|
||||
|
||||
|
||||
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_aiter_decode_topk_matches_per_row_under_mtp() -> None:
|
||||
"""The aiter backend expands (B, 1) seq_lens into per-row ends before
|
||||
launching, so next_n > 1 must agree with the in-tree kernel, including
|
||||
the rows that clamp to a zero-length prefix.
|
||||
"""
|
||||
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
|
||||
|
||||
def prefill_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
next_n = 4
|
||||
top_k = 512
|
||||
num_columns = 1024
|
||||
seq_lens = torch.tensor([[2], [513], [700]], dtype=torch.int32, device="cuda")
|
||||
num_rows = seq_lens.shape[0] * next_n
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(num_rows, num_columns, dtype=torch.float32, device="cuda")
|
||||
|
||||
outputs = {}
|
||||
for backend in ("aiter", "per_row"):
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
get_indexer_topk(backend)(logits, seq_lens, next_n, indices, top_k, num_columns)
|
||||
outputs[backend] = indices
|
||||
|
||||
row_ends = (
|
||||
(seq_lens - next_n + 1 + torch.arange(next_n, device="cuda"))
|
||||
.clamp_(min=0)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
)
|
||||
assert row_ends[:next_n] == [0, 0, 1, 2]
|
||||
|
||||
for row_idx, row_end in enumerate(row_ends):
|
||||
num_valid = min(top_k, row_end)
|
||||
aiter_row = outputs["aiter"][row_idx]
|
||||
assert set(aiter_row[:num_valid].tolist()) == set(
|
||||
outputs["per_row"][row_idx][:num_valid].tolist()
|
||||
)
|
||||
assert torch.all(aiter_row[num_valid:] == -1)
|
||||
|
||||
|
||||
def _enable_aiter_topk(monkeypatch, enabled: bool = True) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla_sparse,
|
||||
"_get_aiter_topk_ops",
|
||||
lambda: (prefill_kernel, decode_kernel),
|
||||
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: enabled, raising=True
|
||||
)
|
||||
get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel
|
||||
|
||||
|
||||
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_enable_aiter_topk(monkeypatch)
|
||||
is_supported = rocm_aiter_ops.is_indexer_top_k_supported
|
||||
|
||||
eligible_cases = [
|
||||
(
|
||||
dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True),
|
||||
prefill_kernel,
|
||||
dict(is_prefill=True, compress_ratio=4, num_rows=1),
|
||||
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_536),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=257,
|
||||
max_valid_seq_len=250_000,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
max_valid_seq_len=65_536,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=257,
|
||||
max_valid_seq_len=250_000,
|
||||
num_columns=524_288,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=385,
|
||||
max_valid_seq_len=250_000,
|
||||
num_columns=524_288,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=384,
|
||||
max_valid_seq_len=50_000,
|
||||
num_columns=1_048_577,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
]
|
||||
for kwargs, expected_kernel in eligible_cases:
|
||||
assert get_kernel(**kwargs) is expected_kernel
|
||||
|
||||
def fail_if_imported() -> None:
|
||||
pytest.fail("ineligible shapes must not import AITER top-k")
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla_sparse,
|
||||
"_get_aiter_topk_ops",
|
||||
fail_if_imported,
|
||||
)
|
||||
ineligible_cases = [
|
||||
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=384,
|
||||
max_valid_seq_len=1_048_576,
|
||||
num_columns=1_048_576,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
max_valid_seq_len=65_537,
|
||||
on_gfx950=True,
|
||||
),
|
||||
]
|
||||
for kwargs in eligible_cases:
|
||||
assert is_supported(**kwargs) is True
|
||||
|
||||
ineligible_cases = [
|
||||
dict(is_prefill=True, compress_ratio=1, num_rows=1),
|
||||
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_537),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=256,
|
||||
max_valid_seq_len=125_000,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=320,
|
||||
max_valid_seq_len=2_500,
|
||||
on_gfx950=False,
|
||||
),
|
||||
]
|
||||
for kwargs in ineligible_cases:
|
||||
assert get_kernel(**kwargs) is None
|
||||
assert is_supported(**kwargs) is False
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None)
|
||||
assert (
|
||||
get_kernel(
|
||||
is_prefill=True,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
on_gfx950=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
_enable_aiter_topk(monkeypatch, enabled=False)
|
||||
assert is_supported(is_prefill=True, compress_ratio=4, num_rows=1) is False
|
||||
|
||||
|
||||
def test_dsv4_native_top_k_window() -> None:
|
||||
"""The in-tree decode kernel's measured advantage band over AITER, which
|
||||
applies to the DSV4 indexer only."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
prefers_native = rocm_aiter_ops.dsv4_indexer_prefers_native_top_k
|
||||
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=512) is True
|
||||
assert prefers_native(num_rows=385, num_columns=524_288, topk_tokens=512) is False
|
||||
assert prefers_native(num_rows=128, num_columns=1_048_577, topk_tokens=512) is False
|
||||
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=1024) is False
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@@ -1927,3 +1896,141 @@ def test_sparse_indexer_topk_backend_resolution() -> None:
|
||||
if not is_sm100:
|
||||
with pytest.raises(RuntimeError, match="SM100a/SM103a"):
|
||||
resolve("deep_select")
|
||||
|
||||
|
||||
def _patch_aiter_topk(monkeypatch, decode_kernel, enabled: bool = True) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_enable_aiter_topk(monkeypatch, enabled=enabled)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: enabled, raising=True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "indexer_top_k_decode", decode_kernel)
|
||||
|
||||
|
||||
def _make_topk(backend: str):
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
return SparseIndexerTopk()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
def test_sparse_indexer_topk_aiter_requires_rocm() -> None:
|
||||
"""Explicitly requesting the ROCm-only backend must fail fast elsewhere."""
|
||||
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(RuntimeError, match="ROCm"):
|
||||
_make_topk("aiter").resolve_backend(logits, 2048, 8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
def test_sparse_indexer_topk_backend_resolution_aiter(monkeypatch) -> None:
|
||||
"""Aiter is opt-in only: "auto" never selects it (the shape-dependent perf
|
||||
gate lives at the call site), and an explicit request validates hard
|
||||
capability only."""
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel)
|
||||
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
|
||||
|
||||
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
|
||||
assert _make_topk("aiter").resolve_backend(logits, 2048, 8) == "aiter"
|
||||
# Shapes the call-site perf gate would decline still resolve.
|
||||
assert _make_topk("aiter").resolve_backend(logits, 512, 8) == "aiter"
|
||||
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
|
||||
_make_topk("aiter").resolve_backend(logits, 3000, 8)
|
||||
|
||||
_enable_aiter_topk(monkeypatch, enabled=False)
|
||||
with pytest.raises(RuntimeError, match="no tuned indexer top-k kernels"):
|
||||
_make_topk("aiter").resolve_backend(logits, 2048, 8)
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel, enabled=False)
|
||||
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
|
||||
with pytest.raises(RuntimeError, match="VLLM_ROCM_USE_AITER"):
|
||||
_make_topk("aiter").resolve_backend(logits, 2048, 8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_aiter_applies_no_shape_gate(monkeypatch) -> None:
|
||||
"""Shape-dependent perf gating belongs to the call site: once "aiter" is
|
||||
requested the dispatcher must run it for every supported k, including the
|
||||
k=512 band the in-tree kernel wins on DSV4's compressed-KV logits."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
calls: list[tuple] = []
|
||||
_patch_aiter_topk(monkeypatch, lambda *args: calls.append(args))
|
||||
monkeypatch.setattr(
|
||||
indexer_topk.ops,
|
||||
"top_k_per_row_decode",
|
||||
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
|
||||
)
|
||||
|
||||
num_rows = 8
|
||||
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((num_rows, 512), dtype=torch.int32, device="cuda")
|
||||
op = _make_topk("aiter")
|
||||
|
||||
op(logits, seq_lens, 1, indices, 512, 4096)
|
||||
# Past the compressed-context boundary the call-site gate declines on.
|
||||
op(logits, seq_lens, 1, indices, 512, 4 * (64 * 1024 + 1))
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_auto_stays_on_per_row(monkeypatch) -> None:
|
||||
"""The "auto" chain must not reach AITER on its own: the decode top-k runs
|
||||
through top_k_per_row_decode unless the caller asks for the aiter backend."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pytest.fail("AITER is opt-in only")
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel)
|
||||
calls: list[tuple] = []
|
||||
monkeypatch.setattr(
|
||||
indexer_topk.ops, "top_k_per_row_decode", lambda *args: calls.append(args)
|
||||
)
|
||||
|
||||
num_rows = 8
|
||||
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((num_rows, 1024), dtype=torch.int32, device="cuda")
|
||||
|
||||
_make_topk("auto")(logits, seq_lens, 1, indices, 1024, 4096)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_aiter_expands_mtp_row_ends(monkeypatch) -> None:
|
||||
"""AITER's decode kernel only implements the next_n == 1 form, so spec
|
||||
decode batches must be handed the equivalent expanded per-row ends."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
recorded: dict = {}
|
||||
|
||||
def decode_kernel(logits, next_n, seq_lens, *args, **kwargs) -> None:
|
||||
recorded["next_n"] = next_n
|
||||
recorded["row_ends"] = seq_lens.tolist()
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel)
|
||||
monkeypatch.setattr(
|
||||
indexer_topk.ops,
|
||||
"top_k_per_row_decode",
|
||||
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
|
||||
)
|
||||
|
||||
next_n = 2
|
||||
logits = torch.randn(4, 4096, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.tensor([[100], [200]], dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((4, 1024), dtype=torch.int32, device="cuda")
|
||||
|
||||
_make_topk("aiter")(logits, seq_lens, next_n, indices, 1024, 800)
|
||||
|
||||
assert recorded["next_n"] == 1
|
||||
assert recorded["row_ends"] == [99, 100, 199, 200]
|
||||
|
||||
@@ -362,6 +362,83 @@ def test_fused_moe_lora_kernel(
|
||||
torch.testing.assert_close(output, output2, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_lora_rank", [128, 256])
|
||||
@pytest.mark.parametrize("add_inputs", [True, False])
|
||||
@pytest.mark.parametrize("num_slices", [1, 2])
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_fused_moe_lora_kernel_large_rank_fallback(
|
||||
max_lora_rank,
|
||||
add_inputs,
|
||||
num_slices,
|
||||
device,
|
||||
):
|
||||
"""Ranks above the one-shot limit fall back to shrink/expand kernels."""
|
||||
torch.set_default_device(device)
|
||||
set_random_seed(42)
|
||||
|
||||
num_tokens = 8
|
||||
num_sequences = 4
|
||||
top_k_num = 2
|
||||
num_experts = 4
|
||||
max_loras = 2
|
||||
N = 256
|
||||
K = 256
|
||||
block_size = 16
|
||||
dtype = torch.bfloat16
|
||||
|
||||
topk_ids, topk_weights, token_lora_mapping, lora_ids = sample_data(
|
||||
num_tokens, num_sequences, max_loras, num_experts, top_k_num
|
||||
)
|
||||
lora_a_stacked = [
|
||||
torch.rand(
|
||||
(max_loras, num_experts, max_lora_rank, K),
|
||||
dtype=dtype,
|
||||
)
|
||||
for _ in range(num_slices)
|
||||
]
|
||||
lora_b_stacked = [
|
||||
torch.rand(
|
||||
(max_loras, num_experts, N // num_slices, max_lora_rank),
|
||||
dtype=dtype,
|
||||
)
|
||||
for _ in range(num_slices)
|
||||
]
|
||||
hidden_states = torch.rand((num_tokens, K), dtype=dtype)
|
||||
|
||||
residual = torch.randn((num_tokens, top_k_num, N), dtype=dtype)
|
||||
output = residual.clone()
|
||||
use_fused_moe_lora_kernel(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
token_lora_mapping,
|
||||
max_lora_rank,
|
||||
top_k_num,
|
||||
lora_ids,
|
||||
lora_a_stacked,
|
||||
lora_b_stacked,
|
||||
hidden_states,
|
||||
output,
|
||||
max_loras,
|
||||
num_experts,
|
||||
block_size,
|
||||
add_inputs=add_inputs,
|
||||
)
|
||||
|
||||
expected = use_torch(
|
||||
hidden_states,
|
||||
token_lora_mapping,
|
||||
topk_ids,
|
||||
lora_a_stacked,
|
||||
lora_b_stacked,
|
||||
top_k_num,
|
||||
num_slices,
|
||||
)
|
||||
if add_inputs:
|
||||
expected += residual
|
||||
|
||||
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def use_fused_moe_lora_kernel_naive(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
# get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
|
||||
# Ensure the default device is CPU so the rest of the test tensors match.
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
TEXT_TOKEN_ID = 1
|
||||
VISION_START_TOKEN_ID = 100
|
||||
VISION_END_TOKEN_ID = 101
|
||||
IMAGE_TOKEN_ID = 200
|
||||
VIDEO_TOKEN_ID = 201
|
||||
AUDIO_TOKEN_ID = 202
|
||||
AUDIO_START_TOKEN_ID = 300
|
||||
AUDIO_END_TOKEN_ID = 301
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
position_id_per_seconds: int = 25
|
||||
vision_config: DummyVisionConfig = dataclasses.field(
|
||||
default_factory=DummyVisionConfig
|
||||
)
|
||||
|
||||
|
||||
def make_model() -> Qwen3OmniMoeThinkerForConditionalGeneration:
|
||||
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration
|
||||
)
|
||||
model.config = DummyConfig()
|
||||
return model
|
||||
|
||||
|
||||
def _elem(data):
|
||||
return MultiModalFieldElem(
|
||||
data=torch.tensor(data),
|
||||
field=None, # HACK.
|
||||
)
|
||||
|
||||
|
||||
def make_image_feature(offset: int, grid_thw: tuple[int, int, int]):
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem({"image_grid_thw": _elem(grid_thw)}),
|
||||
modality="image",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset, length=int(torch.tensor(grid_thw).prod())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_video_feature(
|
||||
offset: int, grid_thw: tuple[int, int, int], use_audio_in_video: bool = False
|
||||
):
|
||||
data = {"video_grid_thw": _elem(grid_thw)}
|
||||
if use_audio_in_video:
|
||||
data["use_audio_in_video"] = _elem(True)
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(data),
|
||||
modality="video",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset, length=int(torch.tensor(grid_thw).prod())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_audio_feature(offset: int, audio_feature_length: int):
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{"audio_feature_lengths": _elem(audio_feature_length)}
|
||||
),
|
||||
modality="audio",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=0),
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_positions_image_at_prompt_end():
|
||||
"""A prompt ending with <|vision_end|> must still produce seq_len positions."""
|
||||
model = make_model()
|
||||
# [<|vision_start|>, <|image_pad|> x 4, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[VISION_START_TOKEN_ID] + [IMAGE_TOKEN_ID] * 4 + [VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [make_image_feature(offset=1, grid_thw=(1, 2, 2))]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_audio_at_prompt_end():
|
||||
model = make_model()
|
||||
# audio_feature_length=16 -> 2 audio tokens.
|
||||
# [<|audio_start|>, <|audio_pad|> x 2, <|audio_end|>]
|
||||
input_tokens = [AUDIO_START_TOKEN_ID] + [AUDIO_TOKEN_ID] * 2 + [AUDIO_END_TOKEN_ID]
|
||||
mm_features = [make_audio_feature(offset=1, audio_feature_length=16)]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_video_at_prompt_end():
|
||||
model = make_model()
|
||||
# [T, <|vision_start|>, <|video_pad|> x 8, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID]
|
||||
+ [VISION_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 8
|
||||
+ [VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [make_video_feature(offset=2, grid_thw=(2, 2, 2))]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_audio_in_video_at_prompt_end():
|
||||
model = make_model()
|
||||
# video grid (1, 2, 2) -> 4 video tokens; audio_feature_length=16 -> 2
|
||||
# audio tokens, interleaved after the video tokens.
|
||||
# [T, T, <|vision_start|>, <|audio_start|>, V x 4, A x 2,
|
||||
# <|audio_end|>, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 4
|
||||
+ [AUDIO_TOKEN_ID] * 2
|
||||
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [
|
||||
make_video_feature(offset=3, grid_thw=(1, 2, 2), use_audio_in_video=True),
|
||||
make_audio_feature(offset=3, audio_feature_length=16),
|
||||
]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_image_then_text():
|
||||
"""Media followed by text: positions must match the expected layout exactly.
|
||||
|
||||
Before the fix this case passed the length guard but every position from
|
||||
the first <|image_pad|> onward was silently shifted by one slot.
|
||||
"""
|
||||
model = make_model()
|
||||
# [T, T, <|vision_start|>, <|image_pad|> x 4, <|vision_end|>, T, T, T]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [VISION_START_TOKEN_ID]
|
||||
+ [IMAGE_TOKEN_ID] * 4
|
||||
+ [VISION_END_TOKEN_ID]
|
||||
+ [TEXT_TOKEN_ID] * 3
|
||||
)
|
||||
mm_features = [make_image_feature(offset=3, grid_thw=(1, 2, 2))]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 3, 3, 3, 5, 6, 7, 8],
|
||||
[0, 1, 2, 3, 3, 4, 4, 5, 6, 7, 8],
|
||||
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8],
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(positions, expected)
|
||||
assert delta == 9 - len(input_tokens)
|
||||
@@ -11,8 +11,16 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEKernelMonolithicImpl,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
|
||||
MoEPrepareAndFinalizeNoDPEPMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
RoutedExpertsSink,
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
@@ -174,45 +182,139 @@ def test_public_binding_supports_direct_capture_source():
|
||||
assert calls == [(3, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
self.layer_id = 3
|
||||
self.router = _make_router()
|
||||
# Use a concrete monolithic expert and override its capability
|
||||
# instead of instantiating the abstract base class directly.
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsFp8,
|
||||
)
|
||||
_SINK_CONFIG = SimpleNamespace(
|
||||
use_ep=False,
|
||||
dp_size=1,
|
||||
ep_size=1,
|
||||
max_num_tokens=4,
|
||||
experts_per_token=2,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8)
|
||||
self.routed_experts = types.SimpleNamespace(
|
||||
quant_method=types.SimpleNamespace(
|
||||
is_monolithic=True,
|
||||
moe_kernel=types.SimpleNamespace(
|
||||
impl=types.SimpleNamespace(fused_experts=fused_experts)
|
||||
),
|
||||
)
|
||||
)
|
||||
self._quant_method = self.routed_experts.quant_method
|
||||
self._quant_method.moe_kernel.impl.fused_experts = fused_experts
|
||||
fused_experts.supports_routing_replay_capture = lambda: False
|
||||
|
||||
class DummyCapturer:
|
||||
def capture(self, layer_id, topk_ids):
|
||||
pass
|
||||
class _ToyMonolithicExperts(CPUExpertsInt4):
|
||||
"""A monolithic kernel instance, as a weight reload rebuilds it: routes
|
||||
token t to experts (t + offset, t + offset + 1)."""
|
||||
|
||||
dummy_module = DummyFusedMoE()
|
||||
moe_config = _SINK_CONFIG
|
||||
quant_config = None
|
||||
|
||||
def __init__(self, offset: int = 0, supports_capture: bool = True):
|
||||
self.offset = offset
|
||||
self.supports_capture = supports_capture
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return self.supports_capture
|
||||
|
||||
def apply(self, hidden_states, *args, routing_replay_out=None, **kwargs):
|
||||
if routing_replay_out is not None:
|
||||
t = torch.arange(hidden_states.shape[0])
|
||||
routing_replay_out[: len(t)] = torch.stack([t, t + 1], 1) + self.offset
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _MonolithicMoE:
|
||||
"""The parts of a MoE layer that capture binding reads."""
|
||||
|
||||
layer_id = 5
|
||||
|
||||
def __init__(self, experts: _ToyMonolithicExperts):
|
||||
self.router = _make_router()
|
||||
kernel = SimpleNamespace(impl=SimpleNamespace(fused_experts=experts))
|
||||
self._quant_method = SimpleNamespace(is_monolithic=True, moe_kernel=kernel)
|
||||
self.routed_experts = SimpleNamespace(
|
||||
quant_method=self._quant_method, routing_sink=None
|
||||
)
|
||||
|
||||
|
||||
def _bind_monolithic(
|
||||
monkeypatch, experts: _ToyMonolithicExperts, capture
|
||||
) -> _MonolithicMoE:
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", _MonolithicMoE)
|
||||
layer = _MonolithicMoE(experts)
|
||||
bind_routed_experts_capturer(
|
||||
SimpleNamespace(modules=lambda: [layer]), SimpleNamespace(capture=capture)
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def test_public_binding_gives_monolithic_layers_a_sink(monkeypatch):
|
||||
"""A monolithic kernel routes internally, so binding gives the layer, which
|
||||
outlives kernel rebuilds, a sink that captures under its layer id."""
|
||||
calls = []
|
||||
layer = _bind_monolithic(
|
||||
monkeypatch, _ToyMonolithicExperts(), lambda *args: calls.append(args)
|
||||
)
|
||||
ids = torch.tensor([[1, 2]])
|
||||
layer.routed_experts.routing_sink.capture_fn(ids)
|
||||
assert calls == [(_MonolithicMoE.layer_id, ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
with pytest.raises(ValueError, match="monolithic MoE kernel"):
|
||||
bind_routed_experts_capturer(
|
||||
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
|
||||
_bind_monolithic(
|
||||
monkeypatch, _ToyMonolithicExperts(supports_capture=False), Mock()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_ep", "dp_size", "ep_size", "rows"),
|
||||
[(False, 1, 1, 4), (False, 2, 4, 8), (True, 2, 4, 16)],
|
||||
ids=["single-rank", "dp", "ep"],
|
||||
)
|
||||
def test_routed_experts_sink_holds_a_dispatched_batch(use_ep, dp_size, ep_size, rows):
|
||||
"""One int16 buffer, allocated once, large enough for the batch the kernel
|
||||
sees after a DP or EP gather."""
|
||||
config = SimpleNamespace(
|
||||
**{
|
||||
**vars(_SINK_CONFIG),
|
||||
"use_ep": use_ep,
|
||||
"dp_size": dp_size,
|
||||
"ep_size": ep_size,
|
||||
}
|
||||
)
|
||||
sink = RoutedExpertsSink(config, Mock())
|
||||
|
||||
assert sink.buffer.shape == (rows, 2) and sink.buffer.dtype == torch.int16
|
||||
|
||||
|
||||
def test_monolithic_capture_survives_kernel_rebuilds():
|
||||
"""The bug in #59449: a weight reload rebuilds the kernel. Through the
|
||||
production kernel path, the layer's sink captures what the current kernel
|
||||
routed, before and after a rebuild, and a rebuilt kernel that cannot
|
||||
capture is refused instead of leaving stale rows."""
|
||||
captured = []
|
||||
sink = RoutedExpertsSink(_SINK_CONFIG, lambda ids: captured.append(ids.tolist()))
|
||||
|
||||
def forward(experts: _ToyMonolithicExperts) -> None:
|
||||
kernel = FusedMoEKernelMonolithicImpl(
|
||||
MoEPrepareAndFinalizeNoDPEPMonolithic(), experts
|
||||
)
|
||||
kernel.apply(
|
||||
torch.zeros(3, 8),
|
||||
w1=None,
|
||||
w2=None,
|
||||
router_logits=torch.zeros(3, 16),
|
||||
activation=None,
|
||||
global_num_experts=16,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=sink,
|
||||
)
|
||||
|
||||
for offset in (0, 7): # the kernel before and after a reload
|
||||
forward(_ToyMonolithicExperts(offset))
|
||||
assert captured[-1] == [[t + offset, t + offset + 1] for t in range(3)]
|
||||
with pytest.raises(ValueError, match="not supported"):
|
||||
forward(_ToyMonolithicExperts(supports_capture=False))
|
||||
|
||||
|
||||
def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
"""dp_metadata is None: capture writes the full topk_ids rows."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
|
||||
@@ -8,10 +8,6 @@ import torch
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA-only dispatch"
|
||||
)
|
||||
|
||||
|
||||
def _require_deep_gemm() -> None:
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
@@ -20,17 +16,10 @@ def _require_deep_gemm() -> None:
|
||||
pytest.skip("kpool indexer requires DeepGEMM")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
def _build(indexer_cls, backend: str):
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
op = SparseAttnIndexerKpool(
|
||||
return indexer_cls(
|
||||
k_cache=None,
|
||||
quant_block_size=128,
|
||||
scale_fmt="ue8m0",
|
||||
@@ -40,4 +29,73 @@ def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> N
|
||||
max_total_seq_len=8192,
|
||||
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
assert op.topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
|
||||
backend: str,
|
||||
) -> None:
|
||||
"""The AMD kpool indexer must go through the same dispatcher, so the
|
||||
AITER decode top-k is reachable from kernel_config."""
|
||||
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize(
|
||||
"configured,full_cudagraph,expected",
|
||||
[
|
||||
("auto", False, "aiter"),
|
||||
("auto", True, "auto"),
|
||||
("aiter", True, "aiter"),
|
||||
("per_row", False, "per_row"),
|
||||
],
|
||||
)
|
||||
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: str,
|
||||
full_cudagraph: bool,
|
||||
expected: str,
|
||||
) -> None:
|
||||
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
|
||||
use the context length to pick AITER and must stay on the in-tree kernel.
|
||||
Explicit backends keep working, and outside FULL the long-context heuristic
|
||||
still narrows to AITER."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.models.glm5next.amd import sparse_indexer
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_indexer_top_k_supported",
|
||||
lambda **kwargs: True,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
assert (
|
||||
sparse_indexer._kpool_decode_topk_backend(
|
||||
configured,
|
||||
num_rows=8,
|
||||
max_valid_seq_len=64 * 1024,
|
||||
select_k=512,
|
||||
index_kpool=4,
|
||||
full_cudagraph=full_cudagraph,
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import (
|
||||
_moe_weight_override_is_int4,
|
||||
_use_k3_situ_int4_gfx942,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeRocmModule(ModuleType):
|
||||
on_gfx942: Callable[[], bool]
|
||||
|
||||
|
||||
class _FakeAiterOpsModule(ModuleType):
|
||||
rocm_aiter_ops: SimpleNamespace
|
||||
|
||||
|
||||
def _vllm_config(moe_weight: str | None):
|
||||
quantization_config = (
|
||||
None
|
||||
if moe_weight is None
|
||||
else QuantizationConfigArgs(moe={"weight": moe_weight})
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(quantization_config=quantization_config)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("moe_weight", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
("mxfp4", False),
|
||||
("int4_per_group_32", True),
|
||||
],
|
||||
)
|
||||
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.get_current_vllm_config",
|
||||
lambda: _vllm_config(moe_weight),
|
||||
)
|
||||
assert _moe_weight_override_is_int4() is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("int4_requested", [False, True])
|
||||
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
|
||||
moe = SimpleNamespace(
|
||||
activation=MoEActivation.SITU,
|
||||
activation_situ_linear_beta=25.0,
|
||||
)
|
||||
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
|
||||
fake_rocm.on_gfx942 = lambda: True
|
||||
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
|
||||
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"vllm.platforms.rocm": fake_rocm,
|
||||
"vllm._aiter_ops": fake_aiter_ops,
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.mxfp4."
|
||||
"_moe_weight_override_is_int4",
|
||||
return_value=int4_requested,
|
||||
),
|
||||
):
|
||||
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
|
||||
|
||||
|
||||
def test_int4_per_group_32_is_not_an_online_moe_method():
|
||||
# Load-time gfx942 requant. Must not match the online MoE table.
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
|
||||
)
|
||||
assert (
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_online_moe_weight_still_raises():
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
|
||||
)
|
||||
with pytest.raises(ValueError, match="online quantization"):
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
|
||||
)
|
||||
@@ -0,0 +1,375 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multi-turn prefix cache reuse with KV offload, P/D and DCP."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import random
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from math import lcm
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from tests.utils import RemoteOpenAIServer, multi_gpu_marks
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
# Official Kimi-K3 config with 8 layers and no weights.
|
||||
MODEL = "riverclouds/Kimi-K3-8L-dummy"
|
||||
# The recipe's DSpark draft, reading target layers that exist in MODEL.
|
||||
DRAFT = "riverclouds/Kimi-K3-DSpark-for-8L-dummy"
|
||||
|
||||
# Conversation: the first prompt spans FIRST_PROMPT_BLOCKS blocks plus an
|
||||
# unaligned tail, and each later turn appends TURN_TOKENS random tokens.
|
||||
NUM_TURNS = 3
|
||||
FIRST_PROMPT_BLOCKS = 3
|
||||
UNALIGNED_TAIL = 37
|
||||
TURN_TOKENS = 777
|
||||
MAX_TOKENS = 8
|
||||
NUM_LOGPROBS = 5
|
||||
STARTUP_TIMEOUT = 1200
|
||||
|
||||
BASE_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--language-model-only",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--no-enable-flashinfer-autotune",
|
||||
"--enable-prompt-tokens-details",
|
||||
# Default max_num_seqs OOMs one GPU during warmup.
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Kimi-K3 NVIDIA kernels require the SM100 family",
|
||||
)
|
||||
|
||||
SPEC_CONFIG = {
|
||||
"method": "dspark",
|
||||
"model": DRAFT,
|
||||
"num_speculative_tokens": 7,
|
||||
"draft_sample_method": "probabilistic",
|
||||
"rejection_sample_method": "block",
|
||||
}
|
||||
|
||||
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
|
||||
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
|
||||
OFFLOAD = {
|
||||
"kv_connector": "SimpleCPUOffloadConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"cpu_bytes_to_use_per_rank": 32 << 30,
|
||||
"lazy_offload": False,
|
||||
},
|
||||
}
|
||||
NIXL_OFFLOAD = {
|
||||
"kv_connector": "MultiConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"connectors": [NIXL, OFFLOAD]},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mode:
|
||||
# A finer prefix_match_unit enables partial hits inside a Mamba block.
|
||||
prefix_match_unit: int | None
|
||||
spec: bool
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
name = "block" if self.prefix_match_unit is None else "partial"
|
||||
return f"{name}-dspark" if self.spec else name
|
||||
|
||||
|
||||
MODES = [Mode(unit, spec) for spec in (False, True) for unit in (None, 128)]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Instance:
|
||||
tp: int = 1
|
||||
dp: int = 1
|
||||
dcp: int = 1
|
||||
kv_config: dict | None = None
|
||||
prefix_caching: bool = True
|
||||
|
||||
@property
|
||||
def num_gpus(self) -> int:
|
||||
return self.tp * self.dp
|
||||
|
||||
def args(self, mode: Mode) -> list[str]:
|
||||
args = BASE_ARGS + ["--tensor-parallel-size", str(self.tp)]
|
||||
if mode.spec:
|
||||
args += ["--speculative-config", json.dumps(SPEC_CONFIG)]
|
||||
if self.dp > 1:
|
||||
args += ["--data-parallel-size", str(self.dp), "--enable-expert-parallel"]
|
||||
if self.dcp > 1:
|
||||
args += ["--decode-context-parallel-size", str(self.dcp)]
|
||||
if self.kv_config is not None:
|
||||
args += ["--kv-transfer-config", json.dumps(self.kv_config)]
|
||||
if not self.prefix_caching:
|
||||
return args + ["--no-enable-prefix-caching"]
|
||||
args.append("--enable-prefix-caching")
|
||||
if mode.prefix_match_unit is not None:
|
||||
args += ["--prefix-match-unit", str(mode.prefix_match_unit)]
|
||||
return args
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Deployment:
|
||||
decode: Instance
|
||||
prefill: Instance | None = None
|
||||
# Reset GPU prefix caches each turn so hits must come from CPU.
|
||||
offload: bool = False
|
||||
|
||||
@property
|
||||
def instances(self) -> tuple[Instance, ...]:
|
||||
return (self.prefill, self.decode) if self.prefill else (self.decode,)
|
||||
|
||||
@property
|
||||
def num_gpus(self) -> int:
|
||||
return sum(i.num_gpus for i in self.instances)
|
||||
|
||||
|
||||
DEPLOYMENTS = {
|
||||
"plain": Deployment(Instance()),
|
||||
"offload": Deployment(Instance(kv_config=OFFLOAD), offload=True),
|
||||
"dcp2": Deployment(Instance(tp=2, dcp=2)),
|
||||
"dcp2-offload": Deployment(Instance(tp=2, dcp=2, kv_config=OFFLOAD), offload=True),
|
||||
"pd": Deployment(Instance(kv_config=NIXL), prefill=Instance(kv_config=NIXL)),
|
||||
# NIXL rejects prefix caching on a hybrid decoder with a different TP.
|
||||
"pd-tp2-dep2": Deployment(
|
||||
Instance(dp=2, kv_config=NIXL, prefix_caching=False),
|
||||
prefill=Instance(tp=2, kv_config=NIXL),
|
||||
),
|
||||
"pd-offload": Deployment(
|
||||
Instance(kv_config=NIXL_OFFLOAD),
|
||||
prefill=Instance(kv_config=NIXL_OFFLOAD),
|
||||
offload=True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _known_failure(name: str, mode: Mode) -> str | None:
|
||||
deployment = DEPLOYMENTS[name]
|
||||
if mode.spec and deployment.decode.dcp > 1:
|
||||
return "FlashInfer MLA DCP decode breaks on ragged spec batches (#59392)"
|
||||
return None
|
||||
|
||||
|
||||
def _case(name: str, mode: Mode):
|
||||
marks = []
|
||||
if (num_gpus := DEPLOYMENTS[name].num_gpus) > 1:
|
||||
marks += multi_gpu_marks(num_gpus=num_gpus)
|
||||
if reason := _known_failure(name, mode):
|
||||
marks.append(pytest.mark.xfail(reason=reason, strict=True))
|
||||
return pytest.param(name, mode, marks=marks, id=f"{name}-{mode.name}")
|
||||
|
||||
|
||||
def _block_size(url: str) -> int:
|
||||
response = requests.get(f"{url}/metrics", timeout=30)
|
||||
response.raise_for_status()
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:cache_config_info":
|
||||
return lcm(
|
||||
*(
|
||||
int(value)
|
||||
for key in ("block_size", "mamba_block_size")
|
||||
if (value := sample.labels.get(key, "None")) != "None"
|
||||
)
|
||||
)
|
||||
raise AssertionError("missing vllm:cache_config_info metric")
|
||||
|
||||
|
||||
def _complete(url: str, prompt: list[int], salt: str, **extra) -> dict:
|
||||
body = {
|
||||
"model": MODEL,
|
||||
"prompt": prompt,
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"logprobs": NUM_LOGPROBS,
|
||||
"return_tokens_as_token_ids": True,
|
||||
"cache_salt": salt,
|
||||
**extra,
|
||||
}
|
||||
response = requests.post(f"{url}/v1/completions", json=body, timeout=600)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def _pd_complete(
|
||||
prefill_url: str, decode_url: str, prompt: list[int], salt: str
|
||||
) -> tuple[dict, dict]:
|
||||
remote_decode = {
|
||||
"do_remote_decode": True,
|
||||
"do_remote_prefill": False,
|
||||
"remote_engine_id": None,
|
||||
"remote_block_ids": None,
|
||||
"remote_host": None,
|
||||
"remote_port": None,
|
||||
}
|
||||
prefill = _complete(
|
||||
prefill_url, prompt, salt, max_tokens=1, kv_transfer_params=remote_decode
|
||||
)
|
||||
decode = _complete(
|
||||
decode_url, prompt, salt, kv_transfer_params=prefill["kv_transfer_params"]
|
||||
)
|
||||
return prefill, decode
|
||||
|
||||
|
||||
def _cached(response: dict) -> int:
|
||||
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
|
||||
|
||||
|
||||
def _token_id(token: str) -> int:
|
||||
return int(token.removeprefix("token_id:"))
|
||||
|
||||
|
||||
def _tokens_text_logprobs(response: dict):
|
||||
choice = response["choices"][0]
|
||||
ids = [_token_id(t) for t in choice["logprobs"]["tokens"]]
|
||||
top = [
|
||||
{_token_id(t): lp for t, lp in d.items()}
|
||||
for d in choice["logprobs"]["top_logprobs"]
|
||||
]
|
||||
return ids, choice["text"], top
|
||||
|
||||
|
||||
def _reset_gpu_prefix_cache(url: str) -> None:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Turn:
|
||||
prompt_len: int
|
||||
cached: int
|
||||
expected_cached: int
|
||||
recompute_cached: int
|
||||
output: tuple
|
||||
recompute: tuple
|
||||
|
||||
@property
|
||||
def matches_recompute(self) -> bool:
|
||||
return self.output[0] == self.recompute[0]
|
||||
|
||||
|
||||
def _expected_cached(prev_prompt_len: int, hit_unit: int, spec: bool) -> int:
|
||||
"""A turn must reuse the previous prompt up to its last checkpoint.
|
||||
|
||||
EAGLE-style drafts drop the trailing unit, so the checkpoint is one earlier.
|
||||
"""
|
||||
if not prev_prompt_len:
|
||||
return 0
|
||||
checkpoint = (prev_prompt_len - 1) // hit_unit * hit_unit
|
||||
return max(checkpoint - hit_unit, 0) if spec else checkpoint
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _serve(deployment: Deployment, mode: Mode) -> Iterator[list[RemoteOpenAIServer]]:
|
||||
with contextlib.ExitStack() as stack:
|
||||
# Sequential shutdown waits on memory still held by sibling servers.
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
next_gpu = 0
|
||||
for instance in deployment.instances:
|
||||
env = {}
|
||||
if deployment.offload:
|
||||
# Enables /reset_prefix_cache.
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1"
|
||||
if deployment.prefill is not None:
|
||||
gpus = range(next_gpu, next_gpu + instance.num_gpus)
|
||||
next_gpu += instance.num_gpus
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus))
|
||||
env["VLLM_SSM_CONV_STATE_LAYOUT"] = "DS"
|
||||
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(get_open_port())
|
||||
servers.append(
|
||||
RemoteOpenAIServer(
|
||||
MODEL,
|
||||
instance.args(mode),
|
||||
env_dict=env,
|
||||
max_wait_seconds=STARTUP_TIMEOUT,
|
||||
)
|
||||
)
|
||||
yield servers
|
||||
|
||||
|
||||
def _run_conversation(
|
||||
deployment: Deployment,
|
||||
servers: list[RemoteOpenAIServer],
|
||||
mode: Mode,
|
||||
) -> list[Turn]:
|
||||
# Under P/D the prefiller computes the prompt, so hits are read from it.
|
||||
compute_url, decode_url = servers[0].url_root, servers[-1].url_root
|
||||
block_size = _block_size(compute_url)
|
||||
hit_unit = mode.prefix_match_unit or block_size
|
||||
rng = random.Random(0)
|
||||
|
||||
def new_tokens(n: int) -> list[int]:
|
||||
return [rng.randint(1000, 150000) for _ in range(n)]
|
||||
|
||||
prompt = new_tokens(FIRST_PROMPT_BLOCKS * block_size + UNALIGNED_TAIL)
|
||||
prev_len = 0
|
||||
turns = []
|
||||
for turn in range(NUM_TURNS):
|
||||
if deployment.offload:
|
||||
for server in servers:
|
||||
_reset_gpu_prefix_cache(server.url_root)
|
||||
if deployment.prefill is not None:
|
||||
computed, output = _pd_complete(compute_url, decode_url, prompt, "conv")
|
||||
else:
|
||||
computed = output = _complete(decode_url, prompt, "conv")
|
||||
recompute = _complete(decode_url, prompt, f"recompute-{turn}")
|
||||
turns.append(
|
||||
Turn(
|
||||
prompt_len=len(prompt),
|
||||
cached=_cached(computed),
|
||||
expected_cached=_expected_cached(prev_len, hit_unit, mode.spec),
|
||||
recompute_cached=_cached(recompute),
|
||||
output=_tokens_text_logprobs(output),
|
||||
recompute=_tokens_text_logprobs(recompute),
|
||||
)
|
||||
)
|
||||
prev_len = len(prompt)
|
||||
prompt = prompt + new_tokens(TURN_TOKENS)
|
||||
return turns
|
||||
|
||||
|
||||
def _format(turns: list[Turn]) -> str:
|
||||
rows = ["turn prompt cached expected matches_recompute"]
|
||||
rows += [
|
||||
f"{i:>4} {t.prompt_len:>6} {t.cached:>6} {t.expected_cached:>8} "
|
||||
f"{t.matches_recompute}"
|
||||
for i, t in enumerate(turns)
|
||||
]
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
|
||||
)
|
||||
def test_turns_reuse_prefix_and_match_recompute(name: str, mode: Mode) -> None:
|
||||
deployment = DEPLOYMENTS[name]
|
||||
with _serve(deployment, mode) as servers:
|
||||
turns = _run_conversation(deployment, servers, mode)
|
||||
table = _format(turns)
|
||||
print(f"\n{name}-{mode.name}\n{table}")
|
||||
|
||||
assert all(t.recompute_cached == 0 for t in turns), table
|
||||
assert all(t.cached >= t.expected_cached for t in turns), table
|
||||
# Dummy weights differ across TP sizes.
|
||||
if len({i.tp for i in deployment.instances}) == 1:
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[t.recompute for t in turns],
|
||||
outputs_1_lst=[t.output for t in turns],
|
||||
name_0="recompute",
|
||||
name_1=name,
|
||||
)
|
||||
@@ -5,16 +5,15 @@ from transformers import GraniteConfig
|
||||
|
||||
from vllm.model_executor.models.granite import granite_layer_attn_params
|
||||
|
||||
from ...utils import check_logprobs_close, check_transformers_version
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
# model -> minimum transformers version, or None if unconstrained
|
||||
MODELS = {
|
||||
MODELS = [
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
"ibm/PowerLM-3b": None,
|
||||
"ibm/PowerMoE-3b": None,
|
||||
"ibm-granite/granite-swash-2b": "5.15.1",
|
||||
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
|
||||
}
|
||||
"ibm/PowerLM-3b",
|
||||
"ibm/PowerMoE-3b",
|
||||
"ibm-granite/granite-swash-2b",
|
||||
"ibm-granite/granite-swash-3b-a600m",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@@ -31,8 +30,6 @@ def test_models(
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
check_transformers_version(model, min_transformers_version=MODELS[model])
|
||||
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
|
||||
@@ -799,7 +799,6 @@ VLM_TEST_SETTINGS = {
|
||||
dtype="half",
|
||||
num_logprobs=10,
|
||||
patch_hf_runner=model_utils.ovis2_5_patch_hf_runner,
|
||||
hf_model_kwargs={"revision": "refs/pr/5"},
|
||||
),
|
||||
"paddleocr_vl": VLMTestInfo(
|
||||
models=["PaddlePaddle/PaddleOCR-VL"],
|
||||
|
||||
@@ -104,16 +104,7 @@ def run_test(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
|
||||
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
|
||||
# instead of shipping a separate lm_head.weight tensor; this
|
||||
# exercises that path (see tie_weights() in nemotron_parse.py).
|
||||
"nvidia/NVIDIA-Nemotron-Parse-2.0",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"])
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
|
||||
@@ -12,12 +12,22 @@ Run with:
|
||||
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.model_executor.models.deepseek_ocr import DeepseekOCRImagePixelInputs
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
|
||||
from vllm.model_executor.models.deepseek_ocr import (
|
||||
DeepseekOCRForCausalLM,
|
||||
DeepseekOCRImagePixelInputs,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
DeepseekOCRProcessor,
|
||||
ImageTransform,
|
||||
)
|
||||
|
||||
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
|
||||
|
||||
@@ -155,3 +165,80 @@ class TestDeepseekOCRInputValidation:
|
||||
def test_missing_images_raises_value_error(self, processor):
|
||||
with pytest.raises(ValueError, match="prompt and images"):
|
||||
processor(prompt="<image>\nDescribe this image.", images=None)
|
||||
|
||||
|
||||
def _balanced_normalized_pixels(
|
||||
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
|
||||
) -> torch.Tensor:
|
||||
"""Build a pixel tensor whose values cancel to an exact sum of 0.
|
||||
|
||||
Matches the normalized half-black / half-white case from the DeepSeek-OCR
|
||||
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
|
||||
"""
|
||||
assert width % 2 == 0
|
||||
pixel_values = torch.ones(batch, channels, height, width)
|
||||
pixel_values[..., : width // 2] = -1.0
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
return pixel_values
|
||||
|
||||
|
||||
class TestDeepseekOCRZeroSumPixelsAccepted:
|
||||
"""Zero-sum normalized pixels must still produce validated image inputs.
|
||||
|
||||
Returning None from ``_parse_and_validate_image_input`` makes
|
||||
``embed_multimodal`` return None and trips the worker encoder-output
|
||||
sanity check, killing EngineCore. The zero-sum shortcut is therefore
|
||||
removed; balanced images must parse like any other valid tensor.
|
||||
"""
|
||||
|
||||
def test_ocr_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_ocr2_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
|
||||
|
||||
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
|
||||
model,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_image_transform_half_black_half_white_sums_to_zero(self):
|
||||
"""A 1024×1024 half-black/half-white PNG through ImageTransform
|
||||
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
|
||||
must still accept it.
|
||||
"""
|
||||
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
|
||||
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
|
||||
pixel_values = ImageTransform()(image).unsqueeze(0)
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
|
||||
images_crop = torch.zeros(0, 3, 640, 640)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
assert image_input is not None
|
||||
assert torch.equal(image_input.data, pixel_values)
|
||||
|
||||
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def _image_info(**kwargs):
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
**kwargs,
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
).info
|
||||
|
||||
|
||||
def test_image_encoder_cache_covers_full_token_budget():
|
||||
"""The most-features probe must reach the processor's token ceiling.
|
||||
|
||||
The inherited square probe refits to 89x89 = 7921 tokens under the
|
||||
max_image_tokens=8000 budget, so the encoder cache came up short and
|
||||
ordinary non-square images in 7922-8000 tokens were refused with
|
||||
HTTP 400 (#59539).
|
||||
"""
|
||||
info = _image_info()
|
||||
assert info.get_max_image_tokens() == 8000
|
||||
|
||||
# The shapes from the issue, with the token counts the processor
|
||||
# actually produces; every one must fit the cache.
|
||||
for width, height, expected_tokens in [
|
||||
(4032, 3024, 7931), # phone photo, 4:3
|
||||
(3840, 2160, 7973), # 4K frame, 16:9
|
||||
(3508, 2480, 7950), # A4 at 300 dpi
|
||||
(2600, 2400, 7998), # 13:12
|
||||
(3000, 3000, 7921), # square worst case before the fix
|
||||
]:
|
||||
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
|
||||
assert num_tokens == expected_tokens
|
||||
assert num_tokens <= info.get_max_image_tokens()
|
||||
|
||||
# The profiling dummy covers the real worst case, not the square one.
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert size.width * size.height == 2240 * 2800
|
||||
|
||||
|
||||
def test_image_encoder_cache_follows_max_pixels_override():
|
||||
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
|
||||
assert info.get_max_image_tokens() == 100
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert (
|
||||
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
|
||||
== 100
|
||||
)
|
||||
|
||||
@@ -4,11 +4,8 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .transformers_backend import PROCESSOR_CLASSES, create_processor
|
||||
|
||||
AUDIO_MODEL_SETTINGS = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
@@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
@@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = {
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
],
|
||||
)
|
||||
def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
def test_audio_multimodal_processor(model_id):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
mm_data = {"audio": (audio, 16000)}
|
||||
@@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
def test_audio_multiple_inputs(separator):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
@@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls):
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# The legacy path reads placeholders off contiguous runs of the audio token, so
|
||||
# it cannot tell adjacent ones apart and says so instead of merging them
|
||||
if processor_cls is LegacyMultiModalProcessor and not separator:
|
||||
with pytest.raises(ValueError, match="Separate them in the prompt"):
|
||||
process()
|
||||
return
|
||||
|
||||
result = process()
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
|
||||
@@ -5,23 +5,16 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
|
||||
from .transformers_backend import (
|
||||
PROCESSOR_CLASSES,
|
||||
create_cached_processor,
|
||||
create_processor,
|
||||
offsets_only,
|
||||
)
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_multimodal_processor(model_id, processor_cls):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
def test_multimodal_processor(model_id):
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
image_pil = ImageAsset("cherry_blossom").pil_image
|
||||
mm_data = {"image": image_pil}
|
||||
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
|
||||
)
|
||||
|
||||
|
||||
def _process_two_images(processor_cls, separator: str):
|
||||
def _process_two_images(separator: str):
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_multiple_inputs(processor_cls):
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(processor_cls, separator="\n and ")
|
||||
result = _process_two_images(separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_adjacent_inputs(processor_cls):
|
||||
def test_image_adjacent_inputs():
|
||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||
result = _process_two_images(processor_cls, separator="")
|
||||
result = _process_two_images(separator="")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
def test_batch_padding_removed_from_image_items():
|
||||
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||
item's data dependent on what it was processed with and so uncacheable."""
|
||||
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
|
||||
)
|
||||
image_token = mm_processor.info.get_hf_processor().image_token
|
||||
|
||||
images = [
|
||||
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
assert len(shapes) == 2
|
||||
|
||||
|
||||
def _process_one_gemma3_image(processor_cls):
|
||||
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
result = mm_processor(
|
||||
prompt=f"{hf_processor.boi_token} What is this?",
|
||||
def _process_one_gemma3_image():
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
|
||||
)
|
||||
boi_token = mm_processor.info.get_hf_processor().boi_token
|
||||
return mm_processor(
|
||||
prompt=f"{boi_token} What is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
return hf_processor, result
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||
inside the placeholder range but masked out of it."""
|
||||
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||
result = _process_one_gemma3_image()
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.is_embed is not None
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
def test_legacy_placeholders_hold_only_image_tokens():
|
||||
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.length == hf_processor.image_seq_length
|
||||
prompt_ids = result["prompt_token_ids"]
|
||||
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
|
||||
assert set(covered) == {hf_processor.tokenizer.image_token_id}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped():
|
||||
"""SmolVLM splits each image into tiles introduced by tokens carrying no
|
||||
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||
tokenize it before splicing in the expansion."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
holding `<image>` when no images are passed, which is how the prompt has to be
|
||||
tokenized before splicing in the expansion."""
|
||||
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
result = mm_processor(
|
||||
prompt="<image>What is this?",
|
||||
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_missing_replacement_offsets_names_the_processor():
|
||||
"""A processor that reports no replacement offsets cannot be served, which must
|
||||
be said plainly rather than surfacing later as a field config mismatch."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||
hf_call = hf_processor_cls.__call__
|
||||
|
||||
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_text_only_prompt(processor_cls):
|
||||
def test_text_only_prompt():
|
||||
"""An image model still accepts a prompt with no images."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
|
||||
assert not result["mm_placeholders"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_repeated_image_hits_the_processor_cache():
|
||||
"""Check that mm caching is actually working."""
|
||||
mm_processor, cache = create_cached_processor(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
|
||||
model_config = ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
|
||||
)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
def process():
|
||||
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
assert first["mm_hashes"] == second["mm_hashes"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "prompt"),
|
||||
[
|
||||
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
"<image> and more text",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="Emu3Processor prepends its BOS token only when images are "
|
||||
"passed, so the unexpanded prompt the offsets path tokenizes never "
|
||||
"passed, so the unexpanded prompt vLLM tokenizes never "
|
||||
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
|
||||
strict=False,
|
||||
),
|
||||
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
],
|
||||
)
|
||||
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||
processor produces itself, which is what the legacy path returns."""
|
||||
prompt_ids = []
|
||||
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
prompt_ids.append(
|
||||
mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)["prompt_token_ids"]
|
||||
)
|
||||
"""The prompt is tokenized without any multi-modal data and the expansion spliced
|
||||
in, so its token ids have to come out the same as the ones the HF processor
|
||||
produces itself."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
info = mm_processor.info
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
legacy_ids, offsets_ids = prompt_ids
|
||||
assert legacy_ids == offsets_ids
|
||||
hf_processor = info.get_hf_processor()
|
||||
prompt_ids = info.get_tokenizer().encode(
|
||||
prompt, **info.default_tok_params.get_encode_kwargs()
|
||||
)
|
||||
hf_ids = info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=hf_processor.decode(prompt_ids), images=[image]),
|
||||
dict(truncation=False, add_special_tokens=False),
|
||||
)["input_ids"][0].tolist()
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_nested_image_fields_split_per_image(processor_cls):
|
||||
def test_nested_image_fields_split_per_image():
|
||||
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||
belonging to each image one dimension further in. Slicing the batch dimension
|
||||
instead handed the first image every row and the second an empty tensor."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
result = mm_processor(
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers for testing both processors of the Transformers modelling backend."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
MultiModalDummyInputsBuilder,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
offsets_only = pytest.mark.skipif(
|
||||
MultiModalProcessor is not OffsetsMultiModalProcessor,
|
||||
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
|
||||
)
|
||||
|
||||
PROCESSOR_CLASSES = [
|
||||
pytest.param(LegacyMultiModalProcessor, id="legacy"),
|
||||
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
|
||||
]
|
||||
|
||||
|
||||
def create_processor(model_id: str, processor_cls):
|
||||
"""Build a processor directly, because the registry only ever builds the one the
|
||||
installed transformers version selects, leaving the other path untested."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info))
|
||||
|
||||
|
||||
def create_cached_processor(model_id: str, processor_cls):
|
||||
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||
cache back so a test can check it was actually used."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
|
||||
+15
-81
@@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"CohereLabs/North-Mini-Code",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
|
||||
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm"),
|
||||
# FIXME: databricks/dbrx-instruct has been deleted
|
||||
"DbrxForCausalLM": _HfExamplesInfo(
|
||||
"databricks/dbrx-instruct", is_available_online=False
|
||||
@@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
|
||||
),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
|
||||
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
|
||||
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
|
||||
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
|
||||
@@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
|
||||
"Gemma4ForCausalLM": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
|
||||
@@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
|
||||
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.7-Flash",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
|
||||
),
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False),
|
||||
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
|
||||
"GPTBigCodeForCausalLM": _HfExamplesInfo(
|
||||
@@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
|
||||
"ibm-research/moe-7b-1b-active-shared-experts"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"),
|
||||
"HrmTextForCausalLM": _HfExamplesInfo(
|
||||
"sapientinc/HRM-Text-1B",
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
|
||||
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
|
||||
@@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
|
||||
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"InklingForCausalLM": _HfExamplesInfo(
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
@@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"IQuestLoopCoderForCausalLM": _HfExamplesInfo(
|
||||
"IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Jais2ForCausalLM": _HfExamplesInfo(
|
||||
"inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
|
||||
),
|
||||
"Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"),
|
||||
"JambaForCausalLM": _HfExamplesInfo(
|
||||
"ai21labs/AI21-Jamba-1.5-Mini",
|
||||
extras={
|
||||
@@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
|
||||
"Lfm2MoeForCausalLM": _HfExamplesInfo(
|
||||
"LiquidAI/LFM2-8B-A1B",
|
||||
min_transformers_version="5.0.0",
|
||||
use_original_num_layers=True,
|
||||
# Initialize at least one MoE layer
|
||||
hf_overrides={"num_hidden_layers": 4},
|
||||
@@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MiniCPM4ForCausalLM": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM4.1-8B",
|
||||
min_transformers_version="4.56",
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"hf": "HF remote code imports removed `is_torch_fx_available`; "
|
||||
@@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
extras={"tiny-random": "tiny-random/qwen3-next-moe"},
|
||||
min_transformers_version="4.56.3",
|
||||
),
|
||||
"Rnj1ForCausalLM": _HfExamplesInfo(
|
||||
"EssentialAI/rnj-1-instruct",
|
||||
@@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo(
|
||||
"openai/privacy-filter",
|
||||
min_transformers_version="5.6.0.dev0",
|
||||
),
|
||||
"RobertaForTokenClassification": _HfExamplesInfo(
|
||||
"Jean-Baptiste/roberta-large-ner-english"
|
||||
@@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
|
||||
"AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/audio-flamingo-3-hf",
|
||||
min_transformers_version="5.3.0",
|
||||
transformers_version_reason={
|
||||
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
|
||||
},
|
||||
@@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"CohereCompassForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/North-Micro-Vision-Instruct",
|
||||
min_transformers_version="5.16.0",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Nano",
|
||||
extras={"super": "nvidia/Cosmos3-Super"},
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Edge",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="5.15",
|
||||
use_original_num_layers=True,
|
||||
hf_overrides={
|
||||
"text_config": {
|
||||
@@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allendou/FireRedASR2-LLM-vllm",
|
||||
@@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.5.0",
|
||||
),
|
||||
# TODO: update min_transformers_version when Gemma4 Unified lands in
|
||||
# a stable transformers release.
|
||||
"Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
min_transformers_version="5.8.0",
|
||||
is_available_online=False,
|
||||
),
|
||||
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-vision-4.1-4b",
|
||||
@@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-speech-4.1-2b-plus",
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"GLM4VForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/glm-4v-9b",
|
||||
@@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
|
||||
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"H2OVLChatModel": _HfExamplesInfo(
|
||||
@@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"tencent/HunyuanOCR",
|
||||
min_transformers_version="5.13",
|
||||
),
|
||||
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
@@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Lfm2VlForConditionalGeneration": _HfExamplesInfo(
|
||||
"LiquidAI/LFM2-VL-450M",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Llama4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
@@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_6",
|
||||
min_transformers_version="5.7.0",
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
@@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allenai/Molmo2-8B",
|
||||
extras={"olmo": "allenai/Molmo2-O-7B"},
|
||||
min_transformers_version="4.51",
|
||||
trust_remote_code=True,
|
||||
# required by current PrefixLM implementation
|
||||
max_num_batched_tokens=31872,
|
||||
@@ -1296,19 +1260,15 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"vllm": "Custom processor code is not compatible with Transformers v5."
|
||||
"hf": "HF remote code reads `is_parallelizable`, which was removed "
|
||||
"in Transformers v5."
|
||||
},
|
||||
),
|
||||
"Ovis2_6ForCausalLM": _HfExamplesInfo(
|
||||
"AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
|
||||
),
|
||||
"Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
|
||||
"AIDC-AI/Ovis2.6-30B-A3B",
|
||||
trust_remote_code=True,
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"vllm": "Custom processor code is not compatible with Transformers v5."
|
||||
},
|
||||
"AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
|
||||
),
|
||||
"PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"PaddlePaddle/PaddleOCR-VL",
|
||||
@@ -1353,7 +1313,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
|
||||
"baidu/Qianfan-OCR",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen2-Audio-7B-Instruct"
|
||||
@@ -1368,12 +1327,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen3VLForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
@@ -1391,23 +1348,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
|
||||
),
|
||||
"Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ForcedAligner-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
|
||||
),
|
||||
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
|
||||
@@ -1456,7 +1409,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"microsoft/VibeVoice-ASR-HF",
|
||||
min_transformers_version="5.13.0",
|
||||
),
|
||||
"VoxtralForConditionalGeneration": _HfExamplesInfo(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
@@ -1530,7 +1482,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
use_original_num_layers=True, # DFlash requires all layers
|
||||
max_model_len=8192, # Reduce for CI
|
||||
max_num_seqs=32,
|
||||
min_transformers_version="4.56.3", # Required for Qwen3Next
|
||||
),
|
||||
"MuseGlimmerAssistantModel": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
@@ -1576,7 +1527,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Gemma4DSparkModel": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
@@ -1737,7 +1687,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"google/gemma-4-E4B-it",
|
||||
speculative_model="google/gemma-4-E4B-it-assistant",
|
||||
trust_remote_code=True,
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"ErnieMTPModel": _HfExamplesInfo(
|
||||
"baidu/ERNIE-4.5-21B-A3B-PT",
|
||||
@@ -1747,13 +1696,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"ExaoneMoeMTP": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
min_transformers_version="5.1.0",
|
||||
enable_prefix_caching=False,
|
||||
),
|
||||
"Exaone4_5_MTP": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"ExtractHiddenStatesModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
@@ -1766,12 +1713,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.7-Flash",
|
||||
speculative_model="zai-org/GLM-4.7-Flash",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"GlmOcrMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-OCR",
|
||||
speculative_model="zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
@@ -1842,9 +1787,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3NextMTP": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
|
||||
),
|
||||
"Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"),
|
||||
"Qwen3_5MTP": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
speculative_model="Qwen/Qwen3.5-0.8B",
|
||||
@@ -1865,29 +1808,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
"TransformersEmbeddingModel": _HfExamplesInfo(
|
||||
"BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
|
||||
"TransformersForSequenceClassification": _HfExamplesInfo(
|
||||
"papluca/xlm-roberta-base-language-detection",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"TransformersForCausalLM": _HfExamplesInfo(
|
||||
"hmellor/Ilama-3.2-1B", trust_remote_code=True
|
||||
),
|
||||
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
|
||||
"TransformersMoEForCausalLM": _HfExamplesInfo(
|
||||
"allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"),
|
||||
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||
),
|
||||
"TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||
"TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
|
||||
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
|
||||
"google/gemma-3-4b-it"
|
||||
|
||||
@@ -1,116 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the flashinfer moe_ep backend plumbing.
|
||||
|
||||
Everything here runs without a GPU or a flashinfer install: the flashinfer
|
||||
modules the helpers import lazily are replaced with capture fakes.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_BACKENDS,
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
MEGA_MOE_BACKENDS,
|
||||
NATIVE_MEGA_MOE_BACKENDS,
|
||||
PASSTHROUGH_ALL2ALL_BACKEND,
|
||||
bind_passthrough_all2all_backend,
|
||||
validate_flashinfer_moe_ep_model,
|
||||
)
|
||||
from vllm.utils.flashinfer_moe_ep import (
|
||||
_E2M1_LUT,
|
||||
FI_MOE_EP_BACKEND_SPECS,
|
||||
_dequant_fp4_ue8m0_gran32,
|
||||
build_fi_mega_config,
|
||||
fi_moe_ep_backend_spec,
|
||||
make_fi_moe_ep_bootstrap,
|
||||
megakernel_runtime_requirements,
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
epilogue_from_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_quant_config,
|
||||
map_mxfp4_backend,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
backend_to_kernel_cls as mxfp4_backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
map_nvfp4_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
backend_to_kernel_cls as nvfp4_backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
|
||||
PassThroughPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeBootstrapConfig:
|
||||
world_size: int
|
||||
rank: int
|
||||
process_group: Any = None
|
||||
auto_bootstrap: bool = True
|
||||
device: int | None = field(default=None, kw_only=True)
|
||||
def test_flashinfer_backends_are_megakernels_outside_the_native_model_path():
|
||||
assert FLASHINFER_MOE_EP_BACKENDS <= MEGA_MOE_BACKENDS
|
||||
assert FLASHINFER_MOE_EP_BACKENDS.isdisjoint(NATIVE_MEGA_MOE_BACKENDS)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeDeepGemmMegaMoeConfig:
|
||||
intermediate_size: int
|
||||
top_k: int
|
||||
activation_clamp: float | None
|
||||
fast_math: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeNvfp4CutedslMegaMoeConfig:
|
||||
intermediate_size: int
|
||||
top_k: int
|
||||
activation_clamp: float | None
|
||||
fast_math: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeMegaConfig:
|
||||
megakernel: Any
|
||||
preprocess_weights: bool
|
||||
quantize_input: bool
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_flashinfer(monkeypatch):
|
||||
"""Install a minimal fake flashinfer.moe_ep for the lazy imports."""
|
||||
moe_ep = ModuleType("flashinfer.moe_ep")
|
||||
core = ModuleType("flashinfer.moe_ep.core")
|
||||
runtime = ModuleType("flashinfer.moe_ep.core.runtime")
|
||||
flashinfer = ModuleType("flashinfer")
|
||||
fake_attrs: dict[ModuleType, dict[str, Any]] = {
|
||||
moe_ep: {
|
||||
"BootstrapConfig": _FakeBootstrapConfig,
|
||||
"DeepGemmMegaMoeConfig": _FakeDeepGemmMegaMoeConfig,
|
||||
"Nvfp4CutedslMegaMoeConfig": _FakeNvfp4CutedslMegaMoeConfig,
|
||||
"MegaConfig": _FakeMegaConfig,
|
||||
"core": core,
|
||||
},
|
||||
runtime: {"TORCH_DIST": "torch_dist", "NVSHMEM": "nvshmem"},
|
||||
flashinfer: {"moe_ep": moe_ep},
|
||||
core: {"runtime": runtime},
|
||||
}
|
||||
for mod, attrs in fake_attrs.items():
|
||||
for attr, value in attrs.items():
|
||||
setattr(mod, attr, value)
|
||||
|
||||
for name, mod in {
|
||||
"flashinfer": flashinfer,
|
||||
"flashinfer.moe_ep": moe_ep,
|
||||
"flashinfer.moe_ep.core": core,
|
||||
"flashinfer.moe_ep.core.runtime": runtime,
|
||||
}.items():
|
||||
monkeypatch.setitem(sys.modules, name, mod)
|
||||
return moe_ep
|
||||
|
||||
|
||||
def test_fi_backend_strings_are_registered_mega_moe_backends():
|
||||
assert set(FI_MOE_EP_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
|
||||
assert FLASHINFER_MOE_EP_BACKENDS < MEGA_MOE_BACKENDS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
|
||||
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
|
||||
instead of silently falling through to the generic FusedMoE path."""
|
||||
def test_only_deep_gemm_backend_is_dsv4_specific():
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
["MixtralForCausalLM"],
|
||||
)
|
||||
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["DeepseekV4ForCausalLM"])
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
["MixtralForCausalLM"],
|
||||
)
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
["DeepseekV4ForCausalLM"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -118,8 +92,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
|
||||
)
|
||||
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
|
||||
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
|
||||
models validate their own constraints at construction time."""
|
||||
"""Models validate native deep_gemm mega constraints at construction time."""
|
||||
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
|
||||
|
||||
|
||||
@@ -128,11 +101,7 @@ def test_non_fi_backend_ignores_architectures():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(MEGA_MOE_BACKENDS))
|
||||
def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
"""Every mega backend must qualify for sequence-parallel MoE at
|
||||
TP>1/EP: the predicate once matched only the native backend string,
|
||||
which silently ran the fi backends full-batch with an all-reduce on
|
||||
every rank — 0.42-0.65x native e2e at TP8."""
|
||||
def test_token_sharding_backends_enable_dsv4_sequence_parallel(moe_backend: str):
|
||||
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
@@ -147,113 +116,262 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
assert _use_sequence_parallel(vllm_config)
|
||||
|
||||
|
||||
def test_fi_moe_ep_backend_spec_kernel_and_nvshmem_contract():
|
||||
dg = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
|
||||
assert dg.megakernel == "deep_gemm_mega"
|
||||
assert not dg.needs_nvshmem
|
||||
|
||||
cd = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
|
||||
assert cd.megakernel == "nvfp4_cutedsl"
|
||||
assert cd.needs_nvshmem
|
||||
|
||||
with pytest.raises(ValueError, match="not a flashinfer moe_ep backend"):
|
||||
fi_moe_ep_backend_spec("deep_gemm_mega_moe")
|
||||
def test_dsv4_requests_pre_fc2_router_weight_placement():
|
||||
dsv4 = SimpleNamespace(routing_method=RoutingMethodType.DeepseekV4)
|
||||
default = SimpleNamespace(routing_method=RoutingMethodType.Default)
|
||||
assert fi_ep.apply_topk_in_fc1(dsv4)
|
||||
assert not fi_ep.apply_topk_in_fc1(default)
|
||||
|
||||
|
||||
def test_megakernel_runtime_requirements(fake_flashinfer):
|
||||
dg = megakernel_runtime_requirements(
|
||||
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
|
||||
def test_backend_specs_preserve_weight_format_contracts():
|
||||
cutedsl = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert cutedsl.kernel == "cutedsl"
|
||||
assert cutedsl.weight_formats == frozenset({"nvfp4", "mxfp4"})
|
||||
|
||||
deep_gemm = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
assert deep_gemm.kernel == "deep_gemm"
|
||||
assert deep_gemm.weight_formats == frozenset({"mxfp4"})
|
||||
|
||||
|
||||
def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
|
||||
fields = dict(
|
||||
moe_backend=moe_backend,
|
||||
moe_parallel_config=SimpleNamespace(
|
||||
use_ep=True, use_batched_activation_format=False
|
||||
),
|
||||
is_act_and_mul=True,
|
||||
activation=MoEActivation.SILU,
|
||||
has_hash_routing=False,
|
||||
routing_method=RoutingMethodType.DeepseekV4,
|
||||
router_logits_dtype=torch.float32,
|
||||
hidden_dim=256,
|
||||
is_lora_enabled=False,
|
||||
skip_final_all_reduce=False,
|
||||
in_dtype=torch.bfloat16,
|
||||
has_bias=False,
|
||||
swiglu_alpha=None,
|
||||
swiglu_beta=None,
|
||||
)
|
||||
assert dg == frozenset({"torch_dist"})
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
cd = megakernel_runtime_requirements(
|
||||
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
|
||||
|
||||
def _is_supported(moe, weight_key, activation_key) -> tuple[bool, str]:
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
weight_key,
|
||||
activation_key,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert cd == frozenset({"torch_dist", "nvshmem"})
|
||||
return supported, reason or ""
|
||||
|
||||
|
||||
def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
|
||||
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
|
||||
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
|
||||
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
|
||||
device it already bound via BootstrapConfig.device."""
|
||||
import vllm.utils.flashinfer_moe_ep as mod
|
||||
|
||||
pg = object()
|
||||
@pytest.fixture
|
||||
def megakernel_host(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
|
||||
"""A vLLM config without DBO/EPLB/weight transfer, on a supported device."""
|
||||
config = SimpleNamespace(
|
||||
weight_transfer_config=None,
|
||||
parallel_config=SimpleNamespace(enable_dbo=False, enable_eplb=False),
|
||||
)
|
||||
monkeypatch.setattr(fi_ep, "get_current_vllm_config", lambda: config)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(world_size=4, rank_in_group=2, device_group=pg),
|
||||
FlashInferMoeEpExperts, "_supports_current_device", staticmethod(lambda: True)
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 3)
|
||||
|
||||
bootstrap = make_fi_moe_ep_bootstrap()
|
||||
|
||||
assert bootstrap.world_size == 4
|
||||
assert bootstrap.rank == 2
|
||||
assert bootstrap.process_group is pg
|
||||
assert bootstrap.auto_bootstrap is False
|
||||
assert bootstrap.device == 3
|
||||
return config
|
||||
|
||||
|
||||
def test_build_fi_mega_config_selects_kernel_config(fake_flashinfer):
|
||||
dg = build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=7.0,
|
||||
megakernel="deep_gemm_mega",
|
||||
def test_megakernel_requires_expert_parallel(megakernel_host):
|
||||
"""The oracle rejects the megakernel through the experts' support check."""
|
||||
moe = _megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert _is_supported(moe, kNvfp4Static, kNvfp4Dynamic) == (True, "")
|
||||
|
||||
moe.moe_parallel_config.use_ep = False
|
||||
supported, reason = _is_supported(moe, kNvfp4Static, kNvfp4Dynamic)
|
||||
assert not supported and "parallel config" in reason
|
||||
|
||||
|
||||
def test_megakernel_reports_its_own_constraints(megakernel_host):
|
||||
"""Constraints the framework does not know about surface as reasons."""
|
||||
_, reason = _is_supported(
|
||||
_megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL), kNvfp4Static, None
|
||||
)
|
||||
assert isinstance(dg.megakernel, _FakeDeepGemmMegaMoeConfig)
|
||||
assert dg.megakernel.intermediate_size == 2048
|
||||
assert dg.megakernel.top_k == 8
|
||||
assert dg.megakernel.activation_clamp == 7.0
|
||||
assert dg.preprocess_weights and dg.quantize_input
|
||||
assert "A16 activations" in reason
|
||||
|
||||
cd = build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=None,
|
||||
megakernel="nvfp4_cutedsl",
|
||||
deep_gemm = _megakernel_moe(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM, routing_method=RoutingMethodType.Renormalize
|
||||
)
|
||||
assert isinstance(cd.megakernel, _FakeNvfp4CutedslMegaMoeConfig)
|
||||
_, reason = _is_supported(deep_gemm, kMxfp4Static, None)
|
||||
assert "routing method" in reason
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported fi_moe_ep megakernel"):
|
||||
build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=None,
|
||||
megakernel="deep_gemm",
|
||||
megakernel_host.parallel_config.enable_eplb = True
|
||||
_, reason = _is_supported(
|
||||
_megakernel_moe(FLASHINFER_MOE_EP_DEEP_GEMM), kMxfp4Static, None
|
||||
)
|
||||
assert reason.endswith("EPLB")
|
||||
|
||||
|
||||
def test_megakernels_are_oracle_backends():
|
||||
"""Quant methods reach the megakernel through the NVFP4/MXFP4 oracles."""
|
||||
nvfp4 = map_nvfp4_backend(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert nvfp4 is NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
|
||||
assert nvfp4_backend_to_kernel_cls(nvfp4) == [FlashInferMoeEpExperts]
|
||||
|
||||
assert map_mxfp4_backend(FLASHINFER_MOE_EP_DEEP_GEMM) == [
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
]
|
||||
for backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
assert mxfp4_backend_to_kernel_cls(backend) == [FlashInferMoeEpExperts]
|
||||
# The megakernel takes the model's own dimensions.
|
||||
assert mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
backend, 2880, 2880
|
||||
) == (2880, 2880)
|
||||
|
||||
quant_config = make_mxfp4_moe_quant_config(
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL, w1_scale=None, w2_scale=None
|
||||
)
|
||||
assert quant_config is not None and quant_config.use_nvfp4_w4a4
|
||||
|
||||
|
||||
def test_megakernel_is_a_modular_kernel_with_pass_through_stages():
|
||||
"""The megakernel plugs into ``FusedMoEKernel`` like any other experts impl.
|
||||
|
||||
Routing stays outside (top-k ids in), dispatch and combine happen inside,
|
||||
so prepare passes tokens through untouched and finalize has nothing to do.
|
||||
"""
|
||||
moe = make_dummy_moe_config(
|
||||
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
|
||||
kernel = mk.FusedMoEKernel(
|
||||
PassThroughPrepareAndFinalize(),
|
||||
FlashInferMoeEpExperts(moe, quant_config),
|
||||
)
|
||||
|
||||
assert not kernel.is_monolithic
|
||||
assert kernel.prepare_finalize.output_is_reduced()
|
||||
# Generic stages: the int32 top-k ids the megakernel wants are its concern.
|
||||
assert kernel.prepare_finalize.topk_indices_dtype() is None
|
||||
assert kernel.fused_experts.expects_unquantized_inputs
|
||||
assert isinstance(
|
||||
kernel.fused_experts.finalize_weight_and_reduce_impl(), TopKWeightAndReduceNoOP
|
||||
)
|
||||
assert kernel.fused_experts.workspace_shapes(
|
||||
16, 256, 256, 2, 8, 8, None, MoEActivation.SILU
|
||||
) == ((0,), (0,), (16, 256))
|
||||
|
||||
tokens = torch.zeros(4, 256, dtype=torch.bfloat16)
|
||||
prepared = kernel.prepare_finalize.prepare(
|
||||
tokens,
|
||||
torch.ones(4, 2),
|
||||
torch.zeros(4, 2, dtype=torch.int32),
|
||||
8,
|
||||
None,
|
||||
False,
|
||||
quant_config,
|
||||
)
|
||||
assert prepared[0] is tokens and all(item is None for item in prepared[1:])
|
||||
|
||||
|
||||
def test_epilogue_takes_the_weight_global_scales_from_the_quant_config():
|
||||
"""The canonical ``g1/g2_alphas`` become the per-expert fc1/fc2 alphas and
|
||||
keep aliasing the layer's tensors, so EPLB permutations reach the kernel."""
|
||||
g1_alphas = torch.tensor([1.0, 2.0])
|
||||
g2_alphas = torch.tensor([3.0, 4.0])
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=g1_alphas,
|
||||
g2_alphas=g2_alphas,
|
||||
a1_gscale=torch.ones(2),
|
||||
a2_gscale=torch.ones(2),
|
||||
w1_scale=torch.ones(2, 4, 1),
|
||||
w2_scale=torch.ones(2, 2, 1),
|
||||
)
|
||||
|
||||
epilogue = epilogue_from_quant_config(quant_config)
|
||||
|
||||
assert epilogue.fc1_alpha is not None and epilogue.fc2_alpha is not None
|
||||
assert epilogue.fc1_alpha.data_ptr() == g1_alphas.data_ptr()
|
||||
assert epilogue.fc2_alpha.data_ptr() == g2_alphas.data_ptr()
|
||||
# Activations are quantized dynamically: no norm constants.
|
||||
assert epilogue.input_norm_const == 1.0 and epilogue.fc1_norm_const is None
|
||||
|
||||
|
||||
def test_epilogue_rejects_unloaded_weight_global_scales():
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=torch.tensor([1.0, float("nan")]),
|
||||
g2_alphas=torch.ones(2),
|
||||
a1_gscale=torch.ones(2),
|
||||
a2_gscale=torch.ones(2),
|
||||
w1_scale=torch.ones(2, 4, 1),
|
||||
w2_scale=torch.ones(2, 2, 1),
|
||||
)
|
||||
with pytest.raises(ValueError, match="g1_alphas"):
|
||||
epilogue_from_quant_config(quant_config)
|
||||
|
||||
|
||||
def test_mxfp4_checkpoints_keep_the_default_epilogue():
|
||||
"""MXFP4 has no per-tensor global scales; the megakernel needs no alphas."""
|
||||
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
|
||||
assert epilogue_from_quant_config(quant_config) == fi_ep.FlashInferMoeEpEpilogue()
|
||||
|
||||
|
||||
def test_nvfp4_kernel_format_passes_megakernel_weights_through():
|
||||
"""The megakernel takes the canonical tensors as they are; the oracle must
|
||||
not reject or reshape them."""
|
||||
tensors = tuple(torch.arange(4.0).reshape(2, 2) + i for i in range(8))
|
||||
converted = convert_to_nvfp4_moe_kernel_format(
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
SimpleNamespace(),
|
||||
*tensors,
|
||||
is_act_and_mul=True,
|
||||
)
|
||||
assert all(out is src for out, src in zip(converted, tensors))
|
||||
|
||||
|
||||
def test_passthrough_all2all_backend_selects_the_pass_through_stages():
|
||||
"""``all2all_backend="passthrough"`` is how the oracles get the
|
||||
pass-through prepare/finalize."""
|
||||
moe = make_dummy_moe_config(
|
||||
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
moe.moe_parallel_config = replace(
|
||||
moe.moe_parallel_config, all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND
|
||||
)
|
||||
assert not moe.moe_parallel_config.use_all2all_kernels
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
moe, FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
|
||||
)
|
||||
|
||||
assert isinstance(prepare_finalize, PassThroughPrepareAndFinalize)
|
||||
|
||||
|
||||
def test_passthrough_all2all_backend_is_not_a_cli_choice():
|
||||
"""Users cannot pick the passthrough all2all backend explicitly."""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["--all2all-backend", PASSTHROUGH_ALL2ALL_BACKEND])
|
||||
parser.parse_args(["--all2all-backend", "deepep_low_latency"])
|
||||
|
||||
|
||||
def test_megakernel_moe_backend_binds_the_all2all_backend():
|
||||
"""A megakernel deployment gets its all2all backend by default."""
|
||||
parallel = SimpleNamespace(all2all_backend="allgather_reducescatter")
|
||||
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
|
||||
assert parallel.all2all_backend == PASSTHROUGH_ALL2ALL_BACKEND
|
||||
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot be paired"):
|
||||
bind_passthrough_all2all_backend(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
SimpleNamespace(all2all_backend="deepep_low_latency"),
|
||||
)
|
||||
with pytest.raises(ValueError, match="only valid with"):
|
||||
bind_passthrough_all2all_backend(
|
||||
"flashinfer_trtllm",
|
||||
SimpleNamespace(all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND),
|
||||
)
|
||||
|
||||
|
||||
def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
|
||||
from vllm.models.deepseek_v4.nvidia.fi_moe import ckpt_uses_nvfp4_experts
|
||||
|
||||
nvfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo="NVFP4"))
|
||||
assert ckpt_uses_nvfp4_experts(nvfp4)
|
||||
|
||||
mxfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo=None))
|
||||
assert not ckpt_uses_nvfp4_experts(mxfp4)
|
||||
|
||||
no_algo = SimpleNamespace(quant_config=SimpleNamespace())
|
||||
assert not ckpt_uses_nvfp4_experts(no_algo)
|
||||
|
||||
|
||||
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
|
||||
"""One 32-element scale group per row: low nibble is the even element,
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group."""
|
||||
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
|
||||
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
|
||||
|
||||
out = _dequant_fp4_ue8m0_gran32(packed, sf)
|
||||
|
||||
assert out.shape == (2, 32)
|
||||
assert out.dtype == torch.bfloat16
|
||||
expected = torch.empty(2, 32)
|
||||
for row in range(2):
|
||||
for col in range(16):
|
||||
byte = int(packed[row, col])
|
||||
expected[row, 2 * col] = _E2M1_LUT[byte & 0x0F]
|
||||
expected[row, 2 * col + 1] = _E2M1_LUT[byte >> 4]
|
||||
expected[row] *= 2.0**row
|
||||
assert torch.equal(out, expected.to(torch.bfloat16))
|
||||
untouched = SimpleNamespace(all2all_backend="deepep_low_latency")
|
||||
bind_passthrough_all2all_backend("flashinfer_trtllm", untouched)
|
||||
assert untouched.all2all_backend == "deepep_low_latency"
|
||||
|
||||
@@ -657,7 +657,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.use_mega_moe = True
|
||||
moe.use_native_mega_moe = True
|
||||
moe.gate = FakeGate()
|
||||
moe.experts = FakeExperts()
|
||||
moe.shared_experts = FakeSharedExperts()
|
||||
|
||||
@@ -241,7 +241,7 @@ def test_v41_dspark_loads_linear_scales(
|
||||
linear_scale_name=scale_name,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [(runtime_name, param)],
|
||||
|
||||
@@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None
|
||||
|
||||
|
||||
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
installed = Version(transformers.__version__)
|
||||
required = Version("5.15.0.dev0")
|
||||
if installed < required:
|
||||
pytest.skip(
|
||||
"MLA models with the Transformers modeling backend require "
|
||||
f"transformers>={required}, but got {installed}"
|
||||
)
|
||||
|
||||
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
|
||||
args = (example_prompts, 32, 5)
|
||||
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
|
||||
|
||||
@@ -609,17 +609,3 @@ def dummy_hf_overrides(
|
||||
)
|
||||
|
||||
return hf_config
|
||||
|
||||
|
||||
def check_transformers_version(
|
||||
model: str,
|
||||
min_transformers_version: str | None = None,
|
||||
max_transformers_version: str | None = None,
|
||||
):
|
||||
from .registry import _HfExamplesInfo
|
||||
|
||||
return _HfExamplesInfo(
|
||||
model,
|
||||
min_transformers_version=min_transformers_version,
|
||||
max_transformers_version=max_transformers_version,
|
||||
).check_transformers_version(on_fail="skip")
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
@@ -46,6 +47,12 @@ def test_quant_spec_string_representation_uses_quantization_name():
|
||||
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
|
||||
|
||||
|
||||
def test_quant_spec_resolves_groupwise_int4_weight():
|
||||
spec = QuantSpec(weight="int4_per_group_32")
|
||||
assert spec.weight == kInt4Static32
|
||||
assert spec.activation is None
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
quant_spec(weight="not_a_real_format")
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the `VLLM_ROCM_MOE_PADDING` weight memory-layout padding trick.
|
||||
|
||||
`VLLM_ROCM_MOE_PADDING` (see `UnquantizedFusedMoEMethod._maybe_pad_weight`)
|
||||
is a memory-stride trick, not a logical-shape one: when eligible, it enlarges
|
||||
a weight tensor's storage so per-expert allocations land farther apart in
|
||||
HBM. Shape and values are unaffected, but `stride(-2)` increases by
|
||||
`num_pad` elements -- that's the whole mechanism. For the **AITER** path
|
||||
tested here, the padded storage never persists on `w13_weight`/`w2_weight`:
|
||||
`convert_to_unquantized_kernel_format` reallocates via
|
||||
`rocm_aiter_ops.shuffle_weights`. This is AITER-specific -- on TRITON-on-ROCm,
|
||||
`.contiguous()` is skipped specifically to keep this same padding, so its
|
||||
persisted parameters *do* stay padded.
|
||||
|
||||
This file verifies:
|
||||
1. `_maybe_pad_weight` in isolation (any ROCm backend, AITER not required):
|
||||
grows storage/`stride(-2)` only when eligible (512-byte-aligned) and the
|
||||
flag is on -- covering no padding, hidden-only, intermediate-only, and
|
||||
both-dimensions-padded, since w13 and w2 key their alignment gate off
|
||||
different dims; a true no-op (identical object) otherwise.
|
||||
2. End to end, AITER only (per #54966's "Test padding" ask):
|
||||
`AiterExperts.apply()` matches an independent reference regardless of
|
||||
the flag, and persisted parameter storage size is unaffected by it.
|
||||
|
||||
Logical `hidden_dim_unpadded`/`intermediate_size_per_partition_unpadded`
|
||||
padding and HIP-graph token-padding are separate mechanisms, covered in
|
||||
#59333 and #59334.
|
||||
|
||||
See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
|
||||
ExpertMapManager,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
aiter_available = is_aiter_found_and_supported()
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="VLLM_ROCM_MOE_PADDING only takes effect on ROCm",
|
||||
)
|
||||
requires_aiter = pytest.mark.skipif(
|
||||
not aiter_available,
|
||||
reason="requires AITER to exercise AiterExperts.apply()",
|
||||
)
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
DTYPE = torch.bfloat16
|
||||
NUM_EXPERTS = 4
|
||||
TOPK = 2
|
||||
NUM_TOKENS = 16
|
||||
# Mirrors `_maybe_pad_weight`'s `num_pad` constant in unquantized_fused_moe_method.py.
|
||||
PAD_BYTES = 256
|
||||
# 256 bf16 elems * 2 bytes = 512 bytes: eligible for `_maybe_pad_weight` on
|
||||
# both w13 and w2.
|
||||
HIDDEN_SIZE = 256
|
||||
INTERMEDIATE_SIZE = 256
|
||||
|
||||
|
||||
def _set_padding_env(monkeypatch: pytest.MonkeyPatch, padding: bool) -> None:
|
||||
"""Sets the flag only; `_maybe_pad_weight` reads it lazily, no AITER needed."""
|
||||
monkeypatch.setenv("VLLM_ROCM_MOE_PADDING", "1" if padding else "0")
|
||||
|
||||
|
||||
def _set_padding_env_with_aiter(monkeypatch: pytest.MonkeyPatch, padding: bool) -> None:
|
||||
"""Also enables AITER, for tests routing through RoutedExperts's backend
|
||||
selection."""
|
||||
_set_padding_env(monkeypatch, padding)
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _make_moe_config(
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
dtype: torch.dtype = DTYPE,
|
||||
) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=NUM_EXPERTS,
|
||||
experts_per_token=TOPK,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=NUM_EXPERTS,
|
||||
num_logical_experts=NUM_EXPERTS,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=dtype,
|
||||
device=DEVICE,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=NUM_TOKENS,
|
||||
)
|
||||
|
||||
|
||||
def _make_routed_experts(hidden_size: int, intermediate_size: int) -> RoutedExperts:
|
||||
moe_config = _make_moe_config(hidden_size, intermediate_size)
|
||||
expert_map_manager = ExpertMapManager(
|
||||
max_num_batched_tokens=NUM_TOKENS,
|
||||
top_k=TOPK,
|
||||
global_num_experts=NUM_EXPERTS,
|
||||
num_redundant_experts=0,
|
||||
num_expert_group=None,
|
||||
moe_parallel_config=moe_config.moe_parallel_config,
|
||||
placement_strategy="linear",
|
||||
enable_eplb=False,
|
||||
)
|
||||
return RoutedExperts(
|
||||
"experts",
|
||||
DTYPE,
|
||||
moe_config,
|
||||
quant_config=None,
|
||||
expert_map_manager=expert_map_manager,
|
||||
)
|
||||
|
||||
|
||||
def _expert_weight_iterator(seed: int, hidden_size: int, intermediate_size: int):
|
||||
set_random_seed(seed)
|
||||
for expert_id in range(NUM_EXPERTS):
|
||||
yield (
|
||||
f"{expert_id}.gate_proj.weight",
|
||||
torch.randn(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
device=DEVICE,
|
||||
dtype=DTYPE,
|
||||
),
|
||||
)
|
||||
yield (
|
||||
f"{expert_id}.up_proj.weight",
|
||||
torch.randn(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
device=DEVICE,
|
||||
dtype=DTYPE,
|
||||
),
|
||||
)
|
||||
yield (
|
||||
f"{expert_id}.down_proj.weight",
|
||||
torch.randn(
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
device=DEVICE,
|
||||
dtype=DTYPE,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _load_and_process_weights(
|
||||
layer: RoutedExperts, seed: int, hidden_size: int, intermediate_size: int
|
||||
) -> None:
|
||||
loaded = set(
|
||||
layer.load_weights(
|
||||
_expert_weight_iterator(seed, hidden_size, intermediate_size)
|
||||
)
|
||||
)
|
||||
assert {"w13_weight", "w2_weight"} <= loaded
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
def _make_static_inputs(
|
||||
hidden_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
set_random_seed(2026)
|
||||
x = torch.randn(
|
||||
NUM_TOKENS,
|
||||
hidden_size,
|
||||
device=DEVICE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
topk_ids = torch.arange(NUM_TOKENS, device=DEVICE, dtype=torch.int64).unsqueeze(1)
|
||||
topk_ids = torch.cat(
|
||||
[topk_ids % NUM_EXPERTS, (topk_ids + 1) % NUM_EXPERTS],
|
||||
dim=1,
|
||||
)
|
||||
topk_weights = torch.full(
|
||||
(NUM_TOKENS, TOPK),
|
||||
1.0 / TOPK,
|
||||
device=DEVICE,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
return x, topk_weights, topk_ids
|
||||
|
||||
|
||||
def _forward(
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return layer.forward_modular(
|
||||
x,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
shared_experts=None,
|
||||
shared_experts_input=None,
|
||||
)
|
||||
|
||||
|
||||
def _assert_backend_is_aiter(layer: RoutedExperts) -> None:
|
||||
selected = layer.quant_method.unquantized_backend
|
||||
assert selected is UnquantizedMoeBackend.AITER, (
|
||||
f"expected AiterExperts to be selected, got backend={selected!r}"
|
||||
)
|
||||
|
||||
|
||||
# --- 1. `_maybe_pad_weight` in isolation -----------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"padding,hidden_size,intermediate_size,dtype,expect_padded",
|
||||
[
|
||||
# eligible shape (512 bytes): padding fires iff the env var is on.
|
||||
(True, HIDDEN_SIZE, INTERMEDIATE_SIZE, DTYPE, True),
|
||||
(False, HIDDEN_SIZE, INTERMEDIATE_SIZE, DTYPE, False),
|
||||
# ineligible shape (400 bytes, not 512-aligned): never fires.
|
||||
(True, 200, 200, DTYPE, False),
|
||||
# fp32: alignment threshold and num_pad both scale with
|
||||
# element_size() (128 fp32 elems * 4 bytes == 512).
|
||||
(True, 128, 128, torch.float32, True),
|
||||
],
|
||||
ids=["eligible-on", "eligible-off", "ineligible-on", "eligible-on-fp32"],
|
||||
)
|
||||
def test_maybe_pad_weight_transparent(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
default_vllm_config,
|
||||
padding: bool,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
dtype: torch.dtype,
|
||||
expect_padded: bool,
|
||||
) -> None:
|
||||
"""Shape/values never change; storage only grows (`stride(-2) += num_pad`)
|
||||
when eligible and the flag is on. Otherwise it's a true no-op (identical
|
||||
object)."""
|
||||
assert default_vllm_config is not None
|
||||
_set_padding_env(monkeypatch, padding)
|
||||
assert envs.VLLM_ROCM_MOE_PADDING is padding
|
||||
|
||||
moe_config = _make_moe_config(hidden_size, intermediate_size, dtype=dtype)
|
||||
method = UnquantizedFusedMoEMethod(moe_config)
|
||||
|
||||
original = torch.randn(
|
||||
NUM_EXPERTS,
|
||||
2 * intermediate_size,
|
||||
hidden_size,
|
||||
device=DEVICE,
|
||||
dtype=dtype,
|
||||
)
|
||||
result = method._maybe_pad_weight(original)
|
||||
|
||||
assert result.shape == original.shape
|
||||
assert torch.equal(result, original), "padding must not alter weight values"
|
||||
|
||||
if expect_padded:
|
||||
num_pad = PAD_BYTES // original.element_size()
|
||||
assert result.stride(-1) == 1
|
||||
assert result.stride(-2) == original.stride(-2) + num_pad
|
||||
assert result.untyped_storage().nbytes() > original.untyped_storage().nbytes()
|
||||
assert result.data_ptr() != original.data_ptr()
|
||||
else:
|
||||
assert result is original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,intermediate_size,expect_w13_padded,expect_w2_padded",
|
||||
[
|
||||
# only hidden_size aligned: only w13 (last dim hidden_size) eligible.
|
||||
(HIDDEN_SIZE, 200, True, False),
|
||||
# only intermediate_size aligned: only w2 (last dim
|
||||
# intermediate_size) eligible.
|
||||
(200, HIDDEN_SIZE, False, True),
|
||||
],
|
||||
ids=["only-w13-eligible", "only-w2-eligible"],
|
||||
)
|
||||
def test_maybe_pad_weight_asymmetric_w13_w2_eligibility(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
default_vllm_config,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
expect_w13_padded: bool,
|
||||
expect_w2_padded: bool,
|
||||
) -> None:
|
||||
"""w13 and w2 key their alignment gate off different dims (hidden_size
|
||||
vs. intermediate_size), so they're independently eligible."""
|
||||
assert default_vllm_config is not None
|
||||
_set_padding_env(monkeypatch, True)
|
||||
|
||||
moe_config = _make_moe_config(hidden_size, intermediate_size)
|
||||
method = UnquantizedFusedMoEMethod(moe_config)
|
||||
|
||||
w13 = torch.randn(
|
||||
NUM_EXPERTS, 2 * intermediate_size, hidden_size, device=DEVICE, dtype=DTYPE
|
||||
)
|
||||
w2 = torch.randn(
|
||||
NUM_EXPERTS, hidden_size, intermediate_size, device=DEVICE, dtype=DTYPE
|
||||
)
|
||||
|
||||
w13_result = method._maybe_pad_weight(w13)
|
||||
w2_result = method._maybe_pad_weight(w2)
|
||||
|
||||
assert (w13_result is not w13) == expect_w13_padded
|
||||
assert (w2_result is not w2) == expect_w2_padded
|
||||
|
||||
|
||||
def _reference_moe_forward(
|
||||
seed: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""A minimal, independent (no AITER) reference MoE forward, so the E2E
|
||||
test below can't pass merely because both flag settings are wrong the
|
||||
same way."""
|
||||
weights: dict[int, dict[str, torch.Tensor]] = {}
|
||||
for name, tensor in _expert_weight_iterator(seed, hidden_size, intermediate_size):
|
||||
expert_id_str, proj_name = name.split(".", 1)
|
||||
weights.setdefault(int(expert_id_str), {})[proj_name] = tensor
|
||||
|
||||
output = torch.zeros_like(x)
|
||||
for expert_id, proj in weights.items():
|
||||
gate = F.silu(x @ proj["gate_proj.weight"].T)
|
||||
up = x @ proj["up_proj.weight"].T
|
||||
expert_out = (gate * up) @ proj["down_proj.weight"].T
|
||||
|
||||
mask = topk_ids == expert_id
|
||||
weight = (topk_weights * mask).sum(dim=1, keepdim=True)
|
||||
output = output + weight * expert_out
|
||||
return output
|
||||
|
||||
|
||||
# --- 2. End-to-end numerical transparency through AiterExperts -------------
|
||||
|
||||
|
||||
@requires_aiter
|
||||
@torch.inference_mode()
|
||||
def test_aiter_moe_padding_numerically_transparent(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
default_vllm_config,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
"""`AiterExperts.apply()` output must match an independent reference
|
||||
(not just itself across flag settings) for an eligible weight shape,
|
||||
regardless of `VLLM_ROCM_MOE_PADDING`. Persisted parameter storage size
|
||||
must also be unaffected by the flag (see module docstring)."""
|
||||
assert default_vllm_config is not None
|
||||
assert workspace_init is None
|
||||
|
||||
outputs: dict[bool, torch.Tensor] = {}
|
||||
storage_nbytes: dict[bool, tuple[int, int]] = {}
|
||||
for padding in (True, False):
|
||||
_set_padding_env_with_aiter(monkeypatch, padding)
|
||||
|
||||
with torch.device(DEVICE):
|
||||
layer = _make_routed_experts(HIDDEN_SIZE, INTERMEDIATE_SIZE)
|
||||
_load_and_process_weights(
|
||||
layer,
|
||||
seed=1,
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
intermediate_size=INTERMEDIATE_SIZE,
|
||||
)
|
||||
_assert_backend_is_aiter(layer)
|
||||
storage_nbytes[padding] = (
|
||||
layer.w13_weight.data.untyped_storage().nbytes(),
|
||||
layer.w2_weight.data.untyped_storage().nbytes(),
|
||||
)
|
||||
|
||||
x, topk_weights, topk_ids = _make_static_inputs(HIDDEN_SIZE)
|
||||
outputs[padding] = _forward(layer, x, topk_weights, topk_ids).clone()
|
||||
|
||||
assert storage_nbytes[True] == storage_nbytes[False], (
|
||||
"VLLM_ROCM_MOE_PADDING must not change the persisted parameters' "
|
||||
"storage size for the AITER path"
|
||||
)
|
||||
|
||||
reference = _reference_moe_forward(
|
||||
seed=1,
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
intermediate_size=INTERMEDIATE_SIZE,
|
||||
x=x,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
)
|
||||
# Same AITER kernel, byte-identical weights and inputs either way (only
|
||||
# a transient weight-loading buffer differs): bitwise-identical results.
|
||||
torch.testing.assert_close(outputs[True], outputs[False], atol=0, rtol=0)
|
||||
|
||||
# Cosine similarity, not elementwise assert_close, for the reference
|
||||
# comparison: two independently computed bf16 GEMM chains can have
|
||||
# near-zero elements where rounding noise blows up elementwise relative
|
||||
# error. Same pattern as test_b12x.py / test_mxfp4_moe.py.
|
||||
for padded, output in outputs.items():
|
||||
cos_sim = F.cosine_similarity(
|
||||
output.flatten().float(), reference.flatten().float(), dim=0
|
||||
)
|
||||
assert cos_sim > 0.99, (
|
||||
f"padding={padded} output diverged from reference "
|
||||
f"(cosine similarity {cos_sim:.4f})"
|
||||
)
|
||||
+43
-1
@@ -2,8 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.outputs import (
|
||||
ClassificationRequestOutput,
|
||||
EmbeddingRequestOutput,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestError,
|
||||
RequestOutput,
|
||||
ScoringRequestOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -19,3 +28,36 @@ def test_request_output_forward_compatible():
|
||||
example_arg_added_in_new_version="some_value",
|
||||
)
|
||||
assert output is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output_type", "empty_value"),
|
||||
[
|
||||
(EmbeddingRequestOutput, []),
|
||||
(ClassificationRequestOutput, []),
|
||||
(ScoringRequestOutput, 0.0),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_specialized_pooling_outputs_preserve_request_error(output_type, empty_value):
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
base_output = PoolingRequestOutput(
|
||||
request_id="request",
|
||||
outputs=PoolingOutput(torch.empty(0)),
|
||||
prompt_token_ids=[1, 2],
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
error=error,
|
||||
)
|
||||
|
||||
output = output_type.from_base(base_output)
|
||||
|
||||
assert output.error is error
|
||||
value = getattr(output.outputs, "embedding", None)
|
||||
value = getattr(output.outputs, "probs", value)
|
||||
value = getattr(output.outputs, "score", value)
|
||||
assert value == empty_value
|
||||
|
||||
@@ -10,7 +10,6 @@ from transformers import AutoConfig
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.hf import CachedHfTokenizer
|
||||
from vllm.tokenizers.registry import (
|
||||
TokenizerRegistry,
|
||||
cached_get_tokenizer,
|
||||
@@ -93,54 +92,6 @@ def test_resolve_tokenizer_args_rejects_hf_mistral_format_true():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("transformers_version", "model_type", "native_filename", "expect_error"),
|
||||
[
|
||||
pytest.param("5.14.0", "mistral", "tekken.json", True),
|
||||
pytest.param("5.15.0", "mistral", "tekken.json", False),
|
||||
pytest.param("5.14.0", "mistral3", "tekken.json", True),
|
||||
pytest.param("5.15.0", "mistral3", "tekken.json", False),
|
||||
pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
|
||||
pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
|
||||
pytest.param("5.14.0", "mistral", None, False),
|
||||
pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
|
||||
],
|
||||
)
|
||||
def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
|
||||
tmp_path: Path,
|
||||
transformers_version: str,
|
||||
model_type: str,
|
||||
native_filename: str | None,
|
||||
expect_error: bool,
|
||||
):
|
||||
if native_filename is not None:
|
||||
(tmp_path / native_filename).touch()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.tokenizers.registry.get_config",
|
||||
return_value=SimpleNamespace(model_type=model_type),
|
||||
),
|
||||
patch(
|
||||
"vllm.tokenizers.registry.transformers.__version__",
|
||||
transformers_version,
|
||||
),
|
||||
patch.object(
|
||||
CachedHfTokenizer,
|
||||
"from_pretrained",
|
||||
return_value=SimpleNamespace(is_fast=True),
|
||||
) as from_pretrained,
|
||||
):
|
||||
if expect_error:
|
||||
with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
|
||||
get_tokenizer(str(tmp_path), tokenizer_mode="hf")
|
||||
else:
|
||||
tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
|
||||
assert tokenizer.is_fast is True
|
||||
|
||||
assert from_pretrained.called == (not expect_error)
|
||||
|
||||
|
||||
def test_customized_tokenizer():
|
||||
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
|
||||
|
||||
|
||||
@@ -3,11 +3,14 @@
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
||||
from xgrammar import Grammar, StructuralTag
|
||||
from xgrammar import get_model_structural_tag as get_xgrammar_builtin_structural_tag
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -1311,6 +1314,536 @@ def test_tool_strict_level_from_name():
|
||||
ToolStrictLevel.from_name("off")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GLM-4.7 structural tag: builtin content for strict tools, shallow otherwise
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _glm47_tools() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"days": {"type": "integer"},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "run_command",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"command": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _glm47_arg(key: str, value: str) -> str:
|
||||
return f"<arg_key>{key}</arg_key><arg_value>{value}</arg_value>"
|
||||
|
||||
|
||||
def _glm47_call(name: str, *args: str) -> str:
|
||||
return f"<tool_call>{name}{''.join(args)}</tool_call>"
|
||||
|
||||
|
||||
def _glm47_shallow_grammar(
|
||||
tool_choice, tools=None, reasoning: bool = False, *, auto: bool = False
|
||||
):
|
||||
# auto needs the FUNCTION level to lift the non-strict auto gate.
|
||||
tag = get_model_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=tools if tools is not None else _glm47_tools(),
|
||||
tool_choice=tool_choice,
|
||||
reasoning=reasoning,
|
||||
strict_level=ToolStrictLevel.FUNCTION if auto else ToolStrictLevel.AUTO,
|
||||
)
|
||||
assert isinstance(tag, StructuralTag)
|
||||
return Grammar.from_structural_tag(tag)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
# plain text, no tool call
|
||||
"Hello there",
|
||||
# text followed by a call
|
||||
"Sure." + _glm47_call("get_weather", _glm47_arg("city", "Paris")),
|
||||
# call only, required arg present
|
||||
_glm47_call("get_weather", _glm47_arg("city", "Paris")),
|
||||
# all arguments omitted
|
||||
_glm47_call("get_weather"),
|
||||
# repeated argument
|
||||
_glm47_call(
|
||||
"get_weather", _glm47_arg("city", "Paris"), _glm47_arg("city", "Rome")
|
||||
),
|
||||
# arguments in reverse declaration order
|
||||
_glm47_call(
|
||||
"get_weather", _glm47_arg("days", "3"), _glm47_arg("city", "Paris")
|
||||
),
|
||||
# enum-constrained value
|
||||
_glm47_call("get_weather", _glm47_arg("unit", "celsius")),
|
||||
# string values with spaces and a bare '<'
|
||||
_glm47_call("run_command", _glm47_arg("command", "grep 'a<b' x.py")),
|
||||
# second tool, two calls
|
||||
_glm47_call("run_command", _glm47_arg("command", "ls -la"))
|
||||
+ _glm47_call("get_weather", _glm47_arg("city", "Paris")),
|
||||
],
|
||||
)
|
||||
def test_glm47_non_strict_auto_accepts_valid_tool_calls(body: str):
|
||||
grammar = _glm47_shallow_grammar("auto", auto=True)
|
||||
assert _is_grammar_accept_string(grammar, body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
# unknown tool name
|
||||
_glm47_call("unknown_tool", _glm47_arg("city", "Paris")),
|
||||
# undeclared argument key
|
||||
_glm47_call("get_weather", _glm47_arg("zzz", "x")),
|
||||
# structural marker inside a string value
|
||||
_glm47_call("get_weather", _glm47_arg("city", "a<arg_key>b")),
|
||||
_glm47_call("get_weather", _glm47_arg("city", "a</tool_call>b")),
|
||||
# integer-typed value given non-numeric text
|
||||
_glm47_call("get_weather", _glm47_arg("days", "abc")),
|
||||
# enum-typed value outside the enum
|
||||
_glm47_call("get_weather", _glm47_arg("unit", "kelvin")),
|
||||
# unterminated tool call
|
||||
"<tool_call>get_weather" + _glm47_arg("city", "Paris"),
|
||||
],
|
||||
)
|
||||
def test_glm47_non_strict_auto_rejects_invalid(body: str):
|
||||
grammar = _glm47_shallow_grammar("auto", auto=True)
|
||||
assert not _is_grammar_accept_string(grammar, body)
|
||||
|
||||
|
||||
def test_glm47_non_strict_required_requires_at_least_one_call():
|
||||
grammar = _glm47_shallow_grammar("required")
|
||||
|
||||
assert not _is_grammar_accept_string(grammar, "Just answering.")
|
||||
assert _is_grammar_accept_string(
|
||||
grammar, _glm47_call("get_weather", _glm47_arg("city", "Paris"))
|
||||
)
|
||||
assert _is_grammar_accept_string(
|
||||
grammar,
|
||||
_glm47_call("run_command", _glm47_arg("command", "ls"))
|
||||
+ _glm47_call("get_weather", _glm47_arg("city", "Paris")),
|
||||
)
|
||||
|
||||
|
||||
def test_glm47_non_strict_forced_emits_single_named_call():
|
||||
grammar = _glm47_shallow_grammar(
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
type="function",
|
||||
function=ChatCompletionNamedFunction(name="get_weather"),
|
||||
)
|
||||
)
|
||||
|
||||
assert _is_grammar_accept_string(
|
||||
grammar, _glm47_call("get_weather", _glm47_arg("city", "Paris"))
|
||||
)
|
||||
assert not _is_grammar_accept_string(grammar, "No call here")
|
||||
assert not _is_grammar_accept_string(grammar, _glm47_call("run_command"))
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar,
|
||||
_glm47_call("get_weather", _glm47_arg("city", "Paris"))
|
||||
+ _glm47_call("run_command"),
|
||||
)
|
||||
|
||||
|
||||
def test_glm47_non_strict_reasoning_gates_on_think_close():
|
||||
grammar = _glm47_shallow_grammar("required", reasoning=True)
|
||||
call = _glm47_call("get_weather", _glm47_arg("city", "Paris"))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, "thinking...</think>" + call)
|
||||
assert not _is_grammar_accept_string(grammar, call)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parameters", "key"),
|
||||
[
|
||||
(
|
||||
{
|
||||
"$ref": "#/$defs/args",
|
||||
"$defs": {
|
||||
"args": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
},
|
||||
"city",
|
||||
),
|
||||
(
|
||||
{
|
||||
"anyOf": [
|
||||
{"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
{"type": "null"},
|
||||
]
|
||||
},
|
||||
"city",
|
||||
),
|
||||
(
|
||||
{"allOf": [{"type": "object", "properties": {"city": {"type": "string"}}}]},
|
||||
"city",
|
||||
),
|
||||
({"type": "object", "patternProperties": {"^arg": {"type": "string"}}}, "city"),
|
||||
({"type": "object"}, "city"),
|
||||
({}, "city"),
|
||||
(
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"additionalProperties": True,
|
||||
},
|
||||
"undeclared_key",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_glm47_non_plain_object_schemas_keep_free_arguments(parameters, key):
|
||||
# Schemas that are not a plain object with declared properties fall back
|
||||
# to the builtin's non-strict envelope content, which accepts any key.
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
type="function", function={"name": "f", "parameters": parameters}
|
||||
)
|
||||
]
|
||||
tag = get_model_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=tools,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
assert isinstance(tag, StructuralTag)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
call = _glm47_call("f", _glm47_arg(key, "Paris"))
|
||||
assert _is_grammar_accept_string(grammar, call)
|
||||
|
||||
|
||||
def _glm47_strict_tools() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"description": "Get the current weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "run_command",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"command": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={"name": "no_params", "strict": True, "parameters": None},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "reasoning"),
|
||||
[
|
||||
("auto", False),
|
||||
("required", False),
|
||||
("named", False),
|
||||
("required", True),
|
||||
("allowed", False),
|
||||
],
|
||||
)
|
||||
def test_glm47_strict_only_matches_xgrammar_builtin(tool_choice: str, reasoning: bool):
|
||||
# Strict-only requests stay byte-identical to the xgrammar builtin.
|
||||
strict_tools = _glm47_strict_tools()
|
||||
dumped = [
|
||||
{"type": "function", "function": t.function.model_dump(exclude_none=True)}
|
||||
for t in strict_tools
|
||||
]
|
||||
if tool_choice == "named":
|
||||
choice = ChatCompletionNamedToolChoiceParam(
|
||||
type="function",
|
||||
function=ChatCompletionNamedFunction(name="run_command"),
|
||||
)
|
||||
builtin_choice: Any = {"type": "function", "function": {"name": "run_command"}}
|
||||
elif tool_choice == "allowed":
|
||||
choice = ToolChoiceAllowed(
|
||||
type="allowed_tools",
|
||||
mode="auto",
|
||||
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
||||
)
|
||||
builtin_choice = {
|
||||
"type": "allowed_tools",
|
||||
"allowed_tools": {
|
||||
"mode": "auto",
|
||||
"tools": [{"type": "function", "function": {"name": "get_weather"}}],
|
||||
},
|
||||
}
|
||||
else:
|
||||
choice = tool_choice
|
||||
builtin_choice = tool_choice
|
||||
|
||||
ours = get_model_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=strict_tools,
|
||||
tool_choice=choice,
|
||||
reasoning=reasoning,
|
||||
)
|
||||
expected = get_xgrammar_builtin_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=dumped,
|
||||
tool_choice=builtin_choice,
|
||||
reasoning=reasoning,
|
||||
)
|
||||
|
||||
assert isinstance(ours, StructuralTag)
|
||||
assert ours.model_dump() == expected.model_dump()
|
||||
|
||||
|
||||
def test_glm47_parameter_level_pins_schemas_like_the_builtin(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
# The operator floor overrides per-tool strictness: every tool is pinned.
|
||||
ours = get_model_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=sample_tools,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
strict_level=ToolStrictLevel.PARAMETER,
|
||||
)
|
||||
expected = get_xgrammar_builtin_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": sample_tools[0].function.parameters,
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
|
||||
assert isinstance(ours, StructuralTag)
|
||||
assert ours.model_dump() == expected.model_dump()
|
||||
|
||||
|
||||
def test_glm47_mixed_tools_dispatch_per_tool(
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
mixed = [sample_tools_strict[0], sample_tools[0]]
|
||||
mixed[1] = mixed[1].model_copy(deep=True)
|
||||
mixed[1].function.name = "run_command"
|
||||
|
||||
tag = get_model_structural_tag(
|
||||
model="glm_4_7",
|
||||
tools=mixed,
|
||||
tool_choice="required",
|
||||
reasoning=False,
|
||||
)
|
||||
|
||||
assert isinstance(tag, StructuralTag)
|
||||
tags = tag.model_dump()["format"]["tags"]
|
||||
assert tags[0]["content"]["json_schema"] == (
|
||||
sample_tools_strict[0].function.parameters
|
||||
)
|
||||
assert tags[1]["content"]["type"] != "json_schema"
|
||||
|
||||
|
||||
def _glm47_parser(
|
||||
tools, monkeypatch: pytest.MonkeyPatch, *, strict_calling: bool = True
|
||||
) -> Glm47MoeModelToolParser:
|
||||
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", strict_calling)
|
||||
return Glm47MoeModelToolParser(MagicMock(), tools=tools)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "tools"),
|
||||
[("auto", "plain"), ("required", "plain"), ("required", "strict")],
|
||||
)
|
||||
def test_glm47_get_structural_tag_disabled_when_flag_off(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
sample_tools_strict: list[ChatCompletionToolsParam],
|
||||
tool_choice: str,
|
||||
tools: str,
|
||||
):
|
||||
parser = _glm47_parser(sample_tools, monkeypatch, strict_calling=False)
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=sample_tools_strict if tools == "strict" else sample_tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
|
||||
assert parser.get_structural_tag(request) is None
|
||||
|
||||
|
||||
def test_glm47_default_strict_level_constrains_auto(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
# --tool-strict-level auto: the parser's FUNCTION default lifts the
|
||||
# non-strict auto gate.
|
||||
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Glm47MoeModelToolParser
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
|
||||
tool_choice="auto",
|
||||
)
|
||||
parser = TestParser(MagicMock(), tools=sample_tools)
|
||||
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
|
||||
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.structured_outputs is not None
|
||||
assert out.structured_outputs.json is None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
call = (
|
||||
"<tool_call>get_weather<arg_key>city</arg_key>"
|
||||
"<arg_value>Paris</arg_value></tool_call>"
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, "Sure. " + call)
|
||||
assert _is_grammar_accept_string(grammar, "Plain answer.")
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, "Sure. " + call.replace("city", "ctiy")
|
||||
)
|
||||
|
||||
|
||||
def test_glm47_responses_request_attaches_shallow_tag(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Glm47MoeModelToolParser
|
||||
|
||||
request = ResponsesRequest.model_validate(
|
||||
{
|
||||
"input": "hi",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
],
|
||||
"tool_choice": "required",
|
||||
}
|
||||
)
|
||||
parser = TestParser(MagicMock(), tools=None)
|
||||
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
|
||||
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.structured_outputs is not None
|
||||
assert out.structured_outputs.structural_tag is not None
|
||||
assert out.text is None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
assert _is_grammar_accept_string(
|
||||
grammar,
|
||||
"<tool_call>get_weather<arg_key>city</arg_key>"
|
||||
"<arg_value>Paris</arg_value></tool_call>",
|
||||
)
|
||||
|
||||
|
||||
def test_glm47_operator_level_overrides_parser_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Glm47MoeModelToolParser
|
||||
tool_strict_level = ToolStrictLevel.PARAMETER
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
|
||||
tool_choice="required",
|
||||
)
|
||||
parser = TestParser(MagicMock(), tools=sample_tools)
|
||||
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
|
||||
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.structured_outputs is not None
|
||||
content = json.loads(out.structured_outputs.structural_tag)["format"]["tags"][0][
|
||||
"content"
|
||||
]
|
||||
assert content["json_schema"] == sample_tools[0].function.parameters
|
||||
|
||||
|
||||
def test_glm47_required_with_non_strict_tools_attaches_shallow_tag(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
tool_parser_cls = Glm47MoeModelToolParser
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
|
||||
tool_choice="required",
|
||||
)
|
||||
parser = TestParser(MagicMock(), tools=sample_tools)
|
||||
parser._reasoning_parser = MagicMock(adjust_request=lambda request: request)
|
||||
|
||||
out = parser.adjust_request(request)
|
||||
|
||||
assert out.structured_outputs is not None
|
||||
assert out.structured_outputs.json is None
|
||||
grammar = Grammar.from_structural_tag(out.structured_outputs.structural_tag)
|
||||
call = (
|
||||
"<tool_call>get_weather<arg_key>city</arg_key>"
|
||||
"<arg_value>Paris</arg_value></tool_call>"
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, call)
|
||||
assert not _is_grammar_accept_string(grammar, "It is sunny in Paris.")
|
||||
assert not _is_grammar_accept_string(grammar, call.replace("city", "ctiy"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("policy", ["auto", "required", "named"])
|
||||
def test_mimo_strict_compact_xml(policy):
|
||||
tools = [
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the agent-file pre-commit check.
|
||||
|
||||
The hook passes silently on a healthy tree, so CI would not notice a rule that
|
||||
stopped firing. Each case breaks one convention in a minimal repo and asserts
|
||||
the check reports it.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.pre_commit.check_agent_files import check
|
||||
|
||||
SKILL_MD = ".agents/skills/demo/SKILL.md"
|
||||
SKILL = "---\nname: demo\ndescription: Demo things. Use when testing.\n---\n"
|
||||
|
||||
|
||||
def write(path: str, text: str) -> None:
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(path).write_text(text)
|
||||
|
||||
|
||||
def tracked() -> dict[str, str]:
|
||||
"""Mimic `git ls-files -s` for every file and symlink under the cwd."""
|
||||
files = {}
|
||||
for root, dirs, names in os.walk("."):
|
||||
for name in dirs + names:
|
||||
path = os.path.normpath(os.path.join(root, name))
|
||||
if os.path.islink(path):
|
||||
files[path] = "120000"
|
||||
elif os.path.isfile(path):
|
||||
files[path] = "100644"
|
||||
return files
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
write("AGENTS.md", "See the [guide](docs/guide.md#some-heading).\n")
|
||||
write("docs/guide.md", "## Some Heading\n")
|
||||
write(SKILL_MD, SKILL)
|
||||
os.makedirs(".claude/skills")
|
||||
os.symlink("../../.agents/skills/demo", ".claude/skills/demo")
|
||||
|
||||
|
||||
def test_valid_repo_passes():
|
||||
assert check(tracked()) == ([], [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("break_repo", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
lambda: os.unlink(".claude/skills/demo"),
|
||||
"must be a symlink",
|
||||
id="skill-not-linked",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(".claude/skills/other/SKILL.md", SKILL),
|
||||
"only symlinks",
|
||||
id="real-file-in-claude-skills",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: os.symlink("../../.agents/skills/gone", ".claude/skills/gone"),
|
||||
"only symlinks",
|
||||
id="dangling-link",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL.replace("demo", "other")),
|
||||
"must be 'demo'",
|
||||
id="name-differs-from-dir",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL.replace("Demo things", "Demo: things")),
|
||||
"not valid YAML",
|
||||
id="unquoted-colon-in-description",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, "# Demo\n"),
|
||||
"missing YAML frontmatter",
|
||||
id="no-frontmatter",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("vllm/CLAUDE.md", "@AGENTS.md\n"),
|
||||
"remove it",
|
||||
id="claude-md",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("AGENTS.md", "line\n" * 201),
|
||||
"200-line budget",
|
||||
id="root-guide-over-budget",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("vllm/AGENTS.md", "line\n" * 301),
|
||||
"300-line budget",
|
||||
id="domain-guide-over-budget",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL + "line\n" * 500),
|
||||
"500-line budget",
|
||||
id="skill-over-budget",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("AGENTS.md", "[gone](docs/gone.md)\n"),
|
||||
"does not exist",
|
||||
id="dead-link",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL + "[guide](../../../docs/guide.md#gone)\n"),
|
||||
"no heading",
|
||||
id="dead-anchor",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_broken_convention_is_reported(break_repo, expected):
|
||||
break_repo()
|
||||
errors, _ = check(tracked())
|
||||
assert len(errors) == 1
|
||||
assert expected in errors[0]
|
||||
|
||||
|
||||
def test_description_without_trigger_is_advisory():
|
||||
write(SKILL_MD, SKILL.replace(" Use when testing.", ""))
|
||||
errors, notes = check(tracked())
|
||||
assert not errors
|
||||
assert "when to use" in notes[0]
|
||||
@@ -23,6 +23,8 @@ from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
BuildPrefillChunkMetadataKernel,
|
||||
DeepseekV4IndexerBackend,
|
||||
DeepSeekV32IndexerDecodeMetadata,
|
||||
DeepseekV32IndexerMetadata,
|
||||
DeepseekV32IndexerMetadataBuilder,
|
||||
DeepseekV41IndexerBackend,
|
||||
get_max_prefill_buffer_size,
|
||||
@@ -99,6 +101,72 @@ def test_fused_indexer_decode_metadata(query_lens, padding):
|
||||
torch.testing.assert_close(per_req[:reqs], lengths)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
if not indexer_module.has_deep_gemm():
|
||||
pytest.skip("requires DeepGEMM")
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 3
|
||||
seq_lens = torch.tensor([17, 33, 65], dtype=torch.int32, device=device)
|
||||
decode_seq_lens = torch.zeros((num_reqs, 1), dtype=torch.int32, device=device)
|
||||
block_table = torch.arange(num_reqs * 4, dtype=torch.int32, device=device).view(
|
||||
num_reqs, 4
|
||||
)
|
||||
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
schedule_metadata = indexer_module.get_paged_mqa_logits_metadata(
|
||||
torch.div(seq_lens, 16, rounding_mode="floor").unsqueeze(1),
|
||||
64,
|
||||
num_sms,
|
||||
).clone()
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = 16
|
||||
builder.num_sms = num_sms
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(num_states=64)
|
||||
metadata = DeepseekV32IndexerMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=128,
|
||||
slot_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
|
||||
num_decodes=num_reqs,
|
||||
num_decode_tokens=num_reqs,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=DeepSeekV32IndexerDecodeMetadata(
|
||||
block_table=block_table,
|
||||
seq_lens=decode_seq_lens,
|
||||
decode_lens=torch.zeros(num_reqs, dtype=torch.int32, device=device),
|
||||
requires_padding=False,
|
||||
schedule_metadata=schedule_metadata,
|
||||
),
|
||||
)
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
seq_lens.copy_(torch.tensor([32, 48, 80], dtype=torch.int32, device=device))
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_decode_seq_lens = torch.div(seq_lens, 16, rounding_mode="floor")
|
||||
torch.testing.assert_close(decode_seq_lens.flatten(), expected_decode_seq_lens)
|
||||
torch.testing.assert_close(
|
||||
schedule_metadata,
|
||||
indexer_module.get_paged_mqa_logits_metadata(
|
||||
expected_decode_seq_lens.unsqueeze(1), 64, num_sms
|
||||
),
|
||||
)
|
||||
assert metadata.decode is not None
|
||||
assert torch.all(metadata.decode.decode_lens == 1)
|
||||
expected_slot_mapping = block_table[:, 0].to(torch.int64) * 64 + torch.tensor(
|
||||
[1, 2, 4], dtype=torch.int64, device=device
|
||||
)
|
||||
torch.testing.assert_close(metadata.slot_mapping, expected_slot_mapping)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0], [0, 0, 0]]
|
||||
|
||||
@@ -300,6 +300,16 @@ def test_builder_build_falls_back_without_positions():
|
||||
assert meta.slot_mapping is cam.slot_mapping
|
||||
|
||||
|
||||
def test_builder_updates_draft_mapping():
|
||||
builder = make_tail_builder()
|
||||
cam = make_common_metadata([[15], [16]], [5, 9])
|
||||
meta = KpoolTailMetadataBuilder.build(builder, 0, cam)
|
||||
assert cam.positions is not None
|
||||
cam.positions.add_(1)
|
||||
builder.update_draft_decode_metadata(meta)
|
||||
assert meta.slot_mapping[:2].tolist() == [5 * KPOOL, 9 * KPOOL + 1]
|
||||
|
||||
|
||||
def test_builder_reuses_slot_mapping_storage():
|
||||
builder = make_tail_builder()
|
||||
first = make_common_metadata([list(range(10))], [5])
|
||||
|
||||
@@ -65,6 +65,9 @@ from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.attention.backends.mla import (
|
||||
flashattn_mla_sparse as flashattn_sparse_module,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla import index_group as index_group_module
|
||||
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
|
||||
FlashAttnMLASparseImpl,
|
||||
@@ -3405,6 +3408,129 @@ def test_flashinfer_hisparse_decode_runs_batched_attention():
|
||||
assert lse is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(90),
|
||||
reason="FlashAttention MLA requires Hopper",
|
||||
)
|
||||
@pytest.mark.parametrize("use_cuda_graph", [False, True])
|
||||
@pytest.mark.parametrize("block_stride_rows", [64, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"query_len,num_decode_tokens,max_decode_rows,num_layers",
|
||||
[
|
||||
pytest.param(1, 2, 2, 4, id="decode"),
|
||||
pytest.param(4, 8, 8, 4, id="mtp4"),
|
||||
pytest.param(6, 12, 12, 4, id="mtp6"),
|
||||
pytest.param(4, 8, 2, 4, id="scratch_overflow"),
|
||||
pytest.param(4, 4, 8, 4, id="mixed"),
|
||||
pytest.param(4, 0, 8, 4, id="prefill"),
|
||||
pytest.param(4, 8, 8, 1, id="single_layer"),
|
||||
],
|
||||
)
|
||||
def test_flashattn_shared_indices_match_independent_conversion(
|
||||
monkeypatch,
|
||||
query_len,
|
||||
num_decode_tokens,
|
||||
max_decode_rows,
|
||||
num_layers,
|
||||
block_stride_rows,
|
||||
use_cuda_graph,
|
||||
):
|
||||
"""Reuse must preserve attention outputs as requests and pages change."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens, block_size, topk, seq_len = 2 * query_len, 64, 2048, 8192
|
||||
blocks_per_req = seq_len // block_size
|
||||
logical = torch.arange(
|
||||
0, seq_len, seq_len // topk, dtype=torch.int32, device=device
|
||||
).repeat(num_tokens, 1)
|
||||
logical[:, ::7] = -1
|
||||
builder = SparseMLAIndexGroupBuilder(logical, max_decode_rows=max_decode_rows)
|
||||
metadata = SimpleNamespace(
|
||||
block_table=torch.randperm(
|
||||
2 * blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(2, blocks_per_req),
|
||||
block_size=block_size,
|
||||
req_id_per_token=torch.arange(
|
||||
2, dtype=torch.int32, device=device
|
||||
).repeat_interleave(query_len),
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
)
|
||||
q_nope = torch.randn(num_tokens, 8, 512, dtype=torch.bfloat16, device=device)
|
||||
q_rope = torch.randn(num_tokens, 8, 64, dtype=torch.bfloat16, device=device)
|
||||
caches = torch.randn(
|
||||
num_layers,
|
||||
2 * blocks_per_req,
|
||||
block_stride_rows,
|
||||
576,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)[:, :, :block_size]
|
||||
impls = []
|
||||
for i in range(num_layers):
|
||||
impl = object.__new__(FlashAttnMLASparseImpl)
|
||||
impl.topk_indices_buffer = logical
|
||||
impl.index_group, impl.index_group_index = builder.register_layer(i == 0)
|
||||
impl.dcp_world_size = 1
|
||||
impl.kv_lora_rank, impl.qk_rope_head_dim = 512, 64
|
||||
impl.scale = 256**-0.5
|
||||
impl.cu_seqlens_q_buffer = torch.arange(
|
||||
num_tokens + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
impls.append(impl)
|
||||
|
||||
convert = triton_convert_req_index_to_global_index
|
||||
convert_spy = MagicMock(wraps=convert)
|
||||
for module in (index_group_module, flashattn_sparse_module):
|
||||
monkeypatch.setattr(
|
||||
module, "triton_convert_req_index_to_global_index", convert_spy
|
||||
)
|
||||
|
||||
def run():
|
||||
impls[0].record_logical_topk_ready()
|
||||
return [
|
||||
impl.forward_mqa((q_nope, q_rope), cache, metadata, None)[0]
|
||||
for impl, cache in zip(impls, caches)
|
||||
]
|
||||
|
||||
stream = torch.cuda.Stream(device=device)
|
||||
stream.wait_stream(current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run() # Warm up the conversion and FA3 kernels before capture.
|
||||
stream.synchronize()
|
||||
convert_spy.reset_mock()
|
||||
if use_cuda_graph:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
outputs = run()
|
||||
else:
|
||||
outputs = run()
|
||||
reuse = num_decode_tokens == num_tokens and num_tokens <= max_decode_rows + 1
|
||||
assert convert_spy.call_count == (1 if reuse else num_layers)
|
||||
|
||||
for _ in range(2):
|
||||
metadata.req_id_per_token.copy_(metadata.req_id_per_token.flip(0))
|
||||
metadata.block_table.copy_(metadata.block_table.roll(1, dims=1))
|
||||
logical.copy_(torch.where(logical >= 0, (logical + 1) % seq_len, logical))
|
||||
if use_cuda_graph:
|
||||
graph.replay()
|
||||
else:
|
||||
outputs = run()
|
||||
physical, counts = convert(
|
||||
metadata.req_id_per_token,
|
||||
metadata.block_table,
|
||||
logical,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
for impl, cache, actual in zip(impls, caches, outputs):
|
||||
expected = impl._run_mqa_kernel(
|
||||
q_nope, q_rope, cache, physical, counts, block_size
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
def test_flashattn_hisparse_decode_uses_index_group():
|
||||
num_tokens = 4
|
||||
q_nope = torch.empty(num_tokens, 2, 3, device=DEVICE_TYPE)
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
the hit to rebuild the non-cacheable sliding-window group, keeps the hit's
|
||||
blocks, and hands the worker the replay start."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -277,3 +279,53 @@ def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||
assert new_req.num_computed_tokens == 0
|
||||
assert new_req.replay_start == 0
|
||||
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
@pytest.mark.parametrize("loads_window", [True, False])
|
||||
def test_kv_load_zeroing_respects_loaded_groups(is_async, loads_window):
|
||||
"""Only groups restored by an async load skip zeroing. P/D loads include
|
||||
the non-cacheable window; stores leave it to be initialized locally."""
|
||||
local = 5 * BLOCK_SIZE
|
||||
scheduler = _replay_scheduler(use_kv_connector=MockKVConfig())
|
||||
if loads_window:
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
|
||||
assert scheduler.needs_kv_cache_zeroing
|
||||
first = create_requests(
|
||||
num_requests=1, num_tokens=local, same_prompt=True, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
_prefill(scheduler, first)
|
||||
scheduler.connector.config = dataclasses.replace(
|
||||
scheduler.connector.config,
|
||||
matched_tokens=NUM_PROMPT_TOKENS - 1 - local,
|
||||
is_async=is_async,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=NUM_PROMPT_TOKENS,
|
||||
same_prompt=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
req_ids=["load"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
|
||||
manager = scheduler.kv_cache_manager
|
||||
hit_ids = {
|
||||
b.block_id for g in manager.get_blocks(first.request_id).blocks for b in g
|
||||
}
|
||||
swa_blocks = manager.get_blocks(request.request_id).blocks[SWA]
|
||||
assert any(
|
||||
not b.is_null and b.block_id not in hit_ids
|
||||
for b in swa_blocks[: local // BLOCK_SIZE]
|
||||
)
|
||||
zeroed = set(out.new_block_ids_to_zero or ())
|
||||
for group_id, blocks in enumerate(manager.get_blocks(request.request_id).blocks):
|
||||
new_ids = {
|
||||
b.block_id for b in blocks if not b.is_null and b.block_id not in hit_ids
|
||||
}
|
||||
assert new_ids
|
||||
if is_async and (group_id == FULL or loads_window):
|
||||
assert not new_ids & zeroed
|
||||
else:
|
||||
assert new_ids <= zeroed
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
KpoolTailManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -332,6 +334,37 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
assert manager.req_to_blocks[request_id] == blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("record_for_zeroing", [True, False])
|
||||
def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
"""A transferred ring at index zero skips zeroing even with a local hit."""
|
||||
spec = KpoolTailSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
manager = KpoolTailManager(
|
||||
spec,
|
||||
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=4,
|
||||
needs_kv_cache_zeroing=True,
|
||||
max_admission_blocks_per_request=1,
|
||||
)
|
||||
manager.allocate_external_computed_blocks(
|
||||
"request", 16, 4, record_for_zeroing=record_for_zeroing
|
||||
)
|
||||
blocks = manager.req_to_blocks["request"]
|
||||
assert len(blocks) == 1
|
||||
assert manager.take_new_block_ids() == (
|
||||
[blocks[0].block_id] if record_for_zeroing else []
|
||||
)
|
||||
assert manager.allocate_new_blocks("request", 20, 20) == []
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
|
||||
def test_sliding_window_records_new_blocks_for_zeroing():
|
||||
block_size = 2
|
||||
spec = SlidingWindowSpec(
|
||||
|
||||
@@ -192,6 +192,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
|
||||
full_sequence_must_fit: bool = False,
|
||||
reserved_blocks: int = 0,
|
||||
has_scheduled_reqs: bool = True,
|
||||
skip_zeroing_group_ids: tuple[int, ...] = (),
|
||||
):
|
||||
ret = original_allocate_slots_fn(
|
||||
self,
|
||||
@@ -206,6 +207,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
|
||||
full_sequence_must_fit,
|
||||
reserved_blocks,
|
||||
has_scheduled_reqs,
|
||||
skip_zeroing_group_ids,
|
||||
)
|
||||
if cur_step_action is not None:
|
||||
cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[
|
||||
|
||||
@@ -491,7 +491,7 @@ async def test_streaming_input_per_chunk_sampling_params(engine: AsyncLLM):
|
||||
async def test_streaming_input_empty_generator(engine: AsyncLLM):
|
||||
"""Test behavior when the input generator yields nothing.
|
||||
|
||||
An empty generator should still produce a finished output.
|
||||
An empty generator should finish without producing output.
|
||||
"""
|
||||
request_id = "test_empty_generator"
|
||||
sampling_params = get_sampling_params(max_tokens=10)
|
||||
@@ -505,9 +505,7 @@ async def test_streaming_input_empty_generator(engine: AsyncLLM):
|
||||
async for output in engine.generate(empty_generator(), sampling_params, request_id):
|
||||
outputs.append(output)
|
||||
|
||||
# Should still get a finished marker
|
||||
assert len(outputs) >= 1, "Should receive at least one output"
|
||||
assert outputs[-1].finished, "Should have a finished output"
|
||||
assert outputs == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="module")
|
||||
|
||||
@@ -150,6 +150,21 @@ def test_draft_model_quantization(
|
||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_draft_model_transformers_backend(vllm_runner):
|
||||
"""Ensure that Attention prefixes don't collide between the target and
|
||||
draft models when both models use the Transformers modelling backend."""
|
||||
sd_case = ArgsTest(
|
||||
target_model="allenai/OLMo-2-0425-1B-Instruct",
|
||||
draft_model="allenai/OLMo-2-0425-1B-Instruct",
|
||||
sampling_config=greedy_sampling(),
|
||||
num_speculative_tokens=3,
|
||||
expected_acceptance_len=0.98 * (3 + 1), # epsilon discount of K + 1
|
||||
expected_acceptance_rate=0.98, # slight epsilon
|
||||
)
|
||||
assert_draft_model_correctness(sd_case, vllm_runner)
|
||||
|
||||
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
def test_draft_model_tensor_parallelism(vllm_runner):
|
||||
"""Ensure spec decode works when running with TP > 1."""
|
||||
|
||||
@@ -48,16 +48,6 @@ def check_mtp_correctness(
|
||||
elif "gemma-4" in model_name:
|
||||
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
|
||||
|
||||
if draft_model is not None and "gemma-4" in draft_model:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
if Version(transformers.__version__) < Version("5.8.0"):
|
||||
pytest.skip(
|
||||
"Gemma4 MTP assistant requires transformers>=5.8.0, "
|
||||
f"got {transformers.__version__}"
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
block_size=None,
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -18,6 +20,38 @@ MODEL = "facebook/opt-125m"
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_step_invalidates_mm_processor_cache_misses():
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.should_execute_dummy_batch = False
|
||||
engine.log_stats = False
|
||||
engine.logger_manager = None
|
||||
|
||||
mm_processor_cache = MagicMock()
|
||||
engine.renderer = MagicMock(mm_processor_cache=mm_processor_cache)
|
||||
|
||||
engine_core_output = MagicMock(
|
||||
mm_cache_miss_hashes=["missing-hash-1", "missing-hash-2"]
|
||||
)
|
||||
outputs = MagicMock(
|
||||
outputs=[engine_core_output],
|
||||
timestamp=0.0,
|
||||
scheduler_stats=None,
|
||||
)
|
||||
engine.engine_core = MagicMock()
|
||||
engine.engine_core.get_output.return_value = outputs
|
||||
|
||||
processed_outputs = MagicMock(reqs_to_abort=[], request_outputs=[])
|
||||
engine.output_processor = MagicMock()
|
||||
engine.output_processor.process_outputs.return_value = processed_outputs
|
||||
|
||||
assert engine.step() == []
|
||||
assert mm_processor_cache.invalidate.call_args_list == [
|
||||
call("missing-hash-1"),
|
||||
call("missing-hash-2"),
|
||||
]
|
||||
|
||||
|
||||
def _vllm_model(
|
||||
apc: bool,
|
||||
vllm_runner: type[VllmRunner],
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.engine.utils import (
|
||||
NUM_PROMPT_LOGPROBS_UNDER_TEST,
|
||||
@@ -18,7 +19,12 @@ from tests.v1.engine.utils import (
|
||||
from vllm import PoolingParams
|
||||
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
SamplingMask,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.engine import (
|
||||
@@ -1630,3 +1636,92 @@ def test_request_output_add_merges_delta_sampling_masks():
|
||||
|
||||
assert merged.outputs[0].token_ids == [10, 20]
|
||||
assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_pooling_cache_miss_is_a_request_error_and_batch_continues():
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
requests = [
|
||||
EngineCoreRequest(
|
||||
request_id=f"request-{idx}-int",
|
||||
external_req_id=f"request-{idx}",
|
||||
prompt_token_ids=[1, 2],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
for idx in range(2)
|
||||
]
|
||||
for request in requests:
|
||||
output_processor.add_request(request, prompt=None)
|
||||
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=requests[0].request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=["missing-hash"],
|
||||
),
|
||||
EngineCoreOutput(
|
||||
request_id=requests[1].request_id,
|
||||
new_token_ids=[],
|
||||
pooling_output=torch.tensor([1.0, 2.0]),
|
||||
finish_reason=FinishReason.STOP,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(processed.request_outputs) == 2
|
||||
failed, succeeded = processed.request_outputs
|
||||
assert isinstance(failed, PoolingRequestOutput)
|
||||
assert failed.error is not None
|
||||
assert failed.error.code == "multimodal_cache_miss"
|
||||
assert failed.error.retryable
|
||||
assert isinstance(succeeded, PoolingRequestOutput)
|
||||
assert succeeded.error is None
|
||||
assert torch.equal(succeeded.outputs.data, torch.tensor([1.0, 2.0]))
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_async_pooling_cache_miss_does_not_fail_output_processor():
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
request = EngineCoreRequest(
|
||||
request_id="request-int",
|
||||
external_req_id="request",
|
||||
prompt_token_ids=[1, 2],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
queue = RequestOutputCollector(
|
||||
output_kind=RequestOutputKind.FINAL_ONLY,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
output_processor.add_request(request, prompt=None, queue=queue)
|
||||
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=request.request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=["missing-hash"],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processed.request_outputs == []
|
||||
output = queue.get_nowait()
|
||||
assert isinstance(output, PoolingRequestOutput)
|
||||
assert output.error is not None
|
||||
assert output.error.retryable
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
@@ -111,6 +111,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
|
||||
}
|
||||
|
||||
fail_scheduler.connector = Mock()
|
||||
fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
|
||||
)
|
||||
|
||||
@@ -327,6 +327,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
|
||||
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
|
||||
)
|
||||
recompute_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
recompute_scheduler.connector.request_finished.return_value = (False, None)
|
||||
recompute_scheduler.connector.take_events.return_value = ()
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ def test_failed_receive_completion_honors_failure_policy(policy):
|
||||
request = create_request(num_tokens=3 * scheduler.block_size)
|
||||
scheduler.add_request(request)
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = [
|
||||
(2 * scheduler.block_size, True),
|
||||
(0, False),
|
||||
@@ -103,6 +104,7 @@ def test_async_load_failure(
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
|
||||
)
|
||||
@@ -332,6 +334,7 @@ def test_async_progressive_load_failure(
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
|
||||
)
|
||||
|
||||
@@ -1549,6 +1549,40 @@ async def test_kv_producer(monkeypatch):
|
||||
prefill_worker.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_block_ids", "expected_finished"),
|
||||
[
|
||||
([], set()),
|
||||
([[]], set()),
|
||||
([[100]], {"d-req-1"}),
|
||||
],
|
||||
)
|
||||
def test_pull_completion_requires_local_blocks(
|
||||
local_block_ids: list[list[int]],
|
||||
expected_finished: set[str],
|
||||
):
|
||||
worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker)
|
||||
worker.shutdown = MagicMock()
|
||||
worker.finished_recving_reqs = set()
|
||||
pull_meta = PullReqMeta(
|
||||
d_req_id="d-req-1",
|
||||
transfer_id="xfer-req-1",
|
||||
local_block_ids=local_block_ids,
|
||||
remote_engine_id="p-engine",
|
||||
remote_bootstrap_addr="http://bootstrap:33333",
|
||||
pull_tasks_count=1,
|
||||
)
|
||||
response = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH,
|
||||
ok_reqs=["d-req-1"],
|
||||
)
|
||||
|
||||
worker.process_pulling_result(response, {"d-req-1": pull_meta})
|
||||
|
||||
assert pull_meta.pull_tasks_count == 0
|
||||
assert worker.finished_recving_reqs == expected_finished
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kv_consumuer(monkeypatch):
|
||||
"""Simulates a Consumer Worker (Decoder) initiating a pull from a Producer.
|
||||
|
||||
@@ -22,14 +22,20 @@ Role shapes below are exactly those in the RFC (#46107) deployments:
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import ctypes
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import msgpack
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
@@ -203,3 +209,127 @@ def test_consecutive_requests_alternate_instances(route):
|
||||
# than filling one instance's ranks before moving on.
|
||||
insts = [inst for inst, _ in _route_n(route, _instances(3, 8), 3)]
|
||||
assert insts == [0, 1, 2]
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="GIL probe uses libc.usleep")
|
||||
def test_discovery_heartbeat_progresses_while_worker_holds_gil():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
|
||||
MoRIIOHeartbeat,
|
||||
)
|
||||
|
||||
payload = {
|
||||
"type": "P",
|
||||
"http_address": "127.0.0.1:8000",
|
||||
"zmq_address": "host:127.0.0.1,handshake:6301,notify:61005",
|
||||
"dp_size": 1,
|
||||
"tp_size": 8,
|
||||
"transfer_mode": "READ",
|
||||
}
|
||||
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
|
||||
port = socket.bind_to_random_port("tcp://127.0.0.1")
|
||||
heartbeat = MoRIIOHeartbeat(f"tcp://127.0.0.1:{port}", payload, 0.02)
|
||||
try:
|
||||
assert socket.poll(5000), "Discovery did not register"
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
# PyDLL retains the GIL throughout the native call.
|
||||
ctypes.PyDLL(None).usleep(300_000)
|
||||
received = 0
|
||||
while socket.poll(0):
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
received += 1
|
||||
assert received >= 3, "Discovery stalled while the worker held the GIL"
|
||||
finally:
|
||||
heartbeat.shutdown()
|
||||
heartbeat.shutdown()
|
||||
while socket.poll(0):
|
||||
socket.recv_multipart()
|
||||
assert not socket.poll(100), "Heartbeats continued after shutdown"
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="Checks child exit via procfs")
|
||||
def test_discovery_heartbeat_exits_when_worker_dies():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_heartbeat
|
||||
|
||||
owner_code = """
|
||||
import runpy
|
||||
import sys
|
||||
Heartbeat = runpy.run_path(sys.argv[1])["MoRIIOHeartbeat"]
|
||||
heartbeat = Heartbeat(sys.argv[2], {"type": "D"}, 0.02)
|
||||
print(heartbeat._process.pid, flush=True)
|
||||
sys.stdin.read()
|
||||
"""
|
||||
with zmq.Context() as context, context.socket(zmq.ROUTER) as socket:
|
||||
port = socket.bind_to_random_port("tcp://127.0.0.1")
|
||||
owner = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
owner_code,
|
||||
moriio_heartbeat.__file__,
|
||||
f"tcp://127.0.0.1:{port}",
|
||||
],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
child_pid = None
|
||||
child_exited = False
|
||||
try:
|
||||
assert socket.poll(5000), "Discovery did not register"
|
||||
assert owner.stdout is not None
|
||||
child_pid = int(owner.stdout.readline())
|
||||
owner.kill()
|
||||
owner.wait(timeout=5)
|
||||
stat = Path(f"/proc/{child_pid}/stat")
|
||||
deadline = time.monotonic() + 5
|
||||
while stat.exists():
|
||||
# A terminated child may await reaping by the container's init.
|
||||
try:
|
||||
if stat.read_text().split(") ")[1].startswith("Z "):
|
||||
break
|
||||
except FileNotFoundError:
|
||||
break
|
||||
assert time.monotonic() < deadline, "Orphan heartbeat survived worker"
|
||||
time.sleep(0.01)
|
||||
child_exited = True
|
||||
finally:
|
||||
if owner.poll() is None:
|
||||
owner.kill()
|
||||
owner.communicate(timeout=5)
|
||||
if child_pid is not None and not child_exited:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(child_pid, 9)
|
||||
|
||||
|
||||
def test_discovery_heartbeat_recovers_after_router_outage(capfd):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
|
||||
MoRIIOHeartbeat,
|
||||
)
|
||||
|
||||
payload = {"type": "P", "http_address": "127.0.0.1:8000"}
|
||||
with zmq.Context() as context:
|
||||
with context.socket(zmq.ROUTER) as socket:
|
||||
port = socket.bind_to_random_port("tcp://127.0.0.1")
|
||||
endpoint = f"tcp://127.0.0.1:{port}"
|
||||
heartbeat = MoRIIOHeartbeat(endpoint, payload, 0.001, max_retries=1)
|
||||
try:
|
||||
assert socket.poll(5000), "Discovery did not register"
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
except BaseException:
|
||||
heartbeat.shutdown()
|
||||
raise
|
||||
try:
|
||||
# Fill the real DEALER queue while discovery is unavailable.
|
||||
deadline = time.monotonic() + 10
|
||||
errors = ""
|
||||
while "heartbeat send failed" not in errors:
|
||||
errors += capfd.readouterr().err
|
||||
assert time.monotonic() < deadline, "No send timeout during outage"
|
||||
time.sleep(0.01)
|
||||
with context.socket(zmq.ROUTER) as socket:
|
||||
socket.bind(endpoint)
|
||||
assert socket.poll(5000), "Discovery did not recover after outage"
|
||||
assert msgpack.unpackb(socket.recv_multipart()[1]) == payload
|
||||
assert heartbeat._process.poll() is None
|
||||
finally:
|
||||
heartbeat.shutdown()
|
||||
|
||||
@@ -269,6 +269,34 @@ def test_worker_get_finished_counts_structured_release_fan_in():
|
||||
assert worker._completed_consumer_notifications == {"tx-fanin"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exit_kind", ["killed", "eof"])
|
||||
def test_worker_reports_unexpected_heartbeat_exit(exit_kind):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_heartbeat import (
|
||||
MoRIIOHeartbeat,
|
||||
)
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.is_producer = True
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.moriio_wrapper = SimpleNamespace(
|
||||
pop_finished_req_ids=lambda: [], shutdown=lambda: None
|
||||
)
|
||||
worker._pending_unmapped_acks = []
|
||||
worker._heartbeat = MoRIIOHeartbeat("tcp://127.0.0.1:1", {"type": "P"}, 0.02)
|
||||
process = worker._heartbeat._process
|
||||
try:
|
||||
assert worker.get_finished() == (set(), set())
|
||||
if exit_kind == "killed":
|
||||
process.kill()
|
||||
else:
|
||||
process.stdin.close()
|
||||
returncode = process.wait(timeout=5)
|
||||
with pytest.raises(RuntimeError, match=f"heartbeat.*{returncode}"):
|
||||
worker.get_finished()
|
||||
finally:
|
||||
worker.shutdown()
|
||||
|
||||
|
||||
def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
class DoneStatus:
|
||||
def Succeeded(self):
|
||||
|
||||
@@ -202,6 +202,20 @@ def mc() -> MultiConnector:
|
||||
return mc
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chosen", [0, 1])
|
||||
def test_loaded_groups_follow_the_connector_serving_the_request(mc, chosen):
|
||||
request = MagicMock(request_id="request")
|
||||
for i, connector in enumerate(mc._connectors):
|
||||
connector.get_num_new_matched_tokens.return_value = (
|
||||
(16, True) if i == chosen else (0, False)
|
||||
)
|
||||
connector.get_loaded_kv_cache_group_ids.return_value = (i,)
|
||||
|
||||
assert mc.get_num_new_matched_tokens(request, 0) == (16, True)
|
||||
assert mc.get_loaded_kv_cache_group_ids(request) == (chosen,)
|
||||
mc._connectors[1 - chosen].get_loaded_kv_cache_group_ids.assert_not_called()
|
||||
|
||||
|
||||
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
|
||||
assert mc.get_mem_pool_context() is None
|
||||
|
||||
|
||||
@@ -1522,34 +1522,6 @@ def _make_fake_kv_cache_manager():
|
||||
return manager
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
|
||||
"""Only zero-recorded (attention) groups contribute, sliced to the
|
||||
externally-loaded token range; Mamba state blocks are never zeroed."""
|
||||
manager = _make_fake_kv_cache_manager()
|
||||
|
||||
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
|
||||
assert manager.get_zeroing_block_ids_in_range("req-1", 16, 56) == [11, 12, 13]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
|
||||
"""Blocks that will be loaded by the connector must not be zeroed."""
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
class FakeKVCacheManager:
|
||||
def take_new_block_ids(self):
|
||||
return [9, 10, 11, 12]
|
||||
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler.needs_kv_cache_zeroing = True
|
||||
scheduler.kv_cache_manager = FakeKVCacheManager()
|
||||
scheduler._skip_zero_block_ids = {10, 12}
|
||||
|
||||
assert scheduler._get_new_block_ids_to_zero() == [9, 11]
|
||||
assert not scheduler._skip_zero_block_ids
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
|
||||
@@ -1575,7 +1547,6 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
|
||||
# Attention blocks covering tokens >= 48 are re-recorded for zeroing
|
||||
# and flow into the next step's zero list; Mamba blocks are not.
|
||||
scheduler._skip_zero_block_ids = set()
|
||||
assert scheduler._get_new_block_ids_to_zero() == [13, 14, 15]
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import patch
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from tests.v1.sample.utils import (
|
||||
@@ -20,7 +21,7 @@ from tests.v1.sample.utils import (
|
||||
compute_correct_cumulative_logprob,
|
||||
get_test_batch,
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
@@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
|
||||
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
|
||||
return SimpleNamespace(
|
||||
max_logprobs=max_logprobs,
|
||||
max_model_len=16,
|
||||
logits_processors=None,
|
||||
is_diffusion=False,
|
||||
get_vocab_size=lambda: vocab_size,
|
||||
@@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
|
||||
"""The candidate count is bounded by max_logprobs."""
|
||||
def test_prompt_logprob_token_ids_validation():
|
||||
"""Rows may be ragged (-1 padded); the width is bounded by max_logprobs."""
|
||||
model_config = _model_config(vocab_size=100, max_logprobs=4)
|
||||
|
||||
def verify(**kwargs):
|
||||
SamplingParams(**kwargs).verify(
|
||||
params = SamplingParams(**kwargs)
|
||||
params.verify(
|
||||
model_config,
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
)
|
||||
return params.prompt_logprob_token_ids
|
||||
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
|
||||
assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [
|
||||
[-1] * 4,
|
||||
[1, -1, -1, -1],
|
||||
[4, 3, 2, 1],
|
||||
]
|
||||
# Any integer layout is normalized to what the worker uploads:
|
||||
# C-contiguous native int32.
|
||||
odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8"))
|
||||
ids = verify(prompt_logprob_token_ids=odd)
|
||||
assert ids.dtype == np.int32 and ids.flags.c_contiguous
|
||||
assert ids.tolist() == [[1, 2], [3, -1]]
|
||||
# The pydantic path (token-in/token-out API, /docs) takes strict integer lists.
|
||||
adapter = TypeAdapter(SamplingParams)
|
||||
for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"):
|
||||
with pytest.raises(ValidationError):
|
||||
adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}')
|
||||
|
||||
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
|
||||
verify(prompt_logprob_token_ids=np.array([1, 2]))
|
||||
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
|
||||
verify(prompt_logprob_token_ids=[1, 2])
|
||||
for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]):
|
||||
with pytest.raises(VLLMValidationError, match="integer token ids"):
|
||||
verify(prompt_logprob_token_ids=bad_ids)
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64))
|
||||
# The error names the value to raise max_logprobs to.
|
||||
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
|
||||
|
||||
verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=[[1], [100]])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=[[1], [-2]])
|
||||
with pytest.raises(VLLMValidationError, match="non-empty"):
|
||||
verify(prompt_logprob_token_ids=[[], []])
|
||||
# Rows are bounded by max_model_len before the table is padded.
|
||||
with pytest.raises(VLLMValidationError, match="at most 15 scored rows"):
|
||||
verify(prompt_logprob_token_ids=[[]] * 15 + [[1]])
|
||||
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
|
||||
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
|
||||
verify(prompt_logprob_start=3)
|
||||
@@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
params = SamplingParams(prompt_logprob_token_ids=[[1, 2]])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
@@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
|
||||
"""Per-row scores stay row-aligned across chunked prefill and preemption."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompts = [
|
||||
@@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
] + [f"Tell me about the number {i}: " for i in range(32)]
|
||||
|
||||
start = 2
|
||||
candidate_ids = [10, 100, 1000, 10000]
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
|
||||
def candidate_ids(j):
|
||||
return [10 + j, 100 + j, 1000 + j, 10000 + j]
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
@@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
token_prompts = [
|
||||
TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts
|
||||
]
|
||||
|
||||
def make_params(row_ids):
|
||||
return [
|
||||
SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=[
|
||||
row_ids(j)
|
||||
for j in range(len(p["prompt_token_ids"]) - 1 - start)
|
||||
],
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
for p in token_prompts
|
||||
]
|
||||
|
||||
metrics_before = vllm_model.llm.get_metrics()
|
||||
outputs = vllm_model.llm.generate(prompts, sampling_params)
|
||||
full_params = make_params(candidate_ids)
|
||||
for params in full_params: # the array form, as a top-k tensor would be
|
||||
params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids)
|
||||
outputs = vllm_model.llm.generate(token_prompts, full_params)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
assert scores is not None, f"Output {i} missing fixed-ID scores"
|
||||
assert scores is not None, f"Output {i} missing per-row scores"
|
||||
expected_shape = (
|
||||
len(output.prompt_token_ids) - 1 - start,
|
||||
len(candidate_ids),
|
||||
4,
|
||||
)
|
||||
assert scores.shape == expected_shape, (
|
||||
f"Output {i} scored {scores.shape}, expected {expected_shape}"
|
||||
@@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
assert math.isfinite(float(scores.min()))
|
||||
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
|
||||
|
||||
# Ragged rows: each row keeps a prefix of the candidates and must
|
||||
# reproduce that prefix of the full scores, with -inf padding.
|
||||
ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4])
|
||||
ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params)
|
||||
for output, ragged_output in zip(outputs, ragged_outputs):
|
||||
expected = np.full_like(output.prompt_token_id_logprobs, -np.inf)
|
||||
for j in range(len(expected)):
|
||||
n = 1 + j % 4
|
||||
expected[j, :n] = output.prompt_token_id_logprobs[j, :n]
|
||||
np.testing.assert_allclose(
|
||||
ragged_output.prompt_token_id_logprobs, expected, atol=1e-2
|
||||
)
|
||||
|
||||
# The row count is checked against the prompt at admission.
|
||||
with pytest.raises(VLLMValidationError, match="scored rows"):
|
||||
vllm_model.llm.generate(token_prompts[0], ragged_params[1])
|
||||
|
||||
metrics_after = vllm_model.llm.get_metrics()
|
||||
preemptions_before = next(
|
||||
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
|
||||
@@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
# unchunked, unpreempted run of the same requests. Batch composition moves
|
||||
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_logprobs=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
"Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25
|
||||
) as reference:
|
||||
reference_outputs = reference.llm.generate(prompts, sampling_params)
|
||||
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
|
||||
reference_outputs = reference.llm.generate(token_prompts, full_params)
|
||||
self_scored = reference.llm.generate(
|
||||
prompts,
|
||||
token_prompts,
|
||||
[
|
||||
SamplingParams(
|
||||
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
|
||||
max_tokens=1,
|
||||
prompt_logprobs=0,
|
||||
prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]],
|
||||
)
|
||||
for ids in own_ids
|
||||
for p in token_prompts
|
||||
],
|
||||
)
|
||||
for output, ref in zip(outputs, reference_outputs):
|
||||
np.testing.assert_allclose(
|
||||
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
|
||||
output.prompt_token_id_logprobs,
|
||||
ref.prompt_token_id_logprobs,
|
||||
rtol=0.1,
|
||||
atol=0.2,
|
||||
)
|
||||
for output, ids in zip(self_scored, own_ids):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
for output in self_scored:
|
||||
scores = output.prompt_token_id_logprobs[:, 0]
|
||||
for row, target in enumerate(output.prompt_token_ids[1:]):
|
||||
expected = output.prompt_logprobs[row + 1][target].logprob
|
||||
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
|
||||
assert scores[row] == pytest.approx(expected, abs=1e-3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
@@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompt = "The capital of France is Paris. " * 20
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=[10, 100, 1000],
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
text = "The capital of France is Paris. " * 20
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
@@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
enable_prefix_caching=True,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
token_ids = vllm_model.llm.get_tokenizer()(text).input_ids
|
||||
prompt = TokensPrompt(prompt_token_ids=token_ids)
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)),
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
|
||||
first = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert first.prompt_token_id_logprobs is not None
|
||||
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
|
||||
|
||||
@@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list():
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_rejects_prompt_scoring():
|
||||
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
|
||||
params = SamplingParams(
|
||||
trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]]
|
||||
)
|
||||
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
|
||||
@@ -39,6 +39,7 @@ def _make_fake_speculator(
|
||||
and a draft model.
|
||||
"""
|
||||
fake_input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(max_num_tokens, dtype=torch.int64),
|
||||
query_start_loc=torch.zeros(max_num_reqs + 1, dtype=torch.int32),
|
||||
seq_lens=torch.zeros(max_num_reqs, dtype=torch.int32),
|
||||
dcp_local_seq_lens=torch.zeros(max_num_reqs, dtype=torch.int32),
|
||||
@@ -113,6 +114,7 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound():
|
||||
|
||||
captured = _run_build(fake, num_reqs=3, num_reqs_padded=4, base=base, step=2)
|
||||
|
||||
assert torch.equal(captured["positions"], fake.input_buffers.positions[:4])
|
||||
bound = captured["seq_lens_cpu_upper_bound"]
|
||||
assert isinstance(bound, torch.Tensor), (
|
||||
"seq_lens_cpu_upper_bound must be a tensor, not None"
|
||||
|
||||
@@ -8,7 +8,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.outputs import STREAM_FINISHED, RequestOutput
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.output_processor import RequestOutputCollector
|
||||
@@ -170,3 +170,31 @@ async def test_generate_with_async_generator():
|
||||
assert outputs[2].finished is True
|
||||
# Both inputs were processed
|
||||
assert inputs_received == ["Hello", " world"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_input_stream_finishes_without_engine_request():
|
||||
llm = AsyncLLM.__new__(AsyncLLM)
|
||||
llm._validate_streaming_input_sampling_params = MagicMock()
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
|
||||
final_req = MagicMock()
|
||||
final_req.request_id = "request-int"
|
||||
llm.input_processor = MagicMock()
|
||||
llm.input_processor.process_inputs.return_value = final_req
|
||||
|
||||
async def empty_stream():
|
||||
return
|
||||
yield # pragma: no cover
|
||||
|
||||
queue = await llm._add_streaming_input_request(
|
||||
"request", empty_stream(), SamplingParams(max_tokens=4)
|
||||
)
|
||||
task = queue._input_stream_task
|
||||
assert task is not None
|
||||
await task
|
||||
|
||||
assert queue.get_nowait() is STREAM_FINISHED
|
||||
llm._add_request.assert_not_awaited()
|
||||
|
||||
@@ -85,6 +85,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
speculator.max_model_len = speculator.draft_max_seq_len = 32
|
||||
speculator.draft_is_prefilling = torch.zeros(4, dtype=torch.bool)
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(4, dtype=torch.int64),
|
||||
query_start_loc=torch.tensor([0, 1, 2, 2, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([11, 21, 0, 0], dtype=torch.int32),
|
||||
)
|
||||
|
||||
@@ -137,6 +137,52 @@ def test_finished_request_releases_unscored_doc_use():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_release_queries_is_targeted_and_idempotent():
|
||||
runner = LateInteractionRunner()
|
||||
query_emb = torch.tensor([[1.0, 0.0], [0.0, 1.0]], dtype=torch.float32)
|
||||
|
||||
for query_key in ("query-release", "query-keep"):
|
||||
query_params = _make_pooling_params(
|
||||
build_late_interaction_query_params(query_key=query_key, query_uses=1)
|
||||
)
|
||||
runner.postprocess_pooler_output(
|
||||
raw_pooler_output=[query_emb],
|
||||
pooling_params=[query_params],
|
||||
req_ids=[f"{query_key}-req"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
|
||||
released_doc_params = _make_pooling_params(
|
||||
build_late_interaction_doc_params(query_key="query-release")
|
||||
)
|
||||
kept_doc_params = _make_pooling_params(
|
||||
build_late_interaction_doc_params(query_key="query-keep")
|
||||
)
|
||||
runner.register_request("released-doc", released_doc_params)
|
||||
runner.register_request("kept-doc", kept_doc_params)
|
||||
|
||||
runner.release_queries(["query-release"])
|
||||
runner.release_queries(["query-release"])
|
||||
|
||||
with pytest.raises(ValueError, match="query cache miss"):
|
||||
runner.postprocess_pooler_output(
|
||||
raw_pooler_output=[query_emb],
|
||||
pooling_params=[released_doc_params],
|
||||
req_ids=["released-doc"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
|
||||
kept_output = runner.postprocess_pooler_output(
|
||||
raw_pooler_output=[query_emb],
|
||||
pooling_params=[kept_doc_params],
|
||||
req_ids=["kept-doc"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
assert isinstance(kept_output, list)
|
||||
assert kept_output[0] is not None
|
||||
|
||||
|
||||
def test_invalid_query_uses_raises():
|
||||
runner = LateInteractionRunner()
|
||||
bad_meta = LateInteractionParams(
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.v1.watermarking.spec_decode import (
|
||||
speculative_target_watermark_key,
|
||||
)
|
||||
from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample
|
||||
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.sample.watermark import (
|
||||
draft_watermarking_mask,
|
||||
@@ -1499,6 +1500,53 @@ def test_draft_watermarker_deduplicates_across_a_block_on_cuda(
|
||||
assert not draft_logits[:, :, -1].any()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator"
|
||||
)
|
||||
def test_draft_watermarker_cuda_graph_reads_current_prompt_lens():
|
||||
draft_watermarker = DraftWatermarker(
|
||||
GumbelWatermarker(key=42, context_width=2),
|
||||
max_num_reqs=1,
|
||||
device=torch.device("cuda"),
|
||||
num_speculative_steps=1,
|
||||
deduplicate_contexts="single_turn",
|
||||
deduplicate_contexts_max_history=None,
|
||||
)
|
||||
prompt_lens = UvaBackedTensor(1, dtype=torch.int32)
|
||||
contexts = torch.tensor([[5, 6]]).cuda()
|
||||
enabled = torch.ones(1, dtype=torch.bool).cuda()
|
||||
# [5, 6] repeats only while the prompt counts as history.
|
||||
all_token_ids = torch.tensor([[5, 6, 9, 5, 6]]).cuda()
|
||||
total_lens = torch.tensor([5]).cuda()
|
||||
logits = torch.zeros(1, 8).cuda()
|
||||
idx_mapping = torch.zeros(1, dtype=torch.int32).cuda()
|
||||
temperature = torch.ones(1).cuda()
|
||||
draft_step = torch.tensor(0).cuda()
|
||||
|
||||
def prepare():
|
||||
draft_watermarker.prepare(
|
||||
contexts, enabled, all_token_ids, prompt_lens.gpu, total_lens
|
||||
)
|
||||
|
||||
def watermark_mask():
|
||||
_, mask = draft_watermarker._sampling_state(
|
||||
logits, idx_mapping, temperature, draft_step, contexts
|
||||
)
|
||||
return mask
|
||||
|
||||
prepare()
|
||||
assert not watermark_mask().item()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
mask = watermark_mask()
|
||||
# A step that adds requests moves prompt_lens.gpu to the next pool buffer.
|
||||
prompt_lens.np[:] = 3
|
||||
prompt_lens.copy_to_uva()
|
||||
prepare()
|
||||
graph.replay()
|
||||
assert mask.item()
|
||||
|
||||
|
||||
def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch):
|
||||
speculator = object.__new__(DSparkSpeculator)
|
||||
speculator.draft_logits = torch.zeros(2, 1, 8)
|
||||
|
||||
Reference in New Issue
Block a user