[Mini-VLLM] Repair test suite after SM86 cut

- Restore example connector, test helpers, and dropped patch tests
- Guard MLA metadata import in the example connector
- Point the PLE test at common.ops.ple after upstream #60021 move
- Cut tests for removed features: tools, DRY, epd proxy, STT batch,
  diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 01:06:30 +07:00
co-authored by Hinata
parent 0152af1336
commit 54a396586e
112 changed files with 55726 additions and 10371 deletions
+2
View File
@@ -0,0 +1,2 @@
{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\n'}}{% endif %}{% endfor %}
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\n' }}{% endif %}
+144
View File
@@ -0,0 +1,144 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Callable, Iterable
from typing import Any, NamedTuple
import pytest
import regex as re
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
class Matches(NamedTuple):
# simple pointwise
aiter_rms_quant_fusion: int = 0
rms_quant_fusion: int = 0
act_quant_fusion: int = 0
norm_rope_fusion: int = 0
attn_quant_fusion: int = 0
# distributed
ar_rms_fusion: int = 0
aiter_ar_rms_fusion: int = 0
class ModelFusionInfo(NamedTuple):
model_name: str
matches: Callable[[int], Matches]
"""Given number of hidden layers, produces the matches object"""
model_kwargs: dict[str, Any] = {}
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
class AttentionBackendCase(NamedTuple):
backend: AttentionBackendEnum
model_kwargs: dict[str, Any] = {}
"""Additional args required for attn+quant fusion"""
is_blackwell = lambda: current_platform.is_device_capability_family(100)
"""Are we running on Blackwell, a lot of tests depend on it"""
def nvfp4_kernel_exposes_input_quant_key() -> bool:
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
output format and the cutedsl backend's expected input format.
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
manual fusion.
"""
if not current_platform.is_cuda():
return False
try:
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCuteDslNvFp4LinearKernel,
)
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
if is_supported:
return False
except ImportError:
pass
try:
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCutlassNvFp4LinearKernel,
)
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
if is_supported:
return True
except ImportError:
pass
# Fallback: assume no manual fusion support
return False
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
"""Generate all combinations of custom ops for parametrization."""
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
for op_list in itertools.product(*custom_ops_lists):
yield ",".join(op_list)
# Quick inline validation
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
"-quant_fp8,-rms_norm",
"-quant_fp8,+rms_norm",
"+quant_fp8,-rms_norm",
"+quant_fp8,+rms_norm",
]
def has_cuda_graph_wrapper_metadata() -> bool:
from importlib import import_module
try:
module = import_module("torch._inductor.utils")
module.CUDAGraphWrapperMetadata # noqa B018
except AttributeError:
return False
return True
INDUCTOR_GRAPH_PARTITION = [
pytest.param(
True,
marks=pytest.mark.skipif(
not has_cuda_graph_wrapper_metadata(),
reason="torch version does not support Inductor partition",
),
id="inductor_partition",
),
pytest.param(False, id="dynamo_partition"),
]
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
"aiter_rms_quant_fusion": re.compile(
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
),
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
# The combined pass can consume these sites before the standalone pass.
"norm_rope_fusion": re.compile(
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
r"qk_norm_rope_kvcache_fusion.py:\d+] "
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
),
"attn_quant_fusion": re.compile(
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
),
"ar_rms_fusion": re.compile(
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
),
"aiter_ar_rms_fusion": re.compile(
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
),
}
+408
View File
@@ -0,0 +1,408 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
import msgspec
import pytest
from vllm.distributed.kv_events import (
EventBatch,
EventPublisherFactory,
NullEventPublisher,
)
DP_RANK = 0
class EventSample(
msgspec.Struct,
tag=True, # type: ignore
array_like=True, # type: ignore
):
"""Test event for publisher testing."""
id: int
value: str
class SampleBatch(EventBatch):
"""Test event batch for publisher testing."""
events: list[EventSample]
def create_test_events(count: int) -> SampleBatch:
"""Create a batch of test events."""
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
return SampleBatch(ts=time.time(), events=events)
def test_basic_publishing(publisher, subscriber):
"""Test basic event publishing works."""
test_batch = create_test_events(5)
publisher.publish(test_batch)
result = subscriber.receive_one(timeout=1000)
assert result is not None, "No message received"
seq, received = result
assert seq == 0, "Sequence number mismatch"
assert received.ts == pytest.approx(test_batch.ts, abs=0.1), "Timestamp mismatch"
assert len(received.events) == len(test_batch.events), "Number of events mismatch"
for i, event in enumerate(received.events):
assert event.id == i, "Event id mismatch"
assert event.value == f"test-{i}", "Event value mismatch"
def test_multiple_events(publisher, subscriber):
"""Test publishing and receiving multiple event batches."""
for _ in range(10):
batch = create_test_events(2)
publisher.publish(batch)
received = []
for _ in range(10):
data = subscriber.receive_one(timeout=100)
if data:
received.append(data)
assert len(received) == 10, "Number of messages mismatch"
seqs = [seq for seq, _ in received]
assert seqs == list(range(10)), "Sequence numbers mismatch"
def test_replay_mechanism(publisher, subscriber):
"""Test the replay mechanism works correctly."""
for _ in range(19):
batch = create_test_events(1)
publisher.publish(batch)
# Drain live events to ensure publisher has buffered them.
for _ in range(19):
assert subscriber.receive_one(timeout=1000) is not None
subscriber.request_replay(10)
replayed = subscriber.receive_replay()
assert len(replayed) == 9, (
f"Expected 9 replayed messages (seq 10-18), got {len(replayed)}"
)
seqs = [seq for seq, _ in replayed]
assert seqs == list(range(10, 19)), "Replayed sequences should be 10-18"
def test_replay_includes_topic(publisher, subscriber, publisher_config):
"""Test that replay responses include the topic, matching PUB format."""
for _ in range(5):
publisher.publish(create_test_events(1))
# Drain live events to ensure publisher has processed them.
for _ in range(5):
assert subscriber.receive_one(timeout=1000) is not None
subscriber.request_replay(0)
# receive_replay unpacks (topic, seq, payload) and asserts
# topic == publisher topic for each message.
replayed = subscriber.receive_replay()
assert len(replayed) == 5, f"Expected 5 replayed messages, got {len(replayed)}"
seqs = [seq for seq, _ in replayed]
assert seqs == list(range(5)), "Replayed sequences should be 0-4"
def test_buffer_limit(publisher, subscriber, publisher_config):
"""Test buffer limit behavior."""
buffer_size = publisher_config.buffer_steps
# Publish more events than the buffer can hold
for i in range(buffer_size + 10):
batch = create_test_events(1)
publisher.publish(batch)
time.sleep(0.5) # Need publisher to process above requests
subscriber.request_replay(0)
replayed = subscriber.receive_replay()
assert len(replayed) == buffer_size, (
f"Expected {buffer_size} replayed messages, got {len(replayed)}"
)
seqs = [seq for seq, _ in replayed]
assert seqs == list(range(10, buffer_size + 10)), (
"Should replay seq 11 through buffer_size+10"
)
def test_topic_filtering(publisher_config):
"""Test that a subscriber only receives messages matching its topic filter."""
publisher_config.replay_endpoint = None
publisher_config.topic = "foo"
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
from .conftest import MockSubscriber
sub_foo = MockSubscriber(publisher_config.endpoint, None, "foo")
sub_bar = MockSubscriber(publisher_config.endpoint, None, "bar")
try:
time.sleep(0.1)
for _ in range(3):
pub.publish(create_test_events(1))
foo_received = [sub_foo.receive_one(timeout=200) for _ in range(3)]
assert all(msg is not None for msg in foo_received), (
"Subscriber with matching topic should receive messages"
)
bar_received = [sub_bar.receive_one(timeout=200) for _ in range(3)]
assert all(msg is None for msg in bar_received), (
"Subscriber with non-matching topic should receive no messages"
)
finally:
pub.shutdown()
sub_foo.close()
sub_bar.close()
def test_high_volume(publisher, subscriber):
"""Test publishing and receiving a high volume of events."""
num_batches = 10_000
events_per_batch = 100
# Publish events in a separate thread to not block
def publish_events():
for i in range(num_batches):
batch = create_test_events(events_per_batch)
publisher.publish(batch)
# Small delay to avoid overwhelming
if i % 100 == 0:
time.sleep(0.01)
received: list[tuple[int, SampleBatch]] = []
publisher_thread = threading.Thread(target=publish_events)
publisher_thread.start()
start_time = time.time()
while len(received) < num_batches:
if time.time() - start_time > 10: # Timeout after 10 seconds
break
result = subscriber.receive_one(timeout=100)
if result:
received.append(result)
publisher_thread.join()
assert len(received) >= num_batches * 0.9, "We should have received most messages"
seqs = [seq for seq, _ in received]
assert sorted(seqs) == seqs, "Sequence numbers should be in order"
def test_null_publisher():
"""Test that NullEventPublisher can be used without errors."""
publisher = NullEventPublisher(DP_RANK)
# This should not raise any errors
batch = create_test_events(5)
publisher.publish(batch)
publisher.shutdown()
def test_data_parallel_rank_tagging(publisher_config):
"""Test that events are properly tagged with their data parallel rank."""
publisher_config.topic = "foo"
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
# Hardcode the expected endpoints based on port offsetting behavior
# Both ranks get offsets according to _offset_endpoint_port function
base_endpoint = publisher_config.endpoint
if "tcp://" in base_endpoint:
# For TCP endpoints: tcp://localhost:5557 -> tcp://localhost:5557, tcp://localhost:5558
expected_endpoint_0 = base_endpoint # rank 0 gets port + 0 = same port
expected_endpoint_1 = base_endpoint.replace(
":5557", ":5558"
) # rank 1 gets port + 1
else:
# For inproc endpoints: inproc://test -> inproc://test_dp0, inproc://test_dp1
expected_endpoint_0 = base_endpoint # rank 0 gets base
expected_endpoint_1 = base_endpoint + "_dp1" # rank 1 gets _dp1
from .conftest import MockSubscriber
sub_0 = MockSubscriber(expected_endpoint_0, None, publisher_config.topic)
sub_1 = MockSubscriber(expected_endpoint_1, None, publisher_config.topic)
try:
time.sleep(0.1) # Let publishers start up
# Publish events from different ranks
batch_0 = create_test_events(2)
batch_1 = create_test_events(3)
pub_0.publish(batch_0)
pub_1.publish(batch_1)
# Receive events from rank 0
result_0 = sub_0.receive_one(timeout=200)
assert result_0 is not None, "No message received from rank 0"
seq_0, received_0 = result_0
# Receive events from rank 1
result_1 = sub_1.receive_one(timeout=200)
assert result_1 is not None, "No message received from rank 1"
seq_1, received_1 = result_1
# Verify DP rank tagging
assert received_0.data_parallel_rank == 0, (
f"Expected DP rank 0, got {received_0.data_parallel_rank}"
)
assert received_1.data_parallel_rank == 1, (
f"Expected DP rank 1, got {received_1.data_parallel_rank}"
)
# Verify event content is correct
assert len(received_0.events) == 2, "Wrong number of events from rank 0"
assert len(received_1.events) == 3, "Wrong number of events from rank 1"
finally:
pub_0.shutdown()
pub_1.shutdown()
sub_0.close()
sub_1.close()
def test_event_publisher_factory(random_port):
"""Test event publisher factory creation behavior under different configurations."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.kv_events import ZmqEventPublisher
# test config is None
publisher = EventPublisherFactory.create(None, DP_RANK)
assert isinstance(publisher, NullEventPublisher)
publisher.shutdown()
# test disable kv cache events
config = KVEventsConfig(
enable_kv_cache_events=False,
publisher="zmq", # Even if zmq is specified, should return NullEventPublisher
endpoint="tcp://localhost:5557",
)
publisher = EventPublisherFactory.create(config, DP_RANK)
assert isinstance(publisher, NullEventPublisher)
publisher.shutdown()
# test zmq publisher
config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint=f"tcp://*:{random_port}",
replay_endpoint=f"tcp://*:{random_port + 100}",
)
publisher = EventPublisherFactory.create(config, DP_RANK + 1)
assert isinstance(publisher, ZmqEventPublisher)
resolved_config = publisher.get_publisher_config()
assert resolved_config.endpoint == f"tcp://*:{random_port + 1}"
assert resolved_config.replay_endpoint == f"tcp://*:{random_port + 101}"
assert config.endpoint == f"tcp://*:{random_port}"
publisher.shutdown()
# test unknown publisher
with pytest.raises(ValueError, match="Input should be"):
KVEventsConfig(
enable_kv_cache_events=True,
publisher="unknown_publisher",
endpoint="tcp://localhost:5557",
)
# test publisher not specified
config = KVEventsConfig(
enable_kv_cache_events=True,
# publisher not specified, should default to "zmq"
endpoint="tcp://localhost:5557",
)
publisher = EventPublisherFactory.create(config, DP_RANK)
assert isinstance(publisher, ZmqEventPublisher)
publisher.shutdown()
@pytest.mark.parametrize(
("endpoint", "expected"),
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
)
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
"""Port 0 is not offset: every rank asks the OS for its own port."""
from vllm.distributed.kv_events import ZmqEventPublisher
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
node's IP and report endpoints a subscriber can use for events and replay."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed import kv_events
from vllm.utils.network_utils import split_zmq_path
from .conftest import MockSubscriber
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint="tcp://*:0",
replay_endpoint="tcp://*:0",
topic="kv",
)
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
try:
resolved = [pub.get_publisher_config() for pub in pubs]
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
assert {h for h, _ in hosts_ports} == {node_ip}
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
assert config.endpoint == "tcp://*:0"
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
try:
# PUB/SUB drops messages sent before the subscription lands, so
# keep publishing until both ranks are heard.
ranks: set[int] = set()
deadline = time.time() + 10
while ranks != {0, 1} and time.time() < deadline:
for pub in pubs:
pub.publish(create_test_events(1))
while (result := sub.receive_one(timeout=100)) is not None:
ranks.add(result[1].data_parallel_rank)
assert ranks == {0, 1}
sub.request_replay(0, socket_idx=1)
assert sub.receive_replay(socket_idx=1)
finally:
sub.close()
finally:
for pub in pubs:
pub.shutdown()
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
publisher refuses instead of reporting a wildcard."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed import kv_events
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
config = KVEventsConfig(
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
)
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
EventPublisherFactory.create(config, DP_RANK)
File diff suppressed because it is too large Load Diff
@@ -16,12 +16,9 @@ from vllm.assets.audio import AudioAsset
from vllm.connections import HTTPConnection
from vllm.entrypoints.launchers.run_batch import (
BatchRequestOutput,
BatchTranscriptionRequest,
download_bytes_from_url,
make_transcription_wrapper,
upload_data,
)
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import VLLMValidationError
from vllm.utils.mem_constants import MiB_bytes
@@ -644,122 +641,10 @@ def test_reasoning_parser():
assert len(reasoning) > 0
def test_transcription():
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_TRANSCRIPTION_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
SPEECH_LARGE_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
print(f"\n\ncontents: {contents}\n\n")
for line in contents.strip().split("\n"):
BatchRequestOutput.model_validate_json(line)
line_dict = json.loads(line)
assert isinstance(line_dict, dict)
assert line_dict["error"] is None
response_body = line_dict["response"]["body"]
assert response_body is not None
assert "text" in response_body
assert "usage" in response_body
def test_transcription_http_url():
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_TRANSCRIPTION_HTTP_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
SPEECH_LARGE_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
BatchRequestOutput.model_validate_json(line)
line_dict = json.loads(line)
assert isinstance(line_dict, dict)
assert line_dict["error"] is None
response_body = line_dict["response"]["body"]
assert response_body is not None
assert "text" in response_body
assert "usage" in response_body
transcription_text = response_body["text"]
assert "Mary had a little lamb" in transcription_text
def test_translation():
with (
tempfile.NamedTemporaryFile("w") as input_file,
tempfile.NamedTemporaryFile("r") as output_file,
):
input_file.write(INPUT_TRANSLATION_BATCH)
input_file.flush()
proc = subprocess.Popen(
[
"vllm",
"run-batch",
"-i",
input_file.name,
"-o",
output_file.name,
"--model",
SPEECH_SMALL_MODEL_NAME,
],
)
proc.communicate()
proc.wait()
assert proc.returncode == 0, f"{proc=}"
contents = output_file.read()
for line in contents.strip().split("\n"):
BatchRequestOutput.model_validate_json(line)
line_dict = json.loads(line)
assert isinstance(line_dict, dict)
assert line_dict["error"] is None
response_body = line_dict["response"]["body"]
assert response_body is not None
assert "text" in response_body
translation_text = response_body["text"]
translation_text_lower = str(translation_text).strip().lower()
assert "mary" in translation_text_lower or "lamb" in translation_text_lower
def test_tool_calling():
@@ -974,28 +859,6 @@ async def test_download_bytes_backslash_bypass():
)
@pytest.mark.asyncio
async def test_transcription_wrapper_rejects_oversized_data_url_before_decode(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
handler = AsyncMock()
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
request = BatchTranscriptionRequest.model_validate(
{
"model": SPEECH_LARGE_MODEL_NAME,
"file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}",
"response_format": "json",
}
)
with patch("vllm.entrypoints.launchers.run_batch.base64.b64decode") as decode:
response = await wrapped_handler(request)
assert isinstance(response, ErrorResponse)
assert "Maximum file size exceeded" in response.error.message
decode.assert_not_called()
handler.assert_not_awaited()
@pytest.mark.asyncio
@@ -1019,35 +882,6 @@ async def test_download_bytes_allows_http_body_at_audio_limit(
await _close_async_connection(connection)
@pytest.mark.asyncio
async def test_transcription_wrapper_rejects_oversized_http_before_handler(
monkeypatch: pytest.MonkeyPatch,
batch_http_server: _BatchHTTPServer,
):
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
connection = HTTPConnection()
monkeypatch.setattr(
"vllm.entrypoints.launchers.run_batch.global_http_connection",
connection,
)
handler = AsyncMock()
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
request = BatchTranscriptionRequest.model_validate(
{
"model": SPEECH_LARGE_MODEL_NAME,
"file_url": _batch_http_url(batch_http_server, "/oversized-chunked"),
"response_format": "json",
}
)
try:
response = await wrapped_handler(request)
finally:
await _close_async_connection(connection)
assert isinstance(response, ErrorResponse)
assert "Maximum file size exceeded" in response.error.message
handler.assert_not_awaited()
# ---------------------------------------------------------------------------
@@ -11,7 +11,6 @@ import pytest
import pytest_asyncio
from transformers import AutoTokenizer
import vllm.envs as envs
from tests.utils import RemoteOpenAIServer
from vllm.config import ModelConfig
from vllm.v1.engine.detokenizer import check_stop_strings
@@ -140,7 +139,7 @@ async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="sampling mask output is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
@@ -218,7 +217,7 @@ async def test_generate_defaults_max_tokens_when_omitted(client):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
)
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
@@ -485,7 +484,7 @@ async def test_stop_string_workflow(client, tokenizer, messages):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
@@ -617,7 +616,7 @@ async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
# either reasoning field is dropped between render and generate.
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
@@ -638,7 +637,7 @@ async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, mes
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="the Rust frontend does not forward reasoning fields to the engine",
)
@pytest.mark.parametrize(
@@ -679,7 +678,7 @@ def _text_mode_payload(token_ids, **sampling_params) -> dict:
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
@@ -707,7 +706,7 @@ async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="output_mode is not supported by the Rust frontend",
)
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
@@ -734,7 +733,7 @@ async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="output_mode is not supported by the Rust frontend",
)
@pytest.mark.parametrize(
@@ -759,7 +758,7 @@ async def test_text_mode_invalid_requests_return_400(client, overrides, expected
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="--tokens-only is not supported by the Rust frontend",
)
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
@@ -778,7 +777,7 @@ async def test_text_mode_rejected_when_tokens_only(client):
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
@@ -825,7 +824,7 @@ async def test_stream_delivers_abort_finish_chunk(
@pytest.mark.asyncio
@pytest.mark.skipif(
envs.VLLM_USE_RUST_FRONTEND,
False,
reason="the Rust frontend does not serve /inference/v1/abort_requests",
)
async def test_abort_requests_is_served_without_tokens_only(client):
+507
View File
@@ -0,0 +1,507 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Isolated GSM8K evaluation script for vLLM serve endpoint."""
import argparse
import ast
import asyncio
import json
import os
import tempfile
import time
from collections.abc import Generator
import aiohttp
import numpy as np
import regex as re
import requests
from tqdm.asyncio import tqdm
from vllm.assets.base import VLLM_S3_BUCKET_URL
INVALID = -9999999
def download_and_cache_file(url: str, filename: str | None = None) -> str:
"""Download and cache a file from a URL."""
if filename is None:
filename = os.path.join(tempfile.gettempdir(), url.split("/")[-1])
if os.path.exists(filename):
return filename
print(f"Downloading from {url} to {filename}")
response = requests.get(url, stream=True)
response.raise_for_status()
with open(filename, "wb") as f:
for chunk in response.iter_content(chunk_size=1024):
f.write(chunk)
return filename
def load_gsm8k_data() -> tuple[list[dict], list[dict]]:
"""Load GSM8K train and test data."""
train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl"
test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl"
train_file = download_and_cache_file(train_url)
test_file = download_and_cache_file(test_url)
train_data = list(read_jsonl(train_file))
test_data = list(read_jsonl(test_file))
return train_data, test_data
def read_jsonl(filename: str) -> Generator[dict, None, None]:
"""Read a JSONL file."""
with open(filename) as fin:
for line in fin:
if not line.startswith("#"):
yield json.loads(line)
def get_answer_value(answer_str: str) -> int:
"""Extract the numerical answer from the response."""
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def _optional_params(**params: object) -> dict[str, object]:
return {k: v for k, v in params.items() if v is not None}
async def call_vllm_api(
session: aiohttp.ClientSession,
prompt: str,
temperature: float | None,
max_tokens: int,
stop: list[str] | None = None,
url: str | None = None,
seed: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible completions endpoint.
Returns:
Tuple of (response_text, completion_tokens)
"""
data = {
"prompt": prompt,
"max_tokens": max_tokens,
"stop": stop,
}
data.update(
_optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k)
)
try:
async with session.post(f"{url}/v1/completions", json=data) as response:
response.raise_for_status()
result = await response.json()
text = result["choices"][0]["text"]
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
return text, completion_tokens
except Exception as e:
print(f"Error calling vLLM API ({type(e).__name__}): {e}")
return "", 0
async def call_vllm_chat_api(
session: aiohttp.ClientSession,
model: str | None,
prompt: str,
temperature: float | None,
max_tokens: int,
stop: list[str] | None = None,
url: str | None = None,
seed: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
reasoning_effort: str | None = None,
chat_template_kwargs: dict[str, object] | None = None,
) -> tuple[str, int]:
"""Call vLLM's OpenAI-compatible chat completions endpoint.
``model``, ``temperature``, ``stop`` and the other optional fields are
omitted from the request when ``None``, so the server defaults apply (vLLM
uses its served model when ``model`` is omitted).
Returns:
Tuple of (final answer content, completion_tokens). Reasoning returned
separately by a reasoning parser is not part of the answer.
"""
data = {
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
}
data.update(
_optional_params(
stop=stop,
model=model,
temperature=temperature,
seed=seed,
top_p=top_p,
top_k=top_k,
reasoning_effort=reasoning_effort,
chat_template_kwargs=chat_template_kwargs,
)
)
try:
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
response.raise_for_status()
result = await response.json()
text = result["choices"][0]["message"]["content"] or ""
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
return text, completion_tokens
except Exception as e:
print(f"Error calling vLLM chat API ({type(e).__name__}): {e}")
return "", 0
def _build_gsm8k_prompts(
num_questions: int = 1319,
num_shots: int = 5,
gen_prefix: str = "",
) -> tuple[list[str], list[int]]:
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
if num_questions == 0:
return [], []
train_data, test_data = load_gsm8k_data()
num_questions = min(num_questions, len(test_data))
few_shot_examples = ""
for i in range(num_shots):
few_shot_examples += (
f"Question: {train_data[i]['question']}\n"
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
)
prompts = []
labels = []
for i in range(num_questions):
prompts.append(
few_shot_examples
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
)
labels.append(get_answer_value(test_data[i]["answer"]))
assert all(label != INVALID for label in labels), "Some labels are invalid"
return prompts, labels
def _score_gsm8k(
states: list[str],
output_tokens: list[int],
labels: list[int],
num_shots: int,
max_tokens: int,
latency: float,
) -> dict[str, float | int]:
"""Score GSM8K responses and return a results dict."""
num_questions = len(labels)
preds = [get_answer_value(state) for state in states]
accuracy = np.mean(np.array(preds) == np.array(labels))
invalid_rate = np.mean(np.array(preds) == INVALID)
total_output_tokens = sum(output_tokens)
tokens_per_second = total_output_tokens / latency if latency > 0 else 0.0
return {
"accuracy": accuracy,
"invalid_rate": invalid_rate,
"latency": latency,
"questions_per_second": num_questions / latency if latency > 0 else 0.0,
"total_output_tokens": total_output_tokens,
"tokens_per_second": tokens_per_second,
"num_questions": num_questions,
"num_shots": num_shots,
"max_tokens": max_tokens,
"timestamp": time.time(),
}
def evaluate_gsm8k(
num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
model: str | None = None,
use_chat_completions: bool = False,
host: str = "http://127.0.0.1",
port: int = 8000,
temperature: float | None = 0.0,
seed: int | None = 42,
request_timeout_seconds: float = 600,
gen_prefix: str = "",
max_concurrency: int | None = None,
top_p: float | None = None,
top_k: int | None = None,
reasoning_effort: str | None = None,
chat_template_kwargs: dict[str, object] | None = None,
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using vLLM serve endpoint.
``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``;
otherwise the server default applies. ``model`` is used only in chat mode,
where it may be ``None`` (the server uses its served model).
``reasoning_effort`` and ``chat_template_kwargs`` require
``use_chat_completions=True``. Stop strings are sent only in completions
mode.
Returns dict with accuracy, invalid_rate, latency, etc.
"""
if not use_chat_completions and (
reasoning_effort is not None or chat_template_kwargs is not None
):
raise ValueError(
"reasoning_effort and chat_template_kwargs require "
"use_chat_completions=True"
)
base_url = f"{host}:{port}"
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
num_questions = len(prompts)
async def run_async_evaluation():
states: list[str] = [""] * num_questions
output_tokens: list[int] = [0] * num_questions
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
if use_chat_completions:
# No stop strings: the chat template delimits the turn, and they
# would also cut off reasoning.
answer, tokens = await call_vllm_chat_api(
session=session,
model=model,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
url=base_url,
seed=seed,
top_p=top_p,
top_k=top_k,
reasoning_effort=reasoning_effort,
chat_template_kwargs=chat_template_kwargs,
)
else:
answer, tokens = await call_vllm_api(
session=session,
prompt=prompts[i],
temperature=temperature,
max_tokens=max_tokens,
stop=["Question", "Assistant:", "<|separator|>"],
url=base_url,
seed=seed,
top_p=top_p,
top_k=top_k,
)
states[i] = answer
output_tokens[i] = tokens
return answer, tokens
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
connector = (
aiohttp.TCPConnector(limit=max_concurrency)
if max_concurrency is not None
else None
)
async with aiohttp.ClientSession(
timeout=timeout, connector=connector
) as session:
tasks = [get_answer(session, i) for i in range(num_questions)]
await tqdm.gather(*tasks, desc="Evaluating")
return states, output_tokens
print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot")
tic = time.perf_counter()
states, output_tokens = asyncio.run(run_async_evaluation())
latency = time.perf_counter() - tic
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
def evaluate_gsm8k_offline(
llm,
num_questions: int = 1319,
num_shots: int = 5,
max_tokens: int = 256,
temperature: float = 0.0,
gen_prefix: str = "",
use_chat_completions: bool = False,
chat_template_kwargs: dict[str, object] | None = None,
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
Same prompts and scoring as evaluate_gsm8k(), but runs generation
directly via llm.generate() instead of calling a server over HTTP.
When ``use_chat_completions=True``, prompts go through the chat template via
``llm.chat()`` instead of raw completion (for instruction-tuned models).
``chat_template_kwargs`` are forwarded to ``llm.chat()`` when provided.
"""
from vllm import SamplingParams
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
sampling_params = SamplingParams(
temperature=temperature,
max_tokens=max_tokens,
stop=["Question", "Assistant:", "<|separator|>"],
)
mode = "chat" if use_chat_completions else "completion"
print(
f"Running offline GSM8K evaluation: {len(prompts)} questions, "
f"{num_shots}-shot, {mode}"
)
tic = time.perf_counter()
if use_chat_completions:
conversations = [[{"role": "user", "content": p}] for p in prompts]
outputs = llm.chat(
conversations,
sampling_params,
chat_template_kwargs=chat_template_kwargs,
)
else:
outputs = llm.generate(prompts, sampling_params)
latency = time.perf_counter() - tic
states = [o.outputs[0].text for o in outputs]
output_tokens = [len(o.outputs[0].token_ids) for o in outputs]
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
def main() -> None:
parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve")
parser.add_argument(
"--num-shots", type=int, default=5, help="Number of few-shot examples"
)
parser.add_argument(
"--num-questions",
type=int,
default=1319,
help="Number of questions to evaluate",
)
parser.add_argument(
"--max-tokens", type=int, default=256, help="Max tokens for generation"
)
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
parser.add_argument("--port", type=int, default=8000, help="Port number")
parser.add_argument(
"--temperature",
type=float,
help="Temperature for generation (default: 0 for completions; "
"server default with --use-chat-completions)",
)
parser.add_argument(
"--top-p", type=float, help="Top-p for generation (server default if unset)"
)
parser.add_argument(
"--top-k", type=int, help="Top-k for generation (server default if unset)"
)
parser.add_argument(
"--use-chat-completions",
action="store_true",
help="Send each few-shot prompt as a user message to /v1/chat/completions "
"instead of /v1/completions (for chat-only models)",
)
parser.add_argument(
"--model",
type=str,
help="Model name to send with chat requests (default: omitted, so the "
"server uses its served model)",
)
parser.add_argument(
"--reasoning-effort",
type=str,
help="reasoning_effort for chat completions, e.g. none, low, high",
)
parser.add_argument(
"--chat-template-kwargs",
type=json.loads,
help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'",
)
parser.add_argument(
"--seed", type=int, default=42, help="Random seed for reproducibility"
)
parser.add_argument(
"--max-concurrency",
type=int,
help="Maximum number of concurrent requests",
)
parser.add_argument(
"--request-timeout-seconds",
type=float,
default=600,
help="Timeout for each request, including time waiting for a connection",
)
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
args = parser.parse_args()
temperature = args.temperature
if temperature is None and not args.use_chat_completions:
temperature = 0.0
result = evaluate_gsm8k(
num_questions=args.num_questions,
num_shots=args.num_shots,
max_tokens=args.max_tokens,
host=args.host,
port=args.port,
temperature=temperature,
seed=args.seed,
max_concurrency=args.max_concurrency,
request_timeout_seconds=args.request_timeout_seconds,
model=args.model,
use_chat_completions=args.use_chat_completions,
top_p=args.top_p,
top_k=args.top_k,
reasoning_effort=args.reasoning_effort,
chat_template_kwargs=args.chat_template_kwargs,
)
# Print results to terminal
print("\nResults:")
print(f"Accuracy: {result['accuracy']:.3f}")
print(f"Invalid responses: {result['invalid_rate']:.3f}")
print(f"Total latency: {result['latency']:.3f} s")
print(f"Questions per second: {result['questions_per_second']:.3f}")
print(f"Total output tokens: {result['total_output_tokens']}")
print(f"Output tokens per second: {result['tokens_per_second']:.3f}")
# Optional file saving
if args.save_results:
# None means the field was not sent and the server default applied.
result["request_params"] = {
"endpoint": "chat" if args.use_chat_completions else "completions",
"model": args.model,
"temperature": temperature,
"top_p": args.top_p,
"top_k": args.top_k,
"seed": args.seed,
"stop": None
if args.use_chat_completions
else ["Question", "Assistant:", "<|separator|>"],
"reasoning_effort": args.reasoning_effort,
"chat_template_kwargs": args.chat_template_kwargs,
}
with open(args.save_results, "w") as f:
json.dump(result, f, indent=2)
print(f"Results saved to {args.save_results}")
if __name__ == "__main__":
main()
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.utils.import_utils import has_helion
if not has_helion():
pytest.skip("Helion is not installed", allow_module_level=True)
from tools import benchmark_helion_kernels
check_correctness = benchmark_helion_kernels.check_correctness
class _FakeKernel:
helion_settings = None
def __init__(self, offset: float):
self.calls = 0
self.offset = offset
def __call__(self, output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
self.calls += 1
output.copy_(input + self.offset)
return input * 2
def test_check_correctness_runs_once_without_mutating_benchmark_inputs():
kernel = _FakeKernel(offset=1)
baseline_calls = 0
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
nonlocal baseline_calls
baseline_calls += 1
output.copy_(input + 1)
return input * 2
output = torch.zeros(4)
inputs = (output, torch.arange(4))
check_correctness(kernel, baseline, inputs, "matching")
assert kernel.calls == 1
assert baseline_calls == 1
torch.testing.assert_close(output, torch.zeros(4))
def test_check_correctness_reports_mutated_input_mismatch():
kernel = _FakeKernel(offset=2)
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
output.copy_(input + 1)
return input * 2
inputs = (torch.zeros(4), torch.arange(4))
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
check_correctness(kernel, baseline, inputs, "bad")
def test_check_correctness_reports_return_value_mismatch():
kernel = _FakeKernel(offset=1)
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
output.copy_(input + 1)
return input * 3
inputs = (torch.zeros(4), torch.arange(4))
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
check_correctness(kernel, baseline, inputs, "bad")
def test_log_versions(monkeypatch):
messages = []
def record(message: str, *args):
messages.append(message % args)
monkeypatch.setattr(benchmark_helion_kernels.logger, "info", record)
benchmark_helion_kernels.log_versions()
assert [message.split(":", 1)[0] for message in messages] == [
"torch",
"helion",
"triton",
]
-140
View File
@@ -1,140 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import unittest
import pytest
import torch
from tests.utils import ensure_current_vllm_config, multi_gpu_test
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.mamba.mamba_mixer2 import Mixer2RMSNormGated
from vllm.utils.system_utils import update_environment_variables
from vllm.utils.torch_utils import set_random_seed
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [128])
@pytest.mark.parametrize(
"hidden_size_n_groups",
[
(64, 1),
(64, 2),
(64, 4), # hidden_size be divisible by num_gpus
],
)
@pytest.mark.parametrize("dtype", [torch.float16])
def test_mixer2_gated_norm_multi_gpu(
batch_size: int,
seq_len: int,
hidden_size_n_groups: tuple[int, int],
dtype: torch.dtype,
device: str = "cuda",
):
hidden_size, n_groups = hidden_size_n_groups
num_processes = 2
def run_torch_spawn(fn, nprocs):
# need to use torch.mp.spawn otherwise will have problems with
# torch.distributed and cuda
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
batch_size,
seq_len,
hidden_size,
n_groups,
dtype,
device,
),
nprocs=nprocs,
)
run_torch_spawn(mixer2_gated_norm_tensor_parallel, 2)
def mixer2_gated_norm_tensor_parallel(
local_rank: int,
world_size: int,
batch_size: int,
seq_len: int,
hidden_size: int,
n_groups: int,
dtype: torch.dtype,
device: str,
):
set_random_seed(0)
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
}
)
# initialize distributed
init_distributed_environment()
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=world_size)
# create random weights an inputs
weight = torch.rand((hidden_size,), dtype=dtype, device=device)
hidden_states = torch.randn(batch_size, seq_len, hidden_size)
gate_states = torch.randn(batch_size, seq_len, hidden_size)
# create gated-norm with TP
with ensure_current_vllm_config():
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
mixer.weight.weight_loader(mixer.weight, weight) # load
# create gated-norm without TP to compute reference
# - utilize mock patching to disable TP when
with (
ensure_current_vllm_config(),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_world_size",
return_value=1,
),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_rank",
return_value=0,
),
):
mixer_single_gpu = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
# assign weight to single-gpu mixer
mixer_single_gpu.weight.data = weight
# generate and compare
N = hidden_size // world_size
output = mixer(
hidden_states[..., local_rank * N : (local_rank + 1) * N],
gate_states[..., local_rank * N : (local_rank + 1) * N],
)
ref_output = mixer_single_gpu(hidden_states, gate_states)
torch.testing.assert_close(
output,
ref_output[..., local_rank * N : (local_rank + 1) * N],
atol=5e-3,
rtol=1e-3,
)
+1 -1
View File
@@ -13,7 +13,7 @@ import pytest
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_conv
from vllm.models.qwen4_exp.common.ops.ple import ple_conv
from vllm.v1.attention.backends.ple_recoverssm import _PleConvCommit
C, KSIZE, DIL, SQ, NB = 512, 4, 3, 4, 8
+634
View File
@@ -0,0 +1,634 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
import importlib
import math
import pytest
import torch
# vLLM fused-expert reference (Triton fallback + DeepGEMM option)
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.activation import (
MoEActivation,
)
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
deepgemm_moe_permute,
ep_gather,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
_fp8_workspace_shape,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.utils.deep_gemm import (
calc_diff,
is_deep_gemm_supported,
per_block_cast_to_fp8,
)
BLOCK_SIZE = [128, 128]
def test_ep_gather_uses_64_bit_row_offsets():
hidden_size = 4096
source_row = 1 << 18
input_tensor = torch.empty(
(source_row + 1, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
input_tensor[source_row].fill_(1)
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
ep_gather(
input_tensor=input_tensor,
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
def test_ep_gather_uses_64_bit_output_row_offsets():
hidden_size = 6144
output_row = (1 << 31) // hidden_size + 1
num_rows = output_row + 1
output = torch.empty(
(num_rows, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
ep_gather(
input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16),
recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32),
input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
def test_fp8_workspace_shape(workspace_dtype, num_columns):
num_rows = 17
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
assert allocated_bytes >= required_bytes
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
activations, scales = per_token_group_quant_fp8(
hidden_states,
group_size=128,
use_ue8m0=True,
)
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
aq=activations,
aq_scale=scales,
topk_ids=topk_ids,
local_num_experts=2,
expert_map=None,
expert_tokens_meta=None,
)
padding = expert_ids < 0
assert padding.any()
torch.testing.assert_close(
permuted_scales[padding],
torch.zeros_like(permuted_scales[padding]),
rtol=0,
atol=0,
)
def make_block_quant_fp8_weights(
e: int,
n: int,
k: int,
block_size: list[int],
):
"""Generate (w1, w2) expert weights and their per-block scale tensors
in FP8 block-quantized format.
w1 shape: (E, 2N, K)
w2 shape: (E, K, N)
"""
dtype = torch.bfloat16
fp8_max, fp8_min = (
torch.finfo(torch.float8_e4m3fn).max,
torch.finfo(torch.float8_e4m3fn).min,
)
# bf16 reference weights
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) / 10
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) / 10
w1_bf16.clamp_(fp8_min, fp8_max)
w2_bf16.clamp_(fp8_min, fp8_max)
block_n, block_k = block_size
n_tiles_w1 = math.ceil((2 * n) / block_n)
k_tiles_w1 = math.ceil(k / block_k)
n_tiles_w2 = math.ceil(k / block_n)
k_tiles_w2 = math.ceil(n / block_k)
w1 = torch.empty_like(w1_bf16, dtype=torch.float8_e4m3fn)
w2 = torch.empty_like(w2_bf16, dtype=torch.float8_e4m3fn)
w1_s = torch.empty(e, n_tiles_w1, k_tiles_w1, device="cuda", dtype=torch.float32)
w2_s = torch.empty(e, n_tiles_w2, k_tiles_w2, device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_block_cast_to_fp8(
w1_bf16[i], block_size=block_size, use_ue8m0=True
)
w2[i], w2_s[i] = per_block_cast_to_fp8(
w2_bf16[i], block_size=block_size, use_ue8m0=True
)
return w1, w2, w1_s, w2_s
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
@pytest.mark.parametrize("with_metadata", [False, True])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
if not (
impl.current_platform.is_device_capability_family(90)
or (
impl.current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
)
):
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
torch.manual_seed(0)
e, n, k, topk = 8, 1024, 512, 2
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
shapes = experts.workspace_shapes(
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
)
def route(mode):
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
ids[:, 1] = (ids[:, 0] + 1) % e
if mode == 1:
ids.fill_(-1)
ids[:1, 0] = e - 1
elif mode == 2:
ids.fill_(-1)
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
route(0)
variants = []
for dense in (True, False):
ws1, ws2, output = [
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
]
def invoke(output=output, ws1=ws1, ws2=ws2):
experts.apply(
output,
aq,
w1,
w2,
weights,
ids,
MoEActivation.SILU,
e,
None,
scales,
None,
ws1,
ws2,
metadata,
False,
)
with monkeypatch.context() as mp:
if dense:
platform = Mock(wraps=impl.current_platform)
platform.is_device_capability_family.return_value = False
mp.setattr(impl, "current_platform", platform)
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
calls: list[int] = []
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
assert kwargs.get("use_psum_layout", False) == (not dense)
calls.append(args[3].numel())
return gemm(*args, **kwargs)
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
invoke()
assert len(calls) == 2
if not dense:
assert calls == [e, e]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
variants.append((graph, output, ws1, ws2))
for mode in (0, 1, 2, 0):
route(mode)
for graph, _, ws1, ws2 in variants:
ws1.fill_(float("nan"))
ws2.fill_(float("nan"))
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
assert torch.isfinite(variants[1][1]).all()
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
"""Both projection shapes skip unused tiles as counts change on replay."""
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
if not (
current_platform.is_device_capability_family(90)
or current_platform.is_device_capability_family(100)
):
pytest.skip("Requires SM90 or SM10.x")
capacity, experts, alignment = 512, 4, 128
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
def invoke():
with mk_alignment_scope(alignment):
m_grouped_fp8_gemm_nt_contiguous(
(aq, scales), (w, ws), output, ends, use_psum_layout=True
)
invoke()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
output.fill_(91)
graph.replay()
torch.accelerator.synchronize()
assert torch.all(output[allocated_end:] == 91)
if allocated_end:
assert torch.all(output[[0, 128]] != 91)
def run_single_case(m, n, k, topk, num_experts, block_size):
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
Triton baseline within tolerance.
"""
tokens_bf16 = (
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
.clamp_min_(-1)
.clamp_max_(1)
)
_, a1_scale = per_token_group_quant_fp8(tokens_bf16, block_size[1])
# expert weight tensors
w1, w2, w1_s, w2_s = make_block_quant_fp8_weights(num_experts, n, k, block_size)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
quant_config = fp8_w8a8_moe_quant_config(
w1_scale=w1_s,
w2_scale=w2_s,
a1_scale=a1_scale,
block_shape=block_size,
)
moe_config = make_dummy_moe_config()
deep_gemm_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=TritonOrDeepGemmExperts(
moe_config=moe_config,
quant_config=quant_config,
),
)
# triton reference
out_triton = fused_experts(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
quant_config=quant_config,
)
# DeepGemm
out_deepgemm = deep_gemm_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
diff = calc_diff(out_deepgemm, out_triton)
assert diff < 0.001, f"Diff exceeded 1%: {diff}"
# Note: N <= 512 will disable the deepgemm path due to performance issues.
MNKs = [
(1024, 768, 128),
(2048, 768, 512),
(512, 1024, 1024),
(4096, 4096, 1024),
]
TOPKS = [2, 6]
NUM_EXPERTS = [32]
@pytest.mark.parametrize(("m", "n", "k"), MNKs)
@pytest.mark.parametrize("topk", TOPKS)
@pytest.mark.parametrize("num_experts", NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_init):
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmExperts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmExperts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmExperts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmExperts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
block_size=BLOCK_SIZE,
)
# ensure that the DeepGEMM path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
# ---------------------------------------------------------------------------
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
# ---------------------------------------------------------------------------
def make_mxfp4_weights(
e: int,
n: int,
k: int,
):
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
w2 shape: (E, K, N//2) uint8 — packed FP4
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
w1_bf16: (E, 2N, K) — original BF16 for reference
w2_bf16: (E, K, N) — original BF16 for reference
"""
from deep_gemm.utils.math import per_token_cast_to_fp4
dtype = torch.bfloat16
gran_k = 32 # MXFP4 block size
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
# Quantize per-expert to FP4
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
w1_s = torch.empty(
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
)
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_token_cast_to_fp4(
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
w2[i], w2_s[i] = per_token_cast_to_fp4(
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
"""BF16 token-loop MoE reference for correctness testing."""
import torch.nn.functional as F
num_tokens, hidden_size = x.shape
intermediate = w1.shape[1] // 2
top_k = topk_ids.shape[1]
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
for t in range(num_tokens):
for kk in range(top_k):
e = topk_ids[t, kk].item()
w = topk_weights[t, kk].item()
fc1 = x[t : t + 1].float() @ w1[e].float().T
linear = fc1[:, :intermediate]
gate = fc1[:, intermediate:]
act = F.silu(gate) * linear
fc2 = act @ w2[e].float().T
output[t] += w * fc2[0]
return output.to(torch.bfloat16)
def run_single_fp4_case(m, n, k, topk, num_experts):
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
# FP4 expert weight tensors + BF16 originals for reference
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
)
from vllm.platforms import current_platform
_fp8_dtype = current_platform.fp8_dtype()
_block_shape = GroupShape(128, 128)
quant_config = FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
)
moe_config = make_dummy_moe_config()
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
DeepGemmFP4Experts,
)
deep_gemm_fp4_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=DeepGemmFP4Experts(
moe_config=moe_config,
quant_config=quant_config,
),
)
# DeepGEMM FP4 path
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# BF16 reference using the same original weights
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
diff = calc_diff(out_deepgemm_fp4, out_ref)
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
# FP4 quantization with block_k=32 needs large K for good accuracy.
FP4_MNKs = [
(128, 4096, 4096), # DeepSeek V4 shape
(256, 2048, 2048), # Half-size variant
(128, 384, 3584), # Kimi-K3 TP8 latent-MoE shape
]
FP4_TOPKS = [2]
FP4_NUM_EXPERTS = [8]
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
@pytest.mark.parametrize("topk", FP4_TOPKS)
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_fp4_vs_triton(
m, n, k, topk, num_experts, monkeypatch, workspace_init
):
pytest.importorskip("deep_gemm.utils.math")
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmFP4Experts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmFP4Experts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmFP4Experts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_fp4_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
)
# ensure that the DeepGEMM FP4 path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM FP4 path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
+658
View File
@@ -0,0 +1,658 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import per_block_cast_to_int8
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
NaiveBatchedExperts,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
TritonExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
BatchedPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
ref_nvfp4_quant,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import per_block_cast_to_fp8
from vllm.utils.math_utils import round_up
DEVICE = current_platform.device_type
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
shape = w.shape
n = shape[-1]
first = w[..., : n // 2]
second = w[..., n // 2 :]
stacked = torch.stack((first, second), dim=-1)
return stacked.reshape(shape)
def make_dummy_moe_config(
num_experts: int = 1,
num_local_experts: int | None = None,
experts_per_token: int = 1,
hidden_dim: int = 1,
intermediate_size: int = 1,
in_dtype: torch.dtype = torch.bfloat16,
max_num_tokens: int = 512,
activation: MoEActivation = MoEActivation.SILU,
) -> FusedMoEConfig:
"""This is a dummy config for the mk constructor interface
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
do not actually use this config.
CUTLASSFp8 needs to set some params for workshapes.
"""
return FusedMoEConfig(
num_experts=num_experts,
experts_per_token=experts_per_token,
hidden_dim=hidden_dim,
intermediate_size=intermediate_size,
num_local_experts=num_local_experts
if num_local_experts is not None
else num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=activation,
in_dtype=in_dtype,
device=DEVICE,
routing_method=RoutingMethodType.TopK,
max_num_tokens=max_num_tokens,
)
def triton_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant=False,
block_shape: list[int] | None = None,
) -> torch.Tensor:
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
return fused_experts(a, w1, w2, topk_weight, topk_ids, quant_config=quant_config)
def batched_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
moe_config: FusedMoEConfig | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
if moe_config is None:
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
),
BatchedTritonExperts(
max_num_tokens=max_num_tokens,
num_dispatchers=1,
quant_config=quant_config,
moe_config=moe_config,
),
)
return fused_experts.apply(
a,
w1,
w2,
topk_weight,
topk_ids,
global_num_experts=w1.shape[0],
activation=moe_config.activation,
apply_router_weight_on_input=False,
expert_map=None,
)
def naive_batched_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
a1_scale: torch.Tensor | None = None,
a2_scale: torch.Tensor | None = None,
quant_dtype: torch.dtype | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
) -> torch.Tensor:
max_num_tokens = round_up(a.shape[0], 64)
quant_config = FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
)
moe_config = make_dummy_moe_config()
fused_experts = FusedMoEKernel(
BatchedPrepareAndFinalize(
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
),
NaiveBatchedExperts(
max_num_tokens=max_num_tokens,
num_dispatchers=1,
quant_config=quant_config,
moe_config=moe_config,
),
)
return fused_experts.apply(
a,
w1,
w2,
topk_weight,
topk_ids,
global_num_experts=w1.shape[0],
activation=moe_config.activation,
apply_router_weight_on_input=False,
expert_map=None,
)
def chunk_scales(
scales: torch.Tensor | None, start: int, end: int
) -> torch.Tensor | None:
if scales is not None:
if scales.numel() == 1:
return scales
else:
return scales[start:end]
return None
def make_quantized_test_activations(
E: int,
m: int,
k: int,
in_dtype: torch.dtype,
quant_dtype: torch.dtype | None = None,
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
a_q = a
a_scale = None
if quant_dtype is not None:
assert quant_dtype == torch.float8_e4m3fn or quant_dtype == torch.int8, (
"only fp8/int8 supported"
)
a_q = torch.zeros_like(a, dtype=quant_dtype)
a_scale_l = [None] * E
for e in range(E):
a_q[e], a_scale_l[e] = moe_kernel_quantize_input(
a[e], None, quant_dtype, per_act_token_quant, block_shape
)
a_scale = torch.stack(a_scale_l)
if not per_act_token_quant and block_shape is None:
a_scale = a_scale.view(E, 1, 1)
return a, a_q, a_scale
def moe_quantize_weights_2d(
w: torch.Tensor,
w_s: torch.Tensor | None,
quant_dtype: torch.dtype | str | None,
per_token_quant: bool,
block_shape: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
is_fp8_dtype = is_fp8(quant_dtype)
assert is_fp8_dtype or quant_dtype == torch.int8 or quant_dtype == "nvfp4", (
"only fp8/int8/nvfp4 supported"
)
w_gs = None
if block_shape is not None:
assert not per_token_quant
if quant_dtype == torch.int8:
w, w_s = per_block_cast_to_int8(w, block_shape)
elif is_fp8_dtype:
w, w_s = per_block_cast_to_fp8(w, block_shape)
elif quant_dtype == "nvfp4":
raise RuntimeError("blocked quantization not supported for nvfp4")
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
else:
if quant_dtype == torch.int8:
w, w_s = ops.scaled_int8_quant(
w, w_s, use_per_token_if_dynamic=per_token_quant
)
elif is_fp8_dtype:
w, w_s = ops.scaled_fp8_quant(
w, w_s, use_per_token_if_dynamic=per_token_quant
)
elif quant_dtype == "nvfp4":
assert not per_token_quant
w_amax = torch.abs(w).max().to(torch.float32)
w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax
if current_platform.is_rocm():
w, w_s = _scaled_fp4_quant_emulated(w, w_gs)
else:
w, w_s = ops.scaled_fp4_quant(w, w_gs)
else:
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
return w, w_s, w_gs
def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor:
assert fp4_values.shape[-1] % 2 == 0
abs_values = fp4_values.abs()
codes = torch.empty_like(abs_values, dtype=torch.uint8)
for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)):
codes[abs_values == value] = code
codes = codes | ((fp4_values < 0).to(torch.uint8) << 3)
return codes[..., 0::2] | (codes[..., 1::2] << 4)
def _scaled_fp4_quant_emulated(
w: torch.Tensor, w_gs: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16)
return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn)
def moe_quantize_weights(
w: torch.Tensor,
w_s: torch.Tensor | None,
quant_dtype: torch.dtype | str | None,
per_token_quant: bool,
block_shape: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
assert w.dim() == 3
e, rows, cols = w.shape
w_l = [None] * e
w_s_l = [None] * e
w_gs_l = [None] * e
for idx in range(e):
w_l[idx], w_s_l[idx], w_gs_l[idx] = moe_quantize_weights_2d(
w[idx], None, quant_dtype, per_token_quant, block_shape
)
w = torch.stack(w_l)
w_s = torch.stack(w_s_l)
w_gs = torch.stack(w_gs_l) if e > 0 and w_gs_l[0] is not None else None
if w_s.ndim == 2:
assert w_s.shape[-1] == 1
w_s = w_s.view(-1, 1, 1)
if block_shape is not None:
block_n, block_k = block_shape
n_tiles = (rows + block_n - 1) // block_n
k_tiles = (cols + block_k - 1) // block_k
assert w_s.shape == (e, n_tiles, k_tiles)
return w, w_s, w_gs
def make_test_weight(
e: int,
rows: int,
cols: int,
in_dtype: torch.dtype = torch.bfloat16,
quant_dtype: torch.dtype | str | None = None,
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
if quant_dtype is not None:
w, w_s, w_gs = moe_quantize_weights(
w_16, None, quant_dtype, per_out_ch_quant, block_shape
)
else:
w = w_16
w_s = None
w_gs = None
return w_16, w, w_s, w_gs
def make_test_weights(
e: int,
n: int,
k: int,
in_dtype: torch.dtype = torch.bfloat16,
quant_dtype: torch.dtype | str | None = None,
block_shape: list[int] | None = None,
per_out_ch_quant: bool = False,
make_gate: bool = True,
) -> tuple[
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
]:
return (
make_test_weight(
e,
(2 if make_gate else 1) * n,
k,
in_dtype,
quant_dtype,
block_shape,
per_out_ch_quant,
),
make_test_weight(e, k, n, in_dtype, quant_dtype, block_shape, per_out_ch_quant),
)
def per_token_cast_to_fp8(
x: torch.Tensor, block_size: int = 128
) -> tuple[torch.Tensor, torch.Tensor]:
assert x.dim() == 2
m, n = x.shape
pad_size = (block_size - (n % block_size)) % block_size
x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x
x_view = x.view(m, -1, block_size)
x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn)
return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1)
def make_test_quant_config(
e: int,
n: int,
k: int,
in_dtype: torch.dtype,
quant_dtype: torch.dtype | str | None = None,
per_act_token_quant: bool = False,
block_shape: list[int] | None = None,
make_gate: bool = True,
is_scale_swizzled: bool = True,
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
(_, w1, w1_s, w1_gs), (_, w2, w2_s, w2_gs) = make_test_weights(
e,
n,
k,
in_dtype,
quant_dtype,
per_out_ch_quant=per_act_token_quant,
block_shape=block_shape,
make_gate=make_gate,
)
# Hacky/trivial scales for nvfp4.
a1_gscale: torch.Tensor | None = None
a2_gscale: torch.Tensor | None = None
if quant_dtype == "nvfp4":
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
a1_scale = a1_gscale
a2_scale = a2_gscale
else:
a1_scale = None
a2_scale = None
return (
w1,
w2,
FusedMoEQuantConfig.make(
quant_dtype,
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
w1_scale=w1_s,
w2_scale=w2_s,
a1_gscale=a1_gscale,
a2_gscale=a2_gscale,
a1_scale=a1_scale,
a2_scale=a2_scale,
# TODO: make sure this is handled properly
g1_alphas=(1 / w1_gs) if w1_gs is not None else None,
g2_alphas=(1 / w2_gs) if w2_gs is not None else None,
is_scale_swizzled=is_scale_swizzled,
),
)
def fused_moe(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
score: torch.Tensor,
topk: int,
renormalize: bool = False,
quant_config: FusedMoEQuantConfig | None = None,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
) -> torch.Tensor:
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score.float(), topk, renormalize
)
return fused_experts(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=quant_config,
)
class BaselineMM(torch.nn.Module):
def __init__(
self,
b: torch.Tensor,
out_dtype: torch.dtype,
):
super().__init__()
self.b = torch.nn.Parameter(b.to(dtype=torch.float32))
self.out_dtype = out_dtype
def forward(self, a: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
return torch.mm(a.to(dtype=torch.float32), self.b).to(self.out_dtype), None
class BaselineSiluAndMul(torch.nn.Module):
def __init__(self):
super().__init__()
def forward(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
return torch.nn.functional.silu(x[..., :d]) * x[..., d:]
class TestMLP(torch.nn.Module):
def __init__(
self,
w1: torch.Tensor,
w2: torch.Tensor,
out_dtype: torch.dtype,
):
super().__init__()
self.gate_up_proj = BaselineMM(w1, out_dtype)
self.down_proj = BaselineMM(w2, out_dtype)
self.act_fn = BaselineSiluAndMul()
def forward(self, x):
x, _ = self.gate_up_proj(x)
x = self.act_fn(x)
x, _ = self.down_proj(x)
return x
def modular_triton_fused_moe(
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> FusedMoEKernel:
return FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
TritonExperts(moe_config, quant_config),
)
def check_accuracy(a, b, atol, rtol, percent):
"""Allow a mismatch percentage of 1 - percent."""
if torch.any(torch.isnan(a)):
raise Exception("NaN in reference output")
if torch.any(torch.isnan(b)):
raise Exception("NaN in actual output")
if torch.any(torch.isinf(a)):
raise Exception("Inf in reference output")
if torch.any(torch.isinf(b)):
raise Exception("Inf in actual output")
assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}"
left = torch.abs(a - b)
right = atol + rtol * torch.abs(b)
count = torch.sum(left > right)
mismatch_percent = count / a.numel()
if mismatch_percent > 1 - percent:
raise Exception(
f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} "
f"(threshold: {1 - percent:.4f})"
)
def check_deferred_moe_finalize(
moe_config: FusedMoEConfig,
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
router_weights: torch.Tensor | None = None,
chunked: bool = False,
) -> None:
"""Check a kernel that defers its finalize against the finalize it skips.
``run`` calls the kernel on fixed inputs, first as built and then with
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
report truthfully. A deferred output reduced by ``moe_unpermute``, the
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
bit, and modular experts hand the router's weights back as-is. A call the
experts split across kernel launches finalizes instead.
"""
# The deferred output views the router's buffer, so compare with a copy.
expected_weights = None if router_weights is None else router_weights.clone()
finalized = run()
assert isinstance(finalized, torch.Tensor)
moe_config.defer_moe_finalize()
output = run()
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
if chunked:
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
return
assert isinstance(output, UnfinalizedMoEOutput)
if expected_weights is not None:
torch.testing.assert_close(
output.expert_weights, expected_weights, atol=0, rtol=0
)
reference = torch.empty_like(finalized)
moe_unpermute(
reference,
output.gemm2_permuted,
output.expert_weights.float(),
output.expanded_idx_to_permuted_idx,
# The kernel reads its valid-row count through this pointer either way.
expert_first_token_offset=output.gemm2_permuted.new_full(
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
),
)
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
triton_kernels 3.8 returns layout instances; earlier versions return a
(layout, opts) tuple.
"""
from triton_kernels.tensor_details import layout
from vllm.utils.import_utils import get_triton_kernels_version
if get_triton_kernels_version() == "3.8":
w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
s = layout.make_default_matmul_mxfp4_w_scale_layout(
mx_axis=mx_axis, num_warps=num_warps
)
return w, {}, s, {}
w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout(
mx_axis=mx_axis, num_warps=num_warps
)
return w, w_opts, s, s_opts
+290
View File
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
group_broadcast,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
DEVICE = current_platform.device_type
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
return torch.as_tensor(x, dtype=torch.float32, device=DEVICE)
def ref_dynamic_per_token_quant(
x: torch.Tensor, quant_dtype: torch.dtype, scale_ub: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
assert quant_dtype in [torch.int8, FP8_DTYPE]
if scale_ub is not None:
assert quant_dtype == FP8_DTYPE
if quant_dtype == torch.int8:
qtype_traits = torch.iinfo(quant_dtype)
qtype_traits_min = qtype_traits.min
qtype_traits_max = qtype_traits.max
else:
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
qtype_max = as_float32_tensor(qtype_traits_max)
s_1 = as_float32_tensor(1.0)
s_512 = as_float32_tensor(512.0)
# For fp8, in order to match the cuda kernel output, we have to do exactly
# the same operations as in the corresponding fp8 kernel to prevent
# rounding errors.
# Compute scales
x_token_max, _ = x.abs().max(dim=-1)
x_token_max = as_float32_tensor(x_token_max)
if scale_ub is not None:
x_token_max = x_token_max.clamp(max=scale_ub)
scales = (x_token_max / qtype_max)[:, None]
# Quant
if quant_dtype == torch.int8:
iscales = as_float32_tensor(s_1 / scales)
torch_out = as_float32_tensor(x) * iscales
torch_out = torch_out.round()
torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype)
else:
assert quant_dtype == FP8_DTYPE
min_scaling_factor = s_1 / (qtype_max * s_512)
scales = scales.clamp(min=min_scaling_factor)
torch_out = as_float32_tensor(x) / scales
torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype)
return torch_out, scales
# The int8 version is very similar. Incorporate the int8 version, like in
# ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant
# kernel
def ref_dynamic_per_tensor_fp8_quant(
x: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
fp8_traits_min, fp8_traits_max = get_fp8_min_max()
fp8_max = as_float32_tensor(fp8_traits_max)
one = as_float32_tensor(1.0)
# For fp8, in order to match the cuda kernel output, we have to do exactly
# the same operations as in the corresponding fp8 kernel to prevent
# rounding errors.
x_max = as_float32_tensor(x.abs().max())
ref_scale = x_max / fp8_max
ref_iscale = one / ref_scale
ref_out = (
(as_float32_tensor(x) * ref_iscale)
.clamp(fp8_traits_min, fp8_traits_max)
.to(FP8_DTYPE)
)
return ref_out, ref_scale.view(1)
def native_w8a8_block_matmul(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
block_size: list[int],
output_dtype: torch.dtype,
compute_type: torch.dtype = torch.float32,
) -> torch.Tensor:
"""This function performs matrix multiplication with block-wise
quantization using native torch.
It is agnostic to the input data type and can be used for both int8 and
fp8 data types.
It takes two input tensors `A` and `B` (int8) with scales `As` and
`Bs` (float32).
The output is returned in the specified `output_dtype`.
"""
A = A.to(compute_type)
B = B.to(compute_type)
assert A.shape[-1] == B.shape[-1]
assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
assert len(block_size) == 2
block_n, block_k = block_size[0], block_size[1]
assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1]
assert A.shape[:-1] == As.shape[:-1]
M = A.numel() // A.shape[-1]
N, K = B.shape
origin_C_shape = A.shape[:-1] + (N,)
A = A.reshape(M, A.shape[-1])
As = As.reshape(M, As.shape[-1])
n_tiles = (N + block_n - 1) // block_n
k_tiles = (K + block_k - 1) // block_k
assert n_tiles == Bs.shape[0], f"{n_tiles} == {Bs.shape[0]}"
assert k_tiles == Bs.shape[1], f"{k_tiles} == {Bs.shape[1]}"
C_shape = (M, N)
C = torch.zeros(C_shape, dtype=compute_type, device=A.device)
A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)]
B_tiles = [
[
B[
j * block_n : min((j + 1) * block_n, N),
i * block_k : min((i + 1) * block_k, K),
]
for i in range(k_tiles)
]
for j in range(n_tiles)
]
C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)]
As_tiles = [As[:, i : i + 1] for i in range(k_tiles)]
for i in range(k_tiles):
for j in range(n_tiles):
a = A_tiles[i]
b = B_tiles[j][i]
c = C_tiles[j]
s = As_tiles[i] * Bs[j][i]
c[:, :] += torch.matmul(a, b.t()) * s
C = C.reshape(origin_C_shape).to(output_dtype)
return C
def native_per_token_group_quant_fp8(
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch."""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
)
assert x.is_contiguous(), "`x` is not contiguous"
finfo = torch.finfo(dtype)
fp8_min = finfo.min
fp8_max = finfo.max
x_ = x.reshape(x.numel() // group_size, group_size)
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
x_s = amax / fp8_max
if is_deep_gemm_e8m0_used():
x_s = _ceil_to_ue8m0(x_s)
x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype)
x_q = x_q.reshape(x.shape)
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
return x_q, x_s
def native_per_token_group_quant_int8(
x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch.
It converts the tensor values into int8 values and returns the
quantized tensor along with the scaling factor used for quantization.
Per-token quantization rounds to nearest; block quantization truncates.
"""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
)
assert x.is_contiguous(), "`x` is not contiguous"
iinfo = torch.iinfo(dtype)
int8_min = iinfo.min
int8_max = iinfo.max
x_ = x.reshape(x.numel() // group_size, group_size)
# Use float32 for scale calculation for stability
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
x_s = amax / int8_max
x_q = x_.to(torch.float32) / x_s
if round_to_nearest:
x_q = x_q.round()
x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype)
x_q = x_q.reshape(x.shape)
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
return x_q, x_s
DEFAULT_BLOCK_SHAPE = [128, 128]
def per_block_cast_to_int8(
x: torch.Tensor,
block_shape: list[int] = DEFAULT_BLOCK_SHAPE,
) -> tuple[torch.Tensor, torch.Tensor]:
block_m, block_n = block_shape
assert x.dim() == 2
m, n = x.shape
x_padded = torch.zeros(
(round_up(m, block_m), round_up(n, block_n)), dtype=x.dtype, device=x.device
)
x_padded[:m, :n] = x
x_view = x_padded.view(-1, block_m, x_padded.size(1) // block_n, block_n)
x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
x_scaled = (x_view * (256.0 / x_amax)).to(torch.int8)
x_scaled_sub = x_scaled.view_as(x_padded)[:m, :n].contiguous()
scales = (x_amax / 256.0).view(x_view.size(0), x_view.size(2))
return x_scaled_sub, scales
def dequant(
t: torch.Tensor,
scale: torch.Tensor | None,
block_shape: list[int] | None,
per_act_token_quant: bool,
out_dtype: torch.dtype | None = torch.float32,
) -> torch.Tensor:
if scale is not None:
f32 = torch.float32
if per_act_token_quant or block_shape is None:
return (t.to(f32) * scale).to(out_dtype)
else:
return (t.to(f32) * group_broadcast(scale, t.shape)).to(out_dtype)
else:
return t.to(out_dtype)
def native_batched_masked_quant_matmul(
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
num_expert_tokens: torch.Tensor,
A_scale: torch.Tensor | None = None,
B_scale: torch.Tensor | None = None,
block_shape: list[int] | None = None,
per_act_token_quant: bool = False,
) -> torch.Tensor:
num_expert_tokens_cpu = num_expert_tokens.clone()
num_expert_tokens_cpu = num_expert_tokens_cpu.to(device="cpu")
num_experts = num_expert_tokens.size(0)
for e in range(num_experts):
num_tokens = num_expert_tokens_cpu[e]
if A.dtype.itemsize == 1 and block_shape is not None:
assert A_scale is not None and B_scale is not None
tmp = native_w8a8_block_matmul(
A[e], B[e], A_scale[e], B_scale[e], block_shape, C.dtype
)
C[e, :num_tokens, :] = tmp[:num_tokens, :]
elif A.dtype.itemsize == 1 and block_shape is None:
assert A_scale is not None and B_scale is not None
A_dq = dequant(A[e], A_scale[e], block_shape, per_act_token_quant)
B_dq = dequant(B[e], B_scale[e], block_shape, per_act_token_quant)
C[e, :num_tokens, :] = (A_dq[:num_tokens] @ B_dq.transpose(0, 1)).to(
C.dtype
)
else:
assert A_scale is None
assert B_scale is None
C[e, :num_tokens, :] = A[e, :num_tokens, :] @ B[e].transpose(0, 1)
return C
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
import torch
from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
from safetensors.torch import load_file, save_file
from vllm.engine.arg_utils import EngineArgs
from vllm.transformers_utils.repo_utils import hf_api
from .conftest import RunaiDummyExecutor
load_format = "runai_streamer"
test_model = "openai-community/gpt2"
def test_runai_model_loader_download_files_s3_mocked_with_patch(
vllm_runner,
tmp_path: Path,
monkeypatch,
):
patcher = StreamerPatcher(str(tmp_path))
test_mock_s3_model = "s3://my-mock-bucket/gpt2/"
# Download model from HF
mock_model_dir = f"{tmp_path}/gpt2"
hf_api().snapshot_download(repo_id=test_model, local_dir=mock_model_dir)
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
patcher.shim_list_safetensors,
)
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_pull_files",
patcher.shim_pull_files,
)
monkeypatch.setattr(
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
patcher.create_mock_streamer,
)
engine_args = EngineArgs(
model=test_mock_s3_model,
load_format=load_format,
tensor_parallel_size=1,
)
vllm_config = engine_args.create_engine_config()
executor = RunaiDummyExecutor(vllm_config)
executor.driver_worker.load_model()
def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch):
# An engine started from object storage keeps the bucket URI in
# model_config.model_weights. A reload with weights_path must stream the
# new checkpoint, not re-stream the one the engine started from.
patcher = StreamerPatcher(str(tmp_path))
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
patcher.shim_list_safetensors,
)
monkeypatch.setattr(
"vllm.transformers_utils.runai_utils.runai_pull_files",
patcher.shim_pull_files,
)
monkeypatch.setattr(
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
patcher.create_mock_streamer,
)
base_dir = tmp_path / "gpt2"
hf_api().snapshot_download(
repo_id=test_model,
local_dir=str(base_dir),
allow_patterns=["*.json", "*.txt", "model.safetensors"],
)
# Second checkpoint under another prefix: the final layer norm scaled, so a
# reload that silently re-streams the original bucket is distinguishable.
tensors = load_file(base_dir / "model.safetensors")
original = tensors["ln_f.weight"].clone()
modified = original * 2
tensors["ln_f.weight"] = modified
mod_dir = tmp_path / "gpt2-mod"
mod_dir.mkdir()
save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"})
vllm_config = EngineArgs(
model="s3://my-mock-bucket/gpt2/",
load_format=load_format,
tensor_parallel_size=1,
).create_engine_config()
assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/"
executor = RunaiDummyExecutor(vllm_config)
executor.driver_worker.load_model()
model = executor.driver_worker.model_runner.get_model()
(param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")]
def current() -> torch.Tensor:
return param.detach().float().cpu()
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
# No weights_path: re-stream the checkpoint the engine started from, i.e.
# the bucket kept in model_weights. Zero the live tensor first so a reload
# that did nothing is distinguishable from one that streamed the bucket.
param.data.zero_()
executor.driver_worker.reload_weights()
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
# weights_path: the new prefix must win over the stale model_weights.
executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/")
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
# After the override, a bare reload means the new checkpoint, not the
# one the engine was started with.
param.data.zero_()
executor.driver_worker.reload_weights()
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
@@ -1,18 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import GenerateModelInfo
from .ppl_utils import wikitext_ppl_test
MODELS = [
GenerateModelInfo("google/gemma-2b", hf_ppl=21.48524284362793),
GenerateModelInfo("google/gemma-2-2b", hf_ppl=102.59290313720703),
GenerateModelInfo("google/gemma-3-4b-it", hf_ppl=27.79648208618164),
]
@pytest.mark.parametrize("model_info", MODELS)
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
wikitext_ppl_test(hf_runner, vllm_runner, model_info)
@@ -1,24 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import GenerateModelInfo
from .ppl_utils import wikitext_ppl_test
MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.457056045532227)]
@pytest.mark.parametrize("model_info", MODELS)
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
bf16_ppl = wikitext_ppl_test(hf_runner, vllm_runner, model_info)
fp32_ppl = wikitext_ppl_test(
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"hf_overrides": {"head_dtype": "float32"}},
)
differ = ((fp32_ppl - bf16_ppl) / bf16_ppl) * 100
print("fp32 head difference (%):", differ)
assert differ < 0
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
import openai
import pytest
from tests.conftest import HfRunner
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
embeddings = []
for d in response.data:
assert isinstance(d.embedding, list)
embeddings.append(d.embedding)
return embeddings
def run_embedding_correctness_test(
hf_model: "HfRunner",
inputs: list[str],
vllm_outputs: Sequence[list[float]],
dimensions: int | None = None,
):
hf_outputs = hf_model.encode(inputs)
if dimensions:
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
def correctness_test_embed_models(
hf_runner,
vllm_runner,
model_info: EmbedModelInfo,
example_prompts,
vllm_extra_kwargs=None,
hf_model_callback=None,
):
pytest.skip("Debug only, ci prefers to use mteb test.")
# The example_prompts has ending "\n", for example:
# "Write a short story about a robot that dreams for the first time.\n"
# sentence_transformers will strip the input texts, see:
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
# This makes the input_ids different between hf_model and vllm_model.
# So we need to strip the input texts to avoid test failing.
example_prompts = [str(s).strip() for s in example_prompts]
vllm_extra_kwargs = vllm_extra_kwargs or {}
vllm_extra_kwargs["dtype"] = model_info.dtype
if model_info.hf_overrides is not None:
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
with vllm_runner(
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
) as vllm_model:
vllm_outputs = vllm_model.embed(example_prompts)
with hf_runner(
model_info.name,
dtype=model_info.hf_dtype,
is_sentence_transformer=True,
) as hf_model:
if hf_model_callback is not None:
hf_model_callback(hf_model)
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
async def run_client_embeddings(
client: openai.AsyncOpenAI,
model_name: str,
queries: list[str],
instruction: str = "",
) -> list[list[float]]:
outputs = await client.embeddings.create(
model=model_name,
input=[instruction + q for q in queries],
)
return [data.embedding for data in outputs.data]
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pytest configuration for vLLM pooling MTEB tests."""
import gc
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
@pytest.fixture(autouse=True)
def release_gpu_memory_between_tests():
"""Reclaim cached GPU memory between tests in this shard.
These tests start many vLLM/HF engines sequentially in one pytest
process; without explicit reclamation between tests, VRAM fragmentation
accumulates and later tests fail at engine startup ("Engine core
initialization failed").
"""
yield
gc.collect()
if torch.accelerator.is_available():
torch.accelerator.empty_cache()
if current_platform.is_rocm():
return
try:
wait_for_memory_to_settle()
except ValueError as e:
# Longer-lived (class/module-scoped) engine fixtures may legitimately
# still hold VRAM; nothing further to reclaim in that case.
logger.info("Failed to clean GPU memory: %s", e)
@@ -1,279 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import mteb
import numpy as np
import torch
from mteb.models import ModelMeta
from mteb.types import Array
from torch.utils.data import DataLoader
import tests.ci_envs as ci_envs
from tests.models.utils import (
EmbedModelInfo,
check_embeddings_close,
get_vllm_extra_kwargs,
)
# Most embedding models on the STS12 task (See #17175):
# - Model implementation and minor changes in tensor dtype
# results in differences less than 1e-4
# - Different model results in differences more than 1e-3
# 5e-4 is a good tolerance threshold
MTEB_EMBED_TASKS = ["STS12"]
MTEB_EMBED_TOL = 5e-4
_empty_model_meta = ModelMeta(
loader=None,
name="vllm/model",
revision="1",
release_date=None,
languages=None,
framework=[],
similarity_fn_name=None,
n_parameters=None,
memory_usage_mb=None,
max_tokens=None,
embed_dim=None,
license=None,
open_weights=None,
public_training_code=None,
public_training_data=None,
use_instructions=None,
training_datasets=None,
modalities=["text"], # 'image' can be added to evaluate multimodal models
)
class MtebEmbedMixin(mteb.EncoderProtocol):
mteb_model_meta = _empty_model_meta
def similarity(
self,
embeddings1: np.ndarray,
embeddings2: np.ndarray,
) -> np.ndarray:
# Cosine similarity
norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
sim = np.dot(embeddings1, embeddings2.T) / (norm1 * norm2.T)
return sim
def similarity_pairwise(
self,
embeddings1: Array,
embeddings2: Array,
) -> Array:
# Cosine similarity
norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
sim = np.sum(embeddings1 * embeddings2, axis=1) / (
norm1.flatten() * norm2.flatten()
)
return sim
class HfMtebEncoder(MtebEmbedMixin):
def __init__(self, model):
self.model = model
def encode(
self,
inputs: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
sentences = [text for batch in inputs for text in batch["text"]]
return self.model.encode(sentences)
class VllmMtebEncoder(MtebEmbedMixin):
def __init__(self, vllm_model, prompt_prefix: str | None = None):
self.llm = vllm_model
self.rng = np.random.default_rng(seed=42)
self.prompt_prefix = prompt_prefix
def encode(
self,
inputs: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
# Hoping to discover potential scheduling
# issues by randomizing the order.
sentences = [
self.prompt_prefix + text if self.prompt_prefix else text
for batch in inputs
for text in batch["text"]
]
r = self.rng.permutation(len(sentences))
sentences = [sentences[i] for i in r]
outputs = self.llm.embed(sentences, use_tqdm=False)
embeds = np.array(outputs)
embeds = embeds[np.argsort(r)]
return embeds
class OpenAIClientMtebEncoder(MtebEmbedMixin):
def __init__(self, model_name: str, client):
self.model_name = model_name
self.client = client
self.rng = np.random.default_rng(seed=42)
def encode(
self,
inputs: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
# Hoping to discover potential scheduling
# issues by randomizing the order.
sentences = [text for batch in inputs for text in batch["text"]]
r = self.rng.permutation(len(sentences))
sentences = [sentences[i] for i in r]
embeddings = self.client.embeddings.create(
model=self.model_name, input=sentences
)
outputs = [d.embedding for d in embeddings.data]
embeds = np.array(outputs)
embeds = embeds[np.argsort(r)]
return embeds
def run_mteb_embed_task(encoder: mteb.EncoderProtocol, tasks):
tasks = mteb.get_tasks(tasks=tasks)
results = mteb.evaluate(
encoder,
tasks,
cache=None,
show_progress_bar=False,
)
main_score = results[0].scores["test"][0]["main_score"]
return main_score
def mteb_test_embed_models(
hf_runner,
vllm_runner,
model_info: EmbedModelInfo,
vllm_extra_kwargs=None,
hf_model_callback=None,
atol: float | None = None,
prompt_prefix: str | None = None,
vllm_model_callback=None,
):
if atol is None:
atol = (
model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_EMBED_TOL
)
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
# Test embed_dims, isnan and whether to use normalize
example_prompts = ["The chef prepared a delicious meal." * 1000]
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=model_info.max_model_len,
**vllm_extra_kwargs,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
if vllm_model_callback is not None:
vllm_model_callback(vllm_model)
# Confirm whether vllm is using the correct architecture
if model_info.architecture:
assert model_info.architecture in model_config.architectures
# Confirm whether the important configs in model_config are correct.
pooler_config = model_config.pooler_config
if model_info.seq_pooling_type is not None:
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
if model_info.tok_pooling_type is not None:
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
if model_info.attn_type is not None:
assert model_config.attn_type == model_info.attn_type
if model_info.is_prefix_caching_supported is not None:
assert (
model_config.is_prefix_caching_supported
== model_info.is_prefix_caching_supported
)
if model_info.is_chunked_prefill_supported is not None:
assert (
model_config.is_chunked_prefill_supported
== model_info.is_chunked_prefill_supported
)
vllm_main_score = run_mteb_embed_task(
VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS
)
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
head_dtype = model_config.head_dtype
# Test embedding_size, isnan and whether to use normalize.
# Apply the same prompt_prefix used for scoring so this check compares
# identical effective inputs: HF's SentenceTransformer.encode() applies
# the model's default prompt (e.g. "Document: "), so vLLM must receive it
# too. This mirrors VllmMtebEncoder and changes no production behavior.
consistency_prompts = (
[prompt_prefix + p for p in example_prompts]
if prompt_prefix
else example_prompts
)
vllm_outputs = vllm_model.embed(
consistency_prompts,
tokenization_kwargs=dict(truncate_prompt_tokens=-1),
)
outputs_tensor = torch.tensor(vllm_outputs)
assert not torch.any(torch.isnan(outputs_tensor))
embedding_size = model_config.embedding_size
assert torch.tensor(vllm_outputs).shape[-1] == embedding_size
# Accelerate mteb test by setting
# SentenceTransformers mteb score to a constant
if model_info.mteb_score is None:
with hf_runner(
model_info.name,
revision=model_info.revision,
is_sentence_transformer=True,
dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype,
) as hf_model:
# e.g. setting default parameters for the encode method of hf_runner
if hf_model_callback is not None:
hf_model_callback(hf_model)
st_main_score = run_mteb_embed_task(
HfMtebEncoder(hf_model), MTEB_EMBED_TASKS
)
st_dtype = next(hf_model.model.parameters()).dtype
# Check embeddings close to hf outputs
hf_outputs = hf_model.encode(example_prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
else:
st_main_score = model_info.mteb_score
st_dtype = "Constant"
print("Model:", model_info.name)
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
print("SentenceTransformers:", st_dtype, st_main_score)
print("Difference:", st_main_score - vllm_main_score)
# We are not concerned that the vllm mteb results are better
# than SentenceTransformers, so we only perform one-sided testing.
diff = st_main_score - vllm_main_score
assert diff < atol, (
f"diff={diff:.6g} tol={atol} model={model_info.name} "
f"(st={st_main_score}, vllm={vllm_main_score})"
)
@@ -1,319 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from pathlib import Path
from typing import Any
import mteb
import numpy as np
import requests
import torch
from mteb.models import ModelMeta
from torch.utils.data import DataLoader
from tests.conftest import HfRunner
from tests.models.utils import (
RerankModelInfo,
get_vllm_extra_kwargs,
)
# See #19344
MTEB_RERANK_TASKS = ["NFCorpus"]
MTEB_RERANK_LANGS = ["eng"]
MTEB_RERANK_TOL = 2e-3
template_home = (
Path(__file__).parent.parent.parent.parent.parent
/ "examples/pooling/score/template"
)
_empty_model_meta = ModelMeta(
loader=None,
name="vllm/model",
revision="1",
release_date=None,
languages=None,
framework=[],
similarity_fn_name=None,
n_parameters=None,
memory_usage_mb=None,
max_tokens=None,
embed_dim=None,
license=None,
open_weights=None,
public_training_code=None,
public_training_data=None,
use_instructions=None,
training_datasets=None,
modalities=["text"], # 'image' can be added to evaluate multimodal models
)
class MtebCrossEncoderMixin(mteb.CrossEncoderProtocol):
mteb_model_meta = _empty_model_meta
class VllmMtebCrossEncoder(MtebCrossEncoderMixin):
def __init__(self, vllm_model):
self.llm = vllm_model
self.rng = np.random.default_rng(seed=42)
self.chat_template: str | None = getattr(vllm_model, "chat_template", None)
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
corpus = [text for batch in inputs2 for text in batch["text"]]
# Hoping to discover potential scheduling
# issues by randomizing the order.
r = self.rng.permutation(len(queries))
queries = [queries[i] for i in r]
corpus = [corpus[i] for i in r]
outputs = self.llm.score(
queries,
corpus,
use_tqdm=False,
chat_template=self.chat_template,
tokenization_kwargs={"truncate_prompt_tokens": -1},
)
scores = np.array(outputs)
scores = scores[np.argsort(r)]
return scores
class ScoreClientMtebEncoder(MtebCrossEncoderMixin):
mteb_model_meta = _empty_model_meta
def __init__(self, model_name: str, url):
self.model_name = model_name
self.url = url
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
full_corpus = [text for batch in inputs2 for text in batch["text"]]
outputs = []
for query, corpus in zip(queries, full_corpus):
outputs.append(self.get_score(query, corpus))
scores = np.array(outputs)
return scores
def get_score(self, query, corpus):
response = requests.post(
self.url,
json={
"model": self.model_name,
"queries": query,
"documents": corpus,
"truncate_prompt_tokens": -1,
},
).json()
return response["data"][0]["score"]
class RerankClientMtebEncoder(ScoreClientMtebEncoder):
def get_score(self, query, corpus):
response = requests.post(
self.url,
json={
"model": self.model_name,
"query": query,
"documents": [corpus],
"truncate_prompt_tokens": -1,
},
).json()
return response["results"][0]["relevance_score"]
class HFMtebCrossEncoder(MtebCrossEncoderMixin, HfRunner):
chat_template: str | None = None
def __init__(self, model_name: str, dtype: str = "auto", **kwargs: Any) -> None:
HfRunner.__init__(
self, model_name=model_name, is_cross_encoder=True, dtype=dtype, **kwargs
)
@torch.no_grad
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
corpus = [text for batch in inputs2 for text in batch["text"]]
if self.chat_template is not None:
tokenizer = self.model.tokenizer
prompts = []
for query, document in zip(queries, corpus):
conversation = [
{"role": "query", "content": query},
{"role": "document", "content": document},
]
prompt = tokenizer.apply_chat_template(
conversation=conversation,
tools=None,
chat_template=self.chat_template,
tokenize=False,
)
prompts.append(prompt)
outputs_list = HfRunner.classify(self, prompts)
scores = np.array(outputs_list).squeeze(-1)
return scores
else:
prompts = list(zip(queries, corpus))
outputs_tensor = HfRunner.predict(self, prompts, show_progress_bar=False)
return outputs_tensor.cpu().numpy()
def run_mteb_rerank(cross_encoder: mteb.CrossEncoderProtocol, tasks, languages):
with tempfile.TemporaryDirectory() as prediction_folder:
bm25s = mteb.get_model("bm25s")
eval_splits = ["test"]
mteb_tasks: list[mteb.abstasks.AbsTaskRetrieval] = mteb.get_tasks(
tasks=tasks, languages=languages, eval_splits=eval_splits
)
for task in mteb_tasks:
if not task.data_loaded:
task.load_data()
mteb.evaluate(
bm25s,
mteb_tasks,
prediction_folder=prediction_folder,
show_progress_bar=False,
# don't save results for test runs
cache=None,
overwrite_strategy="always",
)
second_stage_tasks = []
for task in mteb_tasks:
second_stage_tasks.append(
task.convert_to_reranking(
prediction_folder,
top_k=10,
)
)
results = mteb.evaluate(
cross_encoder,
second_stage_tasks,
show_progress_bar=False,
cache=None,
)
main_score = results[0].scores["test"][0]["main_score"]
return main_score
def mteb_test_rerank_models(
vllm_runner,
model_info: RerankModelInfo,
hf_runner=HFMtebCrossEncoder,
vllm_extra_kwargs=None,
vllm_mteb_encoder=VllmMtebCrossEncoder,
atol: float | None = None,
):
if atol is None:
atol = (
model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_RERANK_TOL
)
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
# Maybe load chat_template.
chat_template: str | None = None
if model_info.chat_template_name is not None:
chat_template = (template_home / model_info.chat_template_name).read_text()
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=None,
max_num_seqs=8,
**vllm_extra_kwargs,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
vllm_model.chat_template = chat_template
# Confirm whether vllm is using the correct architecture
if model_info.architecture:
assert model_info.architecture in model_config.architectures
# Score API is only enabled for num_labels == 1
assert model_config.hf_config.num_labels == 1
# Confirm whether the important configs in model_config are correct.
pooler_config = model_config.pooler_config
if model_info.seq_pooling_type is not None:
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
if model_info.tok_pooling_type is not None:
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
if model_info.attn_type is not None:
assert model_config.attn_type == model_info.attn_type
if model_info.is_prefix_caching_supported is not None:
assert (
model_config.is_prefix_caching_supported
== model_info.is_prefix_caching_supported
)
if model_info.is_chunked_prefill_supported is not None:
assert (
model_config.is_chunked_prefill_supported
== model_info.is_chunked_prefill_supported
)
vllm_main_score = run_mteb_rerank(
vllm_mteb_encoder(vllm_model),
tasks=MTEB_RERANK_TASKS,
languages=MTEB_RERANK_LANGS,
)
vllm_dtype = model_config.dtype
head_dtype = model_config.head_dtype
# Accelerate mteb test by setting
# SentenceTransformers mteb score to a constant
if model_info.mteb_score is None:
with hf_runner(
model_info.name, revision=model_info.revision, dtype=model_info.hf_dtype
) as hf_model:
hf_model.chat_template = chat_template
st_main_score = run_mteb_rerank(
hf_model,
tasks=MTEB_RERANK_TASKS,
languages=MTEB_RERANK_LANGS,
)
st_dtype = next(hf_model.model.model.parameters()).dtype
else:
st_main_score = model_info.mteb_score
st_dtype = "Constant"
print("Model:", model_info.name)
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
print("SentenceTransformers:", st_dtype, st_main_score)
print("Difference:", st_main_score - vllm_main_score)
# We are not concerned that the vllm mteb results are better
# than SentenceTransformers, so we only perform one-sided testing.
diff = st_main_score - vllm_main_score
assert diff < atol, (
f"diff={diff:.6g} tol={atol} model={model_info.name} "
f"(st={st_main_score}, vllm={vllm_main_score})"
)
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
from tests.models.utils import (
EmbedModelInfo,
RerankModelInfo,
)
from .mteb_embed_utils import mteb_test_embed_models
from .mteb_score_utils import mteb_test_rerank_models
MODELS = [
########## BertModel
EmbedModelInfo(
"BAAI/bge-base-en",
architecture="BertModel",
mteb_score=0.779336792,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo("BAAI/bge-base-zh", architecture="BertModel", enable_test=False),
EmbedModelInfo("BAAI/bge-small-en", architecture="BertModel", enable_test=False),
EmbedModelInfo("BAAI/bge-small-zh", architecture="BertModel", enable_test=False),
EmbedModelInfo("BAAI/bge-large-en", architecture="BertModel", enable_test=False),
EmbedModelInfo("BAAI/bge-large-zh", architecture="BertModel", enable_test=False),
EmbedModelInfo(
"BAAI/bge-large-zh-noinstruct", architecture="BertModel", enable_test=False
),
EmbedModelInfo(
"BAAI/bge-base-en-v1.5", architecture="BertModel", enable_test=False
),
EmbedModelInfo(
"BAAI/bge-base-zh-v1.5", architecture="BertModel", enable_test=False
),
EmbedModelInfo(
"BAAI/bge-small-en-v1.5", architecture="BertModel", enable_test=False
),
EmbedModelInfo(
"BAAI/bge-small-zh-v1.5", architecture="BertModel", enable_test=False
),
EmbedModelInfo(
"BAAI/bge-large-en-v1.5", architecture="BertModel", enable_test=False
),
EmbedModelInfo(
"BAAI/bge-large-zh-v1.5", architecture="BertModel", enable_test=False
),
########## XLMRobertaModel
EmbedModelInfo(
"BAAI/bge-m3",
architecture="XLMRobertaModel",
mteb_score=0.787343078,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
########## Qwen2Model
EmbedModelInfo(
"BAAI/bge-code-v1",
architecture="Qwen2Model",
mteb_score=0.75724465,
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
# Skip: model's custom tokenizer on HF hub is incompatible with
# transformers v5 (sets attrs before super().__init__, triggering
# AttributeError on 'verbose' in __getattr__).
enable_test=False,
),
]
RERANK_MODELS = [
########## XLMRobertaForSequenceClassification
RerankModelInfo(
"BAAI/bge-reranker-base",
architecture="XLMRobertaForSequenceClassification",
mteb_score=0.32398,
mteb_tol=1e-2,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
RerankModelInfo(
"BAAI/bge-reranker-large",
architecture="XLMRobertaForSequenceClassification",
enable_test=False,
),
RerankModelInfo(
"BAAI/bge-reranker-v2-m3",
architecture="XLMRobertaForSequenceClassification",
enable_test=False,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info)
@@ -1,138 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import mteb
import numpy as np
import pytest
import torch
from torch.utils.data import DataLoader
from tests.conftest import HfRunner
from tests.models.utils import RerankModelInfo
from .mteb_score_utils import (
MtebCrossEncoderMixin,
mteb_test_rerank_models,
)
RERANK_MODELS = [
RerankModelInfo(
"BAAI/bge-reranker-v2-gemma",
architecture="GemmaForSequenceClassification",
hf_overrides={
"architectures": ["GemmaForSequenceClassification"],
"classifier_from_token": ["Yes"],
"method": "no_post_processing",
},
mteb_score=0.33757,
mteb_tol=1e-2,
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
chat_template_name="bge-reranker-v2-gemma.jinja",
),
]
PROMPT = "Given a query A and a passage B, determine whether the passage contains an answer to the query by providing a prediction of either 'Yes' or 'No'." # noqa: E501
class GemmaRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
def __init__(
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
) -> None:
from transformers import AutoModelForCausalLM, AutoTokenizer
HfRunner.__init__(
self,
model_name=model_name,
auto_cls=AutoModelForCausalLM,
dtype=dtype,
**kwargs,
)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
self.yes_loc = self.tokenizer.convert_tokens_to_ids("Yes")
@torch.no_grad
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
corpus = [text for batch in inputs2 for text in batch["text"]]
def get_inputs(pairs, tokenizer, prompt=None):
if prompt is None:
prompt = PROMPT
sep = "\n"
prompt_inputs = tokenizer(
prompt, return_tensors=None, add_special_tokens=False
)["input_ids"]
sep_inputs = tokenizer(sep, return_tensors=None, add_special_tokens=False)[
"input_ids"
]
inputs = []
for query, passage in pairs:
query_inputs = tokenizer(
f"A: {query}",
return_tensors=None,
add_special_tokens=False,
truncation=True,
)
passage_inputs = tokenizer(
f"B: {passage}",
return_tensors=None,
add_special_tokens=False,
truncation=True,
)
item = tokenizer.prepare_for_model(
[tokenizer.bos_token_id] + query_inputs["input_ids"],
sep_inputs + passage_inputs["input_ids"],
truncation="only_second",
padding=False,
return_attention_mask=False,
return_token_type_ids=False,
add_special_tokens=False,
)
item["input_ids"] = item["input_ids"] + sep_inputs + prompt_inputs
item["attention_mask"] = [1] * len(item["input_ids"])
inputs.append(item)
return tokenizer.pad(
inputs,
padding=True,
return_tensors="pt",
)
scores = []
for query, document in zip(queries, corpus):
pairs = [(query, document)]
inputs = get_inputs(pairs, self.tokenizer)
inputs = inputs.to(self.model.device)
_n_tokens = inputs["input_ids"].shape[1]
logits = self.model(**inputs, return_dict=True).logits
_scores = (
logits[:, -1, self.yes_loc]
.view(
-1,
)
.float()
.sigmoid()
)
scores.append(_scores[0].item())
return torch.Tensor(scores)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(
vllm_runner,
model_info,
hf_runner=GemmaRerankerHfRunner,
)
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import (
RerankModelInfo,
)
from .mteb_score_utils import mteb_test_rerank_models
RERANK_MODELS = [
RerankModelInfo(
"cross-encoder/ms-marco-TinyBERT-L-2-v2",
architecture="BertForSequenceClassification",
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
mteb_score=0.32898,
mteb_tol=1e-2,
),
RerankModelInfo(
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
architecture="Qwen3ForSequenceClassification",
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
chat_template_name="qwen3_reranker.jinja",
mteb_score=0.33459,
mteb_tol=1e-2,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info)
@@ -1,63 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
from tests.models.utils import EmbedModelInfo
from .mteb_embed_utils import mteb_test_embed_models
MODELS = [
########## BertModel
EmbedModelInfo(
"intfloat/e5-small",
architecture="BertModel",
mteb_score=0.742285423,
mteb_tol=2e-3,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo("intfloat/e5-base", architecture="BertModel", enable_test=False),
EmbedModelInfo("intfloat/e5-large", architecture="BertModel", enable_test=False),
EmbedModelInfo(
"intfloat/multilingual-e5-small", architecture="BertModel", enable_test=False
),
########## XLMRobertaModel
EmbedModelInfo(
"intfloat/multilingual-e5-base",
architecture="XLMRobertaModel",
mteb_score=0.779325955,
mteb_tol=2e-3,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"intfloat/multilingual-e5-large",
architecture="XLMRobertaModel",
enable_test=False,
),
EmbedModelInfo(
"intfloat/multilingual-e5-large-instruct",
architecture="XLMRobertaModel",
enable_test=False,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
@@ -1,111 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import mteb
import numpy as np
import pytest
import torch
from torch.utils.data import DataLoader
from tests.conftest import HfRunner
from tests.models.utils import RerankModelInfo
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
mxbai_rerank_hf_overrides = {
"architectures": ["Qwen2ForSequenceClassification"],
"classifier_from_token": ["0", "1"],
"method": "from_2_way_softmax",
}
RERANK_MODELS = [
RerankModelInfo(
"mixedbread-ai/mxbai-rerank-base-v2",
architecture="Qwen2ForSequenceClassification",
hf_overrides=mxbai_rerank_hf_overrides,
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
chat_template_name="mxbai_rerank_v2.jinja",
mteb_score=0.33651,
mteb_tol=1e-2,
enable_test=True,
),
RerankModelInfo(
"mixedbread-ai/mxbai-rerank-large-v2",
architecture="Qwen2ForSequenceClassification",
hf_overrides=mxbai_rerank_hf_overrides,
chat_template_name="mxbai_rerank_v2.jinja",
enable_test=False,
),
]
class MxbaiRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
def __init__(
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
) -> None:
from transformers import AutoModelForCausalLM, AutoTokenizer
HfRunner.__init__(
self,
model_name=model_name,
auto_cls=AutoModelForCausalLM,
dtype=dtype,
**kwargs,
)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
self.yes_loc = self.tokenizer.convert_tokens_to_ids("1")
self.no_loc = self.tokenizer.convert_tokens_to_ids("0")
@torch.no_grad
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
corpus = [text for batch in inputs2 for text in batch["text"]]
tokenizer = self.tokenizer
prompts = []
for query, document in zip(queries, corpus):
conversation = [
{"role": "query", "content": query},
{"role": "document", "content": document},
]
prompt = tokenizer.apply_chat_template(
conversation=conversation,
tools=None,
chat_template=self.chat_template,
tokenize=False,
)
prompts.append(prompt)
def compute_logits(inputs):
logits = self.model(**inputs).logits[:, -1, :]
yes_logits = logits[:, self.yes_loc]
no_logits = logits[:, self.no_loc]
logits = yes_logits - no_logits
scores = logits.float().sigmoid()
return scores
scores = []
for prompt in prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = self.wrap_device(inputs)
score = compute_logits(inputs)
scores.append(score[0].item())
return torch.Tensor(scores)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=MxbaiRerankerHfRunner)
@@ -1,54 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling_mteb_test.mteb_embed_utils import (
mteb_test_embed_models,
)
from tests.models.language.pooling_mteb_test.mteb_score_utils import (
mteb_test_rerank_models,
)
from tests.models.utils import (
EmbedModelInfo,
RerankModelInfo,
)
EMBEDDING_MODELS = [
EmbedModelInfo(
"nvidia/llama-nemotron-embed-1b-v2",
architecture="LlamaBidirectionalModel",
mteb_score=0.689164662128673,
mteb_tol=2e-3,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
)
]
RERANK_MODELS = [
RerankModelInfo(
"nvidia/llama-nemotron-rerank-1b-v2",
architecture="LlamaBidirectionalForSequenceClassification",
chat_template_name="nemotron-rerank.jinja",
mteb_score=0.33994,
mteb_tol=1e-2,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info)
@@ -1,59 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
from tests.models.utils import EmbedModelInfo
from .mteb_embed_utils import mteb_test_embed_models
MODELS = [
EmbedModelInfo(
"nomic-ai/nomic-embed-text-v1",
architecture="NomicBertModel",
# Fixme:
# Update nomic-embed code to support the latest
# HF version and remove revision set.
revision="720244025c1a7e15661a174c63cce63c8218e52b",
mteb_score=0.737568559,
mteb_tol=2e-3,
enable_test=True,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
EmbedModelInfo(
"nomic-ai/nomic-embed-text-v1.5",
architecture="NomicBertModel",
enable_test=False,
),
EmbedModelInfo(
"nomic-ai/CodeRankEmbed", architecture="NomicBertModel", enable_test=False
),
EmbedModelInfo(
"nomic-ai/nomic-embed-text-v2-moe",
architecture="NomicBertModel",
mteb_score=0.715488912,
mteb_tol=2e-3,
enable_test=True,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
@@ -1,133 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
from typing import Any
import mteb
import numpy as np
import pytest
import torch
from torch.utils.data import DataLoader
from tests.conftest import HfRunner
from tests.models.utils import RerankModelInfo
from tests.utils import multi_gpu_test
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
qwen3_reranker_hf_overrides = {
"architectures": ["Qwen3ForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
RERANK_MODELS = [
RerankModelInfo(
"Qwen/Qwen3-Reranker-0.6B",
architecture="Qwen3ForSequenceClassification",
hf_overrides=qwen3_reranker_hf_overrides,
chat_template_name="qwen3_reranker.jinja",
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
mteb_score=0.33459,
mteb_tol=1e-2,
enable_test=True,
),
RerankModelInfo(
"Qwen/Qwen3-Reranker-4B",
architecture="Qwen3ForSequenceClassification",
chat_template_name="qwen3_reranker.jinja",
hf_overrides=qwen3_reranker_hf_overrides,
enable_test=False,
),
]
class Qwen3RerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
def __init__(
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
) -> None:
from transformers import AutoModelForCausalLM, AutoTokenizer
HfRunner.__init__(
self,
model_name=model_name,
auto_cls=AutoModelForCausalLM,
dtype=dtype,
**kwargs,
)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
self.token_false_id = self.tokenizer.convert_tokens_to_ids("no")
self.token_true_id = self.tokenizer.convert_tokens_to_ids("yes")
self.max_length = 40960
@torch.no_grad
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
corpus = [text for batch in inputs2 for text in batch["text"]]
tokenizer = self.tokenizer
prompts = []
for query, document in zip(queries, corpus):
conversation = [
{"role": "query", "content": query},
{"role": "document", "content": document},
]
prompt = tokenizer.apply_chat_template(
conversation=conversation,
tools=None,
chat_template=self.chat_template,
tokenize=False,
)
prompts.append(prompt)
def compute_logits(inputs):
batch_scores = self.model(**inputs).logits[:, -1, :]
true_vector = batch_scores[:, self.token_true_id]
false_vector = batch_scores[:, self.token_false_id]
batch_scores = torch.stack([false_vector, true_vector], dim=1)
batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
scores = batch_scores[:, 1].exp()
return scores
scores = []
for prompt in prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = self.wrap_device(inputs)
score = compute_logits(inputs)
scores.append(score[0].item())
return torch.Tensor(scores)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=Qwen3RerankerHfRunner)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
@multi_gpu_test(num_gpus=2)
def test_rerank_models_mteb_tp(vllm_runner, model_info: RerankModelInfo) -> None:
assert model_info.architecture == "Qwen3ForSequenceClassification"
vllm_extra_kwargs: dict[str, Any] = {
"tensor_parallel_size": 2,
}
mteb_test_rerank_models(
vllm_runner,
model_info,
vllm_extra_kwargs=vllm_extra_kwargs,
hf_runner=Qwen3RerankerHfRunner,
)
@@ -1,103 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
from tests.models.utils import EmbedModelInfo
from .mteb_embed_utils import mteb_test_embed_models
MODELS = [
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-xs",
is_matryoshka=False,
architecture="BertModel",
mteb_score=0.714927797,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-s",
is_matryoshka=False,
architecture="BertModel",
enable_test=False,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-m",
is_matryoshka=False,
architecture="BertModel",
enable_test=False,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-m-long",
is_matryoshka=False,
architecture="NomicBertModel",
mteb_score=0.681146831,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-l",
is_matryoshka=False,
architecture="BertModel",
enable_test=False,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-m-v1.5",
is_matryoshka=True,
architecture="BertModel",
mteb_score=0.649088363,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-l-v2.0",
is_matryoshka=True,
architecture="XLMRobertaModel",
mteb_score=0.712258299,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"Snowflake/snowflake-arctic-embed-m-v2.0",
is_matryoshka=True,
architecture="GteModel",
mteb_score=0.706622444,
mteb_tol=2e-3,
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import (
EmbedModelInfo,
)
from .mteb_embed_utils import mteb_test_embed_models
# ST models with projector (Dense) layers
ST_PROJECTOR_MODELS = [
EmbedModelInfo(
"TencentBAC/Conan-embedding-v1",
architecture="BertModel",
mteb_score=0.688611955,
mteb_tol=2e-3,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"google/embeddinggemma-300m",
architecture="Gemma3TextModel",
mteb_score=0.7473819294684156,
mteb_tol=2e-3,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
from tests.models.utils import EmbedModelInfo
from .mteb_embed_utils import mteb_test_embed_models
MODELS = [
EmbedModelInfo(
"voyageai/voyage-4-nano",
architecture="VoyageQwen3BidirectionalEmbedModel",
enable_test=True,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
hf_overrides={
"architectures": ["VoyageQwen3BidirectionalEmbedModel"],
"num_labels": 2048,
},
mteb_score=0.7054,
mteb_tol=2e-3,
# === MTEB Results ===
# STS12: 0.6613
# STS13: 0.6906
# STS14: 0.6556
# STS15: 0.7843
# STS16: 0.7340
# STSBenchmark: 0.7063
# Average score: 0.7054
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
# Encoder-only attention models need enforce_eager=True to avoid
# CUDA graph capture issues with piecewise compilation
mteb_test_embed_models(
hf_runner, vllm_runner, model_info, vllm_extra_kwargs={"enforce_eager": True}
)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
correctness_test_embed_models(
hf_runner,
vllm_runner,
model_info,
example_prompts,
vllm_extra_kwargs={"enforce_eager": True},
)
@@ -1,95 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import mteb
import numpy as np
import pytest
import torch
from torch.utils.data import DataLoader
from tests.conftest import HfRunner
from tests.models.utils import RerankModelInfo
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
RERANK_MODELS = [
RerankModelInfo(
"polaria-tech/zerank-2-reranker-vllm",
architecture="Qwen3ForSequenceClassification",
chat_template_name="zerank2.jinja",
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
mteb_score=0.34305,
mteb_tol=1e-2,
enable_test=True,
),
]
class ZerankHfRunner(MtebCrossEncoderMixin, HfRunner):
# The converted checkpoint has no lm_head, so the reference is the original
# causal LM: the logit of the "Yes" token at the last position. The converted
# checkpoint returns sigmoid(logit / 5), which ranks documents identically.
original_model = "zeroentropy/zerank-2-reranker"
def __init__(
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
) -> None:
from transformers import AutoModelForCausalLM, AutoTokenizer
HfRunner.__init__(
self,
model_name=self.original_model,
auto_cls=AutoModelForCausalLM,
dtype=dtype,
**kwargs,
)
self.tokenizer = AutoTokenizer.from_pretrained(
self.original_model, padding_side="left"
)
self.token_true_id = self.tokenizer.convert_tokens_to_ids("Yes")
@torch.no_grad
def predict(
self,
inputs1: DataLoader[mteb.types.BatchedInput],
inputs2: DataLoader[mteb.types.BatchedInput],
*args,
**kwargs,
) -> np.ndarray:
queries = [text for batch in inputs1 for text in batch["text"]]
corpus = [text for batch in inputs2 for text in batch["text"]]
tokenizer = self.tokenizer
prompts = []
for query, document in zip(queries, corpus):
conversation = [
{"role": "query", "content": query},
{"role": "document", "content": document},
]
prompt = tokenizer.apply_chat_template(
conversation=conversation,
tools=None,
chat_template=self.chat_template,
tokenize=False,
)
prompts.append(prompt)
scores = []
for prompt in prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = self.wrap_device(inputs)
logits = self.model(**inputs).logits[:, -1, :]
scores.append(logits[0, self.token_true_id].item())
return torch.Tensor(scores)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=ZerankHfRunner)
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm.assets.audio import AudioAsset
MODELS = {
"Gemma3nForConditionalGeneration": "google/gemma-3n-E2B-it",
"Gemma4ForConditionalGeneration": "google/gemma-4-E2B-it",
}
# Seconds. Spread wide enough that no two clips can produce the same number of
# mel frames: equal-length clips stack normally and never reach the ragged path.
CLIP_DURATIONS = (1.0, 3.5, 7.0, 12.0)
@pytest.fixture(scope="module")
def ragged_clips():
"""One asset sliced to several durations, so only length varies."""
audio, sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
clips = []
for duration in CLIP_DURATIONS:
num_samples = int(duration * sample_rate)
if num_samples > len(audio):
pytest.skip(f"asset is shorter than {duration}s")
clips.append((audio[:num_samples], sample_rate))
assert len({len(clip) for clip, _ in clips}) == len(clips)
return clips
@pytest.mark.core_model
@pytest.mark.parametrize("arch", sorted(MODELS))
@pytest.mark.parametrize("max_tokens", [8])
def test_variable_length_audio_batching(
vllm_runner,
ragged_clips,
arch: str,
max_tokens: int,
) -> None:
"""Test batching of requests with different audio durations.
Audio features are unpadded per item so a multimodal cache entry does not
depend on the batch it was first processed in, and
`MultiModalFieldConfig.batched` stacks items only when their shapes agree.
A batch of differently sized clips therefore reaches the model as a list,
which used to take EngineCore down with
AttributeError: 'list' object has no attribute 'squeeze'
failing every in-flight request with EngineDeadError.
"""
from transformers import AutoProcessor
model_name = MODELS[arch]
model_info = HF_EXAMPLE_MODELS.get_hf_info(arch)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
# The two architectures spell the audio placeholder differently; ask the
# processor rather than hardcoding either.
processor = AutoProcessor.from_pretrained(model_name)
prompt = processor.apply_chat_template(
[
{
"role": "user",
"content": [
{"type": "audio", "audio": None},
{"type": "text", "text": "Transcribe this audio."},
],
}
],
tokenize=False,
add_generation_prompt=True,
)
with vllm_runner(
model_name,
dtype="bfloat16",
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
) as vllm_model:
# One call so the clips are scheduled into the same forward pass;
# issued separately they never reach the ragged path.
outputs = vllm_model.generate_greedy(
[prompt] * len(ragged_clips),
max_tokens,
audios=[[clip] for clip in ragged_clips],
)
assert len(outputs) == len(ragged_clips)
for output in outputs:
assert len(output[1]) > 0, "Expected non-empty output"
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.image import ImageAsset
from vllm.assets.video import VideoAsset
from vllm.multimodal.image import convert_image_mode
models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
def base_prompt(modalities_str: str) -> str:
return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n" # noqa: E501
INTERLEAVED_PROMPT = base_prompt("<image><video><image>\n")
NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", ["float16"])
@pytest.mark.parametrize("max_tokens", [128])
def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
"""This is a simple test to check if interleaved and non-interleaved prompts
give the same result.
"""
image_cherry = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
image_stop = convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB")
images = [image_cherry, image_stop]
video = VideoAsset(name="baby_reading", num_frames=16).np_ndarrays
inputs = [
(
[INTERLEAVED_PROMPT],
[images],
[video],
),
(
[NONINTERLEAVED_PROMPT],
[images],
[video],
),
]
with vllm_runner(
model,
runner="generate",
dtype=dtype,
limit_mm_per_prompt={"image": 2},
max_model_len=32768,
max_num_seqs=2,
tensor_parallel_size=1,
enforce_eager=True,
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.generate_greedy(
prompts, max_tokens, images=images, videos=videos
)
for prompts, images, videos in inputs
]
all_results = [output[0][1] for output in vllm_outputs_per_case]
outputs = [
(total_str, total_str.find("assistant\n") + len("assistant\n"))
for total_str in all_results
]
prompt_lengths = [prompt_len for _, prompt_len in outputs]
generated_strs = [total_str[prompt_len:] for total_str, prompt_len in outputs]
interleaved_prompt_len, noninterleaved_prompt_len = prompt_lengths
interleaved_output_str, noninterleaved_output_str = generated_strs
# The two prompts are identical except for the order of modality tokens.
assert interleaved_prompt_len == noninterleaved_prompt_len
# The two generated strings should be different because of the
# interleaved modality tokens.
assert interleaved_output_str != noninterleaved_output_str
@@ -1,296 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Sequence
import pytest
import regex as re
from transformers import AutoTokenizer
from vllm.assets.image import ImageAsset
from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.multimodal.image import convert_image_mode, rescale_image_size
from vllm.multimodal.media.audio import load_audio
from vllm.transformers_utils.repo_utils import hf_api
from ....conftest import (
IMAGE_ASSETS,
HfRunner,
PromptAudioInput,
PromptImageInput,
VllmRunner,
)
from ....utils import large_gpu_test
from ...utils import check_logprobs_close
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
"cherry_blossom": "<|user|>\n<|image_1|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
}
)
HF_MULTIIMAGE_IMAGE_PROMPT = (
"<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
)
model_path = hf_api().snapshot_download("microsoft/Phi-4-multimodal-instruct")
# Since the vision-lora and speech-lora co-exist with the base model,
# we have to manually specify the path of the lora weights.
vision_lora_path = os.path.join(model_path, "vision-lora")
speech_question = os.path.join(
model_path, "examples", "what_is_shown_in_this_image.wav"
)
models = [model_path]
def vllm_to_hf_output(
vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
):
"""Sanitize vllm output to be comparable with hf output."""
_, output_str, out_logprobs = vllm_output
output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
assert output_str_without_image[0] == " "
output_str_without_image = output_str_without_image[1:]
hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
tokenizer = AutoTokenizer.from_pretrained(model)
hf_output_ids = tokenizer.encode(output_str_without_image)
assert hf_output_ids[0] == 1
hf_output_ids = hf_output_ids[1:]
return hf_output_ids, hf_output_str, out_logprobs
target_dtype = "half"
IMAGE_SIZE_FACTOR_GROUPS = (
(1.0,),
(1.0, 1.0, 1.0),
(0.25, 0.5, 1.0),
)
def run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]],
model: str,
*,
max_model_len: int,
dtype: str,
max_tokens: int,
num_logprobs: int,
mm_limit: int,
tensor_parallel_size: int,
distributed_executor_backend: str | None = None,
):
"""Inference result should be the same between hf and vllm.
All the image fixtures for the test are from IMAGE_ASSETS.
For huggingface runner, we provide the PIL images as input.
For vllm runner, we provide MultiModalDataDict objects
and corresponding MultiModalConfig as input.
Note, the text input is also adjusted to abide by vllm contract.
The text output is sanitized to be able to compare with hf.
"""
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
# max_model_len should be greater than image_feature_size
with vllm_runner(
model,
runner="generate",
max_model_len=max_model_len,
max_num_seqs=2,
dtype=dtype,
limit_mm_per_prompt={"image": mm_limit},
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enable_lora=True,
max_lora_rank=320,
gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
enforce_eager=True,
) as vllm_model:
lora_request = LoRARequest("vision", 1, vision_lora_path)
vllm_outputs_per_case = [
vllm_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
audios=audios,
lora_request=lora_request,
)
for prompts, images, audios in inputs
]
# This error occurs inside `get_peft_model`
# FIXME: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/75
pytest.skip("HF impl is not compatible with current transformers")
hf_model_kwargs = {"_attn_implementation": "sdpa"}
with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model:
hf_processor = hf_model.processor
eos_token_id = hf_processor.tokenizer.eos_token_id
def patch_hf_processor(
*args, text="", images=None, audio=None, sampling_rate=None, **kwargs
):
audios = None
if audio is not None and sampling_rate is not None:
audios = [(audio, sampling_rate)]
return hf_processor(
*args, text=text, images=images, audios=audios, **kwargs
)
hf_model.processor = patch_hf_processor
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
audios=audios,
eos_token_id=eos_token_id,
num_logits_to_keep=0,
)
for prompts, images, audios in inputs
]
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [12800])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
def test_models(
hf_runner,
vllm_runner,
image_assets,
model,
dtype: str,
max_model_len: int,
max_tokens: int,
num_logprobs: int,
) -> None:
images = [asset.pil_image for asset in image_assets]
inputs_per_image = [
(
[prompt for _ in size_factors],
[rescale_image_size(image, factor) for factor in size_factors],
None,
)
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
]
run_test(
hf_runner,
vllm_runner,
inputs_per_image,
model,
dtype=dtype,
max_model_len=max_model_len,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
mm_limit=1,
tensor_parallel_size=1,
)
@large_gpu_test(min_gb=48)
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [25600])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
def test_multi_images_models(
hf_runner,
vllm_runner,
image_assets,
model,
dtype: str,
max_model_len: int,
max_tokens: int,
num_logprobs: int,
) -> None:
images = [asset.pil_image for asset in image_assets]
inputs_per_case = [
(
[HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
[
[rescale_image_size(image, factor) for image in images]
for factor in size_factors
],
None,
)
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
]
run_test(
hf_runner,
vllm_runner,
inputs_per_case,
model,
dtype=dtype,
max_model_len=max_model_len,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
mm_limit=2,
tensor_parallel_size=1,
)
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_model_len", [12800])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
def test_vision_speech_models(
hf_runner,
vllm_runner,
model,
dtype: str,
max_model_len: int,
max_tokens: int,
num_logprobs: int,
) -> None:
# use the example speech question so that the model outputs are reasonable
audio = load_audio(speech_question, sr=None)
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
inputs_vision_speech = [
(
["<|user|><|image_1|><|audio_1|><|end|><|assistant|>"],
[image],
[audio],
),
]
run_test(
hf_runner,
vllm_runner,
inputs_vision_speech,
model,
dtype=dtype,
max_model_len=max_model_len,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
mm_limit=1,
tensor_parallel_size=1,
)
@@ -1,229 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
import numpy as np
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
from ....conftest import AUDIO_ASSETS, AudioTestAssets, VllmRunner
from ....utils import RemoteOpenAIServer
from ...registry import HF_EXAMPLE_MODELS
MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
AUDIO_PROMPTS = AUDIO_ASSETS.prompts(
{
"mary_had_lamb": "Transcribe this into English.",
"winning_call": "What is happening in this audio clip?",
}
)
MULTI_AUDIO_PROMPT = "Describe each of the audios above."
AudioTuple = tuple[np.ndarray, int]
VLLM_PLACEHOLDER = "<|audio|>"
HF_PLACEHOLDER = "<|audio|>"
CHUNKED_PREFILL_KWARGS = {
"enable_chunked_prefill": True,
"max_num_seqs": 2,
# Use a very small limit to exercise chunked prefill.
"max_num_batched_tokens": 16,
}
def params_kwargs_to_cli_args(params_kwargs: dict[str, Any]) -> list[str]:
"""Convert kwargs to CLI args."""
args = []
for key, value in params_kwargs.items():
if isinstance(value, bool):
if value:
args.append(f"--{key.replace('_', '-')}")
else:
args.append(f"--{key.replace('_', '-')}={value}")
return args
@pytest.fixture(
params=[
pytest.param({}, marks=pytest.mark.cpu_model),
pytest.param(CHUNKED_PREFILL_KWARGS),
]
)
def server(request, audio_assets: AudioTestAssets):
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--enforce-eager",
"--limit-mm-per-prompt",
json.dumps({"audio": len(audio_assets)}),
"--trust-remote-code",
] + params_kwargs_to_cli_args(request.param)
with RemoteOpenAIServer(
MODEL_NAME, args, env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"}
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
def _get_prompt(audio_count, question, placeholder):
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
placeholder = f"{placeholder}\n" * audio_count
return tokenizer.apply_chat_template(
[{"role": "user", "content": f"{placeholder}{question}"}],
tokenize=False,
add_generation_prompt=True,
)
def run_multi_audio_test(
vllm_runner: type[VllmRunner],
prompts_and_audios: list[tuple[str, list[AudioTuple]]],
model: str,
*,
dtype: str,
max_tokens: int,
num_logprobs: int,
**kwargs,
):
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
with vllm_runner(
model,
dtype=dtype,
enforce_eager=True,
limit_mm_per_prompt={
"audio": max((len(audio) for _, audio in prompts_and_audios))
},
**kwargs,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
[prompt for prompt, _ in prompts_and_audios],
max_tokens,
num_logprobs=num_logprobs,
audios=[audios for _, audios in prompts_and_audios],
)
# The HuggingFace model doesn't support multiple audios yet, so
# just assert that some tokens were generated.
assert all(tokens for tokens, *_ in vllm_outputs)
@pytest.mark.core_model
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize(
"vllm_kwargs",
[
pytest.param({}, marks=pytest.mark.cpu_model),
pytest.param(CHUNKED_PREFILL_KWARGS),
],
)
def test_models_with_multiple_audios(
vllm_runner,
audio_assets: AudioTestAssets,
dtype: str,
max_tokens: int,
num_logprobs: int,
vllm_kwargs: dict,
) -> None:
vllm_prompt = _get_prompt(len(audio_assets), MULTI_AUDIO_PROMPT, VLLM_PLACEHOLDER)
run_multi_audio_test(
vllm_runner,
[(vllm_prompt, [audio.audio_and_sample_rate for audio in audio_assets])],
MODEL_NAME,
dtype=dtype,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
**vllm_kwargs,
)
@pytest.mark.core_model
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [32])
def test_variable_length_audio_batching(
vllm_runner,
audio_assets: AudioTestAssets,
dtype: str,
max_tokens: int,
) -> None:
"""Test batching of requests with different audio durations.
This exercises the variable-length tensor handling in
MultiModalFlatField._reduce_data() which was buggy before
https://github.com/vllm-project/vllm/issues/31658 was fixed.
"""
model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
# Create prompts with single audio each (different durations)
prompts_and_audios = []
for audio, question in zip(audio_assets, AUDIO_PROMPTS):
prompt = _get_prompt(1, question, VLLM_PLACEHOLDER)
prompts_and_audios.append((prompt, [audio.audio_and_sample_rate]))
with vllm_runner(
MODEL_NAME,
dtype=dtype,
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
) as vllm_model:
# Generate for all prompts in a single batch
# This triggers the variable-length batching code path
outputs = vllm_model.generate_greedy(
[prompt for prompt, _ in prompts_and_audios],
max_tokens,
audios=[audios for _, audios in prompts_and_audios],
)
# Verify outputs were generated for each request
assert len(outputs) == len(prompts_and_audios)
for output in outputs:
assert len(output[1]) > 0, "Expected non-empty output"
@pytest.mark.asyncio
async def test_online_serving(client, audio_assets: AudioTestAssets):
"""Exercises online serving with/without chunked prefill enabled."""
messages = [
{
"role": "user",
"content": [
*[
{"type": "audio_url", "audio_url": {"url": audio.url}}
for audio in audio_assets
],
{
"type": "text",
"text": f"What's happening in these {len(audio_assets)} audio clips?", # noqa: E501
},
],
}
]
chat_completion = await client.chat.completions.create(
model=MODEL_NAME, messages=messages, max_tokens=10
)
assert len(chat_completion.choices) == 1
choice = chat_completion.choices[0]
assert choice.finish_reason == "length"
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.tokens.tokenizers.audio import Audio
from transformers import VoxtralForConditionalGeneration
from vllm.tokenizers.mistral import MistralTokenizer
from ....conftest import AudioTestAssets
from ....utils import RemoteOpenAIServer
from ...utils import check_logprobs_close
from .test_ultravox import MULTI_AUDIO_PROMPT, run_multi_audio_test
from .vlm_utils import model_utils
MODEL_NAME = "mistralai/Voxtral-Mini-3B-2507"
MISTRAL_FORMAT_ARGS = [
"--tokenizer_mode",
"mistral",
"--config_format",
"mistral",
"--load_format",
"mistral",
]
def _get_prompt(audio_assets: AudioTestAssets, question: str) -> list[int]:
"""Build a token-ID prompt via mistral_common for vLLM offline inference."""
tokenizer = MistralTokenizer.from_pretrained(MODEL_NAME)
audios = [
Audio.from_file(str(asset.get_local_path()), strict=False)
for asset in audio_assets
]
audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
messages = [
UserMessage(content=[*audio_chunks, TextChunk(text=question)]).to_openai()
]
return tokenizer.apply_chat_template(messages=messages)
@pytest.mark.core_model
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models_with_multiple_audios(
vllm_runner,
audio_assets: AudioTestAssets,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
vllm_prompt = _get_prompt(audio_assets, MULTI_AUDIO_PROMPT)
run_multi_audio_test(
vllm_runner,
[(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])], # type: ignore[list-item]
MODEL_NAME,
dtype=dtype,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
tokenizer_mode="mistral",
)
def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
"""Two-layer accuracy and serving validation using Mistral format.
1. Offline vLLM greedy output (runs first to avoid CUDA fork issues
with multiprocessing - see vlm_utils/core.py).
2. Online OpenAI-compatible API output must match offline — validates
that the serving path (chat template, audio encoding, tokenization)
does not corrupt anything.
Steps run sequentially so each releases the GPU before the next starts.
"""
question = f"What's happening in these {len(audio_assets)} audio clips?"
max_tokens = 10
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
vllm_prompt = _get_prompt(audio_assets, question)
with vllm_runner(
MODEL_NAME,
dtype="half",
enforce_eager=True,
tokenizer_mode="mistral",
config_format="mistral",
load_format="mistral",
limit_mm_per_prompt={"audio": len(audio_assets)},
) as vllm_model:
offline_outputs = vllm_model.generate_greedy(
[vllm_prompt],
max_tokens,
audios=[audio_data],
)
offline_text = offline_outputs[0][1]
assert offline_text, "Offline vLLM inference produced empty output"
def _asset_to_openai_chunk(asset):
audio = Audio.from_file(str(asset.get_local_path()), strict=False)
audio.format = "wav"
return AudioChunk.from_audio(audio).to_openai()
messages = [
{
"role": "user",
"content": [
*[_asset_to_openai_chunk(a) for a in audio_assets],
{"type": "text", "text": question},
],
}
]
server_args = [
"--enforce-eager",
"--limit-mm-per-prompt",
json.dumps({"audio": len(audio_assets)}),
*MISTRAL_FORMAT_ARGS,
]
with RemoteOpenAIServer(
MODEL_NAME,
server_args,
env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"},
) as remote_server:
client = remote_server.get_client()
completion = client.chat.completions.create(
model=MODEL_NAME,
messages=messages,
max_tokens=max_tokens,
temperature=0,
)
assert len(completion.choices) == 1
choice = completion.choices[0]
assert choice.finish_reason == "length"
assert choice.message.content == offline_text, (
f"Online serving output does not match offline inference.\n"
f" Online: {choice.message.content!r}\n"
f" Offline: {offline_text!r}"
)
def test_hf_reference(hf_runner, vllm_runner, audio_assets: AudioTestAssets):
"""Compare vLLM Mistral-format output against HF Transformers reference.
Instead of requiring an exact text match (which is brittle across
attention backends), we compare per-token logprobs using the standard
check_logprobs_close helper: when tokens diverge at a position, each
runner's chosen token must appear in the other's top-k logprobs.
"""
question = f"What's happening in these {len(audio_assets)} audio clips?"
max_tokens = 10
num_logprobs = 5
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
vllm_prompt = _get_prompt(audio_assets, question)
with vllm_runner(
MODEL_NAME,
dtype="half",
enforce_eager=True,
tokenizer_mode="mistral",
config_format="mistral",
load_format="mistral",
limit_mm_per_prompt={"audio": len(audio_assets)},
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
[vllm_prompt],
max_tokens,
num_logprobs,
audios=[audio_data],
)
assert vllm_outputs[0][1], "vLLM inference produced empty output"
with hf_runner(
MODEL_NAME,
dtype="half",
auto_cls=VoxtralForConditionalGeneration,
) as hf_model:
hf_model = model_utils.voxtral_patch_hf_runner(hf_model)
hf_outputs = hf_model.generate_greedy_logprobs_limit(
[question],
max_tokens,
num_logprobs,
audios=[audio_data],
)
assert hf_outputs[0][1], "HF Transformers produced empty output"
print(
f"HF Reference Comparison\n"
f" vLLM: {vllm_outputs[0][1]!r}\n"
f" HF: {hf_outputs[0][1]!r}"
)
check_logprobs_close(
outputs_0_lst=vllm_outputs,
outputs_1_lst=hf_outputs,
name_0="vllm",
name_1="hf",
)
@@ -0,0 +1,207 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Core test implementation to be shared across modalities."""
from collections.abc import Callable
from typing import Any
import torch
from transformers.models.auto.auto_factory import _BaseAutoModelClass
from vllm.config.model import RunnerOption
from vllm.tokenizers import TokenizerLike
from .....conftest import HfRunner, VllmRunner
from ....registry import HF_EXAMPLE_MODELS
from .types import PromptWithMultiModalInput, RunnerOutput
def run_test(
*,
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
inputs: list[PromptWithMultiModalInput],
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
enforce_eager: bool,
max_model_len: int,
max_num_seqs: int,
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
auto_cls: type[_BaseAutoModelClass],
use_tokenizer_eos: bool,
comparator: Callable[..., None],
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None,
stop_str: list[str] | None,
limit_mm_per_prompt: dict[str, int],
vllm_runner_kwargs: dict[str, Any] | None,
hf_model_kwargs: dict[str, Any] | None,
hf_processor: Callable[[str], Any] | None,
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
runner: RunnerOption = "auto",
distributed_executor_backend: str | None = None,
tensor_parallel_size: int = 1,
vllm_embeddings: torch.Tensor | None = None,
):
"""Modality agnostic test executor for comparing HF/vLLM outputs."""
# In the case of embeddings, vLLM takes separate input tensors
vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
# Disable other modalities to save memory
default_limits = {"image": 0, "video": 0, "audio": 0}
limit_mm_per_prompt = default_limits | limit_mm_per_prompt
vllm_outputs_per_mm = []
hf_outputs_per_mm = []
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
vllm_runner_kwargs_: dict[str, Any] = {"mm_processor_cache_gb": 0}
if model_info.tokenizer:
vllm_runner_kwargs_["tokenizer_name"] = model_info.tokenizer
if model_info.tokenizer_mode:
vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
if model_info.hf_overrides:
vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
if model_info.require_embed_inputs:
for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
vllm_runner_kwargs_[k] = model_info.require_embed_inputs
if not model_info.enable_prefix_caching:
vllm_runner_kwargs_["enable_prefix_caching"] = False
if vllm_runner_kwargs:
vllm_runner_kwargs_.update(vllm_runner_kwargs)
# Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
# already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
if "limit_mm_per_prompt" in vllm_runner_kwargs_:
limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
with vllm_runner(
model,
max_model_len=max_model_len,
max_num_seqs=max_num_seqs,
dtype=dtype,
limit_mm_per_prompt=limit_mm_per_prompt,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=enforce_eager,
runner=runner,
**vllm_runner_kwargs_,
) as vllm_model:
tokenizer = vllm_model.llm.get_tokenizer()
vllm_kwargs: dict[str, Any] = {}
if get_stop_token_ids is not None:
vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
if stop_str:
vllm_kwargs["stop"] = stop_str
for prompts, image_data, video_data, audio_data in vllm_inputs:
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
vllm_kwargs_with_mm_data = vllm_kwargs | mm_data
vllm_output = vllm_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
**vllm_kwargs_with_mm_data,
)
vllm_outputs_per_mm.append(vllm_output)
hf_runner_kwargs: dict[str, Any] = {}
if model_info.tokenizer:
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
if hf_processor is not None:
hf_runner_kwargs["processor"] = hf_processor(model)
hf_model = hf_runner(
model,
dtype=dtype,
auto_cls=auto_cls,
model_kwargs=hf_model_kwargs,
**hf_runner_kwargs,
)
# Some models need to patch things like the model processor, e.g., internvl
if patch_hf_runner is not None:
hf_model = patch_hf_runner(hf_model)
with hf_model, torch.no_grad():
tokenizer = hf_model.tokenizer
# Some models need to explicitly pass the eos_token_id off the tokenizer
# or processor for a good comparison;
# currently assume processor/tokenizer agree on the EOS, and pull it off
# the tokenizer if requested.
hf_kwargs = {}
if use_tokenizer_eos:
hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
if stop_str:
hf_kwargs["stop_strings"] = stop_str
for prompts, image_data, video_data, audio_data in inputs:
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
hf_kwargs_with_mm_data = hf_kwargs | mm_data
hf_output = hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
tokenizer=tokenizer,
**hf_kwargs_with_mm_data,
)
hf_outputs_per_mm.append(hf_output)
# Apply output processing / sanitation to the vLLM and HF runner results
hf_outputs_per_mm, vllm_outputs_per_mm = process_runner_outputs(
model,
first_runner_outputs=hf_outputs_per_mm,
second_runner_outputs=vllm_outputs_per_mm,
first_runner_processor=hf_output_post_proc,
second_runner_processor=vllm_output_post_proc,
)
for hf_outputs, vllm_outputs in zip(hf_outputs_per_mm, vllm_outputs_per_mm):
# This is usually check_logprobs_close, but it's passed through to
# allow things like check_outputs_equal where needed
comparator(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
def process_runner_outputs(
model,
first_runner_outputs,
second_runner_outputs,
first_runner_processor=None,
second_runner_processor=None,
):
"""Applies the runner processor(s) to the runner outputs, if any."""
if first_runner_processor is not None:
first_runner_outputs = process_outputs(
first_runner_processor, model, first_runner_outputs
)
if second_runner_processor is not None:
second_runner_outputs = process_outputs(
second_runner_processor, model, second_runner_outputs
)
return first_runner_outputs, second_runner_outputs
def process_outputs(output_processor, model, outputs_per_image):
"""Applies a model specific post-processor function to a runner's output."""
return [
[output_processor(res, model) for res in outputs]
for outputs in outputs_per_image
]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,218 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Types for writing multimodal model tests."""
from collections.abc import Callable, Iterable
from enum import Enum
from pathlib import PosixPath
from typing import Any, NamedTuple
import torch
from pytest import MarkDecorator
from transformers import AutoModelForCausalLM
from transformers.models.auto.auto_factory import _BaseAutoModelClass
from vllm.config.model import RunnerOption
from vllm.logprobs import SampleLogprobs
from vllm.tokenizers import TokenizerLike
from .....conftest import (
AUDIO_ASSETS,
IMAGE_ASSETS,
HfRunner,
ImageAsset,
ImageTestAssets,
PromptAudioInput,
PromptImageInput,
PromptVideoInput,
)
from ....utils import check_logprobs_close
# meta image tag; will be replaced by the appropriate tag for the model
TEST_IMG_PLACEHOLDER = "<vlm_image>"
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
}
)
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
{
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
}
)
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
IMAGE_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
EMBEDDING_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0)]
RunnerOutput = tuple[list[int], str, SampleLogprobs | None]
class PromptWithMultiModalInput(NamedTuple):
"""Holds the multimodal input for a single test case."""
prompts: list[str]
image_data: PromptImageInput | None = None
video_data: PromptVideoInput | None = None
audio_data: PromptAudioInput | None = None
class VLMTestType(Enum):
IMAGE = 1
MULTI_IMAGE = 2
EMBEDDING = 3
VIDEO = 4
AUDIO = 5
CUSTOM_INPUTS = 6
class SizeType(Enum):
SIZE_FACTOR = 1
FIXED_SIZE = 2
class CustomTestOptions(NamedTuple):
inputs: list[PromptWithMultiModalInput]
limit_mm_per_prompt: dict[str, int]
class ImageSizeWrapper(NamedTuple):
type: SizeType
# A size factor is a wrapper of 0+ floats,
# while a fixed size contains an iterable of integer pairs
data: Iterable[float] | Iterable[tuple[int, int]]
class VLMTestInfo(NamedTuple):
"""Holds the configuration for 1+ tests for one model architecture."""
models: list[str]
test_type: VLMTestType | Iterable[VLMTestType]
# Should be None only if this is a CUSTOM_INPUTS test
prompt_formatter: Callable[[str], str] | None = None
img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
audio_idx_to_prompt: Callable[[int], str] = lambda idx: "<audio>\n"
# Most models work on the single / multi-image prompts above, but in some
# cases the log prob check fails, e.g., for paligemma. We allow passing
# an override for the single image prompts / multi-image prompt for this
# reason.
single_image_prompts: Iterable[str] = SINGLE_IMAGE_BASE_PROMPTS
multi_image_prompt: str = MULTI_IMAGE_BASE_PROMPT
# Function for converting ImageAssets to image embeddings;
# We need to define this explicitly for embedding tests
convert_assets_to_embeddings: (
Callable[[ImageTestAssets], list[torch.Tensor]] | None
) = None
# Exposed options for vLLM runner; we change these in a several tests,
# but the defaults are derived from VllmRunner & the engine defaults
# These settings are chosen to avoid OOMs when running in the CI
enforce_eager: bool = True
max_model_len: int = 1024
max_num_seqs: int = 256
runner: RunnerOption = "auto"
tensor_parallel_size: int = 1
vllm_runner_kwargs: dict[str, Any] | None = None
# Optional callable which gets a list of token IDs from the model tokenizer
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None
# Optional list of strings to stop generation, useful when stop tokens are
# not special tokens in the tokenizer
stop_str: list[str] | None = None
# Exposed options for HF runner
hf_model_kwargs: dict[str, Any] | None = None
hf_processor: Callable[[str], Any] | None = None
# Indicates we should explicitly pass the EOS from the tokenizer
use_tokenizer_eos: bool = False
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
patch_hf_runner: Callable[[HfRunner], HfRunner] | None = None
# Post processors that if defined, will run oun the outputs of the
# vLLM and HF runner, respectively (useful for sanitization, etc).
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
# Consumes the output of the callables above and checks if they're equal
comparator: Callable[..., None] = check_logprobs_close
# Default expandable params per test; these defaults can be overridden in
# instances of this object; the complete set of test cases for the model
# is all combinations of .models + all fields below
max_tokens: int = 128
num_logprobs: int = 5
dtype: str = "auto"
distributed_executor_backend: str | None = None
# Only expanded in video tests
num_video_frames: int | tuple[int] = 16
needs_video_metadata: bool = False
# Fixed image sizes / image size factors; most tests use image_size_factors.
# Each inner iterable defines one request batch. All batches are run against
# the same model instance.
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
# Hack for updating a prompt to take into a local path; currently only used
# for Qwen-VL, which requires encoding the image path / url into the prompt
# for HF runner
prompt_path_encoder: (
Callable[[PosixPath, str, list[ImageAsset] | ImageTestAssets], str] | None
) = None # noqa: E501
# Allows configuring a test to run with custom inputs
custom_test_opts: list[CustomTestOptions] | None = None
marks: list[MarkDecorator] | None = None
def get_non_parametrized_runner_kwargs(self):
"""Returns a dictionary of expandable kwargs for items that are used
in all test types, which are NOT used when creating the parametrized
test cases.
"""
return {
"enforce_eager": self.enforce_eager,
"max_model_len": self.max_model_len,
"max_num_seqs": self.max_num_seqs,
"runner": self.runner,
"tensor_parallel_size": self.tensor_parallel_size,
"vllm_runner_kwargs": self.vllm_runner_kwargs,
"hf_output_post_proc": self.hf_output_post_proc,
"vllm_output_post_proc": self.vllm_output_post_proc,
"auto_cls": self.auto_cls,
"use_tokenizer_eos": self.use_tokenizer_eos,
"comparator": self.comparator,
"get_stop_token_ids": self.get_stop_token_ids,
"hf_model_kwargs": self.hf_model_kwargs,
"hf_processor": self.hf_processor,
"stop_str": self.stop_str,
"patch_hf_runner": self.patch_hf_runner,
}
class ExpandableVLMTestArgs(NamedTuple):
"""The expanded kwargs which correspond to a single test case."""
model: str
max_tokens: int
num_logprobs: int
dtype: str
distributed_executor_backend: str | None
# Sizes are used for everything except audio and custom input tests.
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
# Video only
num_video_frames: int | None = None
needs_video_metadata: bool = False
# Custom inputs only
custom_test_opts: CustomTestOptions | None = None
@@ -1,40 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import GenerateModelInfo
from .ppl_utils import vqa_ppl_test
MODELS = [
(GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
]
def _pixel_size(factor: int):
# Match max image token size with qwen's PPL test
return {
"size": {
"shortest_edge": 2 * factor**2,
"longest_edge": 2 * 1280 * factor**2,
}
}
@pytest.mark.parametrize("model_info,factor", MODELS)
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
def test_ppl(
hf_runner,
vllm_runner,
model_info: GenerateModelInfo,
factor: int,
mm_device_do_normalize: bool,
):
vqa_ppl_test(
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
mm_processor_kwargs=_pixel_size(factor),
)
@@ -1,140 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
caching.
Tests the use_audio_in_video feature where audio is extracted from video and
processed together with video frames in an interleaved manner.
Regression test: when use_audio_in_video=True and the multimodal processor
cache is warm, the second request goes through MultiModalProcessorSenderCache
which sets mm_kwargs["video"] items to None on a cache hit. The processor
must still detect use_audio_in_video=True (via token-count heuristic) and
produce the same prompt_token_ids as the first (cache-miss) request.
Without the fix the cache-hit path left use_audio_in_video=False, causing
audio placeholder tokens to be inserted separately instead of being derived
from the interleaved video placeholders – yielding a different (wrong) token
sequence on every subsequent request for the same video.
"""
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import LruKeyReplicatedSenderCache
from ....multimodal.utils import random_audio, random_video
from ...utils import build_model_context
MODELS = [
"Qwen/Qwen2.5-Omni-3B",
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
]
def create_mm_data(num_videos: int) -> dict[str, list]:
# Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
# stays fast even without a GPU.
mm_data = dict[str, list](video=[], audio=[])
for i in range(num_videos):
rng = np.random.RandomState(i)
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
mm_data["video"].append(video)
mm_data["audio"].append((audio, sr))
return mm_data
@pytest.mark.parametrize("model_id", MODELS)
@pytest.mark.parametrize("num_videos", [1, 2])
def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
"""Regression test for https://github.com/vllm-project/vllm/pull/36800.
MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
on a cache hit, so mm_kwargs["video"] items become None on the second call.
The Qwen processor override of _maybe_apply_prompt_updates must detect
use_audio_in_video=True via token-count heuristics and re-derive the audio
placeholders correctly.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": num_videos, "image": 0, "video": num_videos},
mm_processor_cache_gb=1,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Sender cache: on a cache hit returns (None, prompt_updates) for each
# item, setting mm_kwargs["video"] = [None] – the exact condition that
# triggered the original bug.
sender_cache = LruKeyReplicatedSenderCache(ctx.model_config)
video_token_id = processor.info.get_hf_config().video_token_id
mm_data = create_mm_data(num_videos)
hf_processor_mm_kwargs = {"use_audio_in_video": True}
def run(processor, cache):
return processor(
[video_token_id] * num_videos,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
cache=cache,
)["prompt_token_ids"]
baseline_ids = run(processor, cache=None)
# First call on the sender-cache processor: cache miss.
# mm_kwargs["video"] items are real tensors; use_audio_in_video is
# detected normally from the item data.
first_ids = run(processor, cache=sender_cache)
assert first_ids == baseline_ids, (
"Cache-miss call produced different prompt_token_ids than baseline.\n"
f" baseline : {baseline_ids}\n"
f" cache-miss: {first_ids}"
)
# Second call on the sender-cache processor: cache hit.
# MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
# so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
# not detected, yielding wrong token ids.
second_ids = run(processor, cache=sender_cache)
assert second_ids == baseline_ids, (
"Cache-hit call produced different prompt_token_ids than baseline.\n"
"This is the regression introduced when use_audio_in_video detection\n"
"fails for None mm_kwargs items on a cache hit.\n"
f" baseline : {baseline_ids}\n"
f" cache-hit: {second_ids}"
)
@pytest.mark.parametrize("model_id", MODELS)
def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
"""A video with no audio track, combined with `use_audio_in_video=True`,
must raise a clear `ValueError` naming the mismatch, rather than an
opaque `StopIteration`-derived error from the underlying HF processor.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
video_token_id = processor.info.get_hf_config().video_token_id
rng = np.random.RandomState(0)
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
# No "audio" key at all in this request's data: this is the "video has
# no audio track" case. (The audio limit above is 1, not 0, so that
# `mm_counts["audio"]` is still declared for this request; a 0 limit
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
# trips an unrelated assertion in `_apply_hf_processor_main`.)
mm_data = {"video": [video]}
with pytest.raises(ValueError, match="doesn't have audio track"):
processor(
[video_token_id],
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"use_audio_in_video": True},
)
@@ -1,113 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from vllm.assets.video import VideoAsset
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
MODEL_ID = "nvidia/Cosmos3-Edge"
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
@pytest.fixture(scope="module")
def processor():
if LOCAL_MODEL_PATH:
model_config = ModelConfig(
LOCAL_MODEL_PATH,
tokenizer=LOCAL_MODEL_PATH,
max_model_len=4096,
limit_mm_per_prompt={"image": 2, "video": 1},
)
else:
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 2, "video": 1},
)
model_config = ctx.model_config
return MULTIMODAL_REGISTRY.create_processor(model_config)
def _assert_image_outputs(processor, processed, num_images: int) -> None:
mm_data = processed["mm_kwargs"].get_data()
grid_thw = mm_data["image_grid_thw"]
pixel_values = mm_data["pixel_values"]
assert grid_thw.shape == (num_images, 3)
assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
image_placeholders = processed["mm_placeholders"]["image"]
assert len(image_placeholders) == num_images
assert [placeholder.length for placeholder in image_placeholders] == (
expected_tokens
)
image_token_id = processor.info.get_hf_processor().image_token_id
assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
def _assert_video_outputs(processor, processed) -> None:
mm_data = processed["mm_kwargs"].get_data()
grid_thw = mm_data["video_grid_thw"]
pixel_values = mm_data["pixel_values_videos"]
assert grid_thw.shape == (1, 3)
assert pixel_values.shape[0] == int(grid_thw.prod())
assert len(processed["mm_placeholders"]["video"]) == 1
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
expected_tokens = int(grid_thw.prod()) // merge_size**2
video_token_id = processor.info.get_hf_config().video_token_id
prompt_token_ids = processed["prompt_token_ids"]
assert prompt_token_ids.count(video_token_id) == expected_tokens
hf_processor = processor.info.get_hf_processor()
expected_frame_wrappers = int(grid_thw[:, 0].sum())
assert (
prompt_token_ids.count(hf_processor.vision_start_token_id)
== expected_frame_wrappers
)
assert (
prompt_token_ids.count(hf_processor.vision_end_token_id)
== expected_frame_wrappers
)
@pytest.mark.parametrize("num_images", [1, 2])
def test_process_images(
processor,
image_assets: ImageTestAssets,
num_images: int,
) -> None:
images = [asset.pil_image for asset in image_assets[:num_images]]
processed = processor(
IMAGE_PLACEHOLDER * num_images,
mm_items=processor.info.parse_mm_data({"image": images}),
hf_processor_mm_kwargs={},
)
_assert_image_outputs(processor, processed, num_images)
def test_process_video(processor) -> None:
video_asset = VideoAsset(name="baby_reading", num_frames=8)
video = (video_asset.np_ndarrays, video_asset.metadata)
processed = processor(
VIDEO_PLACEHOLDER,
mm_items=processor.info.parse_mm_data({"video": [video]}),
hf_processor_mm_kwargs={},
)
_assert_video_outputs(processor, processed)
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for H2OVL's multimodal preprocessing kwargs."""
from collections.abc import Mapping
import pytest
from PIL import Image
from transformers import PreTrainedConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.image import rescale_image_size
from vllm.multimodal.processing import BaseMultiModalProcessor
from ....conftest import ImageTestAssets
from ...utils import build_model_context
def _get_expected_num_patches(
config: PreTrainedConfig,
image: Image.Image,
num_imgs: int,
min_num: int,
max_num: int,
):
from vllm.transformers_utils.processors.h2ovl import (
calculate_h2ovl_targets,
get_h2ovl_target_ratios,
)
width, height = image.size
# Calculate the expected number of blocks
if num_imgs == 1 and config.use_msac:
# First pass
blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_h2ovl_target_ratios(
min_num=1,
max_num=max_num,
prior_aspect_ratio=None,
),
image_size=config.vision_config.image_size,
use_thumbnail=False, # Thumbnail is handled separately
)
# Second pass
blocks2, _, _, _ = calculate_h2ovl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_h2ovl_target_ratios(
min_num=3,
max_num=max_num,
prior_aspect_ratio=aspect_ratio,
),
image_size=config.vision_config.image_size,
use_thumbnail=False,
)
# Add thumbnail if use_thumbnail is True and total_blocks > 1
if config.use_thumbnail:
blocks1 += 1 if blocks1 > 1 else 0
blocks2 += 1 if blocks2 > 1 else 0
# Total blocks is the sum of blocks from both passes minus
# overlapping
total_blocks = blocks1 + blocks2 - 1
return total_blocks
blocks, _, _, _ = calculate_h2ovl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_h2ovl_target_ratios(
min_num,
max_num,
prior_aspect_ratio=None,
),
image_size=config.vision_config.image_size,
use_thumbnail=False,
)
expected_num_patches = blocks
if config.use_thumbnail and expected_num_patches > 1:
expected_num_patches += 1
return expected_num_patches
def _run_check(
processor: BaseMultiModalProcessor,
images: list[Image.Image],
min_num: int,
max_num: int,
mm_processor_kwargs: Mapping[str, object],
):
tokenizer = processor.info.get_tokenizer()
config = processor.info.get_hf_config()
prompt = "<image>" * len(images)
mm_data = {"image": images}
total_expected_num_patches = sum(
_get_expected_num_patches(config, image, len(images), min_num, max_num)
for image in images
)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@pytest.mark.parametrize(
"model_id",
[
"h2oai/h2ovl-mississippi-800m",
"h2oai/h2ovl-mississippi-2b",
],
)
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
[4.0, 2.0, 1.0],
],
)
@pytest.mark.parametrize(
("min_dynamic_patch", "max_dynamic_patch"),
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
)
@pytest.mark.parametrize("dynamic_image_size", [True, False])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
model_id: str,
image_assets: ImageTestAssets,
size_factors: list[int],
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool | None,
kwargs_on_init: bool,
):
mm_processor_kwargs = {
"min_dynamic_patch": min_dynamic_patch,
"max_dynamic_patch": max_dynamic_patch,
"dynamic_image_size": dynamic_image_size,
}
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": len(size_factors)},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
min_num = min_dynamic_patch if dynamic_image_size else 1
max_num = max_dynamic_patch if dynamic_image_size else 1
_run_check(
processor,
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
min_num,
max_num,
hf_processor_mm_kwargs,
)
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Idefics3's multimodal preprocessing kwargs."""
import pytest
from transformers import Idefics3Config
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"size": {"longest_edge": 364}}, 169),
({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
# Same as the previous test - don't initialize mm_processor_kwargs
# in this test and assume that the kwargs will be correctly expanded by
# the partial when calling the custom input processor.
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
placeholders = (
"<image>"
if num_imgs == 1
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
)
prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
# Build mm_data
image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
dummy_image_size = (image_size * 4, image_size * 4)
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
mm_data = {"image": [dummy_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure the placeholders format are correct
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
hf_processed_inputs = hf_processor(
text=prompt,
images=mm_data["image"],
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
)
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
# Ensure we have the right number of placeholders per num_crops size
image_token_id = ctx.get_hf_config().image_token_id
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
assert img_tok_count == expected_toks_per_img * num_imgs
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for InternVL's multimodal preprocessing kwargs."""
from collections.abc import Mapping
import pytest
from PIL import Image
from transformers import PreTrainedConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.image import rescale_image_size
from vllm.multimodal.processing import BaseMultiModalProcessor
from ....conftest import ImageTestAssets
from ...utils import build_model_context
def _get_expected_num_patches(
config: PreTrainedConfig,
image: Image.Image,
num_imgs: int,
min_num: int,
max_num: int,
):
from vllm.transformers_utils.processors.internvl import (
calculate_internvl_targets,
get_internvl_target_ratios,
)
width, height = image.size
blocks, _, _ = calculate_internvl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_internvl_target_ratios(
min_num,
max_num,
),
image_size=config.vision_config.image_size,
use_thumbnail=False,
)
expected_num_patches = blocks
if config.use_thumbnail and expected_num_patches > 1:
expected_num_patches += 1
return expected_num_patches
def _run_check(
processor: BaseMultiModalProcessor,
images: list[Image.Image],
min_num: int,
max_num: int,
mm_processor_kwargs: Mapping[str, object],
):
tokenizer = processor.info.get_tokenizer()
config = processor.info.get_hf_config()
prompt = "<image>" * len(images)
mm_data = {"image": images}
total_expected_num_patches = sum(
_get_expected_num_patches(config, image, len(images), min_num, max_num)
for image in images
)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
[4.0, 2.0, 1.0],
],
)
@pytest.mark.parametrize(
("min_dynamic_patch", "max_dynamic_patch"),
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
)
@pytest.mark.parametrize("dynamic_image_size", [True, False])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
model_id: str,
image_assets: ImageTestAssets,
size_factors: list[int],
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool | None,
kwargs_on_init: bool,
):
mm_processor_kwargs = {
"min_dynamic_patch": min_dynamic_patch,
"max_dynamic_patch": max_dynamic_patch,
"dynamic_image_size": dynamic_image_size,
}
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": len(size_factors)},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
min_num = min_dynamic_patch if dynamic_image_size else 1
max_num = max_dynamic_patch if dynamic_image_size else 1
_run_check(
processor,
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
min_num,
max_num,
hf_processor_mm_kwargs,
)
@@ -1,50 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for LightOnOCR precomputed image embeddings."""
import pytest
import torch
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("tensor_input", [False, True])
@pytest.mark.parametrize("prompt_as_tokens", [False, True])
def test_lightonocr_image_embeddings_preserve_lengths_and_values(
tensor_input: bool, prompt_as_tokens: bool
):
"""Reserve each image's feature count and pass projected embeddings through."""
ctx = build_model_context(
_LIGHTON_MODEL_ID,
limit_mm_per_prompt={"image": 2},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
ctx.model_config.get_multimodal_config().enable_mm_embeds = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
hidden_size = ctx.model_config.get_hidden_size()
feature_sizes = [3, 3] if tensor_input else [3, 7]
embeddings = [
torch.full((size, hidden_size), float(i))
for i, size in enumerate(feature_sizes)
]
image_input = torch.stack(embeddings) if tensor_input else embeddings
prompt = " ".join([hf_processor.image_token] * len(embeddings))
if prompt_as_tokens:
prompt = ctx.get_tokenizer().encode(prompt)
items = processor.info.parse_mm_data({"image": image_input})
processed = processor(prompt, mm_items=items)
assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum(
feature_sizes
)
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Llama4's multimodal preprocessing kwargs."""
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-4-Scout-17B-16E-Instruct"])
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
@pytest.mark.parametrize("num_imgs", [1, 5])
@pytest.mark.parametrize("mm_processor_cache_gb", [0, 4])
@pytest.mark.parametrize("tokenized_prompt", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict,
num_imgs: int,
mm_processor_cache_gb: int,
tokenized_prompt: bool,
):
"""Ensure llama4 processor works properly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": num_imgs},
mm_processor_cache_gb=mm_processor_cache_gb,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
config = processor.info.get_hf_config()
tokenizer = processor.info.get_tokenizer()
hf_processor = processor.info.get_hf_processor()
vocab = tokenizer.get_vocab()
prompt = (
"<|begin_of_text|><|header_start|>user<|header_end|>"
+ "<|image|>" * num_imgs
+ "<|eot|><|header_start|>assistant<|header_end|>"
)
mm_data = {
"image": [
image_assets[(i % len(image_assets))].pil_image for i in range(num_imgs)
]
}
if tokenized_prompt:
prompt = tokenizer.encode(prompt)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
mm_data = processed_inputs["mm_kwargs"].get_data()
# place holder replacements
prompt_token_ids = processed_inputs["prompt_token_ids"]
assert prompt_token_ids.count(config.boi_token_index) == num_imgs
assert prompt_token_ids.count(config.eoi_token_index) == num_imgs
assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs
aspect_ratios = mm_data["aspect_ratios"]
num_x_separators = num_y_separators = 0
for tiles_y, tiles_x in aspect_ratios:
if tiles_x * tiles_y > 1:
num_x_separators += (tiles_x - 1) * tiles_y
num_y_separators += tiles_y
assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators
assert (
prompt_token_ids.count(vocab[hf_processor.tile_global_token])
== num_y_separators
)
# image token offsets
img_locs = processed_inputs["mm_placeholders"].get("image", [])
assert len(img_locs) == num_imgs
assert [img_loc.offset for img_loc in img_locs] == [
i for i, v in enumerate(prompt_token_ids) if v == config.boi_token_index
]
# patch sizes and masks
num_patches_per_chunk = processor.info.get_patch_per_chunk(config.vision_config)
assert (
prompt_token_ids.count(config.image_token_index)
== sum(mm_data["patches_per_image"]) * num_patches_per_chunk
)
assert len(mm_data["pixel_values"]) == sum(mm_data["patches_per_image"])
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from functools import partial
import pytest
from PIL import Image
from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.tokenizers.hf import maybe_make_thread_pool
from ...utils import build_model_context
def _validate_image_max_tokens_one(
processor: BaseMultiModalProcessor,
max_tokens: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
info = processor.info
feature_size = info.get_num_image_tokens(
image_width=image_size.width, image_height=image_size.height
)
try:
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
except Exception as exc:
failed_size_excs.append((image_size, exc))
@pytest.mark.skip(
"This test takes around 5 minutes to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
def test_processor_max_tokens(model_id):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
info = processor.info
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 2
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(32, 4096), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_max_tokens_one,
processor,
info.get_max_image_tokens(), # type: ignore
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
def _validate_image_prompt_replacements_one(
processor: BaseMultiModalProcessor,
num_imgs: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
prompt = "<image>" * num_imgs
image = Image.new("RGB", size=image_size)
mm_data = {"image": [image] * num_imgs}
try:
# The processor will throw an error if there is a mismatch
# in the prompt replacements
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == num_imgs
first_placeholder = image_placeholders[0]
# NOTE: There is a BOS token
assert first_placeholder.offset == 1
assert (
first_placeholder.length
== (len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
)
except Exception as exc:
failed_size_excs.append((image_size, exc))
def _test_image_prompt_replacements(
processor,
*,
num_imgs: int,
image_sizes: list[ImageSize],
) -> None:
"""Ensure LlavaNextMultiModalProcessor
handles prompt replacement properly for input images.
"""
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_prompt_replacements_one,
processor,
num_imgs,
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_processor_prompt_replacements_regression(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
image_ratios = [
(171, 152),
(184, 161),
(198, 176),
(333, 296),
(369, 328),
(488, 183),
(2560, 1669),
]
image_sizes = [
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
]
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
@pytest.mark.skip(
"This test takes around 2 hours to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@pytest.mark.parametrize("num_imgs", [1])
def test_processor_prompt_replacements_all(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 2
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(64, 1024), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
@@ -1,211 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from functools import partial
import pytest
from PIL import Image
from pqdm.threads import pqdm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.tokenizers.hf import maybe_make_thread_pool
from ...utils import build_model_context
def _validate_image_max_tokens_one(
processor: BaseMultiModalProcessor,
max_tokens: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
info = processor.info
feature_size = info.get_num_image_tokens(
image_width=image_size.width, image_height=image_size.height
)
try:
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
except Exception as exc:
failed_size_excs.append((image_size, exc))
@pytest.mark.skip(
"This test takes around 5 minutes to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_processor_max_tokens(model_id):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": 1},
)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
info = processor.info
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 6
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(32, 4096), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_max_tokens_one,
processor,
info.get_max_image_tokens(), # type: ignore
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
def _validate_image_prompt_replacements_one(
processor: BaseMultiModalProcessor,
num_imgs: int,
failed_size_excs: list[tuple[ImageSize, Exception]],
image_size: ImageSize,
) -> None:
prompt = "<image>" * num_imgs
image = Image.new("RGB", size=image_size)
mm_data = {"image": [image] * num_imgs}
try:
# The processor will throw an error if there is a mismatch
# in the prompt replacements
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == num_imgs
first_placeholder = image_placeholders[0]
assert first_placeholder.offset == 0
assert (
first_placeholder.length
== len(processed_inputs["prompt_token_ids"]) // num_imgs
)
except Exception as exc:
failed_size_excs.append((image_size, exc))
def _test_image_prompt_replacements(
processor,
*,
num_imgs: int,
image_sizes: list[ImageSize],
) -> None:
"""Ensure LlavaOnevisionMultiModalProcessor
handles prompt replacement properly for input images.
"""
failed_size_excs = list[tuple[ImageSize, Exception]]()
validate_one = partial(
_validate_image_prompt_replacements_one,
processor,
num_imgs,
failed_size_excs,
)
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
if failed_size_excs:
msg = "Found failing image sizes:" + "\n========\n".join(
f"[{size}]\n{exc}" for size, exc in failed_size_excs
)
raise AssertionError(msg)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_processor_prompt_replacements_regression(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
# Avoid tokenizer already borrowed error
maybe_make_thread_pool(ctx.tokenizer)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
image_ratios = [
(171, 152),
(184, 161),
(198, 176),
(333, 296),
(369, 328),
(488, 183),
(2560, 1669),
]
image_sizes = [
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
]
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
@pytest.mark.skip(
"This test takes around 2 hours to run. Comment this out to run it manually."
)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
@pytest.mark.parametrize("num_imgs", [1])
def test_processor_prompt_replacements_all(model_id, num_imgs):
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
seen_aspect_ratios = set[float]()
image_sizes = list[ImageSize]()
# The aspect ratio of the grid layout is between 1 and 6
# NOTE: Assumes that feature size calculation is the same if we
# swap the width and height of the image
for w, h in itertools.product(range(64, 1024), repeat=2):
aspect_ratio = w / h
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
image_sizes.append(ImageSize(w, h))
seen_aspect_ratios.add(aspect_ratio)
_test_image_prompt_replacements(
processor,
num_imgs=num_imgs,
image_sizes=image_sizes,
)
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MiniCPMV's multimodal preprocessing."""
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
"""Calls with different kwargs must not reuse stale processor instances."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
info = processor.info
processor_1 = info.get_hf_processor(max_slice_nums=1)
processor_2 = info.get_hf_processor(max_slice_nums=2)
assert processor_1.image_processor.max_slice_nums == 1
assert processor_2.image_processor.max_slice_nums == 2
@pytest.mark.parametrize(
"model_ids",
[
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
],
)
def test_image_processor_for_different_models(model_ids):
first_model_id, second_model_id = model_ids
first_ctx = build_model_context(
first_model_id,
limit_mm_per_prompt={"image": 1},
)
first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
first_image_processor = first_processor.info.get_image_processor()
second_ctx = build_model_context(
second_model_id,
limit_mm_per_prompt={"image": 1},
)
second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
second_image_processor = second_processor.info.get_image_processor()
second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
assert type(first_image_processor) is not type(second_image_processor)
assert (
type(first_image_processor).__module__
!= type(second_image_processor).__module__
)
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = ctx.get_tokenizer()
messages = [
{"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
]
prompt = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
image = np.zeros((768, 1024, 3), dtype=np.uint8)
mm_items = processor.info.parse_mm_data({"image": [image]})
processed = processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
image_placeholders = processed["mm_placeholders"].get("image", [])
assert len(image_placeholders) == 1
assert image_placeholders[0].length > 0
@@ -1,138 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
These exercise the vendored processor directly (no checkpoint / GPU needed), so
they validate the long-side resize spec and the resulting prompt-token counts
deterministically.
"""
import pytest
import torch
from vllm.transformers_utils.processors.minimax_m3 import (
IMAGE_MAX_TOTAL_PIXELS,
MIN_SHORT_SIDE_PIXEL,
VIDEO_MAX_TOTAL_PIXELS,
MiniMaxM3VLImageProcessor,
MiniMaxM3VLVideoProcessor,
smart_resize,
)
# Long sides are multiples of patch_size*merge_size (28) so the rounding is
# exact and the expected token counts are unambiguous.
LONG_SIDES = [252, 504, 1008]
MERGE2 = 2**2 # merge_size ** 2
def _image_tokens(grid_thw) -> int:
g = list(grid_thw)
return int(g[0] * g[1] * g[2]) // MERGE2
# --------------------------------------------------------------------------- #
# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
# --------------------------------------------------------------------------- #
def test_smart_resize_long_side_shrink():
# (a) long side exceeds the cap -> shrink so the long side equals the cap.
h, w = smart_resize(
2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
)
assert max(h, w) == 1008
assert (h, w) == (1008, 504) # aspect ratio preserved
def test_smart_resize_short_side_enlarge():
# (b) long side within the cap but short side below the floor -> enlarge so
# the short side reaches min_short_side_pixel.
h, w = smart_resize(
200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
)
assert min(h, w) == MIN_SHORT_SIDE_PIXEL # 112
def test_smart_resize_total_pixels_raises():
# (c) still over the area cap after resizing -> raise instead of inferring.
with pytest.raises(ValueError, match="max_total_pixels"):
smart_resize(
5000,
5000,
factor=28,
max_long_side_pixel=4000,
max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
)
def test_smart_resize_backward_compatible_area_bound():
# Without max_long_side_pixel the original Qwen-style area bound is used.
assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
# --------------------------------------------------------------------------- #
# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
# --------------------------------------------------------------------------- #
def test_image_tokens_increase_with_max_long_side_pixel():
proc = MiniMaxM3VLImageProcessor()
counts = []
for long_side in LONG_SIDES:
patches = proc.get_number_of_image_patches(
2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
)
counts.append(patches // MERGE2)
assert counts == [81, 324, 1296]
assert counts[0] < counts[1] < counts[2]
def test_image_processor_defaults_match_spec():
proc = MiniMaxM3VLImageProcessor()
assert proc.max_long_side_pixel is None # opt-in
assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
def test_image_preprocess_pipeline_monotonic():
proc = MiniMaxM3VLImageProcessor()
image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
counts = []
for long_side in LONG_SIDES:
out = proc.preprocess(
[image],
do_resize=True,
max_long_side_pixel=long_side,
return_tensors="pt",
)
counts.append(_image_tokens(out["image_grid_thw"][0]))
assert counts == [81, 324, 1296]
# --------------------------------------------------------------------------- #
# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
# --------------------------------------------------------------------------- #
def test_video_tokens_increase_with_max_long_side_pixel():
proc = MiniMaxM3VLVideoProcessor()
assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
counts = []
for long_side in LONG_SIDES:
out = proc.preprocess(
videos=[video],
do_resize=True,
max_long_side_pixel=long_side,
return_tensors="pt",
)
counts.append(_image_tokens(out["video_grid_thw"][0]))
assert counts[0] < counts[1] < counts[2]
def test_video_volumetric_cap_raises():
proc = MiniMaxM3VLVideoProcessor()
# 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
with pytest.raises(ValueError, match="max_total_pixels"):
proc.preprocess(
videos=[video],
do_resize=True,
max_long_side_pixel=1008,
return_tensors="pt",
)
@@ -1,53 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for mllama's multimodal preprocessing and profiling."""
import pytest
from torch import prod
from transformers import Llama4Config
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-Guard-4-12B"])
@pytest.mark.parametrize("max_model_len", [4096, 8192, 25600, 131072])
def test_profiling(model_id: str, max_model_len: int):
model_config_kwargs = {
"max_model_len": max_model_len,
}
mm_counts = {"image": 1}
ctx = build_model_context(
model_id,
model_config_kwargs=model_config_kwargs,
limit_mm_per_prompt=mm_counts,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_inputs = processor.get_dummy_mm_inputs(mm_counts)
hf_config = ctx.get_hf_config(Llama4Config)
image_size = hf_config.vision_config.image_size
patch_size = hf_config.vision_config.patch_size
downsample_ratio = int(
round(1.0 / (hf_config.vision_config.pixel_shuffle_ratio**2))
)
tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio
mm_data = mm_inputs["mm_kwargs"].get_data()
chunks_per_image = prod(mm_data["patches_per_image"])
total_num_patches = chunks_per_image * tokens_per_patch
num_tiles = (
mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][1]
) # x-y separator tokens
total_tokens = (
total_num_patches.item() + num_tiles.item() + 3
) # image start, image, image end
assert total_num_patches == sum(
item.get_num_embeds() for item in mm_inputs["mm_placeholders"]["image"]
)
assert total_tokens == sum(
placeholder.length for placeholder in mm_inputs["mm_placeholders"]["image"]
)
@@ -1,553 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Moondream3 multimodal processing.
Includes:
- Processor creation and application tests
- Image tokenization and placeholder expansion tests
- Tiling and cropping logic tests (CPU-based)
- Pixel normalization tests
"""
import numpy as np
import pytest
import torch
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
# Expected multimodal prefix: BOS + 729 image tokens.
EXPECTED_IMAGE_TOKENS = 730
# Vision encoder constants
CROP_SIZE = 378
PATCH_SIZE = 14
MAX_CROPS = 12
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_creation(model_id: str):
"""Test that Moondream3 processor can be created."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
assert processor is not None
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_apply(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that Moondream3 processor can process inputs.
NOTE: The prompt includes the leading BOS token because Moondream3
pre-fills BOS and image embeddings together.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
assert "prompt_token_ids" in processed_inputs
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_pixel_values(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that pixel values are correctly produced."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Check mm_kwargs contains pixel_values
mm_kwargs = processed_inputs.get("mm_kwargs")
assert mm_kwargs is not None
mm_data_result = mm_kwargs.get_data()
assert "pixel_values" in mm_data_result
# Verify pixel_values shape
pixel_values = mm_data_result["pixel_values"]
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
assert pixel_values.shape[2] == 3 # RGB channels
assert pixel_values.shape[3] == 378 # crop height
assert pixel_values.shape[4] == 378 # crop width
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_image_token_expansion(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that <image> placeholder is expanded to correct number of tokens."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_multi_crop_tiling(
model_id: str,
):
"""Test that large images produce correct multi-crop tiling."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Create a large image that requires multiple crops
large_image = Image.new("RGB", (1000, 1000), color="blue")
pixel_values, tiling = processor.preprocess_image(large_image)
# Large images should produce more than 1x1 tiling
assert tiling[0] >= 1 and tiling[1] >= 1
# Check that we have global crop + local crops
expected_crops = tiling[0] * tiling[1] + 1
assert pixel_values.shape[0] == expected_crops
@pytest.mark.parametrize(
"image_size",
[
(500, 500),
(800, 600),
(1920, 1080),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_tiling_various_sizes(
image_size: tuple[int, int],
model_id: str,
):
"""Test tiling with various image sizes."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
width, height = image_size
image = Image.new("RGB", (width, height), color="red")
pixel_values, tiling = processor.preprocess_image(image)
# Basic shape checks
assert pixel_values.dim() == 4 # [num_crops, C, H, W]
assert pixel_values.shape[1] == 3 # RGB
assert pixel_values.shape[2] == 378 # crop height
assert pixel_values.shape[3] == 378 # crop width
# Tiling should respect max_crops (12)
assert tiling[0] * tiling[1] <= 12
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_pixel_normalization(
model_id: str,
):
"""Test that pixel values are normalized to [-1, 1] range."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Create test image
image = Image.new("RGB", (378, 378), color="green")
pixel_values, _ = processor.preprocess_image(image)
# Normalization: (x - 0.5) / 0.5 = 2*x - 1
# For input [0, 1], output should be [-1, 1]
assert pixel_values.min() >= -1.0
assert pixel_values.max() <= 1.0
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_chat_template_with_image(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that chat template correctly formats BOS + image + prompt."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = ctx.tokenizer
# Use the chat template format
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
token_ids = processed_inputs["prompt_token_ids"]
# BOS token (<|endoftext|>) should be token ID 0
bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
assert bos_token_id == 0
# First token should be BOS
assert token_ids[0] == bos_token_id
@pytest.mark.parametrize(
"content",
[
pytest.param(
[
{
"type": "image_url",
"image_url": {"url": "https://example.invalid/image.png"},
},
{"type": "text", "text": "What is in this image?"},
],
id="image-first",
),
pytest.param(
[
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {"url": "https://example.invalid/image.png"},
},
],
id="text-first",
),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_chat_template_content_list_uses_moondream_image_prefix(
image_assets: ImageTestAssets,
content: list[dict[str, object]],
model_id: str,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
prompt = hf_processor.tokenizer.apply_chat_template(
[{"role": "user", "content": content}],
chat_template=hf_processor.chat_template,
tokenize=False,
)
expected_prompt = (
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
"What is in this image?<|md_reserved_2|>"
)
assert prompt == expected_prompt
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_bos_token_always_first(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that BOS token (ID 0) is always at position 0."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Start with BOS token explicitly
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
token_ids = processed_inputs["prompt_token_ids"]
# Token ID 0 (<|endoftext|>) should be the first token
assert token_ids[0] == 0, (
f"Expected BOS token (0) at position 0, got {token_ids[0]}"
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_with_small_image(
model_id: str,
):
"""Test processor with image smaller than crop size."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Small image (smaller than crop size)
small_image = Image.new("RGB", (100, 100), color="yellow")
pixel_values, tiling = processor.preprocess_image(small_image)
# Small images should use 1x1 tiling
assert tiling == (1, 1)
# Should have 2 crops (global + 1 local)
assert pixel_values.shape[0] == 2
@pytest.mark.parametrize(
"image_kind",
[
pytest.param("numpy_hwc", id="numpy-hwc"),
pytest.param("numpy_chw", id="numpy-chw"),
pytest.param("torch_chw", id="torch-chw"),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_preprocess_image_accepts_non_pil_inputs(
image_assets: ImageTestAssets,
image_kind: str,
model_id: str,
):
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
pil_image = image_assets[0].pil_image.convert("RGB")
hwc_array = np.asarray(pil_image)
expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
if image_kind == "numpy_hwc":
image = hwc_array
elif image_kind == "numpy_chw":
image = np.transpose(hwc_array, (2, 0, 1))
else:
image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
pixel_values, tiling = processor.preprocess_image(image)
assert tiling == expected_tiling
assert pixel_values.shape == expected_pixel_values.shape
assert pixel_values.dtype == torch.bfloat16
assert torch.equal(pixel_values, expected_pixel_values)
@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_apply_accepts_non_pil_image_inputs(
image_assets: ImageTestAssets,
image_kind: str,
model_id: str,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data({"image": [image]}),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
class TestMoondream3TilingLogic:
"""CPU-based tests for Moondream3 tiling selection logic.
These tests validate the select_tiling() function which determines
how images are divided into crops for the vision encoder.
"""
def test_small_image_no_tiling(self):
"""Small images should use 1x1 tiling."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
assert tiling == (1, 1)
def test_exact_crop_size(self):
"""Image exactly at crop size should use 1x1."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
assert tiling == (1, 1)
def test_large_square_image(self):
"""Large square image should use multiple tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert h_tiles >= 2
assert w_tiles >= 2
assert h_tiles * w_tiles <= MAX_CROPS
def test_wide_image(self):
"""Wide image should have more width tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert w_tiles >= h_tiles
def test_tall_image(self):
"""Tall image should have more height tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert h_tiles >= w_tiles
def test_respects_max_crops(self):
"""Tiling should not exceed max_crops."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
)
h_tiles, w_tiles = tiling
assert h_tiles * w_tiles <= 4
class TestMoondream3VisionShapes:
"""CPU-based tests for vision encoder expected shapes.
These tests verify the mathematical relationships between
crop size, patch size, and token counts.
"""
def test_expected_patch_count(self):
"""Test 378/14 = 27 patches per side, 729 total."""
patches_per_side = CROP_SIZE // PATCH_SIZE
total_patches = patches_per_side**2
assert patches_per_side == 27
assert total_patches == EXPECTED_IMAGE_TOKENS - 1
def test_patch_embedding_input_dim(self):
"""Test patch embedding input dimension."""
channels = 3
input_dim = PATCH_SIZE * PATCH_SIZE * channels
assert input_dim == 14 * 14 * 3
assert input_dim == 588
class TestMoondream3TauAttention:
"""CPU-based tests for tau attention scaling components.
These tests validate the tau attention formula used in Moondream3:
- Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
- Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
"""
def test_tau_position_range(self):
"""Test tau position scaling produces values in valid range."""
num_heads = 32
seq_len = 100
tau_alpha = torch.randn(num_heads)
positions = torch.arange(seq_len)
pos_float = (positions.float() + 1.0).clamp(min=1e-6)
pos_log = pos_float.log()
tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
assert tau_pos.shape == (num_heads, seq_len)
# tau_pos should be between 0.5 and 1.5
assert tau_pos.min() >= 0.5
assert tau_pos.max() <= 1.5
def test_tau_token_output_range(self):
"""Test tau token scaling output is bounded by tanh."""
import torch.nn.functional as F
seq_len = 100
qkv_dim = 6144 # 2048 * 3
num_heads = 32
qkv = torch.randn(seq_len, qkv_dim)
tau_wq = torch.randn(num_heads, qkv_dim)
tok_feat = F.gelu(qkv)
tok_q = torch.tanh(tok_feat @ tau_wq.t())
assert tok_q.shape == (seq_len, num_heads)
# tanh output is bounded by [-1, 1]
assert tok_q.min() >= -1.0
assert tok_q.max() <= 1.0
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs."""
from collections.abc import Mapping
import pytest
from PIL import Image
from transformers import PreTrainedConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.image import rescale_image_size
from vllm.multimodal.processing import BaseMultiModalProcessor
from ....conftest import ImageTestAssets
from ...utils import build_model_context
def _get_expected_num_patches(
config: PreTrainedConfig,
image: Image.Image,
num_imgs: int,
min_num: int,
max_num: int,
):
from vllm.transformers_utils.processors.nemotron_vl import (
calculate_nemotron_vl_targets,
get_nemotron_vl_target_ratios,
)
width, height = image.size
blocks, _, _ = calculate_nemotron_vl_targets(
orig_width=width,
orig_height=height,
target_ratios=get_nemotron_vl_target_ratios(
min_num,
max_num,
),
image_size=config.force_image_size,
use_thumbnail=False,
)
expected_num_patches = blocks
if config.use_thumbnail and expected_num_patches > 1:
expected_num_patches += 1
return expected_num_patches
def _run_check(
processor: BaseMultiModalProcessor,
images: list[Image.Image],
min_num: int,
max_num: int,
mm_processor_kwargs: Mapping[str, object],
):
tokenizer = processor.info.get_tokenizer()
config = processor.info.get_hf_config()
image_processor = processor.info.get_image_processor()
config.use_thumbnail = image_processor.use_thumbnail
prompt = "<image>" * len(images)
mm_data = {"image": images}
total_expected_num_patches = sum(
_get_expected_num_patches(config, image, len(images), min_num, max_num)
for image in images
)
print(total_expected_num_patches)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("<image>")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@pytest.mark.parametrize("model_id", ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"])
@pytest.mark.parametrize(
"size_factors",
[
# Single-scale
[1.0],
# Single-scale, batched
[1.0, 1.0, 1.0],
# Multi-scale
[0.25, 0.5, 1.0],
[4.0, 2.0, 1.0],
],
)
@pytest.mark.parametrize(
("min_dynamic_patch", "max_dynamic_patch"),
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
)
@pytest.mark.parametrize("dynamic_image_size", [True, False])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
model_id: str,
image_assets: ImageTestAssets,
size_factors: list[int],
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool | None,
kwargs_on_init: bool,
):
mm_processor_kwargs = {
"min_dynamic_patch": min_dynamic_patch,
"max_dynamic_patch": max_dynamic_patch,
"dynamic_image_size": dynamic_image_size,
}
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": len(size_factors)},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
min_num = min_dynamic_patch if dynamic_image_size else 1
max_num = max_dynamic_patch if dynamic_image_size else 1
_run_check(
processor,
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
min_num,
max_num,
hf_processor_mm_kwargs,
)
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs."""
import numpy as np
import pytest
import torch
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen3_asr import Qwen3ASRProcessor
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.transformers_utils.processor import cached_processor_from_config
from .test_common import _assert_inputs_equal
_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>"
@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"])
def asr_processor(request):
config = ModelConfig(
request.param,
dtype="float32",
max_model_len=4096,
limit_mm_per_prompt={"audio": 2},
mm_processor_cache_gb=0,
)
processor = MULTIMODAL_REGISTRY.create_processor(config)
native = Qwen3ASRProcessor.from_pretrained(request.param)
return processor, native
@pytest.mark.parametrize(
"samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001]
)
@pytest.mark.parametrize("num_audios", [1, 2])
def test_audio_processing_matches_hf(asr_processor, samples, num_audios):
"""Each clip must match HF independently of other clips in the request."""
processor, native = asr_processor
rng = np.random.RandomState(42)
audios = [
rng.randn(samples + index * 160).astype(np.float32)
for index in range(num_audios)
]
expected = [
native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios
]
actual = processor(
_AUDIO_PROMPT * num_audios,
mm_items=processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
expected_ids = [
token for item in expected for token in item["input_ids"][0].tolist()
]
assert actual["prompt_token_ids"] == expected_ids
masks = [item["input_features_mask"][0] for item in expected]
expected_features = torch.cat(
[
item["input_features"][0, :, mask.bool()]
for item, mask in zip(expected, masks)
],
dim=1,
)
actual_features = actual["mm_kwargs"].get_data()
torch.testing.assert_close(
actual_features["input_audio_features"], expected_features, rtol=0, atol=0
)
torch.testing.assert_close(
actual_features["audio_feature_lengths"],
torch.stack([mask.sum() for mask in masks]),
)
@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]])
def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices):
"""Partial cache hits must not change features, masks, or prompt expansion."""
processor, _ = asr_processor
config = processor.info.ctx.model_config
config.get_multimodal_config().mm_processor_cache_gb = 1
cache = MultiModalProcessorOnlyCache(config)
rng = np.random.RandomState(42)
audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)]
for index in cached_indices:
processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": audios[index]}),
hf_processor_mm_kwargs={},
cache=cache,
)
for batch in (audios, audios[::-1]):
kwargs = dict(
mm_items=processor.info.parse_mm_data({"audio": batch}),
hf_processor_mm_kwargs={},
)
_assert_inputs_equal(
processor(_AUDIO_PROMPT * 2, **kwargs),
processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache),
)
def test_text_only_processing(asr_processor):
"""Empty media input must not invoke the audio feature extractor."""
processor, native = asr_processor
prompt = "Transcribe the audio."
actual = processor(
prompt,
mm_items=processor.info.parse_mm_data({}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt)
assert actual["mm_kwargs"].get_data() == {}
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}])
def test_hf_processor_without_audio(asr_processor, hf_data):
"""The media hook must not require audio or call HF for absent waveforms."""
processor, _ = asr_processor
assert not processor._call_hf_processor(hf_data, {})
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}])
def test_postprocessing_without_audio_features(asr_processor, hf_data):
"""Non-audio output must pass through even when HF input is nonempty."""
processor, _ = asr_processor
processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])})
actual = processor._postprocess_hf_mm_data(hf_data, {}, processed)
assert actual is processed
assert set(actual) == {"input_ids"}
torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]]))
def test_chat_template_processing(asr_processor):
"""The generic processor loader must still render audio chat prompts."""
processor, native = asr_processor
chat_processor = cached_processor_from_config(processor.info.ctx.model_config)
messages = [
{"role": "system", "content": ""},
{"role": "user", "content": [{"type": "audio", "audio": "unused"}]},
]
prompt = chat_processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
expected_prompt = native.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
assert prompt == expected_prompt
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
expected = native(text=expected_prompt, audio=audio, return_tensors="pt")
actual = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audio}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist()
def test_precomputed_audio_features(asr_processor):
"""Precomputed features must bypass waveform extraction unchanged."""
processor, _ = asr_processor
processor.info.ctx.get_mm_config().enable_mm_embeds = True
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
expected = processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": audio}),
hf_processor_mm_kwargs={},
)
features = expected["mm_kwargs"].get_data()
actual = processor(
_AUDIO_PROMPT,
mm_items=processor.info.parse_mm_data({"audio": features}),
hf_processor_mm_kwargs={},
)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features)
@@ -1,140 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Qwen3 Omni audio processing and sample rate handling."""
from typing import Any
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
@pytest.mark.parametrize(
("audio_sample_rate", "audio_duration_sec"),
[
(16000, 1.0), # Native Whisper sample rate, 1 second
(16000, 2.0), # Native Whisper sample rate, 2 seconds
],
)
def test_processor_with_audio_sample_rate(
model_id: str,
audio_sample_rate: int,
audio_duration_sec: float,
) -> None:
"""Test that vLLM's processor generates expected outputs with audio_sample_rate.
This validates that the processor correctly handles audio_sample_rate
passed via hf_processor_mm_kwargs and generates audio tokens.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
# Create audio data at the specified sample rate
audio_length = int(audio_sample_rate * audio_duration_sec)
rng = np.random.RandomState(42)
audio_data = rng.rand(audio_length).astype(np.float32)
# Build prompt with audio placeholder
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
# Apply processor with audio_sample_rate in mm_kwargs
hf_processor_mm_kwargs: dict[str, Any] = {
"audio_sample_rate": audio_sample_rate,
}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Verify audio tokens are generated
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token)
aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id)
assert aud_tok_count >= 1, (
f"Expected at least 1 audio token but got {aud_tok_count}. "
f"sample_rate: {audio_sample_rate}Hz, duration: {audio_duration_sec}s"
)
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
def test_longer_audio_generates_more_tokens(model_id: str) -> None:
"""Test that longer audio generates more tokens than shorter audio.
This validates that audio_sample_rate is being used correctly by checking
that audio duration affects token count as expected.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
audio_sample_rate = 16000
rng = np.random.RandomState(42)
def get_token_count(duration: float) -> int:
audio_length = int(audio_sample_rate * duration)
audio_data = rng.rand(audio_length).astype(np.float32)
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
hf_processor_mm_kwargs: dict[str, Any] = {
"audio_sample_rate": audio_sample_rate,
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token)
return processed["prompt_token_ids"].count(audio_token_id)
short_tokens = get_token_count(1.0)
long_tokens = get_token_count(2.0)
assert long_tokens > short_tokens, (
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
f"Got short={short_tokens}, long={long_tokens}"
)
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
"""Audio and video tokens interleave by the fps the video was processed
at, as in HF, not by the default of one frame a second."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
rng = np.random.RandomState(42)
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
audio = rng.rand(32000).astype(np.float32)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
mm_kwargs: dict[str, Any] = {
"use_audio_in_video": True,
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [video], "audio": [(audio, 16000)]}
),
hf_processor_mm_kwargs=mm_kwargs,
)
expected = processor.info.get_hf_processor()(
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
)
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
@@ -1,85 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for smolvlm's multimodal preprocessing kwargs."""
import pytest
from transformers import SmolVLMConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"max_image_size": {"longest_edge": 384}}, 1377),
({"max_image_size": {"longest_edge": 768}}, 405),
({"do_image_splitting": False}, 81),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
# Same as the previous test - don't initialize mm_processor_kwargs
# in this test and assume that the kwargs will be correctly expanded by
# the partial when calling the custom input processor.
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
placeholders = (
"<image>"
if num_imgs == 1
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
)
prompt = f"<|im_start|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
# Build mm_data
image_size = ctx.get_hf_config(SmolVLMConfig).vision_config.image_size
dummy_image_size = (image_size * 4, image_size * 4)
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
mm_data = {"image": [dummy_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure the placeholders format are correct
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
num_patches = processor.info.get_num_patches(
image_width=dummy_image.width,
image_height=dummy_image.height,
processor=hf_processor,
mm_kwargs=hf_processor_mm_kwargs,
)
assert num_patches == expected_toks_per_img // hf_processor.image_seq_len
hf_processed_inputs = hf_processor(
text=prompt,
images=mm_data["image"],
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
)
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
# Ensure we have the right number of placeholders per num_crops size
image_token_id = ctx.get_hf_config().image_token_id
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
assert img_tok_count == expected_toks_per_img * num_imgs
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Whisper's multimodal preprocessing."""
import numpy as np
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["openai/whisper-large-v3-turbo"])
@pytest.mark.parametrize("audio_duration_s", [5, 30, 35])
def test_audio_features_fit_encoder_window(
model_id: str,
audio_duration_s: int,
) -> None:
"""The encoder only has positions for one 30 s window, so a longer clip
has to be cut to that window like the HF feature extractor does by default.
"""
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
feature_extractor = processor.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
rng = np.random.RandomState(0)
audio = rng.rand(sampling_rate * audio_duration_s).astype(np.float32)
processed_inputs = processor(
"<|startoftranscript|>",
mm_items=processor.info.parse_mm_data({"audio": [(audio, sampling_rate)]}),
hf_processor_mm_kwargs={},
)
input_features = processed_inputs["mm_kwargs"].get_data()["input_features"]
assert input_features.shape[-1] == feature_extractor.nb_max_frames
-164
View File
@@ -1,164 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.multimodal.image import rescale_image_size
from ...conftest import IMAGE_ASSETS, ImageTestAssets, VllmRunner
from ..utils import check_logprobs_close
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
"cherry_blossom": "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
}
)
IMAGE_SIZE_FACTOR_GROUPS = (
# Single-scale
(1.0,),
# Single-scale, batched
(1.0, 1.0, 1.0),
# Multi-scale
(0.25, 0.5, 1.0),
)
def run_awq_test(
vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets,
source_model: str,
quant_model: str,
*,
size_factor_groups: tuple[tuple[float, ...], ...],
dtype: str,
max_tokens: int,
num_logprobs: int,
tensor_parallel_size: int,
distributed_executor_backend: str | None = None,
):
images = [asset.pil_image for asset in image_assets]
inputs_per_image_and_size_group = [
(
[prompt for _ in size_factors],
[rescale_image_size(image, factor) for factor in size_factors],
)
for size_factors in size_factor_groups
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
]
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
# max_model_len should be greater than image_feature_size
with vllm_runner(
source_model,
max_model_len=4096,
dtype=dtype,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=True,
default_torch_num_threads=1,
) as vllm_model:
source_outputs_per_image = [
vllm_model.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs=num_logprobs, images=images
)
for prompts, images in inputs_per_image_and_size_group
]
with vllm_runner(
quant_model,
quantization="awq",
max_model_len=4096,
dtype=dtype,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=True,
default_torch_num_threads=1,
) as vllm_model:
quant_outputs_per_image = [
vllm_model.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs=num_logprobs, images=images
)
for prompts, images in inputs_per_image_and_size_group
]
for source_outputs, quant_outputs in zip(
source_outputs_per_image, quant_outputs_per_image
):
# TODO: Check whether using original CLIPVisionModel can improve
# consistency against HF
check_logprobs_close(
outputs_0_lst=source_outputs,
outputs_1_lst=quant_outputs,
name_0="source",
name_1="awq",
)
@pytest.mark.parametrize(
("model", "quantization", "dtype"),
[
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
],
ids=[
"gemma4-moe-standard-awq-dot-suffix",
"gemma4-moe-compressed-tensors-underscore-suffix",
],
)
@torch.inference_mode()
def test_awq_load(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: str,
quantization: str,
dtype: str,
) -> None:
"""Regression test: AWQ weight loading must not KeyError."""
with vllm_runner(
model,
quantization=quantization,
dtype=dtype,
max_model_len=128,
enforce_eager=True,
) as vllm_model:
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
assert len(outputs) == 2
@pytest.mark.parametrize(
("source_model", "quant_model"),
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [5])
@torch.inference_mode()
def test_awq_models(
vllm_runner,
image_assets,
source_model,
quant_model,
dtype,
max_tokens,
num_logprobs,
) -> None:
run_awq_test(
vllm_runner,
image_assets,
source_model,
quant_model,
size_factor_groups=IMAGE_SIZE_FACTOR_GROUPS,
dtype=dtype,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
tensor_parallel_size=1,
)
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for online FP8 per-channel quantization.
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
quantization) and compares log-probabilities against the same model served in
BF16 without quantization. This exercises the full pipeline: config parsing,
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
loading, online quantization / shuffling, and inference.
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
diverse prompts from ``tests/prompts/example.txt``.
"""
import pytest
from tests.quantization.utils import is_quant_method_supported
from ..utils import check_logprobs_close
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
# Small dense model (no MoE) to validate the linear-only path.
DENSE_MODEL = "Qwen/Qwen3-0.6B"
MAX_MODEL_LEN = 1024
MAX_TOKENS = 4
NUM_LOG_PROBS = 8
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported on this GPU type.",
)
@pytest.mark.quant_model
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_fp8_per_channel_logprobs(
vllm_runner,
quant_baseline_logprobs,
example_prompts,
model: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Compare BF16 baseline logprobs against online per-channel-quantized
model.
Runs the same model twice -- once in BF16 (baseline) and once with online
FP8 per-channel quantization -- then checks that the top log-probabilities
are close. Only 4 tokens are generated to keep the test fast while still
catching numerical divergence beyond expected per-channel error.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
baseline_outputs = quant_baseline_logprobs(
model,
example_prompts,
max_model_len=MAX_MODEL_LEN,
max_tokens=MAX_TOKENS,
num_logprobs=NUM_LOG_PROBS,
)
with vllm_runner(
model,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
quantization="fp8_per_channel",
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="bf16",
name_1="fp8_per_channel",
)
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
device gate accepts the card, so the checkpoint could not be served natively at
all.
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
a correction bias), so the backend only has to ask for it. This module covers
both halves: the gate accepts the router family, and the routing the wrapper
requests reproduces the model's own math.
"""
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
MIMO_TOP_K = 8
def mimo_routing_method() -> RoutingMethodType:
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
return get_routing_method_type(
scoring_func="sigmoid",
top_k=MIMO_TOP_K,
renormalize=True,
num_expert_group=1,
has_e_score_bias=True,
)
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=num_experts,
experts_per_token=MIMO_TOP_K,
hidden_dim=4096,
intermediate_size=2048,
num_local_experts=num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device=torch.device("cuda"),
routing_method=mimo_routing_method(),
router_logits_dtype=torch.float32,
)
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
def test_aiter_w4a16_supports_mimo_routing() -> None:
"""The gate accepts MiMo's router family — the reported construction abort.
Only the predicate the fix touches is exercised, so this runs on any host;
the full config path is covered by the test below.
"""
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
)
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="only runs on ROCm with a supported AITER install",
)
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
"""The same gate through the real config path, device check included."""
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
AiterW4A16ExpertsMonolithic,
mimo_moe_config(),
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="only runs on ROCm with a supported AITER install",
)
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
"""The routing the wrapper requests must equal MiMo's own router math.
MiMo's router is ungrouped: sigmoid scores, selection on
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
"""
aiter_routing = pytest.importorskip(
"aiter.ops.triton.moe.moe_routing.routing"
).routing
torch.manual_seed(0)
top_k, num_tokens = MIMO_TOP_K, 16
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
bias = torch.randn(num_experts, device="cuda") * 0.5
# Reference: select on the biased score, return the unbiased score, renorm.
scores = torch.sigmoid(logits.float())
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
ref_weights = scores.gather(1, ref_ids)
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
routing_data, topk_indx, _ = aiter_routing(
logits,
top_k,
score_mode=score_mode,
bias=bias,
renorm=True,
routed_scaling_factor=1.0,
use_grouped_topk=False,
)
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
# i-th entry of the expert-sorted array whose weights are gate_scal.
hist = routing_data.expt_hist.to(torch.long).cpu()
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
slot = topk_indx.reshape(-1).to(torch.long).cpu()
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
out = torch.zeros(num_tokens, top_k)
for i in range(slot.numel()):
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
ids[token, position] = int(expert_of_slot[i])
out[token, position] = float(weights[i])
return ids, out
got_ids, got_weights = routed("sigmoid")
# aiter's triton top-k emits each token's selection sorted by *ascending
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
# rank order — align both sides on the expert id so the weight check
# becomes elementwise and still validates the expert<->weight pairing.
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
order = torch.argsort(ids, dim=-1)
return ids.gather(1, order), w.gather(1, order)
got_ids, got_weights = align(got_ids, got_weights)
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
assert torch.equal(got_ids, ref_ids)
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
# Control: the correction bias must steer selection only. Ignoring it would
# pick from the plain sigmoid scores instead.
plain_ids = scores.topk(top_k, dim=-1).indices
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-93
View File
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only tests for AliceAI configuration."""
from types import SimpleNamespace
import pytest
from vllm.model_executor.models.config import AliceAIForCausalLMConfig
def _make_config(**overrides) -> SimpleNamespace:
return SimpleNamespace(
**{
"kda_allow_negative_eigenvalues": False,
"router_score_function": "sigmoid",
"router_bias_correction": True,
**overrides,
}
)
def test_model_config_warns_when_mtp_is_ignored(caplog) -> None:
config = _make_config(mtp_num_hidden_layers=1)
with caplog.at_level("WARNING"):
AliceAIForCausalLMConfig.verify_and_update_model_config(
SimpleNamespace(hf_config=config)
)
assert "ignores the MTP configuration and weights" in caplog.text
@pytest.mark.parametrize(
"overrides,error_type,error",
[
({}, None, None),
(
{"kda_allow_negative_eigenvalues": True},
NotImplementedError,
"kda_allow_negative_eigenvalues=false",
),
(
{"router_score_function": "softmax"},
ValueError,
"router_score_function='sigmoid'",
),
(
{"router_bias_correction": False},
ValueError,
"router_bias_correction=true",
),
],
)
def test_model_config_rejects_unsupported_alice_contract(
overrides, error_type, error
) -> None:
model_config = SimpleNamespace(hf_config=_make_config(**overrides))
if error is None:
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
else:
with pytest.raises(error_type, match=error):
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
@pytest.mark.parametrize(
"pipeline_parallel_size,speculative_method,error",
[
(1, None, None),
(2, None, "pipeline_parallel_size=1"),
(1, "eagle3", "eagle3"),
(1, "dflash", "dflash"),
(1, "dspark", "dspark"),
(1, "extract_hidden_states", "extract_hidden_states"),
],
)
def test_runtime_config_rejects_unsupported_alice_modes(
pipeline_parallel_size, speculative_method, error
) -> None:
vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(pipeline_parallel_size=pipeline_parallel_size),
speculative_config=(
None
if speculative_method is None
else SimpleNamespace(method=speculative_method)
),
)
if error is None:
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
else:
with pytest.raises(NotImplementedError, match=error):
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
-57
View File
@@ -1,57 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import pytest
import torch
from tests.conftest import VllmRunner
from tests.utils import create_new_process_for_each_test
pytestmark = pytest.mark.skipif(
importlib.util.find_spec("terratorch") is None,
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
)
@create_new_process_for_each_test() # Hangs otherwise
@pytest.mark.parametrize(
"model",
[
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars",
],
)
def test_inference(
vllm_runner: type[VllmRunner],
model: str,
) -> None:
pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16)
location_coords = torch.full((1, 2), 1.0, dtype=torch.float16)
prompt = dict(
prompt_token_ids=[1],
multi_modal_data={
"image": {
"pixel_values": pixel_values,
"location_coords": location_coords,
}
},
)
with vllm_runner(
model,
runner="pooling",
dtype="half",
enforce_eager=True,
skip_tokenizer_init=True,
enable_mm_embeds=True,
# Limit the maximum number of sequences to avoid the
# test going OOM during the warmup run
max_num_seqs=32,
default_torch_num_threads=1,
) as vllm_model:
vllm_output = vllm_model.llm.encode(prompt, pooling_task="plugin")
assert torch.equal(
torch.isnan(vllm_output[0].outputs.data).any(), torch.tensor(False)
)
File diff suppressed because it is too large Load Diff
+611
View File
@@ -0,0 +1,611 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import warnings
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any
import torch
import torch.nn.functional as F
from transformers import PreTrainedConfig
from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption
from vllm.config.pooler import SequencePoolingType, TokenPoolingType
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers import cached_tokenizer_from_config
from .. import ci_envs
from .registry import HF_EXAMPLE_MODELS
TokensText = tuple[list[int], str]
def check_outputs_equal(
*,
outputs_0_lst: Sequence[TokensText],
outputs_1_lst: Sequence[TokensText],
name_0: str,
name_1: str,
):
"""Compare the two sequences generated by different models,
which should be equal.
"""
assert len(outputs_0_lst) == len(outputs_1_lst)
for prompt_idx, (outputs_0, outputs_1) in enumerate(
zip(outputs_0_lst, outputs_1_lst)
):
output_ids_0, output_str_0 = outputs_0
output_ids_1, output_str_1 = outputs_1
# The text and token outputs should exactly match
fail_msg = (
f"Test{prompt_idx}:"
f"\n{name_0}:\t{output_str_0!r}"
f"\n{name_1}:\t{output_str_1!r}"
)
assert output_str_0 == output_str_1, fail_msg
assert output_ids_0 == output_ids_1, fail_msg
# Representation of generated sequence as a tuple of
# * Token ID list
# * String
# * List of top sample logprobs for each sampled token
#
# Assumes prompt logprobs were not requested.
TokensTextLogprobs = tuple[
list[int], str, list[dict[int, float]] | SampleLogprobs | None
]
# Allow for tokens to be represented as str's rather than IDs;
# tuple of
# * Token string representations list
# * String
# * Optional list of top sample logprobs for each sampled token
#
# Assumes prompt logprobs were not requested.
TextTextLogprobs = tuple[
list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None
]
# Representation of generated sequence as a tuple of
# * Token ID list
# * String
# * Optional list of top sample logprobs for each sampled token
# * Optional list of top prompt logprobs for each prompt token
#
# Allows prompt logprobs to be requested.
TokensTextLogprobsPromptLogprobs = tuple[
list[int],
str,
list[dict[int, float]] | SampleLogprobs | None,
list[dict[int, float] | None] | PromptLogprobs | None,
]
def check_logprobs_close(
*,
outputs_0_lst: Sequence[
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
],
outputs_1_lst: Sequence[
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
],
name_0: str,
name_1: str,
num_outputs_0_skip_tokens: int = 0,
warn_on_mismatch: bool = True,
always_check_logprobs: bool = False,
) -> None:
"""Compare the logprobs of two sequences generated by different models,
which should be similar but not necessarily equal.
How sample logprobs are compared:
* `always_check_logprobs == True`: set of highest-logprob token ids
must match between seq0 and seq1 at all sampled token offsets
* `always_check_logprobs == False`: highest-logprob token ids are
only compared at sampled token offsets for which generated token
ids don't match
Prompt logprobs must be provided either for both input sequences, or
for neither. If prompt logprobs are provided, then highest-logprob
prompt token ids must match between seq0 and seq1 at all prompt token
offsets.
Args:
outputs_0_lst: First sequence to compare
outputs_0_lst: Second sequence to compare
name_0: sequence #0 name
name_1: sequence #1 name
num_outputs_0_skip_tokens: If > 0, specifies the number of initial
sequence #0 tokens & logprobs to discard
before comparison, i.e. all
of sequence #1 will be compared to
sequence #0 beginning at index
num_outputs_0_skip_tokens
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
mismatch between the two sequences
always_check_logprobs: If true, check logprobs even when tokens match
"""
assert len(outputs_0_lst) == len(outputs_1_lst)
# Loop through responses to each prompt.
for prompt_idx, (outputs_0, outputs_1) in enumerate(
zip(outputs_0_lst, outputs_1_lst)
):
assert len(outputs_0) == len(outputs_1)
if len(outputs_0) == 3:
assert len(outputs_1) == 3
# Break out tokens, text & sample logprobs
# (prompt logprobs were not provided)
output_ids_0, output_str_0, logprobs_0 = outputs_0
output_ids_1, output_str_1, logprobs_1 = outputs_1
elif len(outputs_0) == 4:
assert len(outputs_1) == 4
# Break out tokens, text, sample logprobs & prompt logprobs
(
output_ids_0,
output_str_0,
logprobs_0,
prompt_logprobs_0,
) = outputs_0
(
output_ids_1,
output_str_1,
logprobs_1,
prompt_logprobs_1,
) = outputs_1
# Test prompt logprobs closeness
if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:
# Both sequences' prompt logprobs lists are not `None`
# (although individual list elements may be `None`);
# for each token's logprobs:
for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
zip(prompt_logprobs_0, prompt_logprobs_1)
):
fail_msg = (
f"Prompt logprobs test:"
f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}"
)
if logprobs_elem_0 is None:
# If the seq 0 token's logprobs are `None`,
# the seq 1 token's logprobs must be `None`
assert logprobs_elem_1 is None, fail_msg
else:
# If the seq 0 token's logprobs are not `None`,
# the seq 1 token's logprobs must not be `None`
assert logprobs_elem_1 is not None, fail_msg
# Logprobs check: top-k token choices must be the same
assert set(logprobs_elem_0.keys()) == set(
logprobs_elem_1.keys()
), fail_msg
else:
# Both sequence logprobs lists must be `None`
fail_msg = (
f"Prompt logprobs test:"
f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}"
)
assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg
else:
raise ValueError(
f"Outputs tuple must have 3 or 4 elements but "
f"{len(outputs_0)} elements were provided: "
f"{outputs_0}"
)
if logprobs_0 is None:
logprobs_0 = [None] * len(output_ids_0)
if logprobs_1 is None:
logprobs_1 = [None] * len(output_ids_1)
# Skip specified number of initial sequence #0 tokens
# & logprobs, leaving output text as-is for simplicity
# (text mismatches may generate warnings but do not
# cause the test to fail.)
if num_outputs_0_skip_tokens < 0:
raise ValueError("num_outputs_0_skip_tokens must be non-negative")
output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]
# Loop through generated tokens.
for idx, (output_id_0, output_id_1) in enumerate(
zip(output_ids_0, output_ids_1)
):
is_tok_mismatch = output_id_0 != output_id_1
# If generated tokens don't match
# or it is desired to always check logprobs,
# then
if is_tok_mismatch or always_check_logprobs:
logprobs_elem_0 = logprobs_0[idx]
logprobs_elem_1 = logprobs_1[idx]
# Each predicted token must be in top N logprobs of the other
fail_msg = (
f"Test{prompt_idx}:"
f"\nMatched tokens:\t{output_ids_0[:idx]}"
f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}"
f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}"
)
assert logprobs_elem_0 is not None, fail_msg
assert logprobs_elem_1 is not None, fail_msg
assert output_id_0 in logprobs_elem_1, fail_msg
assert output_id_1 in logprobs_elem_0, fail_msg
if warn_on_mismatch and is_tok_mismatch:
with warnings.catch_warnings():
# This ensures that repeated warnings are shown
# in the output, not just the first occurrence
warnings.simplefilter("always")
warnings.warn(fail_msg, stacklevel=2)
# Break out since sequences will now diverge.
break
else:
if output_str_0 != output_str_1 and warn_on_mismatch:
# The token outputs exactly match,
# so the text outputs should exactly match as well
fail_msg = (
f"Test{prompt_idx}:"
f"\n{name_0}:\t{output_str_0!r}"
f"\n{name_1}:\t{output_str_1!r}"
)
with warnings.catch_warnings():
# This ensures that repeated warnings are shown
# in the output, not just the first occurrence
warnings.simplefilter("always")
warnings.warn(fail_msg, stacklevel=2)
def build_model_context(
model_id: str,
runner: RunnerOption = "auto",
dtype: ModelDType = "auto",
model_config_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None,
mm_processor_cache_gb: int = 0,
):
"""Creates an InputProcessingContext for a given model.
Args:
model_id: ID of the model being considered.
mm_processor_kwargs: optional processor kwargs for to be leveraged
in the input processor, mapper, dummy data creation, etc.
limit_mm_per_prompt: Multimodal limits.
Returns:
InputProcessingContext for the model being considered.
"""
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_config_kwargs = model_config_kwargs or {}
limit_mm_per_prompt = {
modality: dict(limit) if isinstance(limit, Mapping) else limit
for modality, limit in (limit_mm_per_prompt or {}).items()
}
model_config = ModelConfig(
model_id,
runner=runner,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
dtype=dtype,
seed=0,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt=limit_mm_per_prompt,
mm_processor_cache_gb=mm_processor_cache_gb,
hf_overrides=model_info.hf_overrides,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
**model_config_kwargs,
)
return InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
def check_embeddings_close(
*,
embeddings_0_lst: Sequence[list[float]],
embeddings_1_lst: Sequence[list[float]],
name_0: str,
name_1: str,
tol: float = 1e-3,
) -> None:
assert len(embeddings_0_lst) == len(embeddings_1_lst)
for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
zip(embeddings_0_lst, embeddings_1_lst)
):
assert len(embeddings_0) == len(embeddings_1), (
f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}"
)
sim = F.cosine_similarity(
torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0
)
fail_msg = (
f"Test{prompt_idx}:"
f"\nCosine similarity: \t{sim:.4f}"
f"\n{name_0}:\t{embeddings_0[:16]!r}"
f"\n{name_1}:\t{embeddings_1[:16]!r}"
)
assert sim >= 1 - tol, fail_msg
def matryoshka_fy(tensor: torch.Tensor, dimensions: int):
tensor = torch.tensor(tensor)
tensor = tensor[..., :dimensions]
tensor = F.normalize(tensor, p=2, dim=1)
return tensor
def softmax(data):
if data.shape[-1] == 1:
return F.sigmoid(data)
else:
return F.softmax(data, dim=-1)
@dataclass
class ModelInfo:
name: str
revision: str | None = None
architecture: str = ""
dtype: str = "auto"
max_model_len: int | None = None
hf_dtype: str = "float32"
hf_overrides: dict[str, Any] | None = None
seq_pooling_type: SequencePoolingType | None = None
tok_pooling_type: TokenPoolingType | None = None
attn_type: AttnTypeStr | None = None
is_prefix_caching_supported: bool | None = None
is_chunked_prefill_supported: bool | None = None
enable_test: bool = True
@dataclass
class EmbedModelInfo(ModelInfo):
mteb_score: float | None = None
# Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL.
mteb_tol: float | None = None
is_matryoshka: bool = False
matryoshka_dimensions: list[int] | None = None
@dataclass
class RerankModelInfo(ModelInfo):
mteb_score: float | None = None
# Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL.
mteb_tol: float | None = None
chat_template_name: str | None = None
@dataclass
class GenerateModelInfo(ModelInfo):
hf_dtype: str = "auto"
hf_ppl: float | None = None
def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs):
# A model family has many models with the same architecture,
# and we don't need to test each one.
if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test:
import pytest
pytest.skip("Skipping test.")
# Allow vllm to test using the given dtype, such as float32
vllm_extra_kwargs = vllm_extra_kwargs or {}
vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype
# Allow vllm to test using hf_overrides
if model_info.hf_overrides is not None:
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
# Allow changing the head dtype used by vllm in tests
if ci_envs.VLLM_CI_HEAD_DTYPE is not None:
if "hf_overrides" not in vllm_extra_kwargs:
vllm_extra_kwargs["hf_overrides"] = {}
vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE
# Allow control over whether tests use enforce_eager
if ci_envs.VLLM_CI_ENFORCE_EAGER is not None:
vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER
return vllm_extra_kwargs
def dummy_hf_overrides(
hf_config: PreTrainedConfig,
*,
model_arch: str = "",
exist_overrides: dict[str, Any] | None = None,
use_original_num_layers: bool = False,
) -> PreTrainedConfig:
"""Dummy HF overrides function used to create dummy model
with only minimum nums of layer.
"""
# Copy because this helper is called more than once
# while loading config, and we `.pop()`
exist_overrides = (exist_overrides or {}).copy()
text_config_override = exist_overrides.pop("text_config", None)
hf_config.update(exist_overrides)
text_config = hf_config.get_text_config()
if text_config_override is not None:
# multimodal test models may override *some* text-model fields
text_config.update(text_config_override)
# Ensure at least 2 expert per group
# Since `grouped_topk` assumes top-2
n_group = getattr(text_config, "n_group", None)
# Kimi uses `num_expert_group` instead of `n_group`.
if n_group is None:
n_group = getattr(text_config, "num_expert_group", None)
# InternS1Pro uses `router_n_groups` instead of `n_group`.
if n_group is None:
n_group = getattr(text_config, "router_n_groups", None)
num_experts = n_group * 2 if n_group is not None else 2
# we use three layers for Gemma-3n to check
# both normal layer and kv_shared_layer
if use_original_num_layers:
# Use the original number of layers from the config
num_layers = getattr(text_config, "num_layers", 1)
num_hidden_layers = getattr(text_config, "num_hidden_layers", 1)
else:
# Use minimal layers for testing
num_layers = 1
num_hidden_layers = (
3
if model_arch
in (
"Gemma3nForConditionalGeneration",
"Gemma4ForCausalLM",
"Gemma4ForConditionalGeneration",
"Gemma4MTPModel",
"DiffusionGemmaForBlockDiffusion",
)
else 1
)
update_dict = {
"num_layers": num_layers,
# For Gemma-3n
"num_kv_shared_layers": 1,
}
_hf_config = hf_config
class DummyConfig:
hf_config = _hf_config
hf_text_config = text_config
# Keep architecture conversion on the default multimodal path; this
# helper only needs HF-derived fields, not deployment MM limits.
@staticmethod
def _supports_multimodal_for_mm_prefix() -> bool:
return True
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
# Only set MoE related config when the model has MoE layers.
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
if model_arch_config.num_experts > 0:
num_experts_per_tok = 2
if model_arch in (
"Llama4ForConditionalGeneration",
"Llama4ForCausalLM",
"EagleLlama4ForCausalLM",
):
num_experts_per_tok = 1
elif model_arch == "InternS1ProForConditionalGeneration":
assert n_group is not None
num_experts_per_tok = n_group
update_dict.update(
{
"num_experts": num_experts,
"num_experts_per_tok": num_experts_per_tok,
# Kimi uses `num_experts_per_token`.
"num_experts_per_token": num_experts_per_tok,
"num_local_experts": num_experts,
# Otherwise there will not be any expert layers
"first_k_dense_replace": 0,
# To avoid OOM on DeepSeek-V3
"n_routed_experts": num_experts,
}
)
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
# from num_layers for its dual-attention layers).
if model_arch not in (
"LongcatFlashForCausalLM",
"LongCatFlashMTPModel",
"LongcatFlashNgramForCausalLM",
):
update_dict["num_hidden_layers"] = num_hidden_layers
text_config.update(update_dict)
# Keep per-layer metadata consistent with the reduced layer count.
# HY V4 uses this to decide which sparse-attention indexers are shared.
if indexer_types := getattr(text_config, "indexer_types", None):
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
# Update n_layers and moe configs for Moondream3 model
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
text_config.update(
{
"n_layers": num_hidden_layers,
"moe_num_experts": num_experts,
"moe_experts_per_token": 2,
"moe_start_layer": num_hidden_layers,
}
)
if hasattr(hf_config, "vision_config"):
vision_config = hf_config.vision_config
vision_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
}
)
# Keep per-layer metadata consistent with the reduced layer count.
if layer_types := getattr(vision_config, "layer_types", None):
vision_config.update({"layer_types": layer_types[:1]})
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
vision_config.update({"enc_n_layers": 1})
# e.g.: ibm-granite/granite-speech-3.3-2b
if hasattr(hf_config, "encoder_config"):
hf_config.encoder_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
}
)
# e.g.: Qwen/Qwen2-Audio-7B-Instruct
# audio_config may exist but be None (e.g. audio-less Gemma4 variants).
if getattr(hf_config, "audio_config", None) is not None:
hf_config.audio_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
"encoder_layers": 1,
}
)
return hf_config
+131
View File
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the unified Cohere Command parser tests."""
from __future__ import annotations
import itertools
import regex as re
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser import ParserManager
from vllm.parser.abstract_parser import Parser
_SPECIAL_TOKEN_RE = re.compile(r"(<\|[A-Z_]+\|>)")
REPLACEMENT_CHAR = "�"
class MockCohereTokenizer:
"""Byte-level stand-in for the Cohere tokenizer.
Text round-trips through UTF-8 bytes so a multi-byte character can split
across "tokens" (reproducing trailing U+FFFD buffering), while each
``<|...|>`` marker is a single id so token-id reasoning-end gating behaves
as with the real tokenizer.
"""
_SPECIAL_TOKEN_IDS = {
tok: 256 + i
for i, tok in enumerate(
(
"<|START_THINKING|>",
"<|END_THINKING|>",
"<|CHATBOT_TOKEN|>",
"<|START_RESPONSE|>",
"<|END_RESPONSE|>",
"<|START_TEXT|>",
"<|END_TEXT|>",
"<|START_ACTION|>",
"<|END_ACTION|>",
)
)
}
_ID_TO_SPECIAL_TOKEN = {v: k for k, v in _SPECIAL_TOKEN_IDS.items()}
def convert_tokens_to_ids(self, token: str) -> int:
return self._SPECIAL_TOKEN_IDS.get(token, 0)
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
ids: list[int] = []
for part in _SPECIAL_TOKEN_RE.split(text):
if part in self._SPECIAL_TOKEN_IDS:
ids.append(self._SPECIAL_TOKEN_IDS[part])
else:
ids.extend(part.encode("utf-8"))
return ids
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
out: list[str] = []
for special, run in itertools.groupby(
ids, self._ID_TO_SPECIAL_TOKEN.__contains__
):
if not special:
out.append(bytes(run).decode("utf-8", errors="replace"))
elif not skip_special_tokens:
out.extend(self._ID_TO_SPECIAL_TOKEN[i] for i in run)
return "".join(out)
def token_deltas(
tokenizer: MockCohereTokenizer, text: str, chunk_size: int = 1
) -> list[tuple[str, list[int]]]:
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
U+FFFD) into the next step as real streaming does."""
ids = tokenizer.encode(text)
deltas: list[tuple[str, list[int]]] = []
prev = ""
pending: list[int] = []
for start in range(0, len(ids), chunk_size):
end = start + chunk_size
pending.extend(ids[start:end])
current = tokenizer.decode(ids[:end])
if current.endswith(REPLACEMENT_CHAR) and end < len(ids):
continue
deltas.append((current[len(prev) :], pending))
prev, pending = current, []
return deltas
def make_parser(
tokenizer: MockCohereTokenizer,
name: str,
tools: list[dict] | None = None,
chat_template_kwargs: dict | None = None,
model_config=None,
) -> Parser:
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
name (``cohere_command3`` / ``cohere_command4``)."""
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
assert cls is not None
return cls(
tokenizer,
tools,
model_config=model_config,
chat_template_kwargs=chat_template_kwargs or {},
)
def drive_parser(
parser: Parser,
request: ChatCompletionRequest,
deltas: list[tuple[str, list[int]]],
) -> list[DeltaMessage]:
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
return the non-``None`` messages in order."""
out: list[DeltaMessage] = []
for i, (text, ids) in enumerate(deltas):
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
if delta is not None:
out.append(delta)
return out
def stream_parser(parser, request, tokenizer, text: str, chunk_size: int = 1):
"""Stream ``text`` through ``parser`` in ``chunk_size``-token deltas."""
return drive_parser(parser, request, token_deltas(tokenizer, text, chunk_size))
File diff suppressed because it is too large Load Diff
+199
View File
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from typing import Any
import regex as re
import torch
from pydantic import TypeAdapter
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.quantization import (
QuantizationConfigArgs,
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
def quant_spec(**kwargs: Any) -> QuantSpec:
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
return TypeAdapter(QuantSpec).validate_python(kwargs)
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
def _limit_num_hidden_layers(
model: torch.nn.Module, num_hidden_layers: int | None
) -> None:
if num_hidden_layers is None:
return
original_load_weights = model.load_weights
def should_load_weight(name: str) -> bool:
layers_prefix = "layers."
if name.startswith(layers_prefix):
layer_name = name.removeprefix(layers_prefix)
elif ".layers." in name:
layer_name = name.split(".layers.", maxsplit=1)[1]
else:
return True
layer_idx_str = layer_name.split(".", maxsplit=1)[0]
if layer_idx_str.isdigit():
return int(layer_idx_str) < num_hidden_layers
return True
def load_weights(weights):
weights = (
(name, weight) for name, weight in weights if should_load_weight(name)
)
return original_load_weights(weights)
model.load_weights = load_weights
def load_model_without_vllm_runner(
model_path: str,
*,
dtype: str | torch.dtype = "bfloat16",
quantization: str | None = None,
model_config_kwargs: dict[str, Any] | None = None,
vllm_config_kwargs: dict[str, Any] | None = None,
model_loader_cls: type = DefaultModelLoader,
) -> tuple[torch.nn.Module, VllmConfig]:
"""Instantiate a model, load weights, and process them for inference."""
model_config = ModelConfig(
model=model_path,
dtype=dtype,
quantization=quantization,
**(model_config_kwargs or {}),
)
model_config.quantization_config = resolve_quantization_config(
model_config.quantization, model_config.quantization_config
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault(
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
)
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
text_config_overrides = hf_overrides.get("text_config") or {}
num_hidden_layers = text_config_overrides.get(
"num_hidden_layers", hf_overrides.get("num_hidden_layers")
)
with set_current_vllm_config(vllm_config):
model_loader = model_loader_cls(vllm_config.load_config)
if num_hidden_layers is not None:
original_load_weights = model_loader.load_weights
def load_weights(model, model_config):
_limit_num_hidden_layers(model, num_hidden_layers)
original_load_weights(model, model_config)
model_loader.load_weights = load_weights
model = model_loader.load_model(vllm_config, model_config)
return model, vllm_config
def is_quant_method_supported(quant_method: str) -> bool:
# Currently, quantization tests only run GPUs
if current_platform.is_cpu():
return False
try:
current_platform.verify_quantization(quant_method)
except ValueError:
return False
if current_platform.is_xpu():
return True
capability = current_platform.get_device_capability()
assert capability is not None
min_capability = get_quantization_config(quant_method).get_min_capability()
return capability.to_int() >= min_capability
def _test_online_quant_peak_mem_impl(
quantization_arg_value,
vllm_runner,
caplog_mp_spawn,
monkeypatch,
) -> None:
# Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
# 1. it covers both Linear and MoE paths
# 2. it is already used by other tests in CI, so adding it here
# does not increase disk space for CI runners
# I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
# which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
# 1.3 GiB fp8), but could not as adding one more model makes CI
# run out of disk space.
model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
# Force spawn to ensure caplog_mp_spawn works consistently
# (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
with (
caplog_mp_spawn(logging.DEBUG) as log_holder,
vllm_runner(
model_name,
quantization=quantization_arg_value,
enforce_eager=True,
) as llm,
):
outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
print(outputs[0][1])
log_text = log_holder.text
# Parse memory usage from captured logs
model_memory_gib = None
peak_memory_gib = None
for line in log_text.splitlines():
if model_memory_gib is None:
match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
if match:
model_memory_gib = float(match.group(1))
if peak_memory_gib is None:
match = re.search(
r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
)
if match:
peak_memory_gib = float(match.group(1))
assert model_memory_gib is not None, "Could not find model loading memory log"
assert peak_memory_gib is not None, "Could not find peak memory log"
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
expected_model_memory_gib = 6.7
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
# GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
# number is expected as when we load and quantize weights in a streaming
# fashion we need to have individual weights in bf16 + fp8 alive at the
# same time.
expected_peak_memory_gib = expected_model_memory_gib * 1.4
assert model_memory_gib < expected_model_memory_gib, (
f"{model_memory_gib=} higher than {expected_model_memory_gib}"
)
assert peak_memory_gib < expected_peak_memory_gib, (
f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
)
+162
View File
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
from transformers import PreTrainedTokenizerBase
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.reasoning import ReasoningParser
from vllm.utils.mistral import is_mistral_tokenizer
class StreamingReasoningReconstructor:
def __init__(self):
self.reasoning = None
self.other_content = None
def append_delta(self, delta: DeltaMessage):
# content and the reasoning content should not be present
# at the same time
assert delta.content is None or delta.reasoning is None, (
"Both content and reasoning content are present in the delta message"
)
if delta.content is not None:
if self.other_content is None:
self.other_content = delta.content
else:
self.other_content += delta.content
else:
if self.reasoning is None:
self.reasoning = delta.reasoning
else:
self.reasoning += delta.reasoning
def run_reasoning_extraction(
reasoning_parser: ReasoningParser,
model_output: list[str],
request: ChatCompletionRequest | None = None,
streaming: bool = False,
) -> tuple[str | None, str | None]:
if streaming:
reconstructor = run_reasoning_extraction_streaming(
reasoning_parser,
model_output,
request,
)
return (
reconstructor.reasoning,
reconstructor.other_content or None,
)
else:
reasoning, content = run_reasoning_extraction_nonstreaming(
reasoning_parser, model_output, request
)
return reasoning, content
def run_reasoning_extraction_mistral(
reasoning_parser: ReasoningParser,
model_output: list[int],
request: ChatCompletionRequest | None = None,
streaming: bool = False,
) -> tuple[str | None, str | None]:
assert is_mistral_tokenizer(reasoning_parser.model_tokenizer), type(
reasoning_parser.model_tokenizer
)
if streaming:
reconstructor = run_reasoning_extraction_streaming_mistral(
reasoning_parser,
model_output,
request,
)
return (
reconstructor.reasoning,
reconstructor.other_content or None,
)
else:
str_output = reasoning_parser.model_tokenizer.convert_ids_to_tokens(
model_output
)
reasoning, content = run_reasoning_extraction_nonstreaming(
reasoning_parser, str_output, request
)
return reasoning, content
def run_reasoning_extraction_nonstreaming(
reasoning_parser: ReasoningParser,
model_output: list[str],
request: ChatCompletionRequest | None = None,
) -> tuple[str | None, str | None]:
request = request or ChatCompletionRequest(messages=[], model="test-model")
return reasoning_parser.extract_reasoning(
model_output="".join(model_output), request=request
)
def run_reasoning_extraction_streaming(
reasoning_parser: ReasoningParser,
model_deltas: list[str],
request: ChatCompletionRequest | None = None,
) -> StreamingReasoningReconstructor:
request = request or ChatCompletionRequest(messages=[], model="test-model")
reconstructor = StreamingReasoningReconstructor()
previous_text = ""
previous_tokens: list[int] = []
tokenizer = cast(PreTrainedTokenizerBase, reasoning_parser.model_tokenizer)
for delta in model_deltas:
token_delta = [
reasoning_parser.vocab[token]
for token in tokenizer.tokenize(delta)
if token in reasoning_parser.vocab
]
current_text = previous_text + delta
current_tokens = previous_tokens + token_delta
delta_message = reasoning_parser.extract_reasoning_streaming(
previous_text,
current_text,
delta,
previous_tokens,
current_tokens,
token_delta,
)
if delta_message is not None:
reconstructor.append_delta(delta_message)
previous_text = current_text
previous_tokens = current_tokens
return reconstructor
def run_reasoning_extraction_streaming_mistral(
reasoning_parser: ReasoningParser,
model_deltas: list[int],
request: ChatCompletionRequest | None = None,
) -> StreamingReasoningReconstructor:
assert is_mistral_tokenizer(reasoning_parser.model_tokenizer), type(
reasoning_parser.model_tokenizer
)
request = request or ChatCompletionRequest(messages=[], model="test-model")
reconstructor = StreamingReasoningReconstructor()
previous_text = ""
previous_tokens: list[int] = []
for model_delta in model_deltas:
token_delta = [model_delta]
delta = reasoning_parser.model_tokenizer.convert_ids_to_tokens([model_delta])[0]
current_text = previous_text + delta
current_tokens = previous_tokens + token_delta
delta_message = reasoning_parser.extract_reasoning_streaming(
previous_text,
current_text,
delta,
previous_tokens,
current_tokens,
token_delta,
)
if delta_message is not None:
reconstructor.append_delta(delta_message)
previous_text = current_text
previous_tokens = current_tokens
return reconstructor
+4268
View File
File diff suppressed because it is too large Load Diff
+380
View File
@@ -0,0 +1,380 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from types import NoneType
from typing import Any
import pytest
from tests.tool_parsers.utils import run_tool_extraction
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParserManager
@dataclass
class ToolParserTestConfig:
"""Configuration for a tool parser's common tests.
This dataclass contains all the test data and expected results needed
to run the common test suite for a parser. Each parser test file
creates one instance of this config with parser-specific values.
Attributes:
parser_name: Name used with ToolParserManager (e.g., "mistral")
Test data (model outputs):
no_tool_calls_output: Plain text without any tool syntax
single_tool_call_output: One tool call with simple arguments
parallel_tool_calls_output: Multiple tool calls in one response
various_data_types_output: Tool with various data types
empty_arguments_output: Tool call with no parameters
surrounding_text_output: Tool call mixed with regular text
escaped_strings_output: Tool call with escaped chars
malformed_input_outputs: List of invalid inputs
Expected results:
single_tool_call_expected_name: Expected function name
single_tool_call_expected_args: Expected arguments dict
parallel_tool_calls_count: Number of tools in parallel test
parallel_tool_calls_names: Function names in order
single_tool_call_expected_content: Content field when tool called
parallel_tool_calls_expected_content: Content for parallel test
xfail markers:
xfail_streaming: Mapping test name to xfail reason (streaming only)
xfail_nonstreaming: Mapping test name to xfail reason (non-streaming)
Special flags:
allow_empty_or_json_empty_args: True if "" or "{}" both valid for empty args
supports_typed_arguments: True if the parser supports typed function arguments
"""
# Parser identification
parser_name: str
# Test data - model outputs for each common test
no_tool_calls_output: str
single_tool_call_output: str
parallel_tool_calls_output: str
various_data_types_output: str
empty_arguments_output: str
surrounding_text_output: str
escaped_strings_output: str
malformed_input_outputs: list[str]
# Expected results for specific tests (optional overrides)
single_tool_call_expected_name: str = "get_weather"
single_tool_call_expected_args: dict[str, Any] = field(
default_factory=lambda: {"city": "Tokyo"}
)
parallel_tool_calls_count: int = 2
parallel_tool_calls_names: list[str] = field(
default_factory=lambda: ["get_weather", "get_time"]
)
# xfail configuration - maps test name to xfail reason
xfail_streaming: dict[str, str] = field(default_factory=dict)
xfail_nonstreaming: dict[str, str] = field(default_factory=dict)
# Content expectations (some parsers strip content, others don't)
single_tool_call_expected_content: str | None = None
parallel_tool_calls_expected_content: str | None = None
# Special assertions for edge cases
allow_empty_or_json_empty_args: bool = True # "{}" or "" for empty args
supports_typed_arguments: bool = True
class ToolParserTests:
"""Mixin class providing common test suite for tool parsers.
To use this mixin in a parser test file:
1. Create a test_config fixture that returns a ToolParserTestConfig instance
2. Inherit from this class
3. Add parser-specific tests as additional methods
Example:
class TestMistralToolParser(ToolParserTests):
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="mistral",
no_tool_calls_output="Plain text...",
# ... other config ...
)
# Parser-specific tests
def test_mistral_specific_feature(self, tool_parser):
# Custom test logic
pass
"""
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
"""Override this to provide parser-specific configuration."""
raise NotImplementedError(
"Subclass must provide test_config fixture returning ToolParserTestConfig"
)
@pytest.fixture
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
"""Override this to provide parser-specific tokenizer."""
return default_tokenizer
@pytest.fixture
def tool_parser(self, test_config: ToolParserTestConfig, tokenizer: TokenizerLike):
return ToolParserManager.get_tool_parser(test_config.parser_name)(tokenizer)
@pytest.fixture(params=[True, False])
def streaming(self, request: pytest.FixtureRequest) -> bool:
return request.param
def test_no_tool_calls(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles plain text without tool syntax."""
# Apply xfail markers if configured
test_name = "test_no_tool_calls"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.no_tool_calls_output, streaming=streaming
)
assert content == test_config.no_tool_calls_output, (
f"Expected content to match input, got {content}"
)
assert len(tool_calls) == 0, f"Expected no tool calls, got {len(tool_calls)}"
def test_single_tool_call_simple_args(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser extracts one tool with simple arguments."""
# Apply xfail markers if configured
test_name = "test_single_tool_call_simple_args"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.single_tool_call_output, streaming=streaming
)
# Content check (some parsers strip it)
if test_config.single_tool_call_expected_content is not None:
assert content == test_config.single_tool_call_expected_content
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
assert tool_calls[0].type == "function"
assert tool_calls[0].function.name == test_config.single_tool_call_expected_name
args = json.loads(tool_calls[0].function.arguments)
for key, value in test_config.single_tool_call_expected_args.items():
assert args.get(key) == value, (
f"Expected {key}={value}, got {args.get(key)}"
)
def test_parallel_tool_calls(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles multiple tools in one response."""
# Apply xfail markers if configured
test_name = "test_parallel_tool_calls"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser,
test_config.parallel_tool_calls_output,
streaming=streaming,
)
assert len(tool_calls) == test_config.parallel_tool_calls_count, (
f"Expected {test_config.parallel_tool_calls_count} "
f"tool calls, got {len(tool_calls)}"
)
# Verify tool names match expected
for i, expected_name in enumerate(test_config.parallel_tool_calls_names):
assert tool_calls[i].type == "function"
assert tool_calls[i].function.name == expected_name
# Verify unique IDs
ids = [tc.id for tc in tool_calls]
assert len(ids) == len(set(ids)), "Tool call IDs should be unique"
def test_various_data_types(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles all JSON types in arguments."""
# Apply xfail markers if configured
test_name = "test_various_data_types"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser,
test_config.various_data_types_output,
streaming=streaming,
)
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
args = json.loads(tool_calls[0].function.arguments)
# Verify all expected fields present
required_fields_types = {
"string_field": str,
"int_field": int,
"float_field": float,
"bool_field": bool,
"null_field": NoneType,
"array_field": list,
"object_field": dict,
}
for required_field, expected_type in required_fields_types.items():
assert required_field in args, (
f"Expected field '{required_field}' in arguments"
)
if test_config.supports_typed_arguments:
found_type = type(args[required_field])
assert found_type is expected_type, (
f"Expected field '{required_field}' to have type {expected_type}, "
f"got {found_type}"
)
def test_empty_arguments(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles parameterless tool calls."""
# Apply xfail markers if configured
test_name = "test_empty_arguments"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.empty_arguments_output, streaming=streaming
)
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
args = tool_calls[0].function.arguments
if test_config.allow_empty_or_json_empty_args:
assert args in ["{}", ""], f"Expected empty args, got {args}"
else:
assert args == "{}", f"Expected {{}}, got {args}"
def test_surrounding_text(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser extracts tools from mixed content."""
# Apply xfail markers if configured
test_name = "test_surrounding_text"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.surrounding_text_output, streaming=streaming
)
assert len(tool_calls) >= 1, (
f"Expected at least 1 tool call, got {len(tool_calls)}"
)
def test_escaped_strings(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles escaped characters in arguments."""
# Apply xfail markers if configured
test_name = "test_escaped_strings"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.escaped_strings_output, streaming=streaming
)
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
args = json.loads(tool_calls[0].function.arguments)
# At minimum, verify we can parse and have expected fields
# Exact escaping behavior varies by parser
assert len(args) > 0, "Expected some arguments with escaped strings"
def test_malformed_input(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser gracefully handles invalid syntax."""
# Apply xfail markers if configured
test_name = "test_malformed_input"
self.apply_xfail_mark(request, test_config, test_name, streaming)
for malformed_input in test_config.malformed_input_outputs:
# Should not raise exception
content, tool_calls = run_tool_extraction(
tool_parser, malformed_input, streaming=streaming
)
# Parser should handle gracefully (exact behavior varies)
def test_streaming_reconstruction(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
):
"""Verify streaming produces same result as non-streaming."""
test_name = "test_streaming_reconstruction"
self.apply_xfail_mark(request, test_config, test_name, True)
test_output = test_config.single_tool_call_output
# Non-streaming result
content_non, tools_non = run_tool_extraction(
tool_parser, test_output, streaming=False
)
# Streaming result
content_stream, tools_stream = run_tool_extraction(
tool_parser, test_output, streaming=True
)
# Compare results
assert content_non == content_stream, "Content should match between modes"
assert len(tools_non) == len(tools_stream), "Tool count should match"
if len(tools_non) > 0:
assert tools_non[0].function.name == tools_stream[0].function.name
assert tools_non[0].function.arguments == tools_stream[0].function.arguments
def apply_xfail_mark(self, request, test_config, test_name, streaming):
reason = None
if streaming and test_name in test_config.xfail_streaming:
reason = test_config.xfail_streaming[test_name]
elif not streaming and test_name in test_config.xfail_nonstreaming:
reason = test_config.xfail_nonstreaming[test_name]
if reason is not None:
mark = pytest.mark.xfail(reason=reason, strict=True)
request.node.add_marker(mark)
+167
View File
@@ -0,0 +1,167 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
class StreamingToolReconstructor:
def __init__(self, assert_one_tool_per_delta: bool = True):
self.tool_calls: list[ToolCall] = []
self.other_content: str = ""
self._assert_one_tool_per_delta = assert_one_tool_per_delta
def append_delta(self, delta: DeltaMessage):
if delta.content is not None:
self.other_content += delta.content
else:
assert delta.tool_calls, (
"Streaming results should have either content or tool calls (or both)"
)
if self._assert_one_tool_per_delta:
# Note: This isn't strictly required by the API and may not be
# possible to adhere to depending on the token space and number of
# tokens per streamed response from the model, but it is required
# by tool_use tests, so we enforce it here by default also.
assert len(delta.tool_calls) < 2, (
"Streaming should include only one tool call per update."
)
for call_delta in delta.tool_calls:
assert call_delta.type is None or call_delta.type == "function", (
"Streaming tool calls should only emit function calls. Got "
f"{call_delta.type}"
)
current_tool_call = (
self.tool_calls[call_delta.index]
if call_delta.index < len(self.tool_calls)
else None
)
if current_tool_call:
assert not call_delta.function.name, (
"Streaming tool calls should emit the full function name "
f"exactly once. Got {call_delta.function.name}"
)
assert not call_delta.id, (
"Streaming tool calls must emit function id only once. Got "
f"{call_delta.id}"
)
assert call_delta.index == len(self.tool_calls) - 1, (
f"Incorrect index for tool delta. Got {call_delta.index}, "
f"expected {len(self.tool_calls) - 1}"
)
current_tool_call.function.arguments += call_delta.function.arguments
else:
assert call_delta.id is not None, (
"Streaming tool calls must have an id on first appearance"
)
assert call_delta.function.name is not None, (
"Streaming tool calls must have a function name on first appearance"
)
assert call_delta.index == len(self.tool_calls), (
f"Incorrect index for tool delta. Got {call_delta.index}, "
f"expected {len(self.tool_calls)}"
)
self.tool_calls.append(
ToolCall(
id=call_delta.id,
function=FunctionCall(
name=call_delta.function.name,
arguments=call_delta.function.arguments or "",
),
)
)
def run_tool_extraction(
tool_parser: ToolParser,
model_output: str,
request: ChatCompletionRequest | None = None,
streaming: bool = False,
assert_one_tool_per_delta: bool = True,
) -> tuple[str | None, list[ToolCall]]:
if streaming:
reconstructor = run_tool_extraction_streaming(
tool_parser,
model_output,
request,
assert_one_tool_per_delta=assert_one_tool_per_delta,
)
return reconstructor.other_content or None, reconstructor.tool_calls
else:
extracted = run_tool_extraction_nonstreaming(tool_parser, model_output, request)
assert extracted.tools_called == bool(extracted.tool_calls)
return extracted.content, extracted.tool_calls
def run_tool_extraction_nonstreaming(
tool_parser: ToolParser,
model_output: str,
request: ChatCompletionRequest | None = None,
) -> ExtractedToolCallInformation:
request = request or ChatCompletionRequest(messages=[], model="test-model")
return tool_parser.extract_tool_calls(model_output, request)
def split_string_into_token_deltas(tokenizer: TokenizerLike, text: str) -> list[str]:
# Split a string into a series of deltas using the provided tokenizer. Each
# delta will be the string equivalent of a single token.
token_ids = tokenizer.encode(text, add_special_tokens=False)
previously_decoded_text = ""
deltas = []
for i in range(1, len(token_ids) + 1):
current_tokens = token_ids[:i]
current_text = tokenizer.decode(current_tokens)
new_text = current_text[len(previously_decoded_text) :]
previously_decoded_text = current_text
deltas.append(new_text)
return deltas
def run_tool_extraction_streaming(
tool_parser: ToolParser,
model_deltas: Iterable[str],
request: ChatCompletionRequest | None = None,
assert_one_tool_per_delta: bool = True,
) -> StreamingToolReconstructor:
if isinstance(model_deltas, str):
model_deltas = split_string_into_token_deltas(
tool_parser.model_tokenizer, model_deltas
)
request = request or ChatCompletionRequest(messages=[], model="test-model")
reconstructor = StreamingToolReconstructor(
assert_one_tool_per_delta=assert_one_tool_per_delta
)
previous_text = ""
previous_tokens: list[int] = []
for delta in model_deltas:
token_delta = [
tool_parser.vocab.get(token)
for token in tool_parser.model_tokenizer.tokenize(delta)
if token in tool_parser.vocab
]
current_text = previous_text + delta
current_tokens = previous_tokens + token_delta
delta_message = tool_parser.extract_tool_calls_streaming(
previous_text,
current_text,
delta,
previous_tokens,
current_tokens,
token_delta,
request,
)
if delta_message is not None:
reconstructor.append_delta(delta_message)
previous_text = current_text
previous_tokens = current_tokens
return reconstructor
View File
-129
View File
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the agent-file pre-commit check.
The hook passes silently on a healthy tree, so CI would not notice a rule that
stopped firing. Each case breaks one convention in a minimal repo and asserts
the check reports it.
"""
import os
from pathlib import Path
import pytest
from tools.pre_commit.check_agent_files import check
SKILL_MD = ".agents/skills/demo/SKILL.md"
SKILL = "---\nname: demo\ndescription: Demo things. Use when testing.\n---\n"
def write(path: str, text: str) -> None:
Path(path).parent.mkdir(parents=True, exist_ok=True)
Path(path).write_text(text)
def tracked() -> dict[str, str]:
"""Mimic `git ls-files -s` for every file and symlink under the cwd."""
files = {}
for root, dirs, names in os.walk("."):
for name in dirs + names:
path = os.path.normpath(os.path.join(root, name))
if os.path.islink(path):
files[path] = "120000"
elif os.path.isfile(path):
files[path] = "100644"
return files
@pytest.fixture(autouse=True)
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
write("AGENTS.md", "See the [guide](docs/guide.md#some-heading).\n")
write("docs/guide.md", "## Some Heading\n")
write(SKILL_MD, SKILL)
os.makedirs(".claude/skills")
os.symlink("../../.agents/skills/demo", ".claude/skills/demo")
def test_valid_repo_passes():
assert check(tracked()) == ([], [])
@pytest.mark.parametrize(
("break_repo", "expected"),
[
pytest.param(
lambda: os.unlink(".claude/skills/demo"),
"must be a symlink",
id="skill-not-linked",
),
pytest.param(
lambda: write(".claude/skills/other/SKILL.md", SKILL),
"only symlinks",
id="real-file-in-claude-skills",
),
pytest.param(
lambda: os.symlink("../../.agents/skills/gone", ".claude/skills/gone"),
"only symlinks",
id="dangling-link",
),
pytest.param(
lambda: write(SKILL_MD, SKILL.replace("demo", "other")),
"must be 'demo'",
id="name-differs-from-dir",
),
pytest.param(
lambda: write(SKILL_MD, SKILL.replace("Demo things", "Demo: things")),
"not valid YAML",
id="unquoted-colon-in-description",
),
pytest.param(
lambda: write(SKILL_MD, "# Demo\n"),
"missing YAML frontmatter",
id="no-frontmatter",
),
pytest.param(
lambda: write("vllm/CLAUDE.md", "@AGENTS.md\n"),
"remove it",
id="claude-md",
),
pytest.param(
lambda: write("AGENTS.md", "line\n" * 201),
"200-line budget",
id="root-guide-over-budget",
),
pytest.param(
lambda: write("vllm/AGENTS.md", "line\n" * 301),
"300-line budget",
id="domain-guide-over-budget",
),
pytest.param(
lambda: write(SKILL_MD, SKILL + "line\n" * 500),
"500-line budget",
id="skill-over-budget",
),
pytest.param(
lambda: write("AGENTS.md", "[gone](docs/gone.md)\n"),
"does not exist",
id="dead-link",
),
pytest.param(
lambda: write(SKILL_MD, SKILL + "[guide](../../../docs/guide.md#gone)\n"),
"no heading",
id="dead-anchor",
),
],
)
def test_broken_convention_is_reported(break_repo, expected):
break_repo()
errors, _ = check(tracked())
assert len(errors) == 1
assert expected in errors[0]
def test_description_without_trigger_is_advisory():
write(SKILL_MD, SKILL.replace(" Use when testing.", ""))
errors, notes = check(tracked())
assert not errors
assert "when to use" in notes[0]
-879
View File
@@ -1,879 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the test-tethering pre-commit check.
The check answers one question - "does any Buildkite job (in
``.buildkite/test_areas/`` or the legacy ``.buildkite/test-amd.yaml``) collect
this test file?" - by parsing job ``commands`` as shell. The risk that
matters is a *false tether*: a misparse that invents coverage makes the gate
pass while the test still never runs, which is the exact failure the check
exists to catch. So the parser cases below pin both directions, and
:func:`test_no_selection_matches_everything` guards the catastrophic version of
it.
"""
import textwrap
from pathlib import Path
import pytest
import tools.pre_commit.check_test_tethering as checker
from tools.pre_commit.check_test_tethering import (
FindSelection,
PytestSelection,
_parse_command,
_to_repo_relative,
all_test_modules,
allowlist_entries_missing_reason,
is_test_module,
is_tethered,
load_allowlist,
load_selections,
main,
normalize_test_path,
run_changed_files_check,
run_full_scan,
)
REPO_ROOT = Path(__file__).resolve().parents[2]
def runs(command: str, test_file: str) -> bool:
"""True if `command` collects `test_file` (path relative to ``tests/``)."""
return any(sel.runs(test_file) for sel in _parse_command(command))
# --------------------------------------------------------------------------- #
# pytest path arguments
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
("command", "test_file", "expected"),
[
# Directory sweeps cover everything nested underneath.
("pytest -v -s kernels", "kernels/test_attention.py", True),
("pytest -v -s kernels/", "kernels/moe/test_moe.py", True),
("pytest -v -s kernels", "lora/test_lora.py", False),
# An exact file argument covers that file and nothing beside it.
("pytest -v -s kernels/test_a.py", "kernels/test_a.py", True),
("pytest -v -s kernels/test_a.py", "kernels/test_b.py", False),
# Commands run from the repo root and from tests/ alike.
("pytest -v -s tests/kernels/test_a.py", "kernels/test_a.py", True),
("pytest -v -s ./kernels/test_a.py", "kernels/test_a.py", True),
(
"pytest -v -s /vllm-workspace/tests/kernels/test_a.py",
"kernels/test_a.py",
True,
),
# A ::nodeid selector still collects the file.
("pytest -v -s kernels/test_a.py::test_one", "kernels/test_a.py", True),
("pytest -v -s kernels/test_a.py::Cls::test_one", "kernels/test_a.py", True),
# Several positional paths, including a bare root-level file.
("pytest -v -s test_envs.py test_outputs.py", "test_outputs.py", True),
("pytest -v -s test_envs.py test_outputs.py", "test_regression.py", False),
# A bare word is a path only when it is really a directory under tests/.
("pytest -v -s samplers", "samplers/test_sampler.py", True),
# Glob path args do not cross a directory separator, matching the shell.
("pytest -v -s kernels/test_*.py", "kernels/test_a.py", True),
("pytest -v -s kernels/test_*.py", "kernels/moe/test_a.py", False),
("pytest -v -s kernels/test_*.py", "kernels/helper.py", False),
],
)
def test_pytest_path_arguments(command, test_file, expected):
assert runs(command, test_file) is expected
# --------------------------------------------------------------------------- #
# --ignore / --deselect
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
("command", "test_file", "expected"),
[
# --ignore removes a file from an otherwise covering sweep, in both the
# `=` and space-separated spellings.
("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_a.py", False),
("pytest -v -s kernels --ignore kernels/test_a.py", "kernels/test_a.py", False),
# ...and only that file.
("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_b.py", True),
# --ignore of a directory removes the whole subtree.
("pytest -v -s kernels --ignore=kernels/moe", "kernels/moe/test_moe.py", False),
(
"pytest -v -s lora --ignore-glob=lora/*_tp.py",
"lora/test_llama_tp.py",
False,
),
# --ignore-glob is a whole-path fnmatch (pytest semantics): its `*`
# crosses `/`, so a bare `*_tp.py` drops a nested match too. A positional
# path glob would not - that is the distinction the parser has to keep.
("pytest -v -s kernels --ignore-glob=*_tp.py", "kernels/test_a_tp.py", False),
(
"pytest -v -s kernels --ignore-glob=*_tp.py",
"kernels/nested/test_a_tp.py",
False,
),
("pytest -v -s kernels --ignore-glob '*_tp.py'", "kernels/test_a.py", True),
# A node-id deselect leaves the file collected; a bare-path one does not.
(
"pytest -v -s kernels --deselect kernels/test_a.py::test_one",
"kernels/test_a.py",
True,
),
(
"pytest -v -s kernels --deselect kernels/test_a.py",
"kernels/test_a.py",
False,
),
],
)
def test_ignore_and_deselect(command, test_file, expected):
assert runs(command, test_file) is expected
# --------------------------------------------------------------------------- #
# Options, env prefixes and other token noise
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
("command", "test_file", "expected"),
[
# A -k / -m filter narrows which tests run but does not un-run the file.
("pytest -v -s kernels -k 'not slow'", "kernels/test_a.py", True),
("pytest -v -s kernels -m core_model", "kernels/test_a.py", True),
# A value-taking option's value is never mistaken for a path.
("pytest -m core_model", "core_model", False),
("pytest -k kernels", "kernels/test_a.py", False),
("pytest -n 4 kernels/test_a.py", "kernels/test_a.py", True),
# Buildkite shard flags carry a $$VAR and must not swallow the path.
(
(
"pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB "
"--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT"
),
"kernels/test_a.py",
True,
),
# A leading env-var assignment is not a path.
(
"VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s kernels",
"kernels/test_a.py",
True,
),
# `python -m pytest` is the same invocation.
("python3 -m pytest -v -s kernels/test_a.py", "kernels/test_a.py", True),
# Quoted paths, wrapper commands and redirects all survive tokenizing.
("pytest -v -s 'kernels/test_a.py'", "kernels/test_a.py", True),
("timeout 600 pytest kernels/test_a.py", "kernels/test_a.py", True),
("pytest kernels/test_a.py > out.log 2>&1", "kernels/test_a.py", True),
# Each sub-command of a compound line is classified on its own.
("pytest kernels/test_a.py && pytest lora/test_b.py", "lora/test_b.py", True),
("pytest kernels/test_a.py ; pytest lora/test_b.py", "lora/test_b.py", True),
# An unresolvable $VAR path yields no coverage rather than a guess.
("pytest -v -s $TEST_TARGET", "kernels/test_a.py", False),
],
)
def test_option_and_token_handling(command, test_file, expected):
assert runs(command, test_file) is expected
# --------------------------------------------------------------------------- #
# find | xargs pytest pipelines
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
("command", "test_file", "expected"),
[
("find kernels -name 'test_*.py' | xargs pytest -v", "kernels/test_a.py", True),
(
"find kernels -name 'test_*.py' | xargs pytest -v",
"kernels/moe/test_b.py",
True,
),
# A non-matching filename is not collected by the find expression.
(
"find kernels -name 'test_*.py' | xargs pytest -v",
"kernels/helper.py",
False,
),
# -maxdepth 1 keeps direct children and drops nested ones.
(
"find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest",
"kernels/test_a.py",
True,
),
(
"find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest",
"kernels/moe/test_b.py",
False,
),
# -not -name excludes its target and leaves siblings alone.
(
"find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest",
"kernels/test_skip.py",
False,
),
(
"find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest",
"kernels/test_a.py",
True,
),
# The root confines the sweep.
("find kernels -name 'test_*.py' | xargs pytest", "lora/test_a.py", False),
# -maxdepth is honored even when the find root is the tests/ root itself
# ("." or "tests/"), not just a subdir.
(
"find . -maxdepth 1 -name 'test_*.py' | xargs pytest",
"test_regression.py",
True,
),
(
"find . -maxdepth 1 -name 'test_*.py' | xargs pytest",
"v1/test_scheduler.py",
False,
),
(
"find tests/ -maxdepth 1 -name 'test_*.py' | xargs pytest",
"v1/test_scheduler.py",
False,
),
# -exec pytest wires the sweep just as `| xargs pytest` does.
(
"find compile -name 'test_*.py' -exec pytest -s -v {} \\;",
"compile/test_a.py",
True,
),
# A pytest in a *separate* command on the line (not fed by the find) does
# not arm the sweep - the find here is piped to `wc`.
(
"find kernels -name 'test_*.py' | wc -l ; pytest test_regression.py",
"kernels/test_a.py",
False,
),
(
"find kernels -name 'test_*.py' -delete && pytest test_regression.py",
"kernels/test_a.py",
False,
),
# ...but only a *non-filtering* intermediate stage. `sort` reorders,
# every match still reaches pytest.
(
"find kernels -name 'test_*.py' | sort | xargs pytest",
"kernels/test_a.py",
True,
),
# `head` drops all but one match - the sweep is not fully tethered.
(
"find kernels -name 'test_*.py' | head -n 1 | xargs pytest",
"kernels/test_a.py",
False,
),
(
"find kernels -name 'test_*.py' | grep attention | xargs pytest",
"kernels/test_a.py",
False,
),
# `xargs` runs `echo` here, not pytest - the trailing `pytest` is echo's
# argument.
(
"find kernels -name 'test_*.py' | xargs echo pytest",
"kernels/test_a.py",
False,
),
# An `-exec` with no `\;` / `+` terminator is a `find` syntax error - it
# never runs pytest.
(
"find kernels -name 'test_*.py' -exec pytest {}",
"kernels/test_a.py",
False,
),
# An `-exec pytest` with no `{}` runs only the named file for every
# match - the matched files never reach pytest.
(
"find kernels -name 'test_*.py' -exec pytest test_regression.py \\;",
"kernels/test_a.py",
False,
),
# `{}` as an option *value* (`--ignore {}`) doesn't reach pytest as a
# test path either - only `test_regression.py` runs.
(
(
"find kernels -name 'test_*.py' -exec pytest test_regression.py "
"--ignore {} \\;"
),
"kernels/test_a.py",
False,
),
# xargs value-flags with a space don't swallow the command.
(
"find kernels -name 'test_*.py' | xargs -0 -I {} pytest {}",
"kernels/test_a.py",
True,
),
# `xargs env FOO=1 pytest` - xargs runs `env`, which runs pytest.
(
"find kernels -name 'test_*.py' | xargs env FOO=1 pytest",
"kernels/test_a.py",
True,
),
],
)
def test_find_pipelines(command, test_file, expected):
assert runs(command, test_file) is expected
def test_find_pipeline_parses_as_find_selection():
(selection,) = [
s
for s in _parse_command("find kernels -name 'test_*.py' | xargs pytest -v")
if isinstance(s, FindSelection)
]
assert selection.root == "kernels"
assert selection.name_globs == ["test_*.py"]
# --------------------------------------------------------------------------- #
# Direct file runners
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
("command", "test_file", "expected"),
[
(
"python3 standalone_tests/lazy_imports.py",
"standalone_tests/lazy_imports.py",
True,
),
(
"python standalone_tests/lazy_imports.py",
"standalone_tests/lazy_imports.py",
True,
),
# `timeout -k <grace> <dur>` wrapping a runner: the grace value must not
# be consumed as the duration (which would hide `python`).
(
"timeout -k 5s 30s python standalone_tests/lazy_imports.py",
"standalone_tests/lazy_imports.py",
True,
),
(
"torchrun --nproc-per-node 2 distributed/test_a.py",
"distributed/test_a.py",
True,
),
# The option's value is a count, not a path.
("torchrun --nproc-per-node 2 distributed/test_a.py", "2", False),
(
"VLLM_X=1 python3 standalone_tests/lazy_imports.py | grep -q ok",
"standalone_tests/lazy_imports.py",
True,
),
# Only the executed script counts; the tokens after it are that script's
# own argv. `--suffix v1` must not tether the whole tests/v1 subtree.
(
(
"python3 features/tensorize_vllm_model.py --model facebook/opt-125m "
"serialize --serialized-directory /tmp/ --suffix v1"
),
"v1/test_scheduler.py",
False,
),
(
(
"python3 features/tensorize_vllm_model.py --model facebook/opt-125m "
"serialize --serialized-directory /tmp/ --suffix v1"
),
"features/tensorize_vllm_model.py",
True,
),
# A script named by an absolute `$VAR` path still resolves.
(
'python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/e2e/test_x.py"',
"v1/e2e/test_x.py",
True,
),
# `python -m <module>` runs the module, not a script - only `-m pytest`
# collects tests. `compileall kernels` byte-compiles, it does not run.
("python -m compileall kernels", "kernels/test_a.py", False),
("python -m mypy kernels/test_a.py", "kernels/test_a.py", False),
# `coverage run -m pytest` still resolves through to pytest.
("coverage run -m pytest kernels/test_a.py", "kernels/test_a.py", True),
# `python -c '<code>' file.py` runs the code string; `file.py` is argv.
("python -c 'pass' kernels/test_a.py", "kernels/test_a.py", False),
("python -c 'import sys' kernels/test_a.py", "kernels/test_a.py", False),
# a `-m pytest` *after* the `-c` code operand is that program's argv too.
("python -c 'pass' -m pytest kernels/test_a.py", "kernels/test_a.py", False),
# ...but `-c` as the *script's* own arg still tethers the script.
("python kernels/test_a.py -c config.py", "kernels/test_a.py", True),
# ...and pytest's own `-c <cfg>` (after `-m pytest`) is fine.
("python -m pytest kernels/test_a.py -c setup.cfg", "kernels/test_a.py", True),
],
)
def test_direct_runners(command, test_file, expected):
assert runs(command, test_file) is expected
# --------------------------------------------------------------------------- #
# Commands that must contribute no coverage
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
"command",
[
"export VLLM_WORKER_MULTIPROC_METHOD=spawn",
"pip install pytest-timeout pytest-forked",
"uv pip install -r requirements/test/cuda.in",
"bash tests/does_not_exist.sh",
"echo 'pytest kernels/test_a.py'",
# `pytest` as an argument to another command - `echo` runs, not pytest.
"echo pytest kernels/test_a.py",
'pytest -v -s "unbalanced',
# A `.sh` that is only an *argument* (Buildkite printing / copying a
# filename) is not a script invocation.
"echo tests/foo.sh",
"cp scripts/run_tests.sh /tmp/",
# A `find` whose output never reaches pytest is not a test selection.
"find kernels -name 'test_*.py'",
"find kernels -name 'test_*.py' | wc -l",
"find kernels -name 'test_*.py' -delete",
"find kernels -name 'test_*.py' | xargs echo pytest",
"find kernels -name 'test_*.py' -exec pytest {}",
],
)
def test_non_test_commands_contribute_nothing(command):
assert _parse_command(command) == [] or not any(
sel.runs("kernels/test_a.py") for sel in _parse_command(command)
)
# --------------------------------------------------------------------------- #
# Job-level yaml parsing
# --------------------------------------------------------------------------- #
def _write_area(tmp_path, monkeypatch, body: str):
mod = checker
area_dir = tmp_path / "test_areas"
area_dir.mkdir(exist_ok=True)
(area_dir / "synthetic.yaml").write_text(textwrap.dedent(body))
monkeypatch.setattr(mod, "TEST_AREAS_DIR", area_dir)
# Keep the real test-amd.yaml out of these isolated parsing tests.
monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-such-test-amd.yaml")
# The yaml-error path reports a repo-relative name, so REPO_ROOT has to
# contain the synthetic area too.
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
return load_selections()
def test_plain_step_commands_are_collected(tmp_path, monkeypatch):
selections = _write_area(
tmp_path,
monkeypatch,
"""
steps:
- label: ":nvidia: (H200) Kernels"
commands:
- pytest -v -s kernels
""",
)
assert is_tethered("kernels/test_a.py", selections)
def test_singular_command_key_is_collected(tmp_path, monkeypatch):
selections = _write_area(
tmp_path,
monkeypatch,
"""
steps:
- label: "Singular"
command: pytest -v -s kernels/test_a.py
""",
)
assert is_tethered("kernels/test_a.py", selections)
def test_test_amd_yaml_coverage_counts_as_tethered(tmp_path, monkeypatch):
"""A test wired only into the legacy test-amd.yaml is still run by CI."""
mod = checker
(tmp_path / "test_areas").mkdir()
amd_yaml = tmp_path / "test-amd.yaml"
amd_yaml.write_text(
textwrap.dedent(
"""
steps:
- label: ":amd: (MI300) Quantization"
commands:
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/
- pytest -v -s rocm/test_moe_weight_replay.py
"""
)
)
monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas")
monkeypatch.setattr(mod, "TEST_AMD_YAML", amd_yaml)
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
selections = load_selections()
assert is_tethered("rocm/test_moe_weight_replay.py", selections)
assert is_tethered("quantization/test_foo.py", selections)
def test_missing_test_amd_yaml_is_not_fatal(tmp_path, monkeypatch):
"""When the migration finally deletes test-amd.yaml, the checker still runs."""
mod = checker
(tmp_path / "test_areas").mkdir()
monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas")
monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "gone.yaml")
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
assert load_selections() == []
def test_job_dirs_come_from_ci_config(tmp_path, monkeypatch):
"""The pipeline directories are whatever `.buildkite/ci_config*.yaml` lists
under `job_dirs:` - so a test wired only into e.g. `hardware_tests/` counts,
and an unreadable config falls back to `test_areas/` alone."""
mod = checker
bk = tmp_path / ".buildkite"
(bk / "test_areas").mkdir(parents=True)
(bk / "hardware_tests").mkdir()
(bk / "ci_config.yaml").write_text("job_dirs:\n - .buildkite/hardware_tests\n")
(bk / "ci_config_broken.yaml").write_text("job_dirs: [oops\n") # ignored
(bk / "hardware_tests" / "cpu.yaml").write_text(
textwrap.dedent(
"""
steps:
- label: "CPU"
commands:
- pytest -v -s v1/e2e/test_cpu_thing.py
"""
)
)
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
monkeypatch.setattr(mod, "TEST_AREAS_DIR", bk / "test_areas")
monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-amd.yaml")
assert (bk / "hardware_tests") in checker._pipeline_job_dirs()
assert is_tethered("v1/e2e/test_cpu_thing.py", load_selections())
def test_step_level_ignore_is_honored(tmp_path, monkeypatch):
selections = _write_area(
tmp_path,
monkeypatch,
"""
steps:
- label: "Kernels"
commands:
- pytest -v -s kernels --ignore=kernels/test_a.py
""",
)
assert not is_tethered("kernels/test_a.py", selections)
assert is_tethered("kernels/test_b.py", selections)
def test_amd_mirror_only_command_is_collected(tmp_path, monkeypatch):
"""A mirror that overrides `commands` runs tests the base step never does."""
selections = _write_area(
tmp_path,
monkeypatch,
"""
steps:
- label: ":nvidia: (H200) Base"
commands:
- pytest -v -s kernels/test_base.py
mirror:
amd:
label: ":amd: (MI300) Base"
commands:
- pytest -v -s kernels/test_amd_only.py
""",
)
assert is_tethered("kernels/test_base.py", selections)
assert is_tethered("kernels/test_amd_only.py", selections)
def test_shell_script_continuation_vars_and_nesting(tmp_path, monkeypatch):
"""The v1 P/D integration scripts name their test through a `\\`-continued
line, a literal shell var, and a nested `bash "$SCRIPT"`. Each indirection
has to be followed or the check falsely reports the test untethered."""
scripts = tmp_path / "tests" / "v1" / "kv_connector" / "nixl_integration"
scripts.mkdir(parents=True)
(scripts / "inner.sh").write_text(
"python3 -m pytest -s -v \\\n"
' "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_inner.py"\n'
)
(scripts / "sweep.sh").write_text(
'SCRIPT="v1/kv_connector/nixl_integration/inner.sh"\n'
'IMPORT_CANARY="v1/kv_connector/nixl_integration/test_canary.py"\n'
'LITERAL="v1/kv_connector/nixl_integration/test_not_run.py"\n'
'python3 -m pytest -s -x "${IMPORT_CANARY}"\n'
# single-quoted: the shell passes a literal $LITERAL, collecting nothing
"python3 -m pytest -s -x '${LITERAL}'\n"
'if ! env FOO=1 bash "${SCRIPT}"; then exit 1; fi\n'
)
monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests")
selections = _parse_command("bash v1/kv_connector/nixl_integration/sweep.sh")
def tethered(path):
return any(s.runs(path) for s in selections)
# `\`-continuation + $VAR-prefixed absolute path, resolved through a nested
# `env FOO=1 bash "$SCRIPT"`:
assert tethered("v1/kv_connector/nixl_integration/test_inner.py")
# literal `VAR=path` + double-quoted `"$VAR"` (the import-canary pattern):
assert tethered("v1/kv_connector/nixl_integration/test_canary.py")
# single-quoted `'$VAR'` is NOT expanded by the shell, so not tethered:
assert not tethered("v1/kv_connector/nixl_integration/test_not_run.py")
def test_bash_dash_c_does_not_follow_a_script_operand(tmp_path, monkeypatch):
"""`bash -c '<code>' foo.sh` runs the `-c` string; `foo.sh` is `$0` and is
never executed, so its contents must not count as coverage."""
(tmp_path / "tests").mkdir()
(tmp_path / "tests" / "decoy.sh").write_text("pytest -v -s lora/test_decoy.py\n")
monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests")
with_c = _parse_command("bash -c 'echo hi' tests/decoy.sh")
assert not any(s.runs("lora/test_decoy.py") for s in with_c)
# ...but a plain `bash tests/decoy.sh` still follows it.
plain = _parse_command("bash tests/decoy.sh")
assert any(s.runs("lora/test_decoy.py") for s in plain)
# ...and a `-c` that is the *script's* own flag (not bash's) is not `bash -c`.
script_flag = _parse_command("bash tests/decoy.sh -c models.txt")
assert any(s.runs("lora/test_decoy.py") for s in script_flag)
# A `bash -c` trailing arg that looks like a command string is `$0`, not run.
cmd_string_arg = _parse_command("bash -c 'echo ok' 'pytest lora/test_x.py'")
assert not any(s.runs("lora/test_x.py") for s in cmd_string_arg)
# ...but the `-c` code string *itself* is parsed - it is what runs.
code_runs = _parse_command("bash -c 'pytest lora/test_x.py'")
assert any(s.runs("lora/test_x.py") for s in code_runs)
# a combined short-option group resolves the `-c` operand with `c` in any
# position (`-ec`, `-ce`), and its trailing args are still `$0`, not run.
for grp in ("-ec", "-ce", "-euxc"):
assert any(
s.runs("lora/test_x.py")
for s in _parse_command(f"bash {grp} 'pytest lora/test_x.py'")
)
assert not any(
s.runs("lora/test_x.py")
for s in _parse_command("bash -ce 'echo ok' 'pytest lora/test_x.py'")
)
def test_unparsable_yaml_is_fatal(tmp_path, monkeypatch):
"""Silently skipping a bad yaml would drop its coverage and report false
untethered files, so it must fail loudly instead."""
with pytest.raises(SystemExit):
_write_area(tmp_path, monkeypatch, "steps: [oops\n")
def test_non_mapping_yaml_is_fatal(tmp_path, monkeypatch):
"""A yaml that parses but isn't a pipeline mapping (a bare list or scalar)
must raise the actionable error, not crash on ``.get``."""
with pytest.raises(SystemExit):
_write_area(tmp_path, monkeypatch, "- just\n- a\n- list\n")
# --------------------------------------------------------------------------- #
# is_test_module / path normalization
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
("path", "expected"),
[
("tests/kernels/test_a.py", True),
("tests/kernels/a_test.py", True),
("tests/kernels/conftest.py", False),
("tests/kernels/__init__.py", False),
("tests/kernels/utils.py", False),
("tests/kernels/test_a.txt", False),
("vllm/kernels/test_a.py", False),
],
)
def test_is_test_module(path, expected):
assert is_test_module(path) is expected
@pytest.mark.parametrize(
("raw", "expected"),
[
("tests/kernels/test_a.py", "kernels/test_a.py"),
("./kernels/test_a.py", "kernels/test_a.py"),
("/vllm-workspace/tests/kernels/test_a.py", "kernels/test_a.py"),
("'kernels/test_a.py'", "kernels/test_a.py"),
("kernels/test_a.py::test_one", "kernels/test_a.py"),
("kernels/", "kernels"),
("${GIT_ROOT}/tests/kernels/test_a.py", "kernels/test_a.py"),
("$GIT_ROOT/tests/kernels/test_a.py", "kernels/test_a.py"),
],
)
def test_normalize_test_path(raw, expected):
assert normalize_test_path(raw) == expected
# --------------------------------------------------------------------------- #
# Whole-tree invariants
# --------------------------------------------------------------------------- #
def test_real_tree_parses_into_selections():
assert len(load_selections()) > 200
assert len(all_test_modules()) > 1000
def test_no_selection_matches_everything():
"""A path arg that normalizes to '' or '.' would tether every test in the
repo and silently disarm the gate - the worst possible misparse."""
for selection in load_selections():
if isinstance(selection, PytestSelection):
offenders = [
p
for p in selection.included_paths
if normalize_test_path(p) in ("", ".")
]
assert not offenders, f"match-everything pytest arg: {offenders}"
else:
assert normalize_test_path(selection.root) not in ("", ".")
def test_tethering_is_deterministic():
selections = load_selections()
modules = all_test_modules()[:200]
first = [is_tethered(m, selections) for m in modules]
assert first == [is_tethered(m, load_selections()) for m in modules]
def test_allowlist_entries_are_well_formed():
allowlist = load_allowlist()
assert allowlist, "allowlist should not be empty while gaps remain"
assert all(is_test_module(path) for path in allowlist)
assert all(not path.startswith("/") for path in allowlist)
def test_real_allowlist_entries_all_have_a_reason():
assert allowlist_entries_missing_reason() == []
def test_bare_allowlist_entry_is_flagged(tmp_path, monkeypatch):
allowlist = tmp_path / "allowlist.txt"
allowlist.write_text(
"tests/a/test_ok.py # torchrun multi-GPU, no job\n"
"tests/b/test_bare.py\n"
" # a standalone comment line is fine\n"
)
monkeypatch.setattr(checker, "ALLOWLIST_PATH", allowlist)
assert allowlist_entries_missing_reason() == ["tests/b/test_bare.py"]
def test_real_tree_has_no_unallowlisted_gaps():
"""Every test module is either collected by a job or allowlisted.
Deliberately non-strict: a stale allowlist entry is advisory by design, so
asserting `strict=True` here would make unrelated allowlist cleanup block
CI - the exact behaviour the two-mode design avoids.
"""
assert run_full_scan(load_selections(), load_allowlist(), strict=False) == 0
# --------------------------------------------------------------------------- #
# CLI behaviour
# --------------------------------------------------------------------------- #
@pytest.fixture
def fake_repo(tmp_path, monkeypatch):
"""Point the module at a throwaway tree so the changed-files path can be
exercised without writing test files into the real repo."""
mod = checker
(tmp_path / "tests" / "kernels").mkdir(parents=True)
(tmp_path / "tests" / "kernels" / "test_new.py").write_text("def test_x(): pass\n")
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
monkeypatch.setattr(mod, "TESTS_DIR", tmp_path / "tests")
monkeypatch.setattr(mod, "ALLOWLIST_PATH", tmp_path / "allowlist.txt")
# The throwaway tree is not a git checkout, so the tracked-file inventory
# has to be stubbed rather than shelled out.
monkeypatch.setattr(mod, "all_test_modules", lambda: ["tests/kernels/test_new.py"])
return tmp_path
def test_untethered_changed_file_fails(fake_repo, capsys):
rc = run_changed_files_check(["tests/kernels/test_new.py"], [], set())
assert rc == 1
assert "is not run by any Buildkite job" in capsys.readouterr().out
def test_tethered_changed_file_passes(fake_repo):
selections = _parse_command("pytest -v -s kernels")
assert (
run_changed_files_check(["tests/kernels/test_new.py"], selections, set()) == 0
)
def test_allowlisted_changed_file_passes(fake_repo):
allowlist = {"tests/kernels/test_new.py"}
assert run_changed_files_check(["tests/kernels/test_new.py"], [], allowlist) == 0
def test_non_test_and_missing_paths_are_ignored(fake_repo):
paths = ["tests/kernels/conftest.py", "vllm/config.py", "tests/kernels/gone.py"]
assert run_changed_files_check(paths, [], set()) == 0
def test_stale_allowlist_entries_are_advisory_but_reported(fake_repo, capsys):
"""Dangling entries must not block an unrelated PR, but they have to be
printed - the hook sets `verbose: true` so a passing run still shows them."""
allowlist = {"tests/kernels/gone.py"}
rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=False)
out = capsys.readouterr().out
assert rc == 0
assert "no longer exists" in out
def test_stale_allowlist_entries_are_fatal_under_all(fake_repo, capsys):
allowlist = {"tests/kernels/gone.py"}
rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True)
assert rc == 1
assert "no longer exists" in capsys.readouterr().out
def test_now_tethered_allowlist_entry_is_reported(fake_repo, capsys):
allowlist = {"tests/kernels/test_new.py"}
rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True)
assert rc == 1
assert "is now tethered" in capsys.readouterr().out
def test_main_passes_a_tethered_file(monkeypatch):
"""The CLI's normal pre-commit path: a tethered file exits 0.
`--all` is not asserted here on purpose - it implies `strict=True`, so a
stale allowlist entry would turn unrelated cleanup into a CI failure.
"""
monkeypatch.setattr(
"sys.argv",
["check_test_tethering.py", "tests/tools/test_check_test_tethering.py"],
)
assert main() == 0
def test_main_flags_an_untethered_file(monkeypatch):
# Must be named like a test module, or the check correctly skips it.
orphan = REPO_ROOT / "tests" / "test_tethering_probe_orphan.py"
orphan.write_text("def test_x(): pass\n")
monkeypatch.setattr("sys.argv", ["check_test_tethering.py", str(orphan)])
try:
assert main() == 1
finally:
orphan.unlink()
def test_to_repo_relative_accepts_absolute_paths():
absolute = str(REPO_ROOT / "tests" / "kernels" / "test_a.py")
assert _to_repo_relative(absolute) == "tests/kernels/test_a.py"
-42
View File
@@ -1,42 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
import pytest
from tools.pre_commit.validate_config import validate_ast
_TestConfig1 = '''
@config
class _TestConfig1:
a: int
"""docstring"""
'''
_TestConfig2 = """
@config
class _TestConfig2:
a: int = 1
"""
_TestConfig3 = '''
@config
class _TestConfig3:
a: Union[Literal[1], Literal[2]] = 1
"""docstring"""
'''
@pytest.mark.parametrize(
("test_config", "expected_error"),
[
(_TestConfig1, "must have a default"),
(_TestConfig2, "must have a docstring"),
(_TestConfig3, "must use a single Literal"),
],
)
def test_config(test_config, expected_error):
tree = ast.parse(test_config)
with pytest.raises(Exception, match=expected_error):
validate_ast(tree)
+2290
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+153
View File
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import random
from typing import NamedTuple
import pytest
import torch
from vllm.platforms import current_platform
from vllm.transformers_utils.config import get_config
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
from vllm.triton_utils import HAS_TRITON
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
class DeviceConfig(NamedTuple):
available: bool
backends: list[str]
# Maps each device to its availability and supported backends.
DEVICE_BACKENDS: dict[str, DeviceConfig] = {
"cuda": DeviceConfig(
available=current_platform.is_cuda()
and current_platform.has_device_capability(80),
# FlashInfer backend temporarily disabled due to invariant CTA sizes.
# See FlashInfer issue #2424
backends=["FLASH_ATTN", "TRITON_ATTN", "FLEX_ATTENTION"],
),
"xpu": DeviceConfig(
available=current_platform.is_xpu() and HAS_TRITON,
backends=["TRITON_ATTN"],
),
"rocm": DeviceConfig(
available=current_platform.is_rocm() and HAS_TRITON,
backends=["TRITON_ATTN"],
),
}
DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
# Override backends for MLA models.
if os.getenv("VLLM_TEST_MODEL"):
config = get_config(TEST_MODEL, trust_remote_code=False)
if ModelArchConfigConvertorBase(config, config.get_text_config()).is_deepseek_mla():
DEVICE_BACKENDS["cuda"] = DeviceConfig(
available=DEVICE_BACKENDS["cuda"].available,
backends=["TRITON_MLA"]
+ (["FLASH_ATTN_MLA"] if flash_attn_supports_mla() else []),
)
DEVICE_BACKENDS["xpu"] = DeviceConfig(
available=DEVICE_BACKENDS["xpu"].available,
backends=[],
)
DEVICE_BACKENDS["rocm"] = DeviceConfig(
available=DEVICE_BACKENDS["rocm"].available,
backends=["TRITON_MLA"],
)
# Only include backends for devices that are actually available.
BACKENDS: list[str] = sorted(
{b for cfg in DEVICE_BACKENDS.values() if cfg.available for b in cfg.backends}
)
skip_unsupported = pytest.mark.skipif(
not any(cfg.available for cfg in DEVICE_BACKENDS.values()),
reason="Requires CUDA >= Ampere (SM80), ROCm, or Intel XPU with Triton",
)
skip_if_not_cuda = pytest.mark.skipif(
not DEVICE_BACKENDS["cuda"].available,
reason="Requires CUDA >= Ampere (SM80)",
)
skip_if_not_cuda_alike = pytest.mark.skipif(
not (DEVICE_BACKENDS["cuda"].available or DEVICE_BACKENDS["rocm"].available),
reason="Requires CUDA >= Ampere (SM80) or ROCm",
)
requires_mx = pytest.mark.skipif(
not (current_platform.is_rocm() and current_platform.supports_mx()),
reason="requires a ROCm device with native MX support (gfx95x)",
)
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
# Generate more realistic prompts that will actually produce varied tokens
# Use a mix of common English text patterns
prompt_templates = [
# Question-answer style
"Question: What is the capital of France?\nAnswer: The capital of France is",
"Q: How does photosynthesis work?\nA: Photosynthesis is the process by which",
"User: Can you explain quantum mechanics?\nAssistant: Quantum mechanics is",
# Story/narrative style
"Once upon a time in a distant galaxy, there lived",
"The old man walked slowly down the street, remembering",
"In the year 2157, humanity finally discovered",
# Technical/code style
"To implement a binary search tree in Python, first we need to",
"The algorithm works by iterating through the array and",
"Here's how to optimize database queries using indexing:",
# Factual/informative style
"The Renaissance was a period in European history that",
"Climate change is caused by several factors including",
"The human brain contains approximately 86 billion neurons which",
# Conversational style
"I've been thinking about getting a new laptop because",
"Yesterday I went to the store and bought",
"My favorite thing about summer is definitely",
]
# Pick a random template
base_prompt = random.choice(prompt_templates)
if max_words < min_words:
max_words = min_words
target_words = random.randint(min_words, max_words)
if target_words > 50:
# For longer prompts, repeat context
padding_text = (
" This is an interesting topic that deserves more explanation. "
# TODO: Update to * (target_words // 10) to better align with word ratio
* (target_words // 50)
)
base_prompt = padding_text + base_prompt
return base_prompt
def _extract_step_logprobs(request_output):
if getattr(request_output, "outputs", None):
inner = request_output.outputs[0]
if hasattr(inner, "logprobs") and inner.logprobs is not None:
t = torch.tensor(
[
inner.logprobs[i][tid].logprob
for i, tid in enumerate(inner.token_ids)
],
dtype=torch.float32,
)
return t, inner.token_ids
return None, None
def is_device_capability_below_90() -> bool:
return not current_platform.has_device_capability(90)
+601
View File
@@ -0,0 +1,601 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import os
import threading
import time
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AsyncExitStack
from types import SimpleNamespace
from typing import TYPE_CHECKING, cast
import msgspec
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import requests
from tests.utils import RemoteOpenAIServer
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
# Number of data parallel ranks for external LB testing
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
# Default tensor parallel size to use
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class DictStore:
def __init__(self, values: dict[str, bytes]):
self.values = values
def check(self, keys: list[str]) -> bool:
return all(key in self.values for key in keys)
def get(self, key: str) -> bytes:
return self.values[key]
def set(self, key: str, value: bytes) -> None:
self.values[key] = value
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
client = SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_master_port=12345,
_data_parallel_master_port_list=[12346, 12347],
)
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
control_store, reconfig_store = DictStore({}), DictStore({})
coordinator.reconfig_store_ref = reconfig_store
monkeypatch.setattr(
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
)
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
for store in (control_store, reconfig_store):
# DictStore preserves insertion order: metadata must precede the epoch.
assert list(store.values)[-1] == coordinator.key("current_epoch")
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
bootstrap
)
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
monkeypatch.setattr(
coordinator, "_get_reconfig_store", lambda store=store: store
)
assert coordinator.get_status()["requested_data_parallel_size"] == 3
def test_external_elastic_ep_calculates_target_expert_redundancy():
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
eplb_config=SimpleNamespace(num_redundant_experts=0)
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
@pytest.mark.asyncio
@pytest.mark.parametrize("dp_rank", range(4))
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
"""Validate expert capacity on every rank before waiting for an epoch."""
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
data_parallel_rank=dp_rank,
elastic_ep_max_dp_size=4,
data_parallel_master_ip="127.0.0.1",
_coord_store_port=12345,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
store = DictStore({})
monkeypatch.setattr(
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
)
with pytest.raises(ValueError, match="cannot retain all model experts"):
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
assert not store.values
assert coordinator.prepared_scale is None
@pytest.mark.asyncio
async def test_external_elastic_ep_late_rank_observes_epoch_error():
coordinator = ExternalElasticEPScaleCoordinator(
cast("DPAsyncMPClient", SimpleNamespace())
)
epoch = "failed-epoch"
store = DictStore(
{
coordinator.key("current_epoch"): epoch.encode(),
coordinator.key(epoch, "error"): b"scale failed",
}
)
with pytest.raises(RuntimeError, match="scale failed"):
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing."""
def __init__(
self,
model_name: str,
dp_size: int,
api_server_count: int,
base_server_args: list,
tp_size: int = TP_SIZE,
):
self.model_name = model_name
self.dp_size = dp_size
self.tp_size = tp_size
self.api_server_count = api_server_count
self.base_server_args = base_server_args
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
self.server_threads: list[threading.Thread] = []
def start_rank(
self,
rank: int,
dp_size: int | None = None,
elastic_ep_scale_up_launch: bool = False,
) -> tuple[RemoteOpenAIServer, list[str]]:
"""Start one externally managed DP rank."""
dp_size = self.dp_size if dp_size is None else dp_size
server_args = self.base_server_args.copy()
server_args.extend(
[
"--data-parallel-size",
str(dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank),
"--api-server-count",
str(self.api_server_count),
]
)
env_dict = {
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
),
}
if elastic_ep_scale_up_launch:
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
server = RemoteOpenAIServer(
self.model_name,
server_args,
auto_port=False,
env_dict=env_dict,
)
print(
f"Server rank {rank} started successfully with "
f"{self.api_server_count} API servers"
)
server_and_args = (server, server_args)
self.servers.append(server_and_args)
return server_and_args
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
"""Start all initial server instances for external LB mode."""
start_errors: list[Exception] = []
for rank in range(self.dp_size):
# Use a thread to start each server to allow parallel initialization
def start_server(r: int):
try:
self.start_rank(r)
except Exception as e:
print(f"Failed to start server rank {r}: {e}")
start_errors.append(e)
thread = threading.Thread(target=start_server, args=(rank,))
thread.start()
self.server_threads.append(thread)
# Wait for all servers to start
for thread in self.server_threads:
thread.join()
if start_errors:
start_error = start_errors[0]
raise RuntimeError("Failed to start external LB servers") from start_error
# Give servers additional time to fully initialize and coordinate
time.sleep(2)
if len(self.servers) != self.dp_size:
raise Exception("Servers failed to start")
return self.servers
def __exit__(self, exc_type, exc_val, exc_tb):
"""Stop all server instances."""
servers = [s for s, _ in self.servers]
self.servers.clear()
try:
RemoteOpenAIServer.shutdown_many(servers)
except Exception as e:
print(f"Error stopping servers: {e}")
@pytest.fixture(scope="module")
def default_server_args():
return [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"128",
"--enforce-eager",
]
@pytest.fixture(
scope="module",
params=[(1, False), (4, False), (1, True)],
ids=["api-1-standard", "api-4-standard", "api-1-ep-enabled"],
)
def server_manager(request, default_server_args):
api_server_count, enable_expert_parallel = request.param
server_args = default_server_args.copy()
if enable_expert_parallel:
server_args.append("--enable-expert-parallel")
server_manager = ExternalLBServerManager(
MODEL_NAME, DP_SIZE, api_server_count, server_args
)
with server_manager:
yield server_manager
@pytest.fixture
def servers(server_manager):
return server_manager.servers
@pytest_asyncio.fixture
async def clients(servers: list[tuple[RemoteOpenAIServer, list[str]]]):
# Create a client for each server
async with AsyncExitStack() as stack:
yield [
await stack.enter_async_context(server.get_async_client())
for server, _ in servers
]
def _get_parallel_config(server: RemoteOpenAIServer):
response = requests.get(server.url_for("server_info?config_format=json"))
response.raise_for_status()
vllm_config = response.json()["vllm_config"]
return vllm_config["parallel_config"]
def _assert_expert_parallel_config(
servers: list[tuple[RemoteOpenAIServer, list[str]]],
):
for server, server_args in servers:
expected = "--enable-expert-parallel" in server_args
parallel_config = _get_parallel_config(server)
assert parallel_config["enable_expert_parallel"] is expected
def _send_scale_command(
server: RemoteOpenAIServer, new_dp_size: int
) -> requests.Response:
return requests.post(
server.url_for("scale_elastic_ep"),
json={"new_data_parallel_size": new_dp_size},
timeout=600,
)
def _wait_for_scale_requests_to_start(
scale_requests: list[Future[requests.Response]],
timeout: float = 30,
) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for request in scale_requests:
if request.done():
response = request.result()
pytest.fail(
"Scale request finished before the new rank was launched: "
f"{response.status_code} {response.text}"
)
if all(request.running() for request in scale_requests):
return
time.sleep(0.1)
pytest.fail("Timed out waiting for all scale requests to start")
@pytest.mark.distributed(num_gpus=3)
@pytest.mark.skipif(
TP_SIZE != 1 or current_platform.device_count() < 3,
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
)
@pytest.mark.skipif(
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
reason="Elastic EP scaling is not supported by the Rust frontend",
)
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
server_args = [
*default_server_args,
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--attention-backend",
"TRITON_MLA",
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
"3",
"--enable-eplb",
"--eplb-config",
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
]
server_manager = ExternalLBServerManager(
ELASTIC_EP_MODEL_NAME,
dp_size=2,
api_server_count=1,
base_server_args=server_args,
)
with server_manager:
existing_servers = [server for server, _ in server_manager.servers]
with ThreadPoolExecutor(max_workers=2) as executor:
scale_requests = [
executor.submit(_send_scale_command, server, 3)
for server in existing_servers
]
_wait_for_scale_requests_to_start(scale_requests)
server_manager.start_rank(
rank=2,
dp_size=3,
elastic_ep_scale_up_launch=True,
)
for request in scale_requests:
response = request.result(timeout=600)
assert response.status_code == 200, response.text
for server, _ in server_manager.servers:
parallel_config = _get_parallel_config(server)
assert parallel_config["data_parallel_size"] == 3
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
with server.get_client() as client:
completion = client.completions.create(
model=ELASTIC_EP_MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].finish_reason in ("length", "stop")
def test_external_lb_server_info(server_manager):
servers = server_manager.servers
api_server_count = server_manager.api_server_count
for i, (server, _) in enumerate(servers):
print(f"Testing {i=}")
# Each request will hit one of the API servers
# `n_reqs` is set so that there is a good chance each server
# receives at least one request
n_reqs = 2 * api_server_count * api_server_count
parallel_configs = [_get_parallel_config(server) for _ in range(n_reqs)]
api_process_counts = [c["_api_process_count"] for c in parallel_configs]
api_process_ranks = [c["_api_process_rank"] for c in parallel_configs]
assert all(c == api_server_count for c in api_process_counts), (
api_process_counts
)
assert all(0 <= r < api_server_count for r in api_process_ranks), (
api_process_ranks
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_external_lb_single_completion(
clients: list[openai.AsyncOpenAI],
servers: list[tuple[RemoteOpenAIServer, list[str]]],
model_name: str,
) -> None:
async def make_request(client: openai.AsyncOpenAI):
completion = await client.completions.create(
model=model_name, prompt="Hello, my name is", max_tokens=10, temperature=1.0
)
assert completion.id is not None
assert completion.choices is not None and len(completion.choices) == 1
choice = completion.choices[0]
# The exact number of tokens can vary slightly with temperature=1.0,
# so we check for a reasonable minimum length.
assert len(choice.text) >= 1
# Finish reason might not always be 'length' if the model finishes early
# or due to other reasons, especially with high temperature.
# So, we'll accept 'length' or 'stop'.
assert choice.finish_reason in ("length", "stop")
# Token counts can also vary, so we check they are positive.
assert completion.usage.completion_tokens > 0
assert completion.usage.prompt_tokens > 0
assert completion.usage.total_tokens > 0
return completion
# Test single request to each server
for i, client in enumerate(clients):
result = await make_request(client)
assert result is not None
print(f"Server {i} handled single completion request successfully")
await asyncio.sleep(0.5)
# Send requests to all servers in round-robin fashion
num_requests_per_server = 25 # Total 50 requests across 2 servers
all_tasks = []
for i, client in enumerate(clients):
tasks = [make_request(client) for _ in range(num_requests_per_server)]
all_tasks.extend(tasks)
results = await asyncio.gather(*all_tasks)
assert len(results) == num_requests_per_server * len(clients)
assert all(completion is not None for completion in results)
await asyncio.sleep(0.5)
# Second burst of requests
all_tasks = []
for i, client in enumerate(clients):
tasks = [make_request(client) for _ in range(num_requests_per_server)]
all_tasks.extend(tasks)
results = await asyncio.gather(*all_tasks)
assert len(results) == num_requests_per_server * len(clients)
assert all(completion is not None for completion in results)
_, server_args = servers[0]
api_server_count = (
server_args.count("--api-server-count")
and server_args[server_args.index("--api-server-count") + 1]
or 1
)
print(
f"Successfully completed external LB test with {len(clients)} servers "
f"(API server count: {api_server_count})"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[MODEL_NAME],
)
async def test_external_lb_completion_streaming(
clients: list[openai.AsyncOpenAI],
servers: list[tuple[RemoteOpenAIServer, list[str]]],
model_name: str,
) -> None:
prompt = "What is an LLM?"
async def make_streaming_request(client: openai.AsyncOpenAI):
# Perform a non-streaming request to get the expected full output
single_completion = await client.completions.create(
model=model_name,
prompt=prompt,
max_tokens=5,
temperature=0.0,
)
single_output = single_completion.choices[0].text
# Perform the streaming request
stream = await client.completions.create(
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
)
chunks: list[str] = []
finish_reason_count = 0
last_chunk = None
async for chunk in stream:
chunks.append(chunk.choices[0].text)
if chunk.choices[0].finish_reason is not None:
finish_reason_count += 1
last_chunk = chunk # Keep track of the last chunk
# finish reason should only return in the last block for OpenAI API
assert finish_reason_count == 1, "Finish reason should appear exactly once."
assert last_chunk is not None, "Stream should have yielded at least one chunk."
assert last_chunk.choices[0].finish_reason == "length", (
"Finish reason should be 'length'."
)
# Check that the combined text matches the non-streamed version.
assert "".join(chunks) == single_output, (
"Streamed output should match non-streamed output."
)
return True # Indicate success for this request
# Test single request to each server
for i, client in enumerate(clients):
result = await make_streaming_request(client)
assert result is not None
print(f"Server {i} handled single streaming request successfully")
await asyncio.sleep(0.5)
# Send streaming requests to all servers in round-robin fashion
num_requests_per_server = 25 # Total 50 requests across 2 servers
all_tasks = []
for i, client in enumerate(clients):
tasks = [make_streaming_request(client) for _ in range(num_requests_per_server)]
all_tasks.extend(tasks)
results = await asyncio.gather(*all_tasks)
assert len(results) == num_requests_per_server * len(clients)
assert all(results), "Not all streaming requests completed successfully."
await asyncio.sleep(0.5)
# Second burst of streaming requests
all_tasks = []
for i, client in enumerate(clients):
tasks = [make_streaming_request(client) for _ in range(num_requests_per_server)]
all_tasks.extend(tasks)
results = await asyncio.gather(*all_tasks)
assert len(results) == num_requests_per_server * len(clients)
assert all(results), "Not all streaming requests completed successfully."
_, server_args = servers[0]
api_server_count = (
server_args.count("--api-server-count")
and server_args[server_args.index("--api-server-count") + 1]
or 1
)
print(
f"Successfully completed external LB streaming test with "
f"{len(clients)} servers (API server count: {api_server_count})"
)
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from vllm import SamplingParams
from vllm.config import CompilationConfig
from ..utils import compute_acceptance_len
def load_and_process_dataset(data_name: str):
from datasets import load_dataset
if data_name == "gsm8k":
dataset = load_dataset("openai/gsm8k", "main", split="test")
prompt_fmt = (
"{question}\nPlease reason step by step,"
" and put your final answer within \\boxed{{}}."
)
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
elif data_name == "mt-bench":
dataset = load_dataset("HuggingFaceH4/mt_bench_prompts", split="train")
dataset = dataset.map(lambda x: {"turns": x["prompt"]})
elif data_name == "humaneval":
dataset = load_dataset("openai/openai_humaneval", split="test")
prompt_fmt = (
"Write a solution to the following problem and make sure"
" that it passes the tests:\n```python\n{prompt}\n```"
)
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
return dataset
def run_acceptance_length_eval(
monkeypatch: pytest.MonkeyPatch,
vllm_runner,
spec_config: dict[str, Any],
expected_acceptance_lengths: dict[str, float],
chat_template_kwargs: dict[str, Any],
use_mrv2: bool,
):
"""E2E acceptance-rate validation for speculative decoding.
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
through the spec decode engine and asserts the mean acceptance length
stays within tolerance of the reference figure for each dataset.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
runner_config = spec_config.copy()
model = runner_config.pop("model")
runner_config.setdefault("block_size", None)
runner_config.setdefault("enable_chunked_prefill", None)
runner_config.setdefault("compilation_config", CompilationConfig())
with vllm_runner(model, **runner_config) as spec_runner:
spec_llm = spec_runner.llm
# MT-Bench has 80 prompts and HumanEval has 164; truncate GSM8K.
max_prompts_per_dataset = 200
tokenizer = spec_llm.get_tokenizer()
for dataset_name, expected_len in expected_acceptance_lengths.items():
dataset = load_and_process_dataset(dataset_name)
prev_metrics = None
acceptance_lengths = []
for i in range(min(max_prompts_per_dataset, len(dataset))):
user_content = dataset[i]["turns"][0]
prompt_text = tokenizer.apply_chat_template(
[{"role": "user", "content": user_content}],
tokenize=False,
add_generation_prompt=True,
**chat_template_kwargs,
)
# Greedy (temp=0) so acceptance length is deterministic and comparable
# across runs.
spec_llm.generate(
[prompt_text],
SamplingParams(temperature=0, max_tokens=2048),
use_tqdm=False,
)
current_metrics = spec_llm.get_metrics()
acceptance_len = compute_acceptance_len(current_metrics, prev_metrics)
prev_metrics = current_metrics
acceptance_lengths.append(acceptance_len)
mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths)
# Fairly tight tolerance of 95% against the reference figures,
# watching for regressions. Can be relaxed if test is flaky but be sure to
# check for genuine issues such as #40727.
expected_len = expected_len * 0.95
print(
f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
f" (expected at least {expected_len:.2f})"
)
assert mean_acceptance_length >= expected_len, (
f"acceptance_len for {dataset_name} is below expected threshold: "
f"{mean_acceptance_length:.2f} < {expected_len:.2f}"
)
+230
View File
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from collections.abc import Iterable, Sequence
from typing import Any
import pytest
import torch
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts, evaluate_gsm8k_offline
from vllm import LLM, SamplingParams
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.assets.image import VLM_IMAGES_DIR
from vllm.outputs import RequestOutput
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.metrics.reader import Metric
def _skip_if_insufficient_gpus_for_tp(tp_size: int):
"""Skip if fewer than ``tp_size`` accelerators are available."""
available_gpus = torch.accelerator.device_count()
if available_gpus < tp_size:
pytest.skip(
f"Test requires {tp_size} GPUs, but only {available_gpus} available"
)
Messages = list[dict[str, Any]]
def get_test_prompts(mm_enabled: bool, num_prompts: int = 100) -> list[Messages]:
prompt_types = ["repeat", "gsm8k"]
if mm_enabled:
prompt_types.append("mm")
prompts: list[Messages] = []
num_repeat_prompts = num_prompts // len(prompt_types)
if mm_enabled:
num_gsm8k_prompts = num_prompts // len(prompt_types)
num_mm_prompts = num_prompts - num_repeat_prompts - num_gsm8k_prompts
else:
num_mm_prompts = 0
num_gsm8k_prompts = num_prompts - num_repeat_prompts
# Generate a mixed batch of prompts, some of which can be easily
# predicted by n-gram matching and some which likely cannot.
set_random_seed(0)
for _ in range(num_repeat_prompts):
word_choices = ["test", "temp", "hello", "where"]
word = random.choice(word_choices)
prompts.append(
[
{
"role": "user",
"content": f"""
please repeat the word '{word}' 10 times.
give no other output than the word at least ten times in a row,
in lowercase with spaces between each word and without quotes.
""",
}
]
)
prompts.extend(
[{"role": "user", "content": prompt}]
for prompt in _build_gsm8k_prompts(
num_questions=num_gsm8k_prompts, num_shots=5
)[0]
)
for _ in range(num_mm_prompts):
placeholders = [
{
"type": "image_url",
"image_url": {
"url": f"{VLLM_S3_BUCKET_URL}/{VLM_IMAGES_DIR}/stop_sign.jpg"
},
}
]
prompt = [
*placeholders,
{"type": "text", "text": "The meaning of the image is"},
]
prompts.append([{"role": "user", "content": prompt}])
return prompts
def get_instruct_coder_messages(n: int) -> list[Messages]:
from vllm.benchmarks.datasets import InstructCoderDataset
dataset = InstructCoderDataset(
dataset_path="likaixin/InstructCoder", dataset_split="train"
)
prompts: Iterable[str] = dataset.sample_prompts(n=n)
return [[{"role": "user", "content": prompt}] for prompt in prompts]
def greedy_sampling() -> SamplingParams:
return SamplingParams(temperature=0, max_tokens=10, ignore_eos=False)
def stochastic_sampling() -> SamplingParams:
return SamplingParams(temperature=1.0, max_tokens=10, ignore_eos=False)
def evaluate_llm_for_gsm8k(llm: LLM, expected_accuracy_threshold: float = 0.70) -> None:
"""Evaluate the LLM on GSM8K and check that accuracy is above a sanity threshold.
The default threshold assumes the LLM uses the same target model as the "model_name"
fixture, with max model len == 4096. Precomputed reference value is 75% to 80%
on GSM8K with greedy decoding, so we check that it's above a sanity threshold of 70%
to verify that the model is correct.
"""
if expected_accuracy_threshold <= 0.0:
print("Skipping GSM8K evaluation")
return
results = evaluate_gsm8k_offline(llm)
accuracy = results["accuracy"]
print(f"GSM8K accuracy: {accuracy:.3f}")
assert accuracy >= expected_accuracy_threshold, (
f"Expected GSM8K accuracy >= {expected_accuracy_threshold}, got {accuracy:.3f}"
)
def assert_request_outputs_match(
ref_outputs: Sequence[RequestOutput],
spec_outputs: Sequence[RequestOutput],
*,
required_matches: int,
context: str,
max_mismatches: int = 3,
) -> None:
"""Assert a thresholded exact-text match with bounded failure diagnostics."""
assert ref_outputs, f"{context}: no reference outputs"
assert len(ref_outputs) == len(spec_outputs), (
f"{context}: output count differs: "
f"reference={len(ref_outputs)}, speculative={len(spec_outputs)}"
)
assert 0 <= required_matches <= len(ref_outputs), (
f"{context}: invalid required_matches={required_matches} for "
f"{len(ref_outputs)} outputs"
)
mismatches: list[str] = []
matches = 0
for index, (ref_output, spec_output) in enumerate(zip(ref_outputs, spec_outputs)):
assert ref_output.outputs, (
f"{context}: reference output {index} has no candidate"
)
assert spec_output.outputs, (
f"{context}: speculative output {index} has no candidate"
)
ref_candidate = ref_output.outputs[0]
spec_candidate = spec_output.outputs[0]
if ref_candidate.text == spec_candidate.text:
matches += 1
elif len(mismatches) < max_mismatches:
mismatches.append(
f"[{index}] ref_text={ref_candidate.text[:240]!r}, "
f"spec_text={spec_candidate.text[:240]!r}\n"
f" ref_token_ids={list(ref_candidate.token_ids)[:64]}\n"
f" spec_token_ids={list(spec_candidate.token_ids)[:64]}"
)
print(
f"{context}: exact text matches={matches}/{len(ref_outputs)} "
f"(required={required_matches})"
)
mismatch_summary = "\n".join(mismatches) or "no mismatches captured"
assert matches >= required_matches, (
f"{context}: only {matches}/{len(ref_outputs)} outputs matched; "
f"required at least {required_matches}. First mismatches:\n"
f"{mismatch_summary}"
)
def get_spec_decode_metric_value(metrics: Sequence[Metric], metric_name: str) -> float:
"""Get a spec-decode metric with an actionable error when stats are absent."""
name2metric = {metric.name: metric for metric in metrics}
metric = name2metric.get(metric_name)
assert metric is not None, (
f"Missing metric {metric_name!r}. Ensure disable_log_stats=False. "
"Available spec-decode metrics: "
f"{sorted(name for name in name2metric if 'spec_decode' in name) or ['<none>']}"
)
return float(metric.value)
def compute_acceptance_rate(
metrics: list[Metric], prev_metrics: list[Metric] | None = None
) -> float:
n_draft_toks = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_draft_tokens"
)
if n_draft_toks == 0:
return float("nan")
n_accepted_toks = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_accepted_tokens"
)
if prev_metrics is not None:
n_draft_toks -= get_spec_decode_metric_value(
prev_metrics, "vllm:spec_decode_num_draft_tokens"
)
n_accepted_toks -= get_spec_decode_metric_value(
prev_metrics, "vllm:spec_decode_num_accepted_tokens"
)
if n_draft_toks <= 0:
return float("nan")
return n_accepted_toks / n_draft_toks
def compute_acceptance_len(
metrics: list[Metric], prev_metrics: list[Metric] | None = None
) -> float:
n_drafts = get_spec_decode_metric_value(metrics, "vllm:spec_decode_num_drafts")
n_accepted_toks = get_spec_decode_metric_value(
metrics, "vllm:spec_decode_num_accepted_tokens"
)
if n_drafts == 0:
return 1
if prev_metrics is not None:
n_drafts -= get_spec_decode_metric_value(
prev_metrics, "vllm:spec_decode_num_drafts"
)
n_accepted_toks -= get_spec_decode_metric_value(
prev_metrics, "vllm:spec_decode_num_accepted_tokens"
)
if n_drafts <= 0:
return 1
return 1 + (n_accepted_toks / n_drafts)
@@ -1,279 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing and registration behaviour of the EPD proxy."""
import asyncio
from typing import Any
from unittest.mock import AsyncMock, Mock
import msgspec
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
from examples.disaggregated.disaggregated_encoder import (
disagg_epd_proxy as proxy_module,
)
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
EPDProxy,
EPDProxyConfig,
InstanceRecord,
InstanceRegistry,
InstanceRole,
build_app,
extract_mm_items,
)
ENCODE = InstanceRole.ENCODE
PREFILL = InstanceRole.PREFILL
DECODE = InstanceRole.DECODE
PD = InstanceRole.PREFILL_DECODE
IMAGE_ITEM = {"type": "image_url", "image_url": {"url": "http://img/0.png"}}
@pytest.fixture
def proxy():
registry = InstanceRegistry(probe_interval=0)
return EPDProxy(registry)
class TestRouting:
def test_no_decode_instance_is_service_unavailable(self, proxy):
"""The proxy comes up before anything registers, so this is normal."""
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=0)
assert excinfo.value.status_code == 503
def test_media_without_an_encoder_is_service_unavailable(self, proxy):
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
assert proxy.route(num_items=0).decode.url == "http://d0:8000"
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=1)
assert excinfo.value.status_code == 503
def test_prefill_is_optional(self, proxy):
"""An E+PD deployment registers no prefill instance at all."""
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
assert proxy.route(num_items=0).prefill is None
def test_encoder_roster_tracks_new_instances(self, proxy):
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
for index in range(2):
proxy.registry.register(InstanceRecord(ENCODE, f"http://e{index}:8000"))
assert proxy.route(num_items=3).encoder_urls == [
"http://e0:8000",
"http://e1:8000",
]
proxy.registry.unregister("http://e0:8000")
assert proxy.route(num_items=3).encoder_urls == ["http://e1:8000"]
@pytest.mark.parametrize("unregister", [False, True])
def test_standalone_decode_always_requires_healthy_prefill(self, unregister):
app = build_app()
proxy = app.state.proxy
proxy.registry.register(InstanceRecord(DECODE, "http://d0:8000"))
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=0)
assert excinfo.value.status_code == 503
proxy.registry.register(InstanceRecord(PREFILL, "http://p0:8000"))
route = proxy.route(num_items=0)
assert route.decode.url == "http://d0:8000"
assert route.consumer.url == "http://p0:8000"
if unregister:
proxy.registry.unregister("http://p0:8000")
else:
for _ in range(proxy.registry._fail_threshold):
proxy.registry._on_probe_failure(route.prefill, 0)
with pytest.raises(HTTPException) as excinfo:
proxy.route(num_items=0)
assert excinfo.value.status_code == 503
class TestConsumerAddress:
"""Which stage receives the embedding depends on the topology."""
def test_shared_storage_connectors_name_no_target(self, proxy):
"""Rotate replicas without requiring a push endpoint."""
proxy.registry.register(InstanceRecord(PD, "http://d0:8000", dp_size=2))
route = proxy.route(num_items=0)
assert route.consumer_zmq is None
assert route.dp_rank == 0
assert proxy.route(num_items=0).dp_rank == 1
def test_decode_is_the_consumer_when_prefill_is_not_split_out(self, proxy):
proxy.registry.register(
InstanceRecord(PD, "http://d0:8000", ec_zmq_addrs=["tcp://d0:20001"])
)
assert proxy.route(num_items=0).consumer_zmq == "tcp://d0:20001"
def test_prefill_is_the_consumer_when_it_is_split_out(self, proxy):
"""In E+P+D the prefill instance consumes the embedding, not decode.
Reading the address out of whichever list it came from -- rather than
from the record that reported one -- sends the push to the wrong
instance in this topology.
"""
proxy.registry.register(
InstanceRecord(PREFILL, "http://p0:8000", ec_zmq_addrs=["tcp://p0:20001"])
)
proxy.registry.register(InstanceRecord(DECODE, "http://d0:8000"))
assert proxy.route(num_items=0).consumer_zmq == "tcp://p0:20001"
class TestProxyApi:
@pytest.fixture
def client(self, monkeypatch):
monkeypatch.setenv("ADMIN_API_KEY", "test-key")
config = EPDProxyConfig(probe_interval=0)
with TestClient(build_app(config)) as client:
client.headers["X-API-Key"] = "test-key"
yield client
def test_proxy_serves_before_anything_registers(self, client):
assert client.get("/health").status_code == 200
assert client.get("/instances").json()["encode"]["live"] == []
def test_requests_are_refused_until_a_decoder_registers(self, client):
response = client.post(
"/v1/chat/completions", json={"model": "m", "messages": []}
)
assert response.status_code == 503
assert client.get("/v1/models").status_code == 503
def test_http_registration_keeps_consumer_replica_and_address_together(
self, client
):
payload: dict[str, Any] = {
"role": "prefill_decode",
"url": "http://d0:8000/",
"dp_size": 2,
"ec_zmq_addrs": ["tcp://d0:20001", "tcp://d0:20003"],
}
assert client.post("/instances", json=payload).json() == {"registered": True}
assert client.post("/instances", json=payload).json() == {"registered": False}
proxy = client.app.state.proxy
for rank in [0, 1, 0]:
route = proxy.route(0)
assert route.dp_rank == rank
assert route.consumer_zmq == payload["ec_zmq_addrs"][rank]
assert client.delete("/instances", params={"url": payload["url"]}).json() == {
"removed": True
}
assert proxy.registry.pick(PD) is None
def test_registration_rejects_invalid_topology(self, client):
cases: list[tuple[dict[str, Any], int]] = [
({"dp_size": 0}, 422),
({"dp_size": 2, "ec_zmq_addrs": ["tcp://d0:20001"]}, 422),
({"role": "decode", "ec_zmq_addrs": ["tcp://d0:20001"]}, 400),
({"role": "encode", "ec_zmq_addrs": ["tcp://d0:20001"]}, 400),
]
for fields, status in cases:
response = client.post(
"/instances",
json={"url": "http://d0:8000", "role": "prefill_decode", **fields},
)
assert response.status_code == status
def test_registration_requires_admin_key(self, client):
client.headers.pop("X-API-Key")
assert (
client.post(
"/instances", json={"role": "encode", "url": "http://e0:8000"}
).status_code
== 403
)
assert (
client.delete("/instances", params={"url": "http://e0:8000"}).status_code
== 403
)
@pytest.mark.parametrize("split_role", ["prefill", "decode"])
@pytest.mark.parametrize("combined_first", [False, True])
def test_registration_rejects_mixed_topologies(
self, client, split_role, combined_first
):
roles = ["prefill_decode", split_role]
if not combined_first:
roles.reverse()
first = {"role": roles[0], "url": "http://first:8000"}
second = {"role": roles[1], "url": "http://second:8000"}
assert client.post("/instances", json=first).status_code == 200
assert client.post("/instances", json=second).status_code == 409
# Health eviction must not silently permit a topology switch.
registry = client.app.state.registry
record = registry._live[first["url"]]
for _ in range(registry._fail_threshold):
registry._on_probe_failure(record, 0)
assert client.post("/instances", json=second).status_code == 409
assert client.delete("/instances", params={"url": first["url"]}).json() == {
"removed": True
}
assert client.post("/instances", json=second).status_code == 200
def test_extract_mm_items_finds_media_across_messages():
req = {
"messages": [
{"role": "user", "content": "plain text"},
{"role": "user", "content": [IMAGE_ITEM, {"type": "text", "text": "hi"}]},
{"role": "user", "content": [IMAGE_ITEM]},
]
}
assert extract_mm_items(req) == [IMAGE_ITEM, IMAGE_ITEM]
@pytest.mark.parametrize("push", [False, True])
def test_encoder_handles_and_json_metadata_survive_rewrite(proxy, push, monkeypatch):
"""Keep main's hash-keyed handles and per-item push IDs after the proxy move."""
proxy.registry.register(InstanceRecord(ENCODE, "http://encoder"))
proxy.registry.register(
InstanceRecord(
PD,
"http://decode",
ec_zmq_addrs=["tcp://decode:14579"] if push else [],
)
)
handle = {
"metadata": {"image_grid_thw": [[1, 2, 3]]},
"item_indices": [0, 1],
"peer_port": 5601,
}
response = Mock(
status=200,
read=AsyncMock(
return_value=msgspec.json.encode(
{"ec_transfer_params": {"engine-hash": handle}}
)
),
)
session = Mock(post=AsyncMock(return_value=response))
monkeypatch.setattr(proxy_module, "encode_session", session)
original = {"messages": [{"role": "user", "content": [IMAGE_ITEM, IMAGE_ITEM]}]}
route = proxy.route(2)
prepared, _, _ = asyncio.run(
proxy_module.prepare_for_decode(
original, "request", route.encoder_urls, None, route.consumer_zmq
)
)
assert "ec_transfer_params" not in original
assert prepared["ec_transfer_params"]["engine-hash"] == handle
items = prepared["ec_transfer_params"]["ec_items"]
assert [item["mm_hash"] for item in items] == ["engine-hash", "engine-hash"]
assert items[0]["transfer_id"] != items[1]["transfer_id"]
assert session.post.call_count == 1
sent = msgspec.json.decode(session.post.call_args.kwargs["data"]).get(
"ec_transfer_params"
)
if push:
assert sent == {
"consumer_zmq": "tcp://decode:14579",
"ec_items": [{"transfer_id": item["transfer_id"]} for item in items],
}
else:
assert sent is None
for index in range(2):
assert prepared["messages"][0]["content"][index]["image_embeds"] == {
"image_grid_thw": [1, 2, 3]
}
@@ -1,156 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Dynamic registration and liveness for the EPD proxy's instance registry."""
from unittest.mock import patch
import pytest
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
InstanceRecord,
InstanceRegistry,
InstanceRole,
)
ENCODE = InstanceRole.ENCODE
DECODE = InstanceRole.DECODE
@pytest.fixture
def registry():
return InstanceRegistry(probe_interval=0, fail_threshold=3, evicted_ttl=60.0)
def _fill(registry, role, count, prefix="http://h"):
for index in range(count):
registry.register(InstanceRecord(role, f"{prefix}{index}:8000"))
async def _probe_round(registry, healthy: set[str], now: float = 0.0):
"""Run one probe pass where only `healthy` URLs answer."""
async def fake_probe(self, session, url):
return url in healthy
with (
patch.object(InstanceRegistry, "_probe", fake_probe),
patch(
"examples.disaggregated.disaggregated_encoder.disagg_epd_proxy.time.monotonic",
return_value=now,
),
):
await registry._probe_once(session=None)
class TestRegistration:
def test_registering_is_idempotent_and_reports_novelty(self, registry):
record = InstanceRecord(ENCODE, "http://e0:8000")
assert registry.register(record) is True
assert registry.register(record) is False
assert registry.urls(ENCODE) == ["http://e0:8000"]
def test_transfer_addresses_stay_with_the_registered_consumer(self, registry):
"""Roster updates must not shift a consumer's transfer addresses."""
registry.register(InstanceRecord(DECODE, "http://d0:8000"))
registry.register(
InstanceRecord(
DECODE,
"http://d0:8000",
ec_zmq_addrs=["tcp://d0:20001", "tcp://d0:20002"],
dp_size=2,
)
)
picked = registry.pick(DECODE)
assert picked.ec_zmq_addrs == ["tcp://d0:20001", "tcp://d0:20002"]
assert picked.dp_size == 2
def test_unregister_is_terminal(self, registry):
registry.register(InstanceRecord(ENCODE, "http://e0:8000"))
assert registry.unregister("http://e0:8000") is True
assert registry.unregister("http://e0:8000") is False
assert registry.pick(ENCODE) is None
def test_round_robin_survives_registration(self, registry):
_fill(registry, ENCODE, 2)
assert [record.url for record in registry.pick_many(ENCODE, 2)] == [
"http://h0:8000",
"http://h1:8000",
]
registry.register(InstanceRecord(ENCODE, "http://h2:8000"))
assert registry.pick(ENCODE).url == "http://h2:8000"
class TestLiveness:
@pytest.mark.asyncio
async def test_identical_reregistration_preserves_inflight_probe_results(
self, registry
):
"""Repeated registrations must not suppress eviction or recovery."""
url = "http://e0:8000"
registry.register(InstanceRecord(ENCODE, url))
healthy = False
async def probe(self, session, url):
registry.register(InstanceRecord(ENCODE, url))
return healthy
with patch.object(InstanceRegistry, "_probe", probe):
for _ in range(registry._fail_threshold):
await registry._probe_once(None)
assert registry.status()["encode"] == {"live": [], "evicted": [url]}
healthy = True
await registry._probe_once(None)
assert registry.status()["encode"] == {"live": [url], "evicted": []}
@pytest.mark.asyncio
async def test_consecutive_failures_stop_routing(self, registry):
registry.register(InstanceRecord(ENCODE, "http://e0:8000"))
registry.register(InstanceRecord(ENCODE, "http://e1:8000"))
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy={"http://e1:8000"})
assert registry.urls(ENCODE) == ["http://e1:8000"]
assert registry.status()["encode"]["evicted"] == ["http://e0:8000"]
@pytest.mark.asyncio
@pytest.mark.parametrize("role", [ENCODE, DECODE])
async def test_a_recovered_instance_rejoins_without_re_registering(
self, registry, role
):
"""Restarting every encoder to recover from a blip is not acceptable."""
registry.register(InstanceRecord(role, "http://e0:8000"))
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy=set())
assert registry.urls(role) == []
await _probe_round(registry, healthy={"http://e0:8000"}, now=1)
assert registry.urls(role) == ["http://e0:8000"]
@pytest.mark.asyncio
async def test_re_registration_does_not_override_failed_health_checks(
self, registry
):
record = InstanceRecord(ENCODE, "http://e0:8000")
registry.register(record)
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy=set())
registry.register(record)
assert registry.urls(ENCODE) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("healthy", [False, True])
async def test_inflight_probe_cannot_restore_unregistered_instance(
self, registry, healthy
):
record = InstanceRecord(ENCODE, "http://e0:8000")
registry.register(record)
for _ in range(registry._fail_threshold):
await _probe_round(registry, healthy=set())
async def probe(self, session, url):
registry.unregister(url)
return healthy
with patch.object(InstanceRegistry, "_probe", probe):
await registry._probe_once(None)
assert registry.status()["encode"] == {"live": [], "evicted": []}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,712 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from typing import Any
from unittest.mock import MagicMock
import pytest
import torch
from tests.v1.kv_connector.unit.utils import (
EOS_TOKEN_ID,
create_model_runner_output,
create_vllm_config,
)
from vllm import SamplingParams
from vllm.config import (
KVEventsConfig,
KVTransferConfig,
set_current_vllm_config,
)
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
OffloadingConnector,
)
from vllm.forward_context import ForwardContext
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import (
get_request_block_hasher,
init_none_hash,
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
GPULoadStoreSpec,
LoadStoreSpec,
LookupResult,
OffloadingManager,
OffloadingSpec,
OffloadingWorker,
OffloadKey,
PrepareStoreOutput,
RequestOffloadingContext,
TransferResult,
make_offload_key,
)
from vllm.v1.kv_offload.config import OffloadingConfig
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
def to_key(int_hash: int) -> OffloadKey:
return make_offload_key(str(int_hash).encode(), 0)
def to_keys(int_hashes: Iterable[int]) -> list[OffloadKey]:
return [to_key(i) for i in int_hashes]
class MockLoadStoreSpec(LoadStoreSpec):
def __init__(self, offload_keys: Iterable[OffloadKey]):
self.offload_keys: list[OffloadKey] = list(offload_keys)
def __repr__(self) -> str:
return repr(self.offload_keys)
class MockOffloadingWorker(OffloadingWorker):
def __init__(self):
self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {}
self.completed_transfers: list[TransferResult] = []
self.waiting_jobs: set[int] = set()
self.completed_jobs: list[int] = []
self.flushed_jobs: set[int] = set()
def get_finished(self) -> list[TransferResult]:
finished = self.completed_transfers
self.completed_transfers = []
return finished
def submit_store(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool: # type: ignore[override]
self.transfer_specs[job_id] = (src_spec, dst_spec)
self.waiting_jobs.add(job_id)
return True
def submit_load(
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
) -> bool: # type: ignore[override]
self.transfer_specs[job_id] = (src_spec, dst_spec)
self.waiting_jobs.add(job_id)
return True
def complete_jobs(self, job_ids: set[int]) -> None:
for job_id in job_ids:
if job_id in self.waiting_jobs:
self.waiting_jobs.remove(job_id)
self.completed_jobs.append(job_id)
result = TransferResult(
job_id=job_id,
success=True,
transfer_size=None,
transfer_time=None,
)
self.completed_transfers.append(result)
def wait(self, job_ids: set[int]) -> None:
self.flushed_jobs |= job_ids
self.complete_jobs(job_ids)
class MockOffloadingSpec(OffloadingSpec):
def __init__(self, config: OffloadingConfig):
super().__init__(config)
self.manager = MagicMock(spec=OffloadingManager)
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
self.manager.lookup.return_value = LookupResult.MISS
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
self.manager.get_stats.return_value = None
self.manager.config_info.return_value = [{}]
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingWorker()
def get_manager(self) -> OffloadingManager:
return self.manager
def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker:
return self.handler
def complete_transfers(self):
self.handler.complete_jobs(self.handler.waiting_jobs.copy())
def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
specs = [
self.handler.transfer_specs[job_id]
for job_id in self.handler.completed_jobs
]
self.handler.completed_jobs.clear()
return specs
def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
specs = [
self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs
]
self.handler.flushed_jobs.clear()
return specs
@dataclass(frozen=True)
class GPUBlock:
group_idx: int
request_block_offset: int
@dataclass
class TransferSummary:
gpu_blocks: list[GPUBlock]
offload_addresses: list[Any]
class RequestRunner:
def __init__(
self,
block_size: int,
num_gpu_blocks: int,
blocks_per_chunk: int = 1,
async_scheduling: bool = True,
kv_cache_groups: list[KVCacheGroupSpec] | None = None,
extra_config_overrides: dict[str, Any] | None = None,
worker_count: int = 1,
retention_interval: int | None = None,
speculative_config: Any | None = None,
):
assert blocks_per_chunk == 1 or kv_cache_groups is None, (
"blocks_per_chunk > 1 requires all groups to have the same "
"block size, so kv_cache_groups must be None (use default group)"
)
self.blocks_per_chunk: int = blocks_per_chunk
self.block_size: int = block_size
self.num_gpu_blocks: int = num_gpu_blocks
self.async_scheduling: bool = async_scheduling
self.req_id: int = -1
vllm_config = create_vllm_config(
block_size=block_size,
max_num_batched_tokens=1000,
disable_hybrid_kv_cache_manager=False,
)
vllm_config.scheduler_config.async_scheduling = async_scheduling
vllm_config.parallel_config.world_size = worker_count
vllm_config.cache_config.prefix_cache_retention_interval = retention_interval
if speculative_config is not None:
vllm_config.speculative_config = speculative_config
extra_config: dict[str, Any] = {
"spec_name": "MockOffloadingSpec",
"spec_module_path": "tests.v1.kv_connector.unit.offloading_connector.utils", # noqa: E501
# Preserve legacy behavior for tests; new opt-in tests override.
"offload_prompt_only": False,
# Exercise the self-describing KV events path by default;
# opt-out tests override this to cover the legacy placeholders.
"self_describing_kv_events": True,
}
if blocks_per_chunk > 1:
extra_config["block_size"] = block_size * blocks_per_chunk
if extra_config_overrides:
extra_config.update(extra_config_overrides)
vllm_config.kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
vllm_config.kv_events_config = KVEventsConfig(
# Enable so the offloading events tracker is active, but use the
# null publisher: these tests drain take_events directly and a
# real ZMQ publisher would bind a port per test.
enable_kv_cache_events=True,
publisher="null",
)
if kv_cache_groups is None:
kv_cache_groups = [
KVCacheGroupSpec(
["layer"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
]
# Groups overlay one allocation, sized by the largest group.
block_stride = max(
group.kv_cache_spec.page_size_bytes * len(group.layer_names)
for group in kv_cache_groups
)
kv_cache_tensors = [
KVCacheTensor(
size=block_stride * num_gpu_blocks,
layers=list(group.layer_names),
layer_stride=group.kv_cache_spec.page_size_bytes * num_gpu_blocks,
block_stride=group.kv_cache_spec.page_size_bytes,
)
for group in kv_cache_groups
]
kv_cache_config = KVCacheConfig(
num_blocks=num_gpu_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=kv_cache_groups,
)
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
self.num_kv_groups = len(kv_cache_config.kv_cache_groups)
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
self.scheduler = scheduler_cls(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=scheduler_block_size,
hash_block_size=hash_block_size,
)
self.worker_connector = OffloadingConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
# register worker kv_caches to enable OffloadingWorker creations
kv_caches: dict[str, torch.Tensor] = {}
for group in kv_cache_groups:
spec = group.kv_cache_spec
for layer_name in group.layer_names:
# Shape follows FlashAttention layout:
# Shape: (num_blocks, 2, block_size, num_kv_heads, head_size)
kv_caches[layer_name] = torch.empty(
num_gpu_blocks,
2,
spec.block_size,
spec.num_kv_heads,
spec.head_size,
dtype=spec.dtype,
)
with set_current_vllm_config(vllm_config):
self.worker_connector.register_kv_caches(kv_caches)
# extract connector of scheduler
scheduler_connector = self.scheduler.connector
assert scheduler_connector is not None
assert isinstance(scheduler_connector, OffloadingConnector)
self.scheduler_connector: OffloadingConnector = scheduler_connector
# extract mocked OffloadingManager of scheduler connector
self.connector_scheduler = scheduler_connector.connector_scheduler
assert self.connector_scheduler is not None
manager = self.connector_scheduler.manager
assert isinstance(manager, MagicMock)
self.manager: MagicMock = manager
num_kv_groups = len(kv_cache_config.kv_cache_groups)
assert len(self.connector_scheduler.config.kv_group_configs) == num_kv_groups
for group_config, kv_cache_group in zip(
self.connector_scheduler.config.kv_group_configs,
kv_cache_config.kv_cache_groups,
):
tokens_per_block = kv_cache_group.kv_cache_spec.block_size
assert group_config.tokens_per_block == tokens_per_block
assert group_config.tokens_per_chunk == tokens_per_block * blocks_per_chunk
# extract OffloadingSpec of worker_connector
connector_worker = self.worker_connector.connector_worker
assert connector_worker is not None
offloading_spec = connector_worker.spec
assert isinstance(offloading_spec, MockOffloadingSpec)
self.offloading_spec: MockOffloadingSpec = offloading_spec
# mapping (offloading address) -> GPUBlock
self.offloaded: dict[Any, GPUBlock] = {}
self.completed_loads: list[TransferSummary] = []
self.completed_stores: list[TransferSummary] = []
self.flushed_gpu_blocks: set[GPUBlock] = set()
self.kv_connector_stats: list[Any] = []
# block_id -> GPUBlock
self.gpu_blocks: dict[int, GPUBlock] = {}
init_none_hash(sha256)
self._block_hasher = get_request_block_hasher(block_size, sha256)
self._dummy_ctx: ForwardContext = ForwardContext(
no_compile_layers={},
attn_metadata={},
slot_mapping={},
)
def _record_kv_connector_stats(self, engine_outputs: dict[int, Any]) -> None:
for output in engine_outputs.values():
scheduler_stats = output.scheduler_stats
if scheduler_stats is not None and scheduler_stats.kv_connector_stats:
self.kv_connector_stats.append(scheduler_stats.kv_connector_stats)
def new_request(
self,
token_ids: list[int],
kv_transfer_params: dict | None = None,
skip_reading_prefix_cache: bool = False,
):
self.req_id += 1
sampling_params = SamplingParams(
max_tokens=1000,
skip_reading_prefix_cache=skip_reading_prefix_cache or None,
)
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
req = Request(
request_id=str(self.req_id),
prompt_token_ids=token_ids,
sampling_params=sampling_params,
pooling_params=None,
block_hasher=self._block_hasher,
)
if kv_transfer_params is not None:
req.kv_transfer_params = kv_transfer_params
self.scheduler.add_request(req)
def _parse_transfers(self):
for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers():
if isinstance(src_spec, GPULoadStoreSpec):
# store flush
for block_id in src_spec.block_ids:
self.flushed_gpu_blocks.add(self.gpu_blocks[block_id.item()])
else:
# load flush
for block_id in dst_spec.block_ids:
self.flushed_gpu_blocks.add(self.gpu_blocks[block_id.item()])
blocks_per_chunk = self.blocks_per_chunk
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
if isinstance(src_spec, GPULoadStoreSpec):
store = True
gpu_spec = src_spec
offload_spec = dst_spec
else:
store = False
gpu_spec = dst_spec
offload_spec = src_spec
assert isinstance(offload_spec, MockLoadStoreSpec)
assert isinstance(gpu_spec, GPULoadStoreSpec)
assert len(gpu_spec.group_sizes) == self.num_kv_groups
gpu_blocks: list[GPUBlock] = []
for block_id in gpu_spec.block_ids:
gpu_blocks.append(self.gpu_blocks[block_id.item()])
# list of (offload_key, sub_block_offset)
offload_addresses: list[Any] = []
for offload_key in offload_spec.offload_keys:
for sub_block_idx in range(blocks_per_chunk):
offload_addresses.append((offload_key, sub_block_idx))
assert gpu_spec.block_indices is not None
assert len(gpu_spec.block_indices) == self.num_kv_groups
gpu_block_offset = 0
offload_address_offset = 0
for group_size, logical_offset in zip(
gpu_spec.group_sizes, gpu_spec.block_indices
):
gpu_block_end_offset = gpu_block_offset + group_size
assert gpu_block_end_offset <= len(gpu_blocks)
offload_addresses_to_skip = logical_offset % blocks_per_chunk
offload_addresses_end_offset = (
offload_address_offset + offload_addresses_to_skip + group_size
)
assert offload_addresses_end_offset <= len(offload_addresses)
offload_addresses = (
offload_addresses[:offload_address_offset]
+ offload_addresses[
offload_address_offset + offload_addresses_to_skip :
]
)
gpu_block_offset += group_size
offload_address_offset += group_size
assert gpu_block_offset == len(gpu_blocks)
assert offload_address_offset == len(offload_addresses)
transfer_summary = TransferSummary(gpu_blocks, offload_addresses)
if store:
self.completed_stores.append(transfer_summary)
else:
self.completed_loads.append(transfer_summary)
def _update_gpu_blocks(self):
for group_idx, manager in enumerate(
self.scheduler.kv_cache_manager.coordinator.single_type_managers
):
for blocks in manager.req_to_blocks.values():
for block_idx, block in enumerate(blocks):
self.gpu_blocks[block.block_id] = GPUBlock(group_idx, block_idx)
def _run(
self,
decoded_tokens: list[int],
complete_transfers: bool,
post_step_fn: Callable[[], None] | None = None,
):
"""Runs multiple engine (scheduler + worker) steps.
Assumes a single request is running.
Args:
decoded_tokens: the tokens to yield at each step.
complete_transfers: complete transfers immediately
post_step_fn: optional callback invoked after each step's
update_from_output(), before the next schedule().
"""
tokens_iter = iter(decoded_tokens)
token_id = next(tokens_iter, None)
prev_scheduler_output = None
prev_model_runner_output = None
while True:
# Strict-always-False frees the request immediately on EOS, but
# the worker may still have a deferred store queued. In production
# the next request's step drains it; in single-request tests we
# must keep stepping until the scheduler sees no in-flight jobs
# and no pending finished_req_ids awaiting build_connector_meta.
if (
not self.scheduler.requests
and not self.connector_scheduler._jobs
and not self.scheduler.finished_req_ids
):
break
scheduler_output = self.scheduler.schedule()
self._update_gpu_blocks()
kv_connector_metadata = scheduler_output.kv_connector_metadata
assert kv_connector_metadata is not None
assert isinstance(kv_connector_metadata, OffloadingConnectorMetadata)
self.worker_connector.handle_preemptions(kv_connector_metadata)
self.worker_connector.bind_connector_metadata(kv_connector_metadata)
self.worker_connector.start_load_kv(self._dummy_ctx)
if complete_transfers:
self.offloading_spec.complete_transfers()
self.worker_connector.wait_for_save()
finished_sending, finished_recving = self.worker_connector.get_finished(
scheduler_output.finished_req_ids
)
worker_meta = (
self.worker_connector.build_connector_worker_meta()
or OffloadingWorkerMetadata()
)
self.worker_connector.clear_connector_metadata()
model_runner_output = create_model_runner_output(
reqs=self.scheduler.running,
finished_sending=finished_sending,
finished_recving=finished_recving,
token_id=token_id or 0,
kv_connector_worker_meta=worker_meta,
)
prev_token_id = token_id
if self.scheduler.running:
token_id = next(tokens_iter, None)
if self.async_scheduling:
# in async scheduling we update the output of the previous step
if prev_model_runner_output is not None:
engine_outputs = self.scheduler.update_from_output(
prev_scheduler_output, prev_model_runner_output
)
self._record_kv_connector_stats(engine_outputs)
prev_scheduler_output = scheduler_output
prev_model_runner_output = model_runner_output
else:
engine_outputs = self.scheduler.update_from_output(
scheduler_output, model_runner_output
)
self._record_kv_connector_stats(engine_outputs)
if post_step_fn is not None:
post_step_fn()
if (
prev_token_id == EOS_TOKEN_ID
and prev_token_id != token_id
and (
self.scheduler.requests
or self.connector_scheduler._jobs
or self.scheduler.finished_req_ids
)
):
# continue for one more step to allow offloading to kick off
continue
if token_id is None:
if self.async_scheduling:
# Flush the previous step's output.
engine_outputs = self.scheduler.update_from_output(
prev_scheduler_output, prev_model_runner_output
)
self._record_kv_connector_stats(engine_outputs)
prev_model_runner_output = None
if self.scheduler.requests:
# Request still running, just exhausted decoded_tokens.
break
if not self.scheduler.finished_req_ids and (
not complete_transfers or not self.connector_scheduler._jobs
):
break
self._parse_transfers()
if EOS_TOKEN_ID in decoded_tokens:
assert not self.scheduler.running
def _to_gpu_blocks(
self, blocks: tuple[int | tuple[int, int], ...]
) -> list[GPUBlock]:
gpu_blocks: list[GPUBlock] = []
for block in blocks:
if isinstance(block, int):
for group_idx in range(self.num_kv_groups):
gpu_blocks.append(
GPUBlock(group_idx=group_idx, request_block_offset=block)
)
else:
group_idx, offset = block
gpu_blocks.append(
GPUBlock(group_idx=group_idx, request_block_offset=offset)
)
return gpu_blocks
def run(
self,
decoded_tokens: list[int],
complete_transfers: bool = True,
expected_stored: tuple[int | tuple[int, int], ...] = (),
expected_loaded: tuple[int | tuple[int, int], ...] = (),
expected_flushed: tuple[int | tuple[int, int], ...] = (),
post_step_fn: Callable[[], None] | None = None,
):
"""Runs multiple engine (scheduler + worker) steps.
Assumes a single request is running.
Args:
decoded_tokens: the tokens to yield at each step.
complete_transfers: complete transfers immediately
expected_stored: GPU blocks
that are expected to be written during the run.
expected_loaded: GPU blocks
that are expected to be loaded during the run.
expected_flushed: GPU blocks
that are expected to be flushed during the run.
A GPU block is either a (group_idx: int, request_block_offset: int)
or just request_block_offset: int.
The latter case is a convenience for representing all groups.
"""
expected_stored_gpu_blocks = self._to_gpu_blocks(expected_stored)
expected_loaded_gpu_blocks = self._to_gpu_blocks(expected_loaded)
expected_flushed_gpu_blocks = self._to_gpu_blocks(expected_flushed)
self.manager.reset_mock()
self._run(decoded_tokens, complete_transfers, post_step_fn=post_step_fn)
loaded_gpu_blocks: set[GPUBlock] = set()
for transfer in self.completed_loads:
for gpu_block, offloaded_address in zip(
transfer.gpu_blocks, transfer.offload_addresses
):
loaded_gpu_blocks.add(gpu_block)
assert gpu_block == self.offloaded[offloaded_address]
assert set(expected_loaded_gpu_blocks) == loaded_gpu_blocks
self.completed_loads.clear()
stored_gpu_blocks: set[GPUBlock] = set()
for transfer in self.completed_stores:
for gpu_block, offloaded_address in zip(
transfer.gpu_blocks, transfer.offload_addresses
):
stored_gpu_blocks.add(gpu_block)
self.offloaded[offloaded_address] = gpu_block
assert set(expected_stored_gpu_blocks) == stored_gpu_blocks
self.completed_stores.clear()
assert set(expected_flushed_gpu_blocks) == self.flushed_gpu_blocks
self.flushed_gpu_blocks.clear()
@pytest.fixture
def request_runner():
runners = []
def runner_factory(
block_size,
num_gpu_blocks,
async_scheduling,
blocks_per_chunk=1,
kv_cache_groups=None,
extra_config_overrides=None,
worker_count=1,
retention_interval=None,
speculative_config=None,
):
runner = RequestRunner(
block_size=block_size,
num_gpu_blocks=num_gpu_blocks,
blocks_per_chunk=blocks_per_chunk,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
extra_config_overrides=extra_config_overrides,
worker_count=worker_count,
retention_interval=retention_interval,
speculative_config=speculative_config,
)
runners.append(runner)
return runner
yield runner_factory # pass factory to the test
def generate_store_output(keys: Iterable[OffloadKey]):
keys = list(keys)
return PrepareStoreOutput(
keys_to_store=list(keys),
store_spec=MockLoadStoreSpec(keys),
evicted_keys=[],
)
File diff suppressed because it is too large Load Diff
+700
View File
@@ -0,0 +1,700 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from itertools import chain, count
from typing import Any, Literal
from unittest.mock import Mock
import numpy as np
import torch
from vllm import SamplingParams
from vllm.config import (
AttentionConfig,
CacheConfig,
DeviceConfig,
KVTransferConfig,
ModelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
ExampleConnector,
)
from vllm.platforms import current_platform
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
from vllm.v1.core.sched.scheduler import Scheduler, SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
EOS_TOKEN_ID = 50256
def create_mock_connector() -> Mock:
connector = Mock()
connector.get_external_cache_hit_sources.side_effect = lambda request, num_tokens: (
{CacheHitSource.EXTERNAL_UNSPECIFIED: num_tokens} if num_tokens else {}
)
return connector
def maybe_update_block_size(block_size: int) -> int:
return 64 if current_platform.is_xpu() else block_size
def assert_scheduler_empty(scheduler: Scheduler):
"""Confirm the scheduler is "empty" - i.e. no leaks."""
# Scheduler Metadata.
assert len(scheduler.requests) == 0
assert len(scheduler.waiting) == 0
assert len(scheduler.kv_holding_waiting) == 0
assert not scheduler.deferred_waiting
assert len(scheduler.running) == 0
assert len(scheduler.finished_req_ids) == 0
assert len(scheduler.finished_recving_kv_req_ids) == 0
assert len(scheduler._inflight_prefills) == 0
assert not scheduler._kv_fetch_stages
assert not any(scheduler._kv_fetch_counts.values())
# EncoderCacheManager.
assert len(scheduler.encoder_cache_manager.freed) == 0
assert len(scheduler.encoder_cache_manager.cached) == 0
# KVCache Manager.
assert (
len(
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks
)
== 0
)
assert (
len(
scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].num_cached_block
)
== 0
)
num_free_blocks = (
scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks
)
assert num_free_blocks == (scheduler.kv_cache_manager.block_pool.num_gpu_blocks - 1)
# NOTE(rob): just the ref count on blocks will be 0. The hash
# value, etc will remain since we lazily evict for prefix cache.
for block in scheduler.kv_cache_manager.block_pool.blocks:
assert block.ref_cnt == 0
def create_vllm_config(
model: str = "facebook/opt-125m",
max_num_seqs: int = 16,
max_num_batched_tokens: int = 64,
block_size: int = 16,
max_model_len: int = 10000,
enable_chunked_prefill: bool = True,
enable_permute_local_kv: bool = False,
kv_connector_extra_config: dict[str, Any] | None = None,
dtype: str = "float16",
cache_dtype: str = "auto",
hf_overrides: dict[str, Any] | None = None,
attention_backend: str | None = None,
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
kv_connector: str = "NixlConnector",
kv_connector_module_path: str | None = None,
kv_role: str = "kv_consumer",
disable_hybrid_kv_cache_manager: bool | None = None,
num_speculative_tokens: int | None = None,
) -> VllmConfig:
"""Initialize VllmConfig For Testing."""
model_config = ModelConfig(
model=model,
trust_remote_code=True,
dtype=dtype,
seed=42,
hf_overrides=hf_overrides or {},
)
scheduler_config = SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=max_num_batched_tokens,
max_model_len=max_model_len,
enable_chunked_prefill=enable_chunked_prefill,
is_encoder_decoder=model_config.is_encoder_decoder,
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager,
)
# Cache config, optionally force APC
cache_config = CacheConfig(
block_size=block_size,
gpu_memory_utilization=0.9,
cache_dtype=cache_dtype,
enable_prefix_caching=True,
)
# Connectors are constructed after layout resolution; mirror that here.
cache_config.kv_cache_layout = "LBNHC"
kv_transfer_config = KVTransferConfig(
kv_connector=kv_connector,
kv_connector_module_path=kv_connector_module_path,
kv_role=kv_role,
enable_permute_local_kv=enable_permute_local_kv,
kv_connector_extra_config=kv_connector_extra_config or {},
kv_load_failure_policy=kv_load_failure_policy,
)
attention_config = AttentionConfig(backend=attention_backend)
speculative_config = (
SpeculativeConfig(model="ngram", num_speculative_tokens=num_speculative_tokens)
if num_speculative_tokens is not None
else None
)
return VllmConfig(
scheduler_config=scheduler_config,
model_config=model_config,
cache_config=cache_config,
kv_transfer_config=kv_transfer_config,
device_config=DeviceConfig("cpu"),
attention_config=attention_config,
speculative_config=speculative_config,
)
def create_scheduler(
vllm_config: VllmConfig,
num_blocks: int = 10000,
kv_cache_config: KVCacheConfig | None = None,
hash_block_size: int | None = None,
) -> Scheduler | AsyncScheduler:
"""Initialize Scheduler For Testing."""
block_size = vllm_config.cache_config.block_size
if kv_cache_config is None:
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks, # A large number of blocks to hold all requests
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
],
)
vllm_config.cache_config.num_gpu_blocks = num_blocks
scheduler_cls = (
AsyncScheduler if vllm_config.scheduler_config.async_scheduling else Scheduler
)
return scheduler_cls(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=hash_block_size,
)
_request_count = count(1)
_none_hash_initialized = False
def create_request(
request_id: int | None = None,
num_tokens: int = 10,
common_prefix_len=0,
max_tokens: int = 16,
do_remote_decode: bool = False,
do_remote_prefill: bool = False,
num_remote_blocks: int = 3,
block_size: int = 16,
hash_fn: Callable = sha256,
) -> Request:
"""Make dummy request for testing."""
assert num_tokens >= common_prefix_len >= 0
if request_id is None:
request_id = next(_request_count)
global _none_hash_initialized
if not _none_hash_initialized:
init_none_hash(hash_fn)
_none_hash_initialized = True
kv_transfer_params: dict[str, Any] | None = None
if do_remote_decode:
assert not do_remote_prefill
kv_transfer_params = dict(do_remote_prefill=False, do_remote_decode=True)
elif do_remote_prefill:
kv_transfer_params = dict(
do_remote_prefill=True,
do_remote_decode=False,
remote_engine_id="my-engine-id",
remote_request_id=f"prefill-{request_id}",
remote_block_ids=list(range(num_remote_blocks)),
remote_host="my-host",
remote_port=1234,
tp_size=1,
)
max_tokens = 1 if do_remote_decode else max_tokens
sampling_params = SamplingParams(max_tokens=max_tokens)
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
common_prefix = [1] * common_prefix_len if common_prefix_len > 0 else []
suffix = [i * request_id for i in range(num_tokens - common_prefix_len)]
prompt_token_ids = common_prefix + suffix
req = Request(
request_id=f"id-{request_id}",
prompt_token_ids=prompt_token_ids,
sampling_params=sampling_params,
pooling_params=None,
mm_features=None,
block_hasher=get_request_block_hasher(block_size, hash_fn),
)
req.kv_transfer_params = kv_transfer_params
return req
def create_model_runner_output(
reqs: list[Request],
finished_sending: set[str] | None = None,
finished_recving: set[str] | None = None,
invalid_block_ids: set[int] | None = None,
use_eos: bool = False,
token_id: int = 0,
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
) -> ModelRunnerOutput:
"""Make dummy model runner output for testing."""
# Make request data.
req_ids = [req.request_id for req in reqs]
req_id_to_index = {req_id: idx for idx, req_id in enumerate(req_ids)}
# Make sampled tokens.
sampled_token = EOS_TOKEN_ID if use_eos else token_id
sampled_token_ids = [[sampled_token] for _ in req_ids]
kv_connector_output = (
None
if (
finished_sending is None
and finished_recving is None
and invalid_block_ids is None
and kv_connector_worker_meta is None
)
else KVConnectorOutput(
finished_sending=finished_sending,
finished_recving=finished_recving,
invalid_block_ids=invalid_block_ids or set(),
kv_connector_worker_meta=kv_connector_worker_meta,
)
)
# Make output data structure.
return ModelRunnerOutput(
req_ids=req_ids,
req_id_to_index=req_id_to_index,
sampled_token_ids=sampled_token_ids,
logprobs=None,
prompt_logprobs_dict={},
pooler_output=None,
kv_connector_output=kv_connector_output,
)
class TestExampleConnector(ExampleConnector):
def __init__(
self,
config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
):
self.name = config.kv_transfer_config.kv_connector_extra_config["name"]
self._connector = ExampleConnector(config, role, kv_cache_config)
self.call_record: dict[str, int] = defaultdict(int)
# Use a unique temp file per connector
self._event_file = (
tempfile.gettempdir()
+ f"/connector_{self.name}-{self.role.name}_events.log"
)
# Start with an empty file
with open(self._event_file, "w") as _:
pass
def __getattribute__(self, name):
if name in (
"_connector",
"call_record",
"name",
"_event_file",
"__class__",
"__dict__",
"__getattribute__",
"__init__",
): # avoid recursion
return object.__getattribute__(self, name)
if not hasattr(self._connector, name):
return object.__getattribute__(self, name)
attr = getattr(self._connector, name)
# Intercept calls to the connector interface and write an event
# for each one to a file, which can be read back in the main test proc.
if callable(attr):
def wrapper(*args, **kwargs):
self.call_record[name] += 1
# Include args that we're interested in
to_log = [name]
for arg in args:
if isinstance(arg, int):
to_log.append(str(arg))
elif isinstance(arg, KVCacheBlocks):
to_log.append(f"num_blocks={[len(b) for b in arg.blocks]}")
# Log the event as a line to the file
try:
with open(self._event_file, "a") as f:
f.write(" ".join(to_log) + "\n")
except Exception as e:
print(f"[ERROR] Could not log event {name} for {self.name}: {e}")
return attr(*args, **kwargs)
return wrapper
return attr
@dataclass(frozen=True)
class MockKVConfig:
matched_tokens: int = 0
is_async: bool = False
num_defers_before_matching: int = 0
supports_divergent_local_hybrid_hits: bool = False
class MockKVConnectorMetadata(KVConnectorMetadata):
def __init__(self):
# Scheduler tests check metadata.requests
self.requests: list = []
class MockKVConnector(KVConnectorBase_V1):
"""Mock KV connector for scheduler tests, supporting both sync and async mode."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
):
super().__init__(vllm_config, role, kv_cache_config)
extra_config = self._kv_transfer_config.kv_connector_extra_config
self.config = MockKVConfig(
matched_tokens=extra_config["matched_tokens"],
is_async=extra_config["is_async"],
num_defers_before_matching=extra_config.get(
"num_defers_before_matching", 0
),
supports_divergent_local_hybrid_hits=extra_config.get(
"supports_divergent_local_hybrid_hits", False
),
)
self._defers_left: defaultdict[str, int] = defaultdict(
lambda: self.config.num_defers_before_matching
)
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
return self.config.supports_divergent_local_hybrid_hits
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
if self._defers_left[request.request_id] > 0:
self._defers_left[request.request_id] -= 1
return (None, False)
return (self.config.matched_tokens, self.config.is_async)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
pass
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
metadata = MockKVConnectorMetadata()
cached_reqs = scheduler_output.scheduled_cached_reqs
for req_id in chain(
(req.req_id for req in scheduler_output.scheduled_new_reqs),
(
req_id
for req_id in cached_reqs.req_ids
if req_id in cached_reqs.resumed_req_ids
),
):
metadata.requests.append({"req_id": req_id})
return metadata
def start_load_kv(self, kv_caches, finished_req_ids):
pass
def wait_for_layer_load(self, layer_name):
pass
def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs):
pass
def wait_for_save(self):
pass
KVConnectorFactory.register_connector(
"TestExampleConnector", __name__, TestExampleConnector.__name__
)
KVConnectorFactory.register_connector(
"MockKVConnector", __name__, MockKVConnector.__name__
)
def make_kv_cache_config(
block_size: int,
swa_enabled: bool = False,
mamba_enabled: bool = False,
sw_size: int = 128,
num_blocks: int = 100,
mamba_cache_mode: Literal["align", "none"] = "none",
) -> KVCacheConfig:
kv_cache_groups = [
KVCacheGroupSpec(
["layer0", "layer2"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
),
)
]
if swa_enabled:
kv_cache_groups.append(
KVCacheGroupSpec(
["layer1", "layer3"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
sliding_window=sw_size,
),
)
)
if mamba_enabled:
kv_cache_groups.append(
KVCacheGroupSpec(
["mamba0", "mamba1"],
MambaSpec(
block_size=block_size,
shapes=((16,), (16,)),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
),
)
)
return KVCacheConfig(
num_blocks=num_blocks, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
)
def make_nixl_scheduler(
has_mamba: bool = False,
is_hma_required: bool = False,
bounded_replay: bool = False,
heartbeat: bool = False,
kv_lease_duration: int = 30,
):
"""Create a NixlConnectorScheduler via __new__ (skipping __init__).
Only sets the flags needed by the tests. When *heartbeat=True* the
scheduler-side heartbeat bookkeeping fields are also initialised.
"""
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
NixlConnectorScheduler,
)
sched = object.__new__(NixlConnectorScheduler)
sched._has_mamba = has_mamba
sched._bounded_replay = bounded_replay
sched._is_hma_required = is_hma_required
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0)
if heartbeat:
sched._heartbeat_by_engine = {}
sched._heartbeat_req_engine = {}
sched._last_heartbeat_time = 0.0
sched._kv_lease_duration = kv_lease_duration
sched._heartbeat_interval = kv_lease_duration // 6
# Fields touched by build_connector_meta / request_finished:
sched._reqs_need_recv = {}
sched._hisparse_host_blocks_to_recv = {}
sched._reqs_need_send = {}
sched._reqs_in_batch = set()
sched._reqs_not_processed = set()
sched._reqs_need_save = {}
sched.use_host_buffer = False
sched.engine_id = "test-engine"
sched.transfer_tp_size = 1
sched.side_channel_host = "localhost"
sched.side_channel_port = 5555
sched.blocks_per_sw = []
sched.is_bidirectional_kv_xfer_enabled = False
return sched
def make_nixl_push_scheduler(
*,
decoder_kv_blocks_ttl: float = 30.0,
push_registration_timeout: float | None = None,
is_bidirectional_kv_xfer_enabled: bool = False,
has_mamba: bool = False,
):
"""Create a NixlPushConnectorScheduler via __new__ (skipping __init__).
The push scheduler can't reuse :func:`make_nixl_scheduler` because it
is a different class (``NixlPushConnectorScheduler`` vs
``NixlConnectorScheduler``) and carries push-specific state. Only the
fields touched by the unit tests are populated.
"""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
NixlPushConnectorScheduler,
)
sched = object.__new__(NixlPushConnectorScheduler)
# Base scheduler fields (shared with pull / heartbeat path).
sched._reqs_need_recv = {}
sched._reqs_need_send = {}
sched._reqs_in_batch = set()
sched._reqs_not_processed = set()
sched._reqs_need_save = {}
sched._kv_lease_duration = 30
sched.decoder_kv_blocks_ttl = decoder_kv_blocks_ttl
sched.use_host_buffer = False
sched.engine_id = "decode-engine"
sched.transfer_tp_size = 1
sched.side_channel_host = "127.0.0.1"
sched.side_channel_port = 5600
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
sched._has_mamba = has_mamba
sched._bounded_replay = False
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
# vllm_config is consulted for parallel_config.tensor_parallel_size, and by
# `_prefill_backoff` on both the P and D prefill paths.
vllm_config = MagicMock()
vllm_config.parallel_config.tensor_parallel_size = 1
vllm_config.num_prefill_lookahead_tokens = 0
sched.vllm_config = vllm_config
# Push-specific state.
sched._push_pending_registrations = {}
sched._push_registration_deadlines = {}
sched._finished_request_blocks = {}
sched._newly_finished_push_blocks = {}
sched._push_registration_timeout = (
push_registration_timeout
if push_registration_timeout is not None
else decoder_kv_blocks_ttl
)
# Heartbeat fields touched by base request_finished /
# update_connector_output.
sched._heartbeat_by_engine = {}
sched._heartbeat_req_engine = {}
sched._last_heartbeat_time = 0.0
sched.blocks_per_sw = []
return sched
def make_moriio_writer(fake_worker: Any) -> Any:
"""Build a MoRIIOWriter with internals stubbed for unit tests.
Bypasses ``__init__`` and wires only the write/finalize state the tests
touch, including the deferred-task fields used by the routing suite.
"""
import threading
from queue import Queue
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWriter,
)
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
writer._write_task_q = Queue()
writer._write_state_lock = threading.Lock()
writer._scheduled_writes = defaultdict(int)
writer._scheduled_layers = defaultdict(set)
writer._sealed_writes = {}
writer._deferred_tasks = []
writer._defer_timeout = 60.0
writer.ensure_worker_started = lambda: None
return writer
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
Mirrors how NIXL indexes a strided descriptor list, so desc ids computed
by the connector can be resolved to (addr, len, dev) byte ranges.
"""
strided_descs = np.asarray(strided_descs, dtype=np.uint64).reshape(-1, 5)
counts = strided_descs[:, 4].astype(np.intp)
starts = np.cumsum(strided_descs[:, 4]) - strided_descs[:, 4]
out = np.repeat(strided_descs[:, :3], counts, axis=0)
offsets = np.arange(len(out), dtype=np.uint64) - np.repeat(starts, counts)
out[:, 0] += offsets * np.repeat(strided_descs[:, 3], counts)
return out
File diff suppressed because it is too large Load Diff
+277
View File
@@ -0,0 +1,277 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from enum import Enum
from typing import NamedTuple
from unittest.mock import MagicMock
import regex as re
import torch
from vllm import CompletionOutput
from vllm.config import ReasoningConfig
from vllm.utils.torch_utils import make_tensor_with_pad
from vllm.v1.sample.logits_processor import BatchUpdate, LogitsProcessor
from vllm.v1.sample.metadata import SamplingMetadata
def create_mock_reasoning_config(
start_token_ids: list[int], end_token_ids: list[int]
) -> ReasoningConfig:
return MagicMock(
spec=ReasoningConfig,
reasoning_start_token_ids=start_token_ids,
reasoning_end_token_ids=end_token_ids,
)
class BatchLogprobsComposition(Enum):
"""Types of logprobs configs to include in test batch."""
NONE = 0
SAMPLE = 1
PROMPT = 2
SAMPLE_PROMPT = 3
BatchLogprobsSpecType = list[tuple[int | None, int | None]]
def get_test_batch(
batch_logprobs_composition: BatchLogprobsComposition,
) -> BatchLogprobsSpecType:
"""Generate logprobs configs for a batch of requests.
A given request's logprobs configuration is (1) num_sample_logprobs and (2)
num_prompt_logprobs. The batch logprobs configuration is the list of request
logprobs configs.
batch_logprobs_composition == NONE yields a batch with no sample or prompt
logprobs
batch_logprobs_composition == SAMPLE yields a batch with some requests
configured for sample logprobs only, and others configured for no logprobs
batch_logprobs_composition == PROMPT yields a batch with some requests
configured for prompt logprobs only, and others configured for no logprobs
batch_logprobs_composition == SAMPLE_PROMPT yields a batch with some
requests configured for sample logprobs and prompt logprobs, some configured
for only sample logprobs or only prompt logprobs, and some configured for
no logprobs
Args:
batch_logprobs_composition: types of logprobs configs to include in batch
Returns:
list of (Optional[num_sample_logprobs], Optional[num_prompt_logprobs])
tuples
"""
if batch_logprobs_composition == BatchLogprobsComposition.NONE:
# No requests with sample or prompt logprobs
return [(None, None)]
elif batch_logprobs_composition == BatchLogprobsComposition.SAMPLE:
# Requests requiring sample logprobs or no logprobs
return [
(None, None),
(0, None),
(5, None),
(3, None),
]
elif batch_logprobs_composition == BatchLogprobsComposition.PROMPT:
# Requests requiring prompt logprobs or no logprobs
return [
(None, None),
(None, 0),
(None, 6),
(None, 5),
]
elif batch_logprobs_composition == BatchLogprobsComposition.SAMPLE_PROMPT:
# Requests requiring either no logprobs, just
# sample logprobs, just prompt logprobs, or
# both sample and prompt logprobs
return [
(None, None),
(0, None),
(5, None),
(3, None),
(0, 3),
(6, 0),
(6, 3),
(None, 6),
(None, 5),
(None, 0),
]
else:
raise ValueError("Invalid logprobs batch configuration for test.")
def assert_incr_detok_str_matches_non_incr_detok_str(
incremental_detokenization_str: str,
non_incremental_detokenization_str: str,
msg: str,
) -> None:
"""Compare incrementally detok. text to non-incrementally detok. text.
Fail if the strings mismatch after non-alphanumeric characters are stripped
out.
Rationale: incremental detokenization in the text generation process allows
the tokenizer to adjust the next token text output based on the token's
context in the string. However, logprobs detokenization detokenizes each
token individually, and the resultant strings may include some
non-alphanumeric placeholder characters where there could be i.e.
whitespace. So, this function compares only the alphanumeric text
between two strings and fails if there is a mismatch, which helps
with validating logprobs detokenization.
Args:
incremental_detokenization_str: incrementally-detokenized generated text
non_incremental_detokenization_str: non-incrementally-detokenized logprob
tokens
msg: error message if `assert` fails
"""
rgx = r"[^a-zA-Z0-9]+"
assert re.sub(rgx, "", incremental_detokenization_str) == re.sub(
rgx, "", non_incremental_detokenization_str
), msg
def compute_correct_cumulative_logprob(completion_output: CompletionOutput) -> float:
"""Compute known-good value for evaluating cumulative logprob.
Args:
completion_output: completion output from engine
Returns:
Known-good cumulative logprob value
"""
token_ids = completion_output.token_ids
logprobs = completion_output.logprobs
assert logprobs is not None
values: list[float] = []
for tok_id, lp in zip(token_ids, logprobs):
assert lp is not None
values.append(lp[tok_id].logprob)
return sum(values)
def create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor:
fake_logits = torch.full((batch_size, vocab_size), 1e-2, dtype=torch.float)
return fake_logits
def create_penalty_tensor(
batch_size: int, penalty_value: float, device: torch.device
) -> torch.Tensor:
return torch.full(
(batch_size,), fill_value=penalty_value, dtype=torch.float, device=device
)
def create_prompt_tokens_tensor(
prompt_token_ids: list[list[int]],
vocab_size: int,
device: torch.device,
) -> torch.Tensor:
return make_tensor_with_pad(
prompt_token_ids,
pad=vocab_size,
device=device,
dtype=torch.int64,
pin_memory=False,
)
class LogitsprocsTestFakes(NamedTuple):
"""Wraps fake data structures to support testing."""
logits: torch.Tensor
sampling_metadata: SamplingMetadata
def get_logitsprocs_by_cls(
self,
cls: type[LogitsProcessor],
) -> Iterator[LogitsProcessor]:
"""Yield logits processors of a specific class.
Args:
cls: :class:`LogitsProcessor` subclass
Returns:
Iterator over logits processors
"""
return (
lp for lp in self.sampling_metadata.logitsprocs.all if isinstance(lp, cls)
)
def get_logitsprocs(self) -> Iterator[LogitsProcessor]:
"""Iterator over all logits processors."""
return self.sampling_metadata.logitsprocs.all
def fake_update_logitsprocs_state(
test_fakes: LogitsprocsTestFakes,
batch_update: BatchUpdate | None,
) -> None:
"""Imitate logits processors persistent batch state update
in engine core"""
for logitproc in test_fakes.get_logitsprocs():
logitproc.update_state(batch_update)
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
if holder is not None:
holder.sync_batch(batch_update)
def fake_apply_logitsprocs(
test_fakes: LogitsprocsTestFakes,
slice_indices: list[int],
slot_output_token_ids: list[list[int]] | None = None,
) -> torch.Tensor:
"""Imitate application of logits processors in engine core.
When ``thinking_budget_state_holder`` has tracked requests, this mirrors
:meth:`Sampler.apply_logits_processors` by refreshing per-slot
``output_token_ids`` (if ``slot_output_token_ids`` is provided), then
``update_state`` + ``apply_to_logits`` on the holder after built-in logits
processors.
"""
logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone()
for processor in test_fakes.get_logitsprocs():
logits = processor.apply(logits)
md = test_fakes.sampling_metadata
holder = md.thinking_budget_state_holder
if holder is not None and holder.has_tracked_requests():
if slot_output_token_ids is not None:
for i, toks in enumerate(slot_output_token_ids):
if i < len(md.output_token_ids):
md.output_token_ids[i] = list(toks)
holder.update_state(md.output_token_ids, md.spec_token_ids, None)
logits = holder.apply_to_logits(logits, False, md.spec_token_ids)
return logits
def create_allowed_token_ids(
batch_size: int,
vocab_size: int,
num_allowed_token_ids: int,
device: torch.device,
) -> torch.Tensor | None:
mask: torch.Tensor | None = None
for i in range(batch_size):
if i % 2 == 1:
continue
if mask is None:
mask = torch.zeros(
(batch_size, vocab_size), dtype=torch.bool, device=device
)
start = min(i, vocab_size - 1)
end = min(i + num_allowed_token_ids, vocab_size - 1)
mask[i, start:end] = True
return mask
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,461 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.logger import init_logger
try:
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
except ImportError:
MLACommonMetadata = type("AbsentMLACommonMetadata", (), {})
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass
class ReqMeta:
# Request tokens
token_ids: torch.Tensor
# Slot mappings, should have the same length as token_ids
slot_mapping: torch.Tensor
# Is store or load
is_store: bool
mm_hashes: list[str]
@staticmethod
def make_meta(
token_ids: list[int],
block_ids: list[int],
block_size: int,
is_store: bool,
mm_hashes: list[str],
) -> "ReqMeta":
valid_num_tokens = align_to_block_size(len(token_ids), block_size)
token_ids_tensor = torch.tensor(token_ids)[:valid_num_tokens]
block_ids_tensor = torch.tensor(block_ids)
num_blocks = block_ids_tensor.shape[0]
block_offsets = torch.arange(0, block_size)
slot_mapping = (
block_offsets.reshape((1, block_size))
+ block_ids_tensor.reshape((num_blocks, 1)) * block_size
)
slot_mapping = slot_mapping.flatten()[:valid_num_tokens]
return ReqMeta(
token_ids=token_ids_tensor,
slot_mapping=slot_mapping,
is_store=is_store,
mm_hashes=mm_hashes,
)
@dataclass
class ExampleConnectorMetadata(KVConnectorMetadata):
requests: list[ReqMeta] = field(default_factory=list)
def add_request(
self,
token_ids: list[int],
block_ids: list[int],
block_size: int,
is_store: bool,
mm_hashes: list[str],
) -> None:
self.requests.append(
ReqMeta.make_meta(token_ids, block_ids, block_size, is_store, mm_hashes)
)
class ExampleConnector(KVConnectorBase_V1):
# NOTE: This is Simple debug implementation of the KV connector.
# It save / load the KV cache to / from the disk.
# It does extra work which will overwrite the existing prefix-cache in GPU
# - to remove the overhead, need to add some "mask" in the ReqMeta class
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config,
role=role,
kv_cache_config=kv_cache_config,
)
self._block_size = vllm_config.cache_config.block_size
self._requests_need_load: dict[str, Request] = {}
self._storage_path = self._kv_transfer_config.get_from_extra_config(
"shared_storage_path", "/tmp"
)
logger.info(self._kv_transfer_config)
logger.info("Shared storage path is %s", self._storage_path)
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""Start loading the KV cache from the connector buffer to vLLM's
paged KV buffer.
Args:
forward_context (ForwardContext): the forward context.
**kwargs: additional arguments for the load operation
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
def inject_kv_into_layer(
dst_kv_cache_layer: torch.Tensor,
src_kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
attn_metadata: AttentionMetadata,
) -> None:
"""Inject the KV cache into the layer.
Args:
dst_kv_cache_layer (torch.Tensor): the destination KV cache layer,
a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
src_kv_cache (torch.Tensor): the source KV cache.
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
attn_metadata (AttentionMetadata): attention metadata for the
current forward pass.
"""
# Deliberate bulk transfers in this reference connector.
with gpu_sync_allowed():
slot_mapping = slot_mapping.to(
dst_kv_cache_layer.device, non_blocking=True
)
if isinstance(attn_metadata, MLACommonMetadata):
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
-1, dst_kv_cache_layer.shape[-1]
)
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
else:
block_idxs = slot_mapping // self._block_size
offsets = slot_mapping % self._block_size
dst_kv_cache_layer[block_idxs, :, offsets] = src_kv_cache
# Get the metadata
metadata: KVConnectorMetadata = self._get_connector_metadata()
assert isinstance(metadata, ExampleConnectorMetadata)
attn_metadata = forward_context.attn_metadata
if attn_metadata is None:
logger.warning("In connector.start_load_kv, but the attn_metadata is None")
return
# Load the KV for each request each layer
for request in metadata.requests:
if request.is_store:
continue
logger.info(
"Inject KV cache of %d tokens to the paged memory",
len(request.slot_mapping),
)
for layer_name in forward_context.no_compile_layers:
layer = forward_context.no_compile_layers[layer_name]
# Only process layers that have kv_cache
# attribute (attention layers) Skip non-attention
# layers like FusedMoEFactory/MLP etc.
kv_cache_layer = getattr(layer, "kv_cache", None)
if kv_cache_layer is None:
continue
filename = self._generate_filename_debug(
layer_name, request.token_ids, request.mm_hashes
)
# Deliberate bulk load in this reference connector.
with gpu_sync_allowed():
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
if isinstance(attn_metadata, dict):
inject_kv_into_layer(
kv_cache_layer,
kv_cache,
request.slot_mapping,
attn_metadata[layer_name],
)
def wait_for_layer_load(self, layer_name: str) -> None:
"""Blocking until the KV for a specific layer is loaded into vLLM's
paged buffer.
This interface will be useful for layer-by-layer pipelining.
Args:
layer_name: the name of that layer
"""
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""Start saving the KV cache of the layer from vLLM's paged buffer
to the connector.
Args:
layer_name (str): the name of the layer.
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
def extract_kv_from_layer(
layer: torch.Tensor,
slot_mapping: torch.Tensor,
) -> torch.Tensor:
"""Extract the KV cache from the layer.
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
"""
# Deliberate bulk transfer in this reference connector.
with gpu_sync_allowed():
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
block_idxs = slot_mapping // self._block_size
offsets = slot_mapping % self._block_size
return layer[block_idxs, :, offsets]
connector_metadata = self._get_connector_metadata()
assert isinstance(connector_metadata, ExampleConnectorMetadata)
for request in connector_metadata.requests:
if request.is_store:
filename = self._generate_filename_debug(
layer_name, request.token_ids, request.mm_hashes
)
kv_cache = extract_kv_from_layer(kv_layer, request.slot_mapping)
with gpu_sync_allowed():
tensors = {"kv_cache": kv_cache.detach().cpu()}
safetensors.torch.save_file(tensors, filename)
def wait_for_save(self):
return
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# NOTE: in this debug implementation, we assume that the prompt is
# cached_prompt + newly_generated_single_token
# Therefore, we use prompt_token_ids[:-1] to determine the folder name
# NOTE: in current v1 scheduler, the num_computed_tokens is aligned
# with the block granularity. And it expects the returned blocks and
# num_computed_tokens to also be aligned with the block granularity.
if not self._found_match_for_request(request):
return 0, False
logger.info("External Cache Hit!")
# Now, first num_tokens_to_check tokens are hit, we need to prepare
# the metadata for the worker connector to correctly load the KV
token_ids = request.prompt_token_ids or []
num_tokens_to_check = align_to_block_size(len(token_ids) - 1, self._block_size)
return num_tokens_to_check - num_computed_tokens, False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation.
If blocks were allocated, add to _requests_need_load,
such that we load the KVs in the next forward pass.
"""
if num_external_tokens > 0:
self._requests_need_load[request.request_id] = request
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify any fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleConnectorMetadata()
total_need_load = 0
for new_req in scheduler_output.scheduled_new_reqs:
token_ids = new_req.prompt_token_ids or []
mm_hashes = [f.identifier for f in new_req.mm_features]
if new_req.req_id in self._requests_need_load:
meta.add_request(
token_ids=token_ids,
block_ids=new_req.block_ids[0],
block_size=self._block_size,
is_store=False,
mm_hashes=mm_hashes,
)
total_need_load += 1
else:
# NOTE: here, we set the store and load being exclusive,
# but a single request can have both store and load.
# NOTE(rob): for this debug implementation, we only cache
# the original prompt tokens.
if not self._found_match_for_prompt(token_ids, mm_hashes):
meta.add_request(
token_ids=token_ids,
block_ids=new_req.block_ids[0],
block_size=self._block_size,
is_store=True,
mm_hashes=mm_hashes,
)
cached_reqs = scheduler_output.scheduled_cached_reqs
for i, req_id in enumerate(cached_reqs.req_ids):
resumed_from_preemption = req_id in cached_reqs.resumed_req_ids
if not resumed_from_preemption or req_id not in self._requests_need_load:
continue
num_computed_tokens = cached_reqs.num_computed_tokens[i]
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
new_block_ids = cached_reqs.new_block_ids[i]
# NOTE(rob): cached_req_data does not have the full
# list of token ids (only new tokens). So we look it
# up in the actual request object.
request = self._requests_need_load[req_id]
total_tokens = num_computed_tokens + num_new_tokens
token_ids = request.all_token_ids[:total_tokens]
# NOTE(rob): For resumed req, new_block_ids is all
# of the block_ids for the request.
assert new_block_ids is not None
block_ids = new_block_ids[0]
meta.add_request(
token_ids=token_ids,
block_ids=block_ids,
block_size=self._block_size,
is_store=False,
mm_hashes=[f.identifier for f in request.mm_features],
)
total_need_load += 1
assert total_need_load == len(self._requests_need_load)
self._requests_need_load.clear()
return meta
# ==============================
# Helper functions
# ==============================
def _found_match_for_request(
self,
request: "Request",
) -> bool:
"""Check if the cache is hit for the request."""
return self._found_match_for_prompt(
list(request.prompt_token_ids or []),
[f.identifier for f in request.mm_features],
)
def _found_match_for_prompt(
self,
prompt_token_ids: list[int],
mm_hashes: list[str],
) -> bool:
num_tokens_to_check = align_to_block_size(
len(prompt_token_ids) - 1, self._block_size
)
foldername = self._generate_foldername_debug(
torch.tensor(prompt_token_ids)[:num_tokens_to_check],
mm_hashes,
create_folder=False,
)
return os.path.exists(foldername)
def _generate_foldername_debug(
self,
token_ids: torch.Tensor,
mm_hashes: list[str],
create_folder=False,
) -> str:
"""Generate a folder name based on the hash of the bytes of the input
ids.
"""
token_bytes = token_ids.numpy().tobytes()
# Add mm_hashes to the bytes being hashed to avoid path traversal and
# to create a canonical key.
if mm_hashes:
mm_str = "-".join(mm_hashes)
token_bytes += mm_str.encode("utf-8")
input_ids_hash = safe_hash(token_bytes, usedforsecurity=False).hexdigest()
foldername = os.path.join(self._storage_path, input_ids_hash)
if create_folder:
os.makedirs(foldername, exist_ok=True)
return foldername
def _generate_filename_debug(
self,
layer_name: str,
token_ids: torch.Tensor,
mm_hashes: list[str],
) -> str:
"""Generate a file name based on the layer name and the hash
of the bytes of the input ids.
"""
foldername = self._generate_foldername_debug(
token_ids, mm_hashes=mm_hashes, create_folder=True
)
return os.path.join(foldername, f"{layer_name}.safetensors")
def align_to_block_size(num_tokens: int, block_size) -> int:
"""Align the number of tokens to the block size."""
return (num_tokens - 1) // block_size * block_size
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HyperConnection (Gated Residual) utilities for the AMD model variant.
Implements the HyperConnection residual scheme proposed in
"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant
delays each HC combine to the following HC mix boundary. HC glue kernels,
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
standard vLLM Linear modules.
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
(HC outer, HS inner — checkpoint-native layout).
Typical usage inside a transformer decoder layer::
self.attn_hc = GatedResidual(hc_config)
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
attention_output = attention(block_input)
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
hidden_states, attention_output, injection
)
"""
import torch
from torch import nn
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from ..common.hyperconnection import (
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from .ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
# ---------------------------------------------------------------------------
# Gated-residual variant
# ---------------------------------------------------------------------------
class GatedResidual(nn.Module):
"""Gated HyperConnection with learnable low-rank mixing and injection.
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
over the HC streams). When passed a pending block output and an injection,
it fuses their residual combine with the RMSNorm. Final mixers use
``use_combine=False`` and do not produce a new injection.
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
are vLLM Linear modules (merged replicated linear for down+inject), so
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
standard quant_method mechanism.
"""
def __init__(
self,
config: HyperConnectionConfig,
use_combine: bool = True,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
norm_size = (
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
)
group_size = config.hidden_size if config.hc_per_branch_norm else None
# Normalize each H-sized HC stream independently while retaining a
# separate affine weight for every element of the HC*H layout.
self.hc_norm = GroupedGemmaRMSNorm(
norm_size,
eps=config.rms_norm_eps,
group_size=group_size,
dtype=config.params_dtype,
)
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows for
# alignment and efficient backend dispatch.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
return_bias=False,
disable_tp=True,
)
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
return_bias=False,
)
def mix(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
xn = grouped_gemma_rmsnorm(
hidden_states,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine_and_mix(
self,
hidden_states: torch.Tensor,
prev_block_output: torch.Tensor,
prev_injection: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Consume a pending combine, then prepare the next block input.
``hidden_states`` is the multi-stream state from before the pending
block's mix. Its combine with ``block_output`` is fused with this
module's input RMSNorm.
"""
hidden_states, xn = hc_combine_norm(
hidden_states,
prev_block_output,
prev_injection,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine(
self,
hidden_states: torch.Tensor,
block_output: torch.Tensor,
injection: torch.Tensor,
) -> torch.Tensor:
return hc_combine(hidden_states, block_output, injection, self.hc_count)
@property
def hyper_hidden_size(self) -> int:
return self.hc_count * self.hidden_size
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionConfig",
]

Some files were not shown because too many files have changed in this diff Show More