[Mini-VLLM] Repair test suite after SM86 cut
- Restore example connector, test helpers, and dropped patch tests - Guard MLA metadata import in the example connector - Point the PLE test at common.ops.ple after upstream #60021 move - Cut tests for removed features: tools, DRY, epd proxy, STT batch, diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\n'}}{% endif %}{% endfor %}
|
||||
{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\n' }}{% endif %}
|
||||
@@ -0,0 +1,144 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import itertools
|
||||
from collections.abc import Callable, Iterable
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
class Matches(NamedTuple):
|
||||
# simple pointwise
|
||||
aiter_rms_quant_fusion: int = 0
|
||||
rms_quant_fusion: int = 0
|
||||
act_quant_fusion: int = 0
|
||||
norm_rope_fusion: int = 0
|
||||
attn_quant_fusion: int = 0
|
||||
# distributed
|
||||
ar_rms_fusion: int = 0
|
||||
aiter_ar_rms_fusion: int = 0
|
||||
|
||||
|
||||
class ModelFusionInfo(NamedTuple):
|
||||
model_name: str
|
||||
matches: Callable[[int], Matches]
|
||||
"""Given number of hidden layers, produces the matches object"""
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
|
||||
|
||||
|
||||
class AttentionBackendCase(NamedTuple):
|
||||
backend: AttentionBackendEnum
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
"""Additional args required for attn+quant fusion"""
|
||||
|
||||
|
||||
is_blackwell = lambda: current_platform.is_device_capability_family(100)
|
||||
"""Are we running on Blackwell, a lot of tests depend on it"""
|
||||
|
||||
|
||||
def nvfp4_kernel_exposes_input_quant_key() -> bool:
|
||||
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
|
||||
|
||||
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
|
||||
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
|
||||
output format and the cutedsl backend's expected input format.
|
||||
|
||||
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
|
||||
manual fusion.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fallback: assume no manual fusion support
|
||||
return False
|
||||
|
||||
|
||||
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
|
||||
"""Generate all combinations of custom ops for parametrization."""
|
||||
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
|
||||
for op_list in itertools.product(*custom_ops_lists):
|
||||
yield ",".join(op_list)
|
||||
|
||||
|
||||
# Quick inline validation
|
||||
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
|
||||
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
|
||||
"-quant_fp8,-rms_norm",
|
||||
"-quant_fp8,+rms_norm",
|
||||
"+quant_fp8,-rms_norm",
|
||||
"+quant_fp8,+rms_norm",
|
||||
]
|
||||
|
||||
|
||||
def has_cuda_graph_wrapper_metadata() -> bool:
|
||||
from importlib import import_module
|
||||
|
||||
try:
|
||||
module = import_module("torch._inductor.utils")
|
||||
module.CUDAGraphWrapperMetadata # noqa B018
|
||||
except AttributeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
INDUCTOR_GRAPH_PARTITION = [
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not has_cuda_graph_wrapper_metadata(),
|
||||
reason="torch version does not support Inductor partition",
|
||||
),
|
||||
id="inductor_partition",
|
||||
),
|
||||
pytest.param(False, id="dynamo_partition"),
|
||||
]
|
||||
|
||||
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
"aiter_rms_quant_fusion": re.compile(
|
||||
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
# The combined pass can consume these sites before the standalone pass.
|
||||
"norm_rope_fusion": re.compile(
|
||||
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
|
||||
r"qk_norm_rope_kvcache_fusion.py:\d+] "
|
||||
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
|
||||
),
|
||||
"attn_quant_fusion": re.compile(
|
||||
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
|
||||
),
|
||||
"ar_rms_fusion": re.compile(
|
||||
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
|
||||
),
|
||||
"aiter_ar_rms_fusion": re.compile(
|
||||
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
import time
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import (
|
||||
EventBatch,
|
||||
EventPublisherFactory,
|
||||
NullEventPublisher,
|
||||
)
|
||||
|
||||
DP_RANK = 0
|
||||
|
||||
|
||||
class EventSample(
|
||||
msgspec.Struct,
|
||||
tag=True, # type: ignore
|
||||
array_like=True, # type: ignore
|
||||
):
|
||||
"""Test event for publisher testing."""
|
||||
|
||||
id: int
|
||||
value: str
|
||||
|
||||
|
||||
class SampleBatch(EventBatch):
|
||||
"""Test event batch for publisher testing."""
|
||||
|
||||
events: list[EventSample]
|
||||
|
||||
|
||||
def create_test_events(count: int) -> SampleBatch:
|
||||
"""Create a batch of test events."""
|
||||
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
|
||||
return SampleBatch(ts=time.time(), events=events)
|
||||
|
||||
|
||||
def test_basic_publishing(publisher, subscriber):
|
||||
"""Test basic event publishing works."""
|
||||
test_batch = create_test_events(5)
|
||||
publisher.publish(test_batch)
|
||||
|
||||
result = subscriber.receive_one(timeout=1000)
|
||||
assert result is not None, "No message received"
|
||||
|
||||
seq, received = result
|
||||
assert seq == 0, "Sequence number mismatch"
|
||||
assert received.ts == pytest.approx(test_batch.ts, abs=0.1), "Timestamp mismatch"
|
||||
assert len(received.events) == len(test_batch.events), "Number of events mismatch"
|
||||
|
||||
for i, event in enumerate(received.events):
|
||||
assert event.id == i, "Event id mismatch"
|
||||
assert event.value == f"test-{i}", "Event value mismatch"
|
||||
|
||||
|
||||
def test_multiple_events(publisher, subscriber):
|
||||
"""Test publishing and receiving multiple event batches."""
|
||||
for _ in range(10):
|
||||
batch = create_test_events(2)
|
||||
publisher.publish(batch)
|
||||
|
||||
received = []
|
||||
for _ in range(10):
|
||||
data = subscriber.receive_one(timeout=100)
|
||||
if data:
|
||||
received.append(data)
|
||||
|
||||
assert len(received) == 10, "Number of messages mismatch"
|
||||
seqs = [seq for seq, _ in received]
|
||||
assert seqs == list(range(10)), "Sequence numbers mismatch"
|
||||
|
||||
|
||||
def test_replay_mechanism(publisher, subscriber):
|
||||
"""Test the replay mechanism works correctly."""
|
||||
for _ in range(19):
|
||||
batch = create_test_events(1)
|
||||
publisher.publish(batch)
|
||||
|
||||
# Drain live events to ensure publisher has buffered them.
|
||||
for _ in range(19):
|
||||
assert subscriber.receive_one(timeout=1000) is not None
|
||||
|
||||
subscriber.request_replay(10)
|
||||
|
||||
replayed = subscriber.receive_replay()
|
||||
|
||||
assert len(replayed) == 9, (
|
||||
f"Expected 9 replayed messages (seq 10-18), got {len(replayed)}"
|
||||
)
|
||||
seqs = [seq for seq, _ in replayed]
|
||||
assert seqs == list(range(10, 19)), "Replayed sequences should be 10-18"
|
||||
|
||||
|
||||
def test_replay_includes_topic(publisher, subscriber, publisher_config):
|
||||
"""Test that replay responses include the topic, matching PUB format."""
|
||||
for _ in range(5):
|
||||
publisher.publish(create_test_events(1))
|
||||
|
||||
# Drain live events to ensure publisher has processed them.
|
||||
for _ in range(5):
|
||||
assert subscriber.receive_one(timeout=1000) is not None
|
||||
|
||||
subscriber.request_replay(0)
|
||||
|
||||
# receive_replay unpacks (topic, seq, payload) and asserts
|
||||
# topic == publisher topic for each message.
|
||||
replayed = subscriber.receive_replay()
|
||||
assert len(replayed) == 5, f"Expected 5 replayed messages, got {len(replayed)}"
|
||||
seqs = [seq for seq, _ in replayed]
|
||||
assert seqs == list(range(5)), "Replayed sequences should be 0-4"
|
||||
|
||||
|
||||
def test_buffer_limit(publisher, subscriber, publisher_config):
|
||||
"""Test buffer limit behavior."""
|
||||
buffer_size = publisher_config.buffer_steps
|
||||
|
||||
# Publish more events than the buffer can hold
|
||||
for i in range(buffer_size + 10):
|
||||
batch = create_test_events(1)
|
||||
publisher.publish(batch)
|
||||
|
||||
time.sleep(0.5) # Need publisher to process above requests
|
||||
subscriber.request_replay(0)
|
||||
|
||||
replayed = subscriber.receive_replay()
|
||||
|
||||
assert len(replayed) == buffer_size, (
|
||||
f"Expected {buffer_size} replayed messages, got {len(replayed)}"
|
||||
)
|
||||
|
||||
seqs = [seq for seq, _ in replayed]
|
||||
assert seqs == list(range(10, buffer_size + 10)), (
|
||||
"Should replay seq 11 through buffer_size+10"
|
||||
)
|
||||
|
||||
|
||||
def test_topic_filtering(publisher_config):
|
||||
"""Test that a subscriber only receives messages matching its topic filter."""
|
||||
publisher_config.replay_endpoint = None
|
||||
|
||||
publisher_config.topic = "foo"
|
||||
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
sub_foo = MockSubscriber(publisher_config.endpoint, None, "foo")
|
||||
sub_bar = MockSubscriber(publisher_config.endpoint, None, "bar")
|
||||
|
||||
try:
|
||||
time.sleep(0.1)
|
||||
|
||||
for _ in range(3):
|
||||
pub.publish(create_test_events(1))
|
||||
|
||||
foo_received = [sub_foo.receive_one(timeout=200) for _ in range(3)]
|
||||
assert all(msg is not None for msg in foo_received), (
|
||||
"Subscriber with matching topic should receive messages"
|
||||
)
|
||||
|
||||
bar_received = [sub_bar.receive_one(timeout=200) for _ in range(3)]
|
||||
assert all(msg is None for msg in bar_received), (
|
||||
"Subscriber with non-matching topic should receive no messages"
|
||||
)
|
||||
finally:
|
||||
pub.shutdown()
|
||||
sub_foo.close()
|
||||
sub_bar.close()
|
||||
|
||||
|
||||
def test_high_volume(publisher, subscriber):
|
||||
"""Test publishing and receiving a high volume of events."""
|
||||
num_batches = 10_000
|
||||
events_per_batch = 100
|
||||
|
||||
# Publish events in a separate thread to not block
|
||||
def publish_events():
|
||||
for i in range(num_batches):
|
||||
batch = create_test_events(events_per_batch)
|
||||
publisher.publish(batch)
|
||||
# Small delay to avoid overwhelming
|
||||
if i % 100 == 0:
|
||||
time.sleep(0.01)
|
||||
|
||||
received: list[tuple[int, SampleBatch]] = []
|
||||
|
||||
publisher_thread = threading.Thread(target=publish_events)
|
||||
publisher_thread.start()
|
||||
|
||||
start_time = time.time()
|
||||
while len(received) < num_batches:
|
||||
if time.time() - start_time > 10: # Timeout after 10 seconds
|
||||
break
|
||||
|
||||
result = subscriber.receive_one(timeout=100)
|
||||
if result:
|
||||
received.append(result)
|
||||
|
||||
publisher_thread.join()
|
||||
|
||||
assert len(received) >= num_batches * 0.9, "We should have received most messages"
|
||||
|
||||
seqs = [seq for seq, _ in received]
|
||||
assert sorted(seqs) == seqs, "Sequence numbers should be in order"
|
||||
|
||||
|
||||
def test_null_publisher():
|
||||
"""Test that NullEventPublisher can be used without errors."""
|
||||
publisher = NullEventPublisher(DP_RANK)
|
||||
|
||||
# This should not raise any errors
|
||||
batch = create_test_events(5)
|
||||
publisher.publish(batch)
|
||||
publisher.shutdown()
|
||||
|
||||
|
||||
def test_data_parallel_rank_tagging(publisher_config):
|
||||
"""Test that events are properly tagged with their data parallel rank."""
|
||||
publisher_config.topic = "foo"
|
||||
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
|
||||
|
||||
# Hardcode the expected endpoints based on port offsetting behavior
|
||||
# Both ranks get offsets according to _offset_endpoint_port function
|
||||
base_endpoint = publisher_config.endpoint
|
||||
if "tcp://" in base_endpoint:
|
||||
# For TCP endpoints: tcp://localhost:5557 -> tcp://localhost:5557, tcp://localhost:5558
|
||||
expected_endpoint_0 = base_endpoint # rank 0 gets port + 0 = same port
|
||||
expected_endpoint_1 = base_endpoint.replace(
|
||||
":5557", ":5558"
|
||||
) # rank 1 gets port + 1
|
||||
else:
|
||||
# For inproc endpoints: inproc://test -> inproc://test_dp0, inproc://test_dp1
|
||||
expected_endpoint_0 = base_endpoint # rank 0 gets base
|
||||
expected_endpoint_1 = base_endpoint + "_dp1" # rank 1 gets _dp1
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
sub_0 = MockSubscriber(expected_endpoint_0, None, publisher_config.topic)
|
||||
sub_1 = MockSubscriber(expected_endpoint_1, None, publisher_config.topic)
|
||||
|
||||
try:
|
||||
time.sleep(0.1) # Let publishers start up
|
||||
|
||||
# Publish events from different ranks
|
||||
batch_0 = create_test_events(2)
|
||||
batch_1 = create_test_events(3)
|
||||
|
||||
pub_0.publish(batch_0)
|
||||
pub_1.publish(batch_1)
|
||||
|
||||
# Receive events from rank 0
|
||||
result_0 = sub_0.receive_one(timeout=200)
|
||||
assert result_0 is not None, "No message received from rank 0"
|
||||
seq_0, received_0 = result_0
|
||||
|
||||
# Receive events from rank 1
|
||||
result_1 = sub_1.receive_one(timeout=200)
|
||||
assert result_1 is not None, "No message received from rank 1"
|
||||
seq_1, received_1 = result_1
|
||||
|
||||
# Verify DP rank tagging
|
||||
assert received_0.data_parallel_rank == 0, (
|
||||
f"Expected DP rank 0, got {received_0.data_parallel_rank}"
|
||||
)
|
||||
assert received_1.data_parallel_rank == 1, (
|
||||
f"Expected DP rank 1, got {received_1.data_parallel_rank}"
|
||||
)
|
||||
|
||||
# Verify event content is correct
|
||||
assert len(received_0.events) == 2, "Wrong number of events from rank 0"
|
||||
assert len(received_1.events) == 3, "Wrong number of events from rank 1"
|
||||
|
||||
finally:
|
||||
pub_0.shutdown()
|
||||
pub_1.shutdown()
|
||||
sub_0.close()
|
||||
sub_1.close()
|
||||
|
||||
|
||||
def test_event_publisher_factory(random_port):
|
||||
"""Test event publisher factory creation behavior under different configurations."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
# test config is None
|
||||
publisher = EventPublisherFactory.create(None, DP_RANK)
|
||||
assert isinstance(publisher, NullEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
# test disable kv cache events
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=False,
|
||||
publisher="zmq", # Even if zmq is specified, should return NullEventPublisher
|
||||
endpoint="tcp://localhost:5557",
|
||||
)
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK)
|
||||
assert isinstance(publisher, NullEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
# test zmq publisher
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint=f"tcp://*:{random_port}",
|
||||
replay_endpoint=f"tcp://*:{random_port + 100}",
|
||||
)
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK + 1)
|
||||
assert isinstance(publisher, ZmqEventPublisher)
|
||||
resolved_config = publisher.get_publisher_config()
|
||||
assert resolved_config.endpoint == f"tcp://*:{random_port + 1}"
|
||||
assert resolved_config.replay_endpoint == f"tcp://*:{random_port + 101}"
|
||||
assert config.endpoint == f"tcp://*:{random_port}"
|
||||
publisher.shutdown()
|
||||
|
||||
# test unknown publisher
|
||||
with pytest.raises(ValueError, match="Input should be"):
|
||||
KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="unknown_publisher",
|
||||
endpoint="tcp://localhost:5557",
|
||||
)
|
||||
|
||||
# test publisher not specified
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
# publisher not specified, should default to "zmq"
|
||||
endpoint="tcp://localhost:5557",
|
||||
)
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK)
|
||||
assert isinstance(publisher, ZmqEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "expected"),
|
||||
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
|
||||
)
|
||||
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
|
||||
"""Port 0 is not offset: every rank asks the OS for its own port."""
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
|
||||
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
|
||||
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
|
||||
node's IP and report endpoints a subscriber can use for events and replay."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
from vllm.utils.network_utils import split_zmq_path
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint="tcp://*:0",
|
||||
replay_endpoint="tcp://*:0",
|
||||
topic="kv",
|
||||
)
|
||||
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
|
||||
try:
|
||||
resolved = [pub.get_publisher_config() for pub in pubs]
|
||||
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
|
||||
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
|
||||
assert {h for h, _ in hosts_ports} == {node_ip}
|
||||
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
|
||||
assert config.endpoint == "tcp://*:0"
|
||||
|
||||
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
|
||||
try:
|
||||
# PUB/SUB drops messages sent before the subscription lands, so
|
||||
# keep publishing until both ranks are heard.
|
||||
ranks: set[int] = set()
|
||||
deadline = time.time() + 10
|
||||
while ranks != {0, 1} and time.time() < deadline:
|
||||
for pub in pubs:
|
||||
pub.publish(create_test_events(1))
|
||||
while (result := sub.receive_one(timeout=100)) is not None:
|
||||
ranks.add(result[1].data_parallel_rank)
|
||||
assert ranks == {0, 1}
|
||||
sub.request_replay(0, socket_idx=1)
|
||||
assert sub.receive_replay(socket_idx=1)
|
||||
finally:
|
||||
sub.close()
|
||||
finally:
|
||||
for pub in pubs:
|
||||
pub.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
|
||||
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
|
||||
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
|
||||
publisher refuses instead of reporting a wildcard."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
|
||||
)
|
||||
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
|
||||
EventPublisherFactory.create(config, DP_RANK)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,12 +16,9 @@ from vllm.assets.audio import AudioAsset
|
||||
from vllm.connections import HTTPConnection
|
||||
from vllm.entrypoints.launchers.run_batch import (
|
||||
BatchRequestOutput,
|
||||
BatchTranscriptionRequest,
|
||||
download_bytes_from_url,
|
||||
make_transcription_wrapper,
|
||||
upload_data,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
|
||||
@@ -644,122 +641,10 @@ def test_reasoning_parser():
|
||||
assert len(reasoning) > 0
|
||||
|
||||
|
||||
def test_transcription():
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_TRANSCRIPTION_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
SPEECH_LARGE_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
print(f"\n\ncontents: {contents}\n\n")
|
||||
for line in contents.strip().split("\n"):
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
line_dict = json.loads(line)
|
||||
assert isinstance(line_dict, dict)
|
||||
assert line_dict["error"] is None
|
||||
|
||||
response_body = line_dict["response"]["body"]
|
||||
assert response_body is not None
|
||||
assert "text" in response_body
|
||||
assert "usage" in response_body
|
||||
|
||||
|
||||
def test_transcription_http_url():
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_TRANSCRIPTION_HTTP_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
SPEECH_LARGE_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
line_dict = json.loads(line)
|
||||
assert isinstance(line_dict, dict)
|
||||
assert line_dict["error"] is None
|
||||
|
||||
response_body = line_dict["response"]["body"]
|
||||
assert response_body is not None
|
||||
assert "text" in response_body
|
||||
assert "usage" in response_body
|
||||
|
||||
transcription_text = response_body["text"]
|
||||
assert "Mary had a little lamb" in transcription_text
|
||||
|
||||
|
||||
def test_translation():
|
||||
with (
|
||||
tempfile.NamedTemporaryFile("w") as input_file,
|
||||
tempfile.NamedTemporaryFile("r") as output_file,
|
||||
):
|
||||
input_file.write(INPUT_TRANSLATION_BATCH)
|
||||
input_file.flush()
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
"vllm",
|
||||
"run-batch",
|
||||
"-i",
|
||||
input_file.name,
|
||||
"-o",
|
||||
output_file.name,
|
||||
"--model",
|
||||
SPEECH_SMALL_MODEL_NAME,
|
||||
],
|
||||
)
|
||||
proc.communicate()
|
||||
proc.wait()
|
||||
assert proc.returncode == 0, f"{proc=}"
|
||||
|
||||
contents = output_file.read()
|
||||
for line in contents.strip().split("\n"):
|
||||
BatchRequestOutput.model_validate_json(line)
|
||||
|
||||
line_dict = json.loads(line)
|
||||
assert isinstance(line_dict, dict)
|
||||
assert line_dict["error"] is None
|
||||
|
||||
response_body = line_dict["response"]["body"]
|
||||
assert response_body is not None
|
||||
assert "text" in response_body
|
||||
|
||||
translation_text = response_body["text"]
|
||||
translation_text_lower = str(translation_text).strip().lower()
|
||||
assert "mary" in translation_text_lower or "lamb" in translation_text_lower
|
||||
|
||||
|
||||
def test_tool_calling():
|
||||
@@ -974,28 +859,6 @@ async def test_download_bytes_backslash_bypass():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_wrapper_rejects_oversized_data_url_before_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
handler = AsyncMock()
|
||||
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
|
||||
request = BatchTranscriptionRequest.model_validate(
|
||||
{
|
||||
"model": SPEECH_LARGE_MODEL_NAME,
|
||||
"file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}",
|
||||
"response_format": "json",
|
||||
}
|
||||
)
|
||||
|
||||
with patch("vllm.entrypoints.launchers.run_batch.base64.b64decode") as decode:
|
||||
response = await wrapped_handler(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert "Maximum file size exceeded" in response.error.message
|
||||
decode.assert_not_called()
|
||||
handler.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1019,35 +882,6 @@ async def test_download_bytes_allows_http_body_at_audio_limit(
|
||||
await _close_async_connection(connection)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcription_wrapper_rejects_oversized_http_before_handler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
batch_http_server: _BatchHTTPServer,
|
||||
):
|
||||
monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1)
|
||||
connection = HTTPConnection()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.launchers.run_batch.global_http_connection",
|
||||
connection,
|
||||
)
|
||||
handler = AsyncMock()
|
||||
wrapped_handler = make_transcription_wrapper(is_translation=False)(handler)
|
||||
request = BatchTranscriptionRequest.model_validate(
|
||||
{
|
||||
"model": SPEECH_LARGE_MODEL_NAME,
|
||||
"file_url": _batch_http_url(batch_http_server, "/oversized-chunked"),
|
||||
"response_format": "json",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
response = await wrapped_handler(request)
|
||||
finally:
|
||||
await _close_async_connection(connection)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert "Maximum file size exceeded" in response.error.message
|
||||
handler.assert_not_awaited()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -11,7 +11,6 @@ import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.v1.engine.detokenizer import check_stop_strings
|
||||
@@ -140,7 +139,7 @@ async def test_generate_rejects_min_tokens_above_filled_max_tokens(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="sampling mask output is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -218,7 +217,7 @@ async def test_generate_defaults_max_tokens_when_omitted(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
|
||||
@@ -485,7 +484,7 @@ async def test_stop_string_workflow(client, tokenizer, messages):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
@@ -617,7 +616,7 @@ async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
|
||||
# either reasoning field is dropped between render and generate.
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -638,7 +637,7 @@ async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, mes
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="the Rust frontend does not forward reasoning fields to the engine",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -679,7 +678,7 @@ def _text_mode_payload(token_ids, **sampling_params) -> dict:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
|
||||
@@ -707,7 +706,7 @@ async def test_text_mode_matches_chat_completions(client, tokenizer, messages):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
|
||||
@@ -734,7 +733,7 @@ async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="output_mode is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -759,7 +758,7 @@ async def test_text_mode_invalid_requests_return_400(client, overrides, expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="--tokens-only is not supported by the Rust frontend",
|
||||
)
|
||||
@pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True)
|
||||
@@ -778,7 +777,7 @@ async def test_text_mode_rejected_when_tokens_only(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
@pytest.mark.parametrize("output_mode", ["tokens", "text"])
|
||||
@@ -825,7 +824,7 @@ async def test_stream_delivers_abort_finish_chunk(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_USE_RUST_FRONTEND,
|
||||
False,
|
||||
reason="the Rust frontend does not serve /inference/v1/abort_requests",
|
||||
)
|
||||
async def test_abort_requests_is_served_without_tokens_only(client):
|
||||
|
||||
@@ -0,0 +1,507 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Isolated GSM8K evaluation script for vLLM serve endpoint."""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import requests
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def download_and_cache_file(url: str, filename: str | None = None) -> str:
|
||||
"""Download and cache a file from a URL."""
|
||||
if filename is None:
|
||||
filename = os.path.join(tempfile.gettempdir(), url.split("/")[-1])
|
||||
|
||||
if os.path.exists(filename):
|
||||
return filename
|
||||
|
||||
print(f"Downloading from {url} to {filename}")
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(filename, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
f.write(chunk)
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def load_gsm8k_data() -> tuple[list[dict], list[dict]]:
|
||||
"""Load GSM8K train and test data."""
|
||||
train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl"
|
||||
test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl"
|
||||
|
||||
train_file = download_and_cache_file(train_url)
|
||||
test_file = download_and_cache_file(test_url)
|
||||
|
||||
train_data = list(read_jsonl(train_file))
|
||||
test_data = list(read_jsonl(test_file))
|
||||
|
||||
return train_data, test_data
|
||||
|
||||
|
||||
def read_jsonl(filename: str) -> Generator[dict, None, None]:
|
||||
"""Read a JSONL file."""
|
||||
with open(filename) as fin:
|
||||
for line in fin:
|
||||
if not line.startswith("#"):
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def get_answer_value(answer_str: str) -> int:
|
||||
"""Extract the numerical answer from the response."""
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def _optional_params(**params: object) -> dict[str, object]:
|
||||
return {k: v for k, v in params.items() if v is not None}
|
||||
|
||||
|
||||
async def call_vllm_api(
|
||||
session: aiohttp.ClientSession,
|
||||
prompt: str,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
stop: list[str] | None = None,
|
||||
url: str | None = None,
|
||||
seed: int | None = None,
|
||||
top_p: float | None = None,
|
||||
top_k: int | None = None,
|
||||
) -> tuple[str, int]:
|
||||
"""Call vLLM's OpenAI-compatible completions endpoint.
|
||||
|
||||
Returns:
|
||||
Tuple of (response_text, completion_tokens)
|
||||
|
||||
"""
|
||||
data = {
|
||||
"prompt": prompt,
|
||||
"max_tokens": max_tokens,
|
||||
"stop": stop,
|
||||
}
|
||||
data.update(
|
||||
_optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k)
|
||||
)
|
||||
|
||||
try:
|
||||
async with session.post(f"{url}/v1/completions", json=data) as response:
|
||||
response.raise_for_status()
|
||||
result = await response.json()
|
||||
text = result["choices"][0]["text"]
|
||||
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
|
||||
return text, completion_tokens
|
||||
except Exception as e:
|
||||
print(f"Error calling vLLM API ({type(e).__name__}): {e}")
|
||||
return "", 0
|
||||
|
||||
|
||||
async def call_vllm_chat_api(
|
||||
session: aiohttp.ClientSession,
|
||||
model: str | None,
|
||||
prompt: str,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
stop: list[str] | None = None,
|
||||
url: str | None = None,
|
||||
seed: int | None = None,
|
||||
top_p: float | None = None,
|
||||
top_k: int | None = None,
|
||||
reasoning_effort: str | None = None,
|
||||
chat_template_kwargs: dict[str, object] | None = None,
|
||||
) -> tuple[str, int]:
|
||||
"""Call vLLM's OpenAI-compatible chat completions endpoint.
|
||||
|
||||
``model``, ``temperature``, ``stop`` and the other optional fields are
|
||||
omitted from the request when ``None``, so the server defaults apply (vLLM
|
||||
uses its served model when ``model`` is omitted).
|
||||
|
||||
Returns:
|
||||
Tuple of (final answer content, completion_tokens). Reasoning returned
|
||||
separately by a reasoning parser is not part of the answer.
|
||||
|
||||
"""
|
||||
data = {
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
data.update(
|
||||
_optional_params(
|
||||
stop=stop,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
seed=seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
reasoning_effort=reasoning_effort,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
async with session.post(f"{url}/v1/chat/completions", json=data) as response:
|
||||
response.raise_for_status()
|
||||
result = await response.json()
|
||||
text = result["choices"][0]["message"]["content"] or ""
|
||||
completion_tokens = result.get("usage", {}).get("completion_tokens", 0)
|
||||
return text, completion_tokens
|
||||
except Exception as e:
|
||||
print(f"Error calling vLLM chat API ({type(e).__name__}): {e}")
|
||||
return "", 0
|
||||
|
||||
|
||||
def _build_gsm8k_prompts(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
gen_prefix: str = "",
|
||||
) -> tuple[list[str], list[int]]:
|
||||
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
|
||||
if num_questions == 0:
|
||||
return [], []
|
||||
train_data, test_data = load_gsm8k_data()
|
||||
num_questions = min(num_questions, len(test_data))
|
||||
|
||||
few_shot_examples = ""
|
||||
for i in range(num_shots):
|
||||
few_shot_examples += (
|
||||
f"Question: {train_data[i]['question']}\n"
|
||||
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
|
||||
)
|
||||
|
||||
prompts = []
|
||||
labels = []
|
||||
for i in range(num_questions):
|
||||
prompts.append(
|
||||
few_shot_examples
|
||||
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
|
||||
)
|
||||
labels.append(get_answer_value(test_data[i]["answer"]))
|
||||
|
||||
assert all(label != INVALID for label in labels), "Some labels are invalid"
|
||||
return prompts, labels
|
||||
|
||||
|
||||
def _score_gsm8k(
|
||||
states: list[str],
|
||||
output_tokens: list[int],
|
||||
labels: list[int],
|
||||
num_shots: int,
|
||||
max_tokens: int,
|
||||
latency: float,
|
||||
) -> dict[str, float | int]:
|
||||
"""Score GSM8K responses and return a results dict."""
|
||||
num_questions = len(labels)
|
||||
preds = [get_answer_value(state) for state in states]
|
||||
accuracy = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid_rate = np.mean(np.array(preds) == INVALID)
|
||||
total_output_tokens = sum(output_tokens)
|
||||
tokens_per_second = total_output_tokens / latency if latency > 0 else 0.0
|
||||
|
||||
return {
|
||||
"accuracy": accuracy,
|
||||
"invalid_rate": invalid_rate,
|
||||
"latency": latency,
|
||||
"questions_per_second": num_questions / latency if latency > 0 else 0.0,
|
||||
"total_output_tokens": total_output_tokens,
|
||||
"tokens_per_second": tokens_per_second,
|
||||
"num_questions": num_questions,
|
||||
"num_shots": num_shots,
|
||||
"max_tokens": max_tokens,
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_gsm8k(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
model: str | None = None,
|
||||
use_chat_completions: bool = False,
|
||||
host: str = "http://127.0.0.1",
|
||||
port: int = 8000,
|
||||
temperature: float | None = 0.0,
|
||||
seed: int | None = 42,
|
||||
request_timeout_seconds: float = 600,
|
||||
gen_prefix: str = "",
|
||||
max_concurrency: int | None = None,
|
||||
top_p: float | None = None,
|
||||
top_k: int | None = None,
|
||||
reasoning_effort: str | None = None,
|
||||
chat_template_kwargs: dict[str, object] | None = None,
|
||||
) -> dict[str, float | int]:
|
||||
"""Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||
|
||||
``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``;
|
||||
otherwise the server default applies. ``model`` is used only in chat mode,
|
||||
where it may be ``None`` (the server uses its served model).
|
||||
``reasoning_effort`` and ``chat_template_kwargs`` require
|
||||
``use_chat_completions=True``. Stop strings are sent only in completions
|
||||
mode.
|
||||
|
||||
Returns dict with accuracy, invalid_rate, latency, etc.
|
||||
"""
|
||||
if not use_chat_completions and (
|
||||
reasoning_effort is not None or chat_template_kwargs is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"reasoning_effort and chat_template_kwargs require "
|
||||
"use_chat_completions=True"
|
||||
)
|
||||
base_url = f"{host}:{port}"
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||
num_questions = len(prompts)
|
||||
|
||||
async def run_async_evaluation():
|
||||
states: list[str] = [""] * num_questions
|
||||
output_tokens: list[int] = [0] * num_questions
|
||||
|
||||
async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]:
|
||||
if use_chat_completions:
|
||||
# No stop strings: the chat template delimits the turn, and they
|
||||
# would also cut off reasoning.
|
||||
answer, tokens = await call_vllm_chat_api(
|
||||
session=session,
|
||||
model=model,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
reasoning_effort=reasoning_effort,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
answer, tokens = await call_vllm_api(
|
||||
session=session,
|
||||
prompt=prompts[i],
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
url=base_url,
|
||||
seed=seed,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
)
|
||||
states[i] = answer
|
||||
output_tokens[i] = tokens
|
||||
return answer, tokens
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=request_timeout_seconds)
|
||||
connector = (
|
||||
aiohttp.TCPConnector(limit=max_concurrency)
|
||||
if max_concurrency is not None
|
||||
else None
|
||||
)
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=timeout, connector=connector
|
||||
) as session:
|
||||
tasks = [get_answer(session, i) for i in range(num_questions)]
|
||||
await tqdm.gather(*tasks, desc="Evaluating")
|
||||
|
||||
return states, output_tokens
|
||||
|
||||
print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot")
|
||||
|
||||
tic = time.perf_counter()
|
||||
states, output_tokens = asyncio.run(run_async_evaluation())
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
|
||||
|
||||
|
||||
def evaluate_gsm8k_offline(
|
||||
llm,
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
temperature: float = 0.0,
|
||||
gen_prefix: str = "",
|
||||
use_chat_completions: bool = False,
|
||||
chat_template_kwargs: dict[str, object] | None = None,
|
||||
) -> dict[str, float | int]:
|
||||
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
|
||||
|
||||
Same prompts and scoring as evaluate_gsm8k(), but runs generation
|
||||
directly via llm.generate() instead of calling a server over HTTP.
|
||||
|
||||
When ``use_chat_completions=True``, prompts go through the chat template via
|
||||
``llm.chat()`` instead of raw completion (for instruction-tuned models).
|
||||
``chat_template_kwargs`` are forwarded to ``llm.chat()`` when provided.
|
||||
"""
|
||||
from vllm import SamplingParams
|
||||
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||
sampling_params = SamplingParams(
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
)
|
||||
mode = "chat" if use_chat_completions else "completion"
|
||||
print(
|
||||
f"Running offline GSM8K evaluation: {len(prompts)} questions, "
|
||||
f"{num_shots}-shot, {mode}"
|
||||
)
|
||||
|
||||
tic = time.perf_counter()
|
||||
if use_chat_completions:
|
||||
conversations = [[{"role": "user", "content": p}] for p in prompts]
|
||||
outputs = llm.chat(
|
||||
conversations,
|
||||
sampling_params,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
states = [o.outputs[0].text for o in outputs]
|
||||
output_tokens = [len(o.outputs[0].token_ids) for o in outputs]
|
||||
|
||||
return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve")
|
||||
parser.add_argument(
|
||||
"--num-shots", type=int, default=5, help="Number of few-shot examples"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-questions",
|
||||
type=int,
|
||||
default=1319,
|
||||
help="Number of questions to evaluate",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-tokens", type=int, default=256, help="Max tokens for generation"
|
||||
)
|
||||
parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL")
|
||||
parser.add_argument("--port", type=int, default=8000, help="Port number")
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
help="Temperature for generation (default: 0 for completions; "
|
||||
"server default with --use-chat-completions)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top-p", type=float, help="Top-p for generation (server default if unset)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top-k", type=int, help="Top-k for generation (server default if unset)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-chat-completions",
|
||||
action="store_true",
|
||||
help="Send each few-shot prompt as a user message to /v1/chat/completions "
|
||||
"instead of /v1/completions (for chat-only models)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
help="Model name to send with chat requests (default: omitted, so the "
|
||||
"server uses its served model)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reasoning-effort",
|
||||
type=str,
|
||||
help="reasoning_effort for chat completions, e.g. none, low, high",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-template-kwargs",
|
||||
type=json.loads,
|
||||
help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed", type=int, default=42, help="Random seed for reproducibility"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-concurrency",
|
||||
type=int,
|
||||
help="Maximum number of concurrent requests",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--request-timeout-seconds",
|
||||
type=float,
|
||||
default=600,
|
||||
help="Timeout for each request, including time waiting for a connection",
|
||||
)
|
||||
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
|
||||
|
||||
args = parser.parse_args()
|
||||
temperature = args.temperature
|
||||
if temperature is None and not args.use_chat_completions:
|
||||
temperature = 0.0
|
||||
|
||||
result = evaluate_gsm8k(
|
||||
num_questions=args.num_questions,
|
||||
num_shots=args.num_shots,
|
||||
max_tokens=args.max_tokens,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
temperature=temperature,
|
||||
seed=args.seed,
|
||||
max_concurrency=args.max_concurrency,
|
||||
request_timeout_seconds=args.request_timeout_seconds,
|
||||
model=args.model,
|
||||
use_chat_completions=args.use_chat_completions,
|
||||
top_p=args.top_p,
|
||||
top_k=args.top_k,
|
||||
reasoning_effort=args.reasoning_effort,
|
||||
chat_template_kwargs=args.chat_template_kwargs,
|
||||
)
|
||||
|
||||
# Print results to terminal
|
||||
print("\nResults:")
|
||||
print(f"Accuracy: {result['accuracy']:.3f}")
|
||||
print(f"Invalid responses: {result['invalid_rate']:.3f}")
|
||||
print(f"Total latency: {result['latency']:.3f} s")
|
||||
print(f"Questions per second: {result['questions_per_second']:.3f}")
|
||||
print(f"Total output tokens: {result['total_output_tokens']}")
|
||||
print(f"Output tokens per second: {result['tokens_per_second']:.3f}")
|
||||
|
||||
# Optional file saving
|
||||
if args.save_results:
|
||||
# None means the field was not sent and the server default applied.
|
||||
result["request_params"] = {
|
||||
"endpoint": "chat" if args.use_chat_completions else "completions",
|
||||
"model": args.model,
|
||||
"temperature": temperature,
|
||||
"top_p": args.top_p,
|
||||
"top_k": args.top_k,
|
||||
"seed": args.seed,
|
||||
"stop": None
|
||||
if args.use_chat_completions
|
||||
else ["Question", "Assistant:", "<|separator|>"],
|
||||
"reasoning_effort": args.reasoning_effort,
|
||||
"chat_template_kwargs": args.chat_template_kwargs,
|
||||
}
|
||||
with open(args.save_results, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
print(f"Results saved to {args.save_results}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.import_utils import has_helion
|
||||
|
||||
if not has_helion():
|
||||
pytest.skip("Helion is not installed", allow_module_level=True)
|
||||
|
||||
from tools import benchmark_helion_kernels
|
||||
|
||||
check_correctness = benchmark_helion_kernels.check_correctness
|
||||
|
||||
|
||||
class _FakeKernel:
|
||||
helion_settings = None
|
||||
|
||||
def __init__(self, offset: float):
|
||||
self.calls = 0
|
||||
self.offset = offset
|
||||
|
||||
def __call__(self, output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
|
||||
self.calls += 1
|
||||
output.copy_(input + self.offset)
|
||||
return input * 2
|
||||
|
||||
|
||||
def test_check_correctness_runs_once_without_mutating_benchmark_inputs():
|
||||
kernel = _FakeKernel(offset=1)
|
||||
baseline_calls = 0
|
||||
|
||||
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
|
||||
nonlocal baseline_calls
|
||||
baseline_calls += 1
|
||||
output.copy_(input + 1)
|
||||
return input * 2
|
||||
|
||||
output = torch.zeros(4)
|
||||
inputs = (output, torch.arange(4))
|
||||
|
||||
check_correctness(kernel, baseline, inputs, "matching")
|
||||
|
||||
assert kernel.calls == 1
|
||||
assert baseline_calls == 1
|
||||
torch.testing.assert_close(output, torch.zeros(4))
|
||||
|
||||
|
||||
def test_check_correctness_reports_mutated_input_mismatch():
|
||||
kernel = _FakeKernel(offset=2)
|
||||
|
||||
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
|
||||
output.copy_(input + 1)
|
||||
return input * 2
|
||||
|
||||
inputs = (torch.zeros(4), torch.arange(4))
|
||||
|
||||
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
|
||||
check_correctness(kernel, baseline, inputs, "bad")
|
||||
|
||||
|
||||
def test_check_correctness_reports_return_value_mismatch():
|
||||
kernel = _FakeKernel(offset=1)
|
||||
|
||||
def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor:
|
||||
output.copy_(input + 1)
|
||||
return input * 3
|
||||
|
||||
inputs = (torch.zeros(4), torch.arange(4))
|
||||
|
||||
with pytest.raises(AssertionError, match="Numerics check failed for case bad"):
|
||||
check_correctness(kernel, baseline, inputs, "bad")
|
||||
|
||||
|
||||
def test_log_versions(monkeypatch):
|
||||
messages = []
|
||||
|
||||
def record(message: str, *args):
|
||||
messages.append(message % args)
|
||||
|
||||
monkeypatch.setattr(benchmark_helion_kernels.logger, "info", record)
|
||||
|
||||
benchmark_helion_kernels.log_versions()
|
||||
|
||||
assert [message.split(":", 1)[0] for message in messages] == [
|
||||
"torch",
|
||||
"helion",
|
||||
"triton",
|
||||
]
|
||||
@@ -1,140 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import unittest
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import ensure_current_vllm_config, multi_gpu_test
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import Mixer2RMSNormGated
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [128])
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size_n_groups",
|
||||
[
|
||||
(64, 1),
|
||||
(64, 2),
|
||||
(64, 4), # hidden_size be divisible by num_gpus
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16])
|
||||
def test_mixer2_gated_norm_multi_gpu(
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size_n_groups: tuple[int, int],
|
||||
dtype: torch.dtype,
|
||||
device: str = "cuda",
|
||||
):
|
||||
hidden_size, n_groups = hidden_size_n_groups
|
||||
num_processes = 2
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
# need to use torch.mp.spawn otherwise will have problems with
|
||||
# torch.distributed and cuda
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
n_groups,
|
||||
dtype,
|
||||
device,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(mixer2_gated_norm_tensor_parallel, 2)
|
||||
|
||||
|
||||
def mixer2_gated_norm_tensor_parallel(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
n_groups: int,
|
||||
dtype: torch.dtype,
|
||||
device: str,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12345",
|
||||
}
|
||||
)
|
||||
|
||||
# initialize distributed
|
||||
init_distributed_environment()
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
# create random weights an inputs
|
||||
weight = torch.rand((hidden_size,), dtype=dtype, device=device)
|
||||
hidden_states = torch.randn(batch_size, seq_len, hidden_size)
|
||||
gate_states = torch.randn(batch_size, seq_len, hidden_size)
|
||||
|
||||
# create gated-norm with TP
|
||||
with ensure_current_vllm_config():
|
||||
mixer = Mixer2RMSNormGated(
|
||||
full_hidden_size=hidden_size,
|
||||
full_n_groups=n_groups,
|
||||
)
|
||||
mixer.weight.weight_loader(mixer.weight, weight) # load
|
||||
|
||||
# create gated-norm without TP to compute reference
|
||||
# - utilize mock patching to disable TP when
|
||||
with (
|
||||
ensure_current_vllm_config(),
|
||||
unittest.mock.patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_mixer2."
|
||||
"get_tensor_model_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
unittest.mock.patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_mixer2."
|
||||
"get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
):
|
||||
mixer_single_gpu = Mixer2RMSNormGated(
|
||||
full_hidden_size=hidden_size,
|
||||
full_n_groups=n_groups,
|
||||
)
|
||||
# assign weight to single-gpu mixer
|
||||
mixer_single_gpu.weight.data = weight
|
||||
|
||||
# generate and compare
|
||||
N = hidden_size // world_size
|
||||
output = mixer(
|
||||
hidden_states[..., local_rank * N : (local_rank + 1) * N],
|
||||
gate_states[..., local_rank * N : (local_rank + 1) * N],
|
||||
)
|
||||
ref_output = mixer_single_gpu(hidden_states, gate_states)
|
||||
torch.testing.assert_close(
|
||||
output,
|
||||
ref_output[..., local_rank * N : (local_rank + 1) * N],
|
||||
atol=5e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
@@ -13,7 +13,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_conv
|
||||
from vllm.models.qwen4_exp.common.ops.ple import ple_conv
|
||||
from vllm.v1.attention.backends.ple_recoverssm import _PleConvCommit
|
||||
|
||||
C, KSIZE, DIL, SQ, NB = 512, 4, 3, 4, 8
|
||||
|
||||
@@ -0,0 +1,634 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# vLLM fused-expert reference (Triton fallback + DeepGEMM option)
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
deepgemm_moe_permute,
|
||||
ep_gather,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
_fp8_workspace_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
calc_diff,
|
||||
is_deep_gemm_supported,
|
||||
per_block_cast_to_fp8,
|
||||
)
|
||||
|
||||
BLOCK_SIZE = [128, 128]
|
||||
|
||||
|
||||
def test_ep_gather_uses_64_bit_row_offsets():
|
||||
hidden_size = 4096
|
||||
source_row = 1 << 18
|
||||
input_tensor = torch.empty(
|
||||
(source_row + 1, hidden_size),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
input_tensor[source_row].fill_(1)
|
||||
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
ep_gather(
|
||||
input_tensor=input_tensor,
|
||||
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
|
||||
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
|
||||
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
|
||||
expert_map=None,
|
||||
output_tensor=output,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ep_gather_uses_64_bit_output_row_offsets():
|
||||
hidden_size = 6144
|
||||
output_row = (1 << 31) // hidden_size + 1
|
||||
num_rows = output_row + 1
|
||||
output = torch.empty(
|
||||
(num_rows, hidden_size),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
ep_gather(
|
||||
input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16),
|
||||
recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64),
|
||||
recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32),
|
||||
input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32),
|
||||
expert_map=None,
|
||||
output_tensor=output,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
|
||||
def test_fp8_workspace_shape(workspace_dtype, num_columns):
|
||||
num_rows = 17
|
||||
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
|
||||
|
||||
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
|
||||
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
|
||||
|
||||
assert allocated_bytes >= required_bytes
|
||||
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
|
||||
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
|
||||
activations, scales = per_token_group_quant_fp8(
|
||||
hidden_states,
|
||||
group_size=128,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64)
|
||||
|
||||
_, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute(
|
||||
aq=activations,
|
||||
aq_scale=scales,
|
||||
topk_ids=topk_ids,
|
||||
local_num_experts=2,
|
||||
expert_map=None,
|
||||
expert_tokens_meta=None,
|
||||
)
|
||||
|
||||
padding = expert_ids < 0
|
||||
assert padding.any()
|
||||
torch.testing.assert_close(
|
||||
permuted_scales[padding],
|
||||
torch.zeros_like(permuted_scales[padding]),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
def make_block_quant_fp8_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
block_size: list[int],
|
||||
):
|
||||
"""Generate (w1, w2) expert weights and their per-block scale tensors
|
||||
in FP8 block-quantized format.
|
||||
|
||||
w1 shape: (E, 2N, K)
|
||||
w2 shape: (E, K, N)
|
||||
"""
|
||||
dtype = torch.bfloat16
|
||||
fp8_max, fp8_min = (
|
||||
torch.finfo(torch.float8_e4m3fn).max,
|
||||
torch.finfo(torch.float8_e4m3fn).min,
|
||||
)
|
||||
|
||||
# bf16 reference weights
|
||||
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) / 10
|
||||
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) / 10
|
||||
w1_bf16.clamp_(fp8_min, fp8_max)
|
||||
w2_bf16.clamp_(fp8_min, fp8_max)
|
||||
|
||||
block_n, block_k = block_size
|
||||
n_tiles_w1 = math.ceil((2 * n) / block_n)
|
||||
k_tiles_w1 = math.ceil(k / block_k)
|
||||
n_tiles_w2 = math.ceil(k / block_n)
|
||||
k_tiles_w2 = math.ceil(n / block_k)
|
||||
|
||||
w1 = torch.empty_like(w1_bf16, dtype=torch.float8_e4m3fn)
|
||||
w2 = torch.empty_like(w2_bf16, dtype=torch.float8_e4m3fn)
|
||||
w1_s = torch.empty(e, n_tiles_w1, k_tiles_w1, device="cuda", dtype=torch.float32)
|
||||
w2_s = torch.empty(e, n_tiles_w2, k_tiles_w2, device="cuda", dtype=torch.float32)
|
||||
|
||||
for i in range(e):
|
||||
w1[i], w1_s[i] = per_block_cast_to_fp8(
|
||||
w1_bf16[i], block_size=block_size, use_ue8m0=True
|
||||
)
|
||||
w2[i], w2_s[i] = per_block_cast_to_fp8(
|
||||
w2_bf16[i], block_size=block_size, use_ue8m0=True
|
||||
)
|
||||
|
||||
return w1, w2, w1_s, w2_s
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
|
||||
@pytest.mark.parametrize("with_metadata", [False, True])
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
|
||||
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
|
||||
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
|
||||
|
||||
if not (
|
||||
impl.current_platform.is_device_capability_family(90)
|
||||
or (
|
||||
impl.current_platform.is_device_capability_family(100)
|
||||
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
|
||||
)
|
||||
):
|
||||
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
|
||||
|
||||
torch.manual_seed(0)
|
||||
e, n, k, topk = 8, 1024, 512, 2
|
||||
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
|
||||
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
|
||||
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
|
||||
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
|
||||
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
|
||||
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
|
||||
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
|
||||
shapes = experts.workspace_shapes(
|
||||
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
|
||||
)
|
||||
|
||||
def route(mode):
|
||||
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
|
||||
ids[:, 1] = (ids[:, 0] + 1) % e
|
||||
if mode == 1:
|
||||
ids.fill_(-1)
|
||||
ids[:1, 0] = e - 1
|
||||
elif mode == 2:
|
||||
ids.fill_(-1)
|
||||
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
|
||||
|
||||
route(0)
|
||||
variants = []
|
||||
for dense in (True, False):
|
||||
ws1, ws2, output = [
|
||||
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
|
||||
]
|
||||
|
||||
def invoke(output=output, ws1=ws1, ws2=ws2):
|
||||
experts.apply(
|
||||
output,
|
||||
aq,
|
||||
w1,
|
||||
w2,
|
||||
weights,
|
||||
ids,
|
||||
MoEActivation.SILU,
|
||||
e,
|
||||
None,
|
||||
scales,
|
||||
None,
|
||||
ws1,
|
||||
ws2,
|
||||
metadata,
|
||||
False,
|
||||
)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
if dense:
|
||||
platform = Mock(wraps=impl.current_platform)
|
||||
platform.is_device_capability_family.return_value = False
|
||||
mp.setattr(impl, "current_platform", platform)
|
||||
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
|
||||
calls: list[int] = []
|
||||
|
||||
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
|
||||
assert kwargs.get("use_psum_layout", False) == (not dense)
|
||||
calls.append(args[3].numel())
|
||||
return gemm(*args, **kwargs)
|
||||
|
||||
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
|
||||
invoke()
|
||||
assert len(calls) == 2
|
||||
if not dense:
|
||||
assert calls == [e, e]
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
invoke()
|
||||
variants.append((graph, output, ws1, ws2))
|
||||
|
||||
for mode in (0, 1, 2, 0):
|
||||
route(mode)
|
||||
for graph, _, ws1, ws2 in variants:
|
||||
ws1.fill_(float("nan"))
|
||||
ws2.fill_(float("nan"))
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
|
||||
assert torch.isfinite(variants[1][1]).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
|
||||
"""Both projection shapes skip unused tiles as counts change on replay."""
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
mk_alignment_scope,
|
||||
)
|
||||
|
||||
if not (
|
||||
current_platform.is_device_capability_family(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("Requires SM90 or SM10.x")
|
||||
|
||||
capacity, experts, alignment = 512, 4, 128
|
||||
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
|
||||
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
|
||||
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def invoke():
|
||||
with mk_alignment_scope(alignment):
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(aq, scales), (w, ws), output, ends, use_psum_layout=True
|
||||
)
|
||||
|
||||
invoke()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
invoke()
|
||||
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
|
||||
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
|
||||
output.fill_(91)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(output[allocated_end:] == 91)
|
||||
if allocated_end:
|
||||
assert torch.all(output[[0, 128]] != 91)
|
||||
|
||||
|
||||
def run_single_case(m, n, k, topk, num_experts, block_size):
|
||||
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
Triton baseline within tolerance.
|
||||
"""
|
||||
tokens_bf16 = (
|
||||
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
|
||||
.clamp_min_(-1)
|
||||
.clamp_max_(1)
|
||||
)
|
||||
_, a1_scale = per_token_group_quant_fp8(tokens_bf16, block_size[1])
|
||||
|
||||
# expert weight tensors
|
||||
w1, w2, w1_s, w2_s = make_block_quant_fp8_weights(num_experts, n, k, block_size)
|
||||
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_scale=a1_scale,
|
||||
block_shape=block_size,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
deep_gemm_experts = mk.FusedMoEKernel(
|
||||
prepare_finalize=maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts=TritonOrDeepGemmExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
# triton reference
|
||||
out_triton = fused_experts(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
# DeepGemm
|
||||
out_deepgemm = deep_gemm_experts.apply(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
diff = calc_diff(out_deepgemm, out_triton)
|
||||
assert diff < 0.001, f"Diff exceeded 1%: {diff}"
|
||||
|
||||
|
||||
# Note: N <= 512 will disable the deepgemm path due to performance issues.
|
||||
MNKs = [
|
||||
(1024, 768, 128),
|
||||
(2048, 768, 512),
|
||||
(512, 1024, 1024),
|
||||
(4096, 4096, 1024),
|
||||
]
|
||||
|
||||
TOPKS = [2, 6]
|
||||
NUM_EXPERTS = [32]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("m", "n", "k"), MNKs)
|
||||
@pytest.mark.parametrize("topk", TOPKS)
|
||||
@pytest.mark.parametrize("num_experts", NUM_EXPERTS)
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_init):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
_DeepGemmExperts = importlib.import_module(
|
||||
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
|
||||
).DeepGemmExperts
|
||||
|
||||
call_counter = {"cnt": 0}
|
||||
|
||||
orig_fn = _DeepGemmExperts.apply
|
||||
|
||||
def _spy_apply(*args, **kwargs):
|
||||
call_counter["cnt"] += 1
|
||||
return orig_fn(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(_DeepGemmExperts, "apply", _spy_apply)
|
||||
if topk > num_experts:
|
||||
pytest.skip(f"topk={topk} > num_experts={num_experts}")
|
||||
|
||||
run_single_case(
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
# ensure that the DeepGEMM path was indeed taken.
|
||||
assert call_counter["cnt"] == 1, (
|
||||
f"DeepGEMM path was not executed during the test. "
|
||||
f"Call counter: {call_counter['cnt']}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mxfp4_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
):
|
||||
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
plus BF16 reference weights for validation.
|
||||
|
||||
w1 shape: (E, 2N, K//2) uint8 — packed FP4
|
||||
w2 shape: (E, K, N//2) uint8 — packed FP4
|
||||
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
|
||||
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
|
||||
w1_bf16: (E, 2N, K) — original BF16 for reference
|
||||
w2_bf16: (E, K, N) — original BF16 for reference
|
||||
"""
|
||||
from deep_gemm.utils.math import per_token_cast_to_fp4
|
||||
|
||||
dtype = torch.bfloat16
|
||||
gran_k = 32 # MXFP4 block size
|
||||
|
||||
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
|
||||
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
|
||||
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
|
||||
|
||||
# Quantize per-expert to FP4
|
||||
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
|
||||
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
|
||||
w1_s = torch.empty(
|
||||
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
|
||||
)
|
||||
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
|
||||
|
||||
for i in range(e):
|
||||
w1[i], w1_s[i] = per_token_cast_to_fp4(
|
||||
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
|
||||
)
|
||||
w2[i], w2_s[i] = per_token_cast_to_fp4(
|
||||
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
|
||||
)
|
||||
|
||||
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
|
||||
|
||||
|
||||
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
|
||||
"""BF16 token-loop MoE reference for correctness testing."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
num_tokens, hidden_size = x.shape
|
||||
intermediate = w1.shape[1] // 2
|
||||
top_k = topk_ids.shape[1]
|
||||
|
||||
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
|
||||
for t in range(num_tokens):
|
||||
for kk in range(top_k):
|
||||
e = topk_ids[t, kk].item()
|
||||
w = topk_weights[t, kk].item()
|
||||
fc1 = x[t : t + 1].float() @ w1[e].float().T
|
||||
linear = fc1[:, :intermediate]
|
||||
gate = fc1[:, intermediate:]
|
||||
act = F.silu(gate) * linear
|
||||
fc2 = act @ w2[e].float().T
|
||||
output[t] += w * fc2[0]
|
||||
return output.to(torch.bfloat16)
|
||||
|
||||
|
||||
def run_single_fp4_case(m, n, k, topk, num_experts):
|
||||
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
DeepGEMM FP4 == BF16 reference within tolerance.
|
||||
"""
|
||||
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
|
||||
|
||||
# FP4 expert weight tensors + BF16 originals for reference
|
||||
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
|
||||
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_fp8_dtype = current_platform.fp8_dtype()
|
||||
_block_shape = GroupShape(128, 128)
|
||||
quant_config = FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
|
||||
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
DeepGemmFP4Experts,
|
||||
)
|
||||
|
||||
deep_gemm_fp4_experts = mk.FusedMoEKernel(
|
||||
prepare_finalize=maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts=DeepGemmFP4Experts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
)
|
||||
|
||||
# DeepGEMM FP4 path
|
||||
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
# BF16 reference using the same original weights
|
||||
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
|
||||
|
||||
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
|
||||
diff = calc_diff(out_deepgemm_fp4, out_ref)
|
||||
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
|
||||
|
||||
|
||||
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
|
||||
# FP4 quantization with block_k=32 needs large K for good accuracy.
|
||||
FP4_MNKs = [
|
||||
(128, 4096, 4096), # DeepSeek V4 shape
|
||||
(256, 2048, 2048), # Half-size variant
|
||||
(128, 384, 3584), # Kimi-K3 TP8 latent-MoE shape
|
||||
]
|
||||
|
||||
FP4_TOPKS = [2]
|
||||
FP4_NUM_EXPERTS = [8]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
|
||||
@pytest.mark.parametrize("topk", FP4_TOPKS)
|
||||
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_fp4_vs_triton(
|
||||
m, n, k, topk, num_experts, monkeypatch, workspace_init
|
||||
):
|
||||
pytest.importorskip("deep_gemm.utils.math")
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
_DeepGemmFP4Experts = importlib.import_module(
|
||||
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
|
||||
).DeepGemmFP4Experts
|
||||
|
||||
call_counter = {"cnt": 0}
|
||||
|
||||
orig_fn = _DeepGemmFP4Experts.apply
|
||||
|
||||
def _spy_apply(*args, **kwargs):
|
||||
call_counter["cnt"] += 1
|
||||
return orig_fn(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
|
||||
if topk > num_experts:
|
||||
pytest.skip(f"topk={topk} > num_experts={num_experts}")
|
||||
|
||||
run_single_fp4_case(
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
# ensure that the DeepGEMM FP4 path was indeed taken.
|
||||
assert call_counter["cnt"] == 1, (
|
||||
f"DeepGEMM FP4 path was not executed during the test. "
|
||||
f"Call counter: {call_counter['cnt']}"
|
||||
)
|
||||
@@ -0,0 +1,658 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import per_block_cast_to_int8
|
||||
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
NaiveBatchedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
TritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
|
||||
BatchedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
ref_nvfp4_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import per_block_cast_to_fp8
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def shuffle_weight(w: torch.Tensor) -> torch.Tensor:
|
||||
"""Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout."""
|
||||
shape = w.shape
|
||||
n = shape[-1]
|
||||
first = w[..., : n // 2]
|
||||
second = w[..., n // 2 :]
|
||||
stacked = torch.stack((first, second), dim=-1)
|
||||
return stacked.reshape(shape)
|
||||
|
||||
|
||||
def make_dummy_moe_config(
|
||||
num_experts: int = 1,
|
||||
num_local_experts: int | None = None,
|
||||
experts_per_token: int = 1,
|
||||
hidden_dim: int = 1,
|
||||
intermediate_size: int = 1,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
max_num_tokens: int = 512,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> FusedMoEConfig:
|
||||
"""This is a dummy config for the mk constructor interface
|
||||
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
|
||||
do not actually use this config.
|
||||
|
||||
CUTLASSFp8 needs to set some params for workshapes.
|
||||
"""
|
||||
return FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=experts_per_token,
|
||||
hidden_dim=hidden_dim,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_local_experts
|
||||
if num_local_experts is not None
|
||||
else num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=activation,
|
||||
in_dtype=in_dtype,
|
||||
device=DEVICE,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=max_num_tokens,
|
||||
)
|
||||
|
||||
|
||||
def triton_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant=False,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
|
||||
return fused_experts(a, w1, w2, topk_weight, topk_ids, quant_config=quant_config)
|
||||
|
||||
|
||||
def batched_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
moe_config: FusedMoEConfig | None = None,
|
||||
) -> torch.Tensor:
|
||||
max_num_tokens = round_up(a.shape[0], 64)
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
|
||||
if moe_config is None:
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
fused_experts = FusedMoEKernel(
|
||||
BatchedPrepareAndFinalize(
|
||||
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
|
||||
),
|
||||
BatchedTritonExperts(
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=1,
|
||||
quant_config=quant_config,
|
||||
moe_config=moe_config,
|
||||
),
|
||||
)
|
||||
|
||||
return fused_experts.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
global_num_experts=w1.shape[0],
|
||||
activation=moe_config.activation,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
|
||||
def naive_batched_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
max_num_tokens = round_up(a.shape[0], 64)
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
fused_experts = FusedMoEKernel(
|
||||
BatchedPrepareAndFinalize(
|
||||
max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0
|
||||
),
|
||||
NaiveBatchedExperts(
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=1,
|
||||
quant_config=quant_config,
|
||||
moe_config=moe_config,
|
||||
),
|
||||
)
|
||||
|
||||
return fused_experts.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
global_num_experts=w1.shape[0],
|
||||
activation=moe_config.activation,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
|
||||
def chunk_scales(
|
||||
scales: torch.Tensor | None, start: int, end: int
|
||||
) -> torch.Tensor | None:
|
||||
if scales is not None:
|
||||
if scales.numel() == 1:
|
||||
return scales
|
||||
else:
|
||||
return scales[start:end]
|
||||
return None
|
||||
|
||||
|
||||
def make_quantized_test_activations(
|
||||
E: int,
|
||||
m: int,
|
||||
k: int,
|
||||
in_dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10
|
||||
a_q = a
|
||||
a_scale = None
|
||||
|
||||
if quant_dtype is not None:
|
||||
assert quant_dtype == torch.float8_e4m3fn or quant_dtype == torch.int8, (
|
||||
"only fp8/int8 supported"
|
||||
)
|
||||
a_q = torch.zeros_like(a, dtype=quant_dtype)
|
||||
a_scale_l = [None] * E
|
||||
for e in range(E):
|
||||
a_q[e], a_scale_l[e] = moe_kernel_quantize_input(
|
||||
a[e], None, quant_dtype, per_act_token_quant, block_shape
|
||||
)
|
||||
a_scale = torch.stack(a_scale_l)
|
||||
|
||||
if not per_act_token_quant and block_shape is None:
|
||||
a_scale = a_scale.view(E, 1, 1)
|
||||
|
||||
return a, a_q, a_scale
|
||||
|
||||
|
||||
def moe_quantize_weights_2d(
|
||||
w: torch.Tensor,
|
||||
w_s: torch.Tensor | None,
|
||||
quant_dtype: torch.dtype | str | None,
|
||||
per_token_quant: bool,
|
||||
block_shape: list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
is_fp8_dtype = is_fp8(quant_dtype)
|
||||
assert is_fp8_dtype or quant_dtype == torch.int8 or quant_dtype == "nvfp4", (
|
||||
"only fp8/int8/nvfp4 supported"
|
||||
)
|
||||
|
||||
w_gs = None
|
||||
|
||||
if block_shape is not None:
|
||||
assert not per_token_quant
|
||||
if quant_dtype == torch.int8:
|
||||
w, w_s = per_block_cast_to_int8(w, block_shape)
|
||||
elif is_fp8_dtype:
|
||||
w, w_s = per_block_cast_to_fp8(w, block_shape)
|
||||
elif quant_dtype == "nvfp4":
|
||||
raise RuntimeError("blocked quantization not supported for nvfp4")
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
|
||||
else:
|
||||
if quant_dtype == torch.int8:
|
||||
w, w_s = ops.scaled_int8_quant(
|
||||
w, w_s, use_per_token_if_dynamic=per_token_quant
|
||||
)
|
||||
elif is_fp8_dtype:
|
||||
w, w_s = ops.scaled_fp8_quant(
|
||||
w, w_s, use_per_token_if_dynamic=per_token_quant
|
||||
)
|
||||
elif quant_dtype == "nvfp4":
|
||||
assert not per_token_quant
|
||||
w_amax = torch.abs(w).max().to(torch.float32)
|
||||
w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax
|
||||
if current_platform.is_rocm():
|
||||
w, w_s = _scaled_fp4_quant_emulated(w, w_gs)
|
||||
else:
|
||||
w, w_s = ops.scaled_fp4_quant(w, w_gs)
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported quant type {quant_dtype}")
|
||||
|
||||
return w, w_s, w_gs
|
||||
|
||||
|
||||
def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor:
|
||||
assert fp4_values.shape[-1] % 2 == 0
|
||||
|
||||
abs_values = fp4_values.abs()
|
||||
codes = torch.empty_like(abs_values, dtype=torch.uint8)
|
||||
for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)):
|
||||
codes[abs_values == value] = code
|
||||
codes = codes | ((fp4_values < 0).to(torch.uint8) << 3)
|
||||
return codes[..., 0::2] | (codes[..., 1::2] << 4)
|
||||
|
||||
|
||||
def _scaled_fp4_quant_emulated(
|
||||
w: torch.Tensor, w_gs: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16)
|
||||
return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn)
|
||||
|
||||
|
||||
def moe_quantize_weights(
|
||||
w: torch.Tensor,
|
||||
w_s: torch.Tensor | None,
|
||||
quant_dtype: torch.dtype | str | None,
|
||||
per_token_quant: bool,
|
||||
block_shape: list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
assert w.dim() == 3
|
||||
e, rows, cols = w.shape
|
||||
w_l = [None] * e
|
||||
w_s_l = [None] * e
|
||||
w_gs_l = [None] * e
|
||||
for idx in range(e):
|
||||
w_l[idx], w_s_l[idx], w_gs_l[idx] = moe_quantize_weights_2d(
|
||||
w[idx], None, quant_dtype, per_token_quant, block_shape
|
||||
)
|
||||
|
||||
w = torch.stack(w_l)
|
||||
w_s = torch.stack(w_s_l)
|
||||
w_gs = torch.stack(w_gs_l) if e > 0 and w_gs_l[0] is not None else None
|
||||
|
||||
if w_s.ndim == 2:
|
||||
assert w_s.shape[-1] == 1
|
||||
w_s = w_s.view(-1, 1, 1)
|
||||
|
||||
if block_shape is not None:
|
||||
block_n, block_k = block_shape
|
||||
n_tiles = (rows + block_n - 1) // block_n
|
||||
k_tiles = (cols + block_k - 1) // block_k
|
||||
assert w_s.shape == (e, n_tiles, k_tiles)
|
||||
|
||||
return w, w_s, w_gs
|
||||
|
||||
|
||||
def make_test_weight(
|
||||
e: int,
|
||||
rows: int,
|
||||
cols: int,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_out_ch_quant: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15
|
||||
|
||||
if quant_dtype is not None:
|
||||
w, w_s, w_gs = moe_quantize_weights(
|
||||
w_16, None, quant_dtype, per_out_ch_quant, block_shape
|
||||
)
|
||||
else:
|
||||
w = w_16
|
||||
w_s = None
|
||||
w_gs = None
|
||||
|
||||
return w_16, w, w_s, w_gs
|
||||
|
||||
|
||||
def make_test_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
in_dtype: torch.dtype = torch.bfloat16,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_out_ch_quant: bool = False,
|
||||
make_gate: bool = True,
|
||||
) -> tuple[
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None],
|
||||
]:
|
||||
return (
|
||||
make_test_weight(
|
||||
e,
|
||||
(2 if make_gate else 1) * n,
|
||||
k,
|
||||
in_dtype,
|
||||
quant_dtype,
|
||||
block_shape,
|
||||
per_out_ch_quant,
|
||||
),
|
||||
make_test_weight(e, k, n, in_dtype, quant_dtype, block_shape, per_out_ch_quant),
|
||||
)
|
||||
|
||||
|
||||
def per_token_cast_to_fp8(
|
||||
x: torch.Tensor, block_size: int = 128
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert x.dim() == 2
|
||||
m, n = x.shape
|
||||
pad_size = (block_size - (n % block_size)) % block_size
|
||||
x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x
|
||||
x_view = x.view(m, -1, block_size)
|
||||
x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
|
||||
fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn)
|
||||
return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1)
|
||||
|
||||
|
||||
def make_test_quant_config(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
in_dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
block_shape: list[int] | None = None,
|
||||
make_gate: bool = True,
|
||||
is_scale_swizzled: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
|
||||
(_, w1, w1_s, w1_gs), (_, w2, w2_s, w2_gs) = make_test_weights(
|
||||
e,
|
||||
n,
|
||||
k,
|
||||
in_dtype,
|
||||
quant_dtype,
|
||||
per_out_ch_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
make_gate=make_gate,
|
||||
)
|
||||
|
||||
# Hacky/trivial scales for nvfp4.
|
||||
a1_gscale: torch.Tensor | None = None
|
||||
a2_gscale: torch.Tensor | None = None
|
||||
if quant_dtype == "nvfp4":
|
||||
a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
|
||||
a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32)
|
||||
a1_scale = a1_gscale
|
||||
a2_scale = a2_gscale
|
||||
else:
|
||||
a1_scale = None
|
||||
a2_scale = None
|
||||
|
||||
return (
|
||||
w1,
|
||||
w2,
|
||||
FusedMoEQuantConfig.make(
|
||||
quant_dtype,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
a1_gscale=a1_gscale,
|
||||
a2_gscale=a2_gscale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
# TODO: make sure this is handled properly
|
||||
g1_alphas=(1 / w1_gs) if w1_gs is not None else None,
|
||||
g2_alphas=(1 / w2_gs) if w2_gs is not None else None,
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def fused_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool = False,
|
||||
quant_config: FusedMoEQuantConfig | None = None,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states, score.float(), topk, renormalize
|
||||
)
|
||||
return fused_experts(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
|
||||
class BaselineMM(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
b: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
):
|
||||
super().__init__()
|
||||
self.b = torch.nn.Parameter(b.to(dtype=torch.float32))
|
||||
self.out_dtype = out_dtype
|
||||
|
||||
def forward(self, a: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return torch.mm(a.to(dtype=torch.float32), self.b).to(self.out_dtype), None
|
||||
|
||||
|
||||
class BaselineSiluAndMul(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return torch.nn.functional.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
class TestMLP(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
):
|
||||
super().__init__()
|
||||
self.gate_up_proj = BaselineMM(w1, out_dtype)
|
||||
self.down_proj = BaselineMM(w2, out_dtype)
|
||||
self.act_fn = BaselineSiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
def modular_triton_fused_moe(
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> FusedMoEKernel:
|
||||
return FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
TritonExperts(moe_config, quant_config),
|
||||
)
|
||||
|
||||
|
||||
def check_accuracy(a, b, atol, rtol, percent):
|
||||
"""Allow a mismatch percentage of 1 - percent."""
|
||||
if torch.any(torch.isnan(a)):
|
||||
raise Exception("NaN in reference output")
|
||||
if torch.any(torch.isnan(b)):
|
||||
raise Exception("NaN in actual output")
|
||||
if torch.any(torch.isinf(a)):
|
||||
raise Exception("Inf in reference output")
|
||||
if torch.any(torch.isinf(b)):
|
||||
raise Exception("Inf in actual output")
|
||||
assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}"
|
||||
|
||||
left = torch.abs(a - b)
|
||||
right = atol + rtol * torch.abs(b)
|
||||
count = torch.sum(left > right)
|
||||
mismatch_percent = count / a.numel()
|
||||
if mismatch_percent > 1 - percent:
|
||||
raise Exception(
|
||||
f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} "
|
||||
f"(threshold: {1 - percent:.4f})"
|
||||
)
|
||||
|
||||
|
||||
def check_deferred_moe_finalize(
|
||||
moe_config: FusedMoEConfig,
|
||||
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
|
||||
router_weights: torch.Tensor | None = None,
|
||||
chunked: bool = False,
|
||||
) -> None:
|
||||
"""Check a kernel that defers its finalize against the finalize it skips.
|
||||
|
||||
``run`` calls the kernel on fixed inputs, first as built and then with
|
||||
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
|
||||
report truthfully. A deferred output reduced by ``moe_unpermute``, the
|
||||
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
|
||||
bit, and modular experts hand the router's weights back as-is. A call the
|
||||
experts split across kernel launches finalizes instead.
|
||||
"""
|
||||
# The deferred output views the router's buffer, so compare with a copy.
|
||||
expected_weights = None if router_weights is None else router_weights.clone()
|
||||
finalized = run()
|
||||
assert isinstance(finalized, torch.Tensor)
|
||||
moe_config.defer_moe_finalize()
|
||||
output = run()
|
||||
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
|
||||
if chunked:
|
||||
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
|
||||
return
|
||||
|
||||
assert isinstance(output, UnfinalizedMoEOutput)
|
||||
if expected_weights is not None:
|
||||
torch.testing.assert_close(
|
||||
output.expert_weights, expected_weights, atol=0, rtol=0
|
||||
)
|
||||
reference = torch.empty_like(finalized)
|
||||
moe_unpermute(
|
||||
reference,
|
||||
output.gemm2_permuted,
|
||||
output.expert_weights.float(),
|
||||
output.expanded_idx_to_permuted_idx,
|
||||
# The kernel reads its valid-row count through this pointer either way.
|
||||
expert_first_token_offset=output.gemm2_permuted.new_full(
|
||||
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
|
||||
|
||||
|
||||
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
|
||||
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
|
||||
|
||||
triton_kernels 3.8 returns layout instances; earlier versions return a
|
||||
(layout, opts) tuple.
|
||||
"""
|
||||
from triton_kernels.tensor_details import layout
|
||||
|
||||
from vllm.utils.import_utils import get_triton_kernels_version
|
||||
|
||||
if get_triton_kernels_version() == "3.8":
|
||||
w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
|
||||
s = layout.make_default_matmul_mxfp4_w_scale_layout(
|
||||
mx_axis=mx_axis, num_warps=num_warps
|
||||
)
|
||||
return w, {}, s, {}
|
||||
|
||||
w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis)
|
||||
s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout(
|
||||
mx_axis=mx_axis, num_warps=num_warps
|
||||
)
|
||||
return w, w_opts, s, s_opts
|
||||
@@ -0,0 +1,290 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
group_broadcast,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
|
||||
return torch.as_tensor(x, dtype=torch.float32, device=DEVICE)
|
||||
|
||||
|
||||
def ref_dynamic_per_token_quant(
|
||||
x: torch.Tensor, quant_dtype: torch.dtype, scale_ub: torch.Tensor | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert quant_dtype in [torch.int8, FP8_DTYPE]
|
||||
if scale_ub is not None:
|
||||
assert quant_dtype == FP8_DTYPE
|
||||
|
||||
if quant_dtype == torch.int8:
|
||||
qtype_traits = torch.iinfo(quant_dtype)
|
||||
qtype_traits_min = qtype_traits.min
|
||||
qtype_traits_max = qtype_traits.max
|
||||
else:
|
||||
qtype_traits_min, qtype_traits_max = get_fp8_min_max()
|
||||
qtype_max = as_float32_tensor(qtype_traits_max)
|
||||
s_1 = as_float32_tensor(1.0)
|
||||
s_512 = as_float32_tensor(512.0)
|
||||
|
||||
# For fp8, in order to match the cuda kernel output, we have to do exactly
|
||||
# the same operations as in the corresponding fp8 kernel to prevent
|
||||
# rounding errors.
|
||||
|
||||
# Compute scales
|
||||
x_token_max, _ = x.abs().max(dim=-1)
|
||||
x_token_max = as_float32_tensor(x_token_max)
|
||||
if scale_ub is not None:
|
||||
x_token_max = x_token_max.clamp(max=scale_ub)
|
||||
scales = (x_token_max / qtype_max)[:, None]
|
||||
|
||||
# Quant
|
||||
if quant_dtype == torch.int8:
|
||||
iscales = as_float32_tensor(s_1 / scales)
|
||||
torch_out = as_float32_tensor(x) * iscales
|
||||
torch_out = torch_out.round()
|
||||
torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype)
|
||||
else:
|
||||
assert quant_dtype == FP8_DTYPE
|
||||
min_scaling_factor = s_1 / (qtype_max * s_512)
|
||||
scales = scales.clamp(min=min_scaling_factor)
|
||||
torch_out = as_float32_tensor(x) / scales
|
||||
torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype)
|
||||
|
||||
return torch_out, scales
|
||||
|
||||
|
||||
# The int8 version is very similar. Incorporate the int8 version, like in
|
||||
# ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant
|
||||
# kernel
|
||||
def ref_dynamic_per_tensor_fp8_quant(
|
||||
x: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
fp8_traits_min, fp8_traits_max = get_fp8_min_max()
|
||||
fp8_max = as_float32_tensor(fp8_traits_max)
|
||||
one = as_float32_tensor(1.0)
|
||||
|
||||
# For fp8, in order to match the cuda kernel output, we have to do exactly
|
||||
# the same operations as in the corresponding fp8 kernel to prevent
|
||||
# rounding errors.
|
||||
|
||||
x_max = as_float32_tensor(x.abs().max())
|
||||
ref_scale = x_max / fp8_max
|
||||
ref_iscale = one / ref_scale
|
||||
ref_out = (
|
||||
(as_float32_tensor(x) * ref_iscale)
|
||||
.clamp(fp8_traits_min, fp8_traits_max)
|
||||
.to(FP8_DTYPE)
|
||||
)
|
||||
return ref_out, ref_scale.view(1)
|
||||
|
||||
|
||||
def native_w8a8_block_matmul(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
block_size: list[int],
|
||||
output_dtype: torch.dtype,
|
||||
compute_type: torch.dtype = torch.float32,
|
||||
) -> torch.Tensor:
|
||||
"""This function performs matrix multiplication with block-wise
|
||||
quantization using native torch.
|
||||
It is agnostic to the input data type and can be used for both int8 and
|
||||
fp8 data types.
|
||||
|
||||
It takes two input tensors `A` and `B` (int8) with scales `As` and
|
||||
`Bs` (float32).
|
||||
The output is returned in the specified `output_dtype`.
|
||||
"""
|
||||
A = A.to(compute_type)
|
||||
B = B.to(compute_type)
|
||||
assert A.shape[-1] == B.shape[-1]
|
||||
assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2
|
||||
assert len(block_size) == 2
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1]
|
||||
assert A.shape[:-1] == As.shape[:-1]
|
||||
|
||||
M = A.numel() // A.shape[-1]
|
||||
N, K = B.shape
|
||||
origin_C_shape = A.shape[:-1] + (N,)
|
||||
A = A.reshape(M, A.shape[-1])
|
||||
As = As.reshape(M, As.shape[-1])
|
||||
n_tiles = (N + block_n - 1) // block_n
|
||||
k_tiles = (K + block_k - 1) // block_k
|
||||
assert n_tiles == Bs.shape[0], f"{n_tiles} == {Bs.shape[0]}"
|
||||
assert k_tiles == Bs.shape[1], f"{k_tiles} == {Bs.shape[1]}"
|
||||
|
||||
C_shape = (M, N)
|
||||
C = torch.zeros(C_shape, dtype=compute_type, device=A.device)
|
||||
|
||||
A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)]
|
||||
B_tiles = [
|
||||
[
|
||||
B[
|
||||
j * block_n : min((j + 1) * block_n, N),
|
||||
i * block_k : min((i + 1) * block_k, K),
|
||||
]
|
||||
for i in range(k_tiles)
|
||||
]
|
||||
for j in range(n_tiles)
|
||||
]
|
||||
C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)]
|
||||
As_tiles = [As[:, i : i + 1] for i in range(k_tiles)]
|
||||
|
||||
for i in range(k_tiles):
|
||||
for j in range(n_tiles):
|
||||
a = A_tiles[i]
|
||||
b = B_tiles[j][i]
|
||||
c = C_tiles[j]
|
||||
s = As_tiles[i] * Bs[j][i]
|
||||
c[:, :] += torch.matmul(a, b.t()) * s
|
||||
|
||||
C = C.reshape(origin_C_shape).to(output_dtype)
|
||||
return C
|
||||
|
||||
|
||||
def native_per_token_group_quant_fp8(
|
||||
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
|
||||
):
|
||||
"""Function to perform per-token-group quantization on an input tensor
|
||||
`x` using native torch."""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` must be divisible by `group_size`"
|
||||
)
|
||||
assert x.is_contiguous(), "`x` is not contiguous"
|
||||
|
||||
finfo = torch.finfo(dtype)
|
||||
fp8_min = finfo.min
|
||||
fp8_max = finfo.max
|
||||
|
||||
x_ = x.reshape(x.numel() // group_size, group_size)
|
||||
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
|
||||
x_s = amax / fp8_max
|
||||
if is_deep_gemm_e8m0_used():
|
||||
x_s = _ceil_to_ue8m0(x_s)
|
||||
x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype)
|
||||
x_q = x_q.reshape(x.shape)
|
||||
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
|
||||
|
||||
return x_q, x_s
|
||||
|
||||
|
||||
def native_per_token_group_quant_int8(
|
||||
x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True
|
||||
):
|
||||
"""Function to perform per-token-group quantization on an input tensor
|
||||
`x` using native torch.
|
||||
|
||||
It converts the tensor values into int8 values and returns the
|
||||
quantized tensor along with the scaling factor used for quantization.
|
||||
Per-token quantization rounds to nearest; block quantization truncates.
|
||||
"""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` must be divisible by `group_size`"
|
||||
)
|
||||
assert x.is_contiguous(), "`x` is not contiguous"
|
||||
|
||||
iinfo = torch.iinfo(dtype)
|
||||
int8_min = iinfo.min
|
||||
int8_max = iinfo.max
|
||||
|
||||
x_ = x.reshape(x.numel() // group_size, group_size)
|
||||
# Use float32 for scale calculation for stability
|
||||
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
|
||||
x_s = amax / int8_max
|
||||
x_q = x_.to(torch.float32) / x_s
|
||||
if round_to_nearest:
|
||||
x_q = x_q.round()
|
||||
x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype)
|
||||
x_q = x_q.reshape(x.shape)
|
||||
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
|
||||
|
||||
return x_q, x_s
|
||||
|
||||
|
||||
DEFAULT_BLOCK_SHAPE = [128, 128]
|
||||
|
||||
|
||||
def per_block_cast_to_int8(
|
||||
x: torch.Tensor,
|
||||
block_shape: list[int] = DEFAULT_BLOCK_SHAPE,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
block_m, block_n = block_shape
|
||||
assert x.dim() == 2
|
||||
m, n = x.shape
|
||||
x_padded = torch.zeros(
|
||||
(round_up(m, block_m), round_up(n, block_n)), dtype=x.dtype, device=x.device
|
||||
)
|
||||
x_padded[:m, :n] = x
|
||||
x_view = x_padded.view(-1, block_m, x_padded.size(1) // block_n, block_n)
|
||||
x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
|
||||
x_scaled = (x_view * (256.0 / x_amax)).to(torch.int8)
|
||||
x_scaled_sub = x_scaled.view_as(x_padded)[:m, :n].contiguous()
|
||||
scales = (x_amax / 256.0).view(x_view.size(0), x_view.size(2))
|
||||
return x_scaled_sub, scales
|
||||
|
||||
|
||||
def dequant(
|
||||
t: torch.Tensor,
|
||||
scale: torch.Tensor | None,
|
||||
block_shape: list[int] | None,
|
||||
per_act_token_quant: bool,
|
||||
out_dtype: torch.dtype | None = torch.float32,
|
||||
) -> torch.Tensor:
|
||||
if scale is not None:
|
||||
f32 = torch.float32
|
||||
if per_act_token_quant or block_shape is None:
|
||||
return (t.to(f32) * scale).to(out_dtype)
|
||||
else:
|
||||
return (t.to(f32) * group_broadcast(scale, t.shape)).to(out_dtype)
|
||||
else:
|
||||
return t.to(out_dtype)
|
||||
|
||||
|
||||
def native_batched_masked_quant_matmul(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
num_expert_tokens: torch.Tensor,
|
||||
A_scale: torch.Tensor | None = None,
|
||||
B_scale: torch.Tensor | None = None,
|
||||
block_shape: list[int] | None = None,
|
||||
per_act_token_quant: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_expert_tokens_cpu = num_expert_tokens.clone()
|
||||
num_expert_tokens_cpu = num_expert_tokens_cpu.to(device="cpu")
|
||||
num_experts = num_expert_tokens.size(0)
|
||||
|
||||
for e in range(num_experts):
|
||||
num_tokens = num_expert_tokens_cpu[e]
|
||||
if A.dtype.itemsize == 1 and block_shape is not None:
|
||||
assert A_scale is not None and B_scale is not None
|
||||
tmp = native_w8a8_block_matmul(
|
||||
A[e], B[e], A_scale[e], B_scale[e], block_shape, C.dtype
|
||||
)
|
||||
C[e, :num_tokens, :] = tmp[:num_tokens, :]
|
||||
elif A.dtype.itemsize == 1 and block_shape is None:
|
||||
assert A_scale is not None and B_scale is not None
|
||||
A_dq = dequant(A[e], A_scale[e], block_shape, per_act_token_quant)
|
||||
B_dq = dequant(B[e], B_scale[e], block_shape, per_act_token_quant)
|
||||
C[e, :num_tokens, :] = (A_dq[:num_tokens] @ B_dq.transpose(0, 1)).to(
|
||||
C.dtype
|
||||
)
|
||||
else:
|
||||
assert A_scale is None
|
||||
assert B_scale is None
|
||||
C[e, :num_tokens, :] = A[e, :num_tokens, :] @ B[e].transpose(0, 1)
|
||||
|
||||
return C
|
||||
-123
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
from .conftest import RunaiDummyExecutor
|
||||
|
||||
load_format = "runai_streamer"
|
||||
test_model = "openai-community/gpt2"
|
||||
|
||||
|
||||
def test_runai_model_loader_download_files_s3_mocked_with_patch(
|
||||
vllm_runner,
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
patcher = StreamerPatcher(str(tmp_path))
|
||||
|
||||
test_mock_s3_model = "s3://my-mock-bucket/gpt2/"
|
||||
|
||||
# Download model from HF
|
||||
mock_model_dir = f"{tmp_path}/gpt2"
|
||||
hf_api().snapshot_download(repo_id=test_model, local_dir=mock_model_dir)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
|
||||
patcher.shim_list_safetensors,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_pull_files",
|
||||
patcher.shim_pull_files,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
|
||||
patcher.create_mock_streamer,
|
||||
)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model=test_mock_s3_model,
|
||||
load_format=load_format,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
|
||||
executor = RunaiDummyExecutor(vllm_config)
|
||||
executor.driver_worker.load_model()
|
||||
|
||||
|
||||
def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch):
|
||||
# An engine started from object storage keeps the bucket URI in
|
||||
# model_config.model_weights. A reload with weights_path must stream the
|
||||
# new checkpoint, not re-stream the one the engine started from.
|
||||
patcher = StreamerPatcher(str(tmp_path))
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_list_safetensors",
|
||||
patcher.shim_list_safetensors,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.runai_utils.runai_pull_files",
|
||||
patcher.shim_pull_files,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
|
||||
patcher.create_mock_streamer,
|
||||
)
|
||||
|
||||
base_dir = tmp_path / "gpt2"
|
||||
hf_api().snapshot_download(
|
||||
repo_id=test_model,
|
||||
local_dir=str(base_dir),
|
||||
allow_patterns=["*.json", "*.txt", "model.safetensors"],
|
||||
)
|
||||
# Second checkpoint under another prefix: the final layer norm scaled, so a
|
||||
# reload that silently re-streams the original bucket is distinguishable.
|
||||
tensors = load_file(base_dir / "model.safetensors")
|
||||
original = tensors["ln_f.weight"].clone()
|
||||
modified = original * 2
|
||||
tensors["ln_f.weight"] = modified
|
||||
mod_dir = tmp_path / "gpt2-mod"
|
||||
mod_dir.mkdir()
|
||||
save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"})
|
||||
|
||||
vllm_config = EngineArgs(
|
||||
model="s3://my-mock-bucket/gpt2/",
|
||||
load_format=load_format,
|
||||
tensor_parallel_size=1,
|
||||
).create_engine_config()
|
||||
assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/"
|
||||
|
||||
executor = RunaiDummyExecutor(vllm_config)
|
||||
executor.driver_worker.load_model()
|
||||
model = executor.driver_worker.model_runner.get_model()
|
||||
(param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")]
|
||||
|
||||
def current() -> torch.Tensor:
|
||||
return param.detach().float().cpu()
|
||||
|
||||
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# No weights_path: re-stream the checkpoint the engine started from, i.e.
|
||||
# the bucket kept in model_weights. Zero the live tensor first so a reload
|
||||
# that did nothing is distinguishable from one that streamed the bucket.
|
||||
param.data.zero_()
|
||||
executor.driver_worker.reload_weights()
|
||||
torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# weights_path: the new prefix must win over the stale model_weights.
|
||||
executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/")
|
||||
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# After the override, a bare reload means the new checkpoint, not the
|
||||
# one the engine was started with.
|
||||
param.data.zero_()
|
||||
executor.driver_worker.reload_weights()
|
||||
torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2)
|
||||
@@ -1,18 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import wikitext_ppl_test
|
||||
|
||||
MODELS = [
|
||||
GenerateModelInfo("google/gemma-2b", hf_ppl=21.48524284362793),
|
||||
GenerateModelInfo("google/gemma-2-2b", hf_ppl=102.59290313720703),
|
||||
GenerateModelInfo("google/gemma-3-4b-it", hf_ppl=27.79648208618164),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
|
||||
wikitext_ppl_test(hf_runner, vllm_runner, model_info)
|
||||
@@ -1,24 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import wikitext_ppl_test
|
||||
|
||||
MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.457056045532227)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
|
||||
bf16_ppl = wikitext_ppl_test(hf_runner, vllm_runner, model_info)
|
||||
fp32_ppl = wikitext_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"hf_overrides": {"head_dtype": "float32"}},
|
||||
)
|
||||
|
||||
differ = ((fp32_ppl - bf16_ppl) / bf16_ppl) * 100
|
||||
print("fp32 head difference (%):", differ)
|
||||
assert differ < 0
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
|
||||
embeddings = []
|
||||
for d in response.data:
|
||||
assert isinstance(d.embedding, list)
|
||||
embeddings.append(d.embedding)
|
||||
return embeddings
|
||||
|
||||
|
||||
def run_embedding_correctness_test(
|
||||
hf_model: "HfRunner",
|
||||
inputs: list[str],
|
||||
vllm_outputs: Sequence[list[float]],
|
||||
dimensions: int | None = None,
|
||||
):
|
||||
hf_outputs = hf_model.encode(inputs)
|
||||
if dimensions:
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: EmbedModelInfo,
|
||||
example_prompts,
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
):
|
||||
pytest.skip("Debug only, ci prefers to use mteb test.")
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = model_info.dtype
|
||||
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
dtype=model_info.hf_dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
|
||||
|
||||
|
||||
async def run_client_embeddings(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
queries: list[str],
|
||||
instruction: str = "",
|
||||
) -> list[list[float]]:
|
||||
outputs = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=[instruction + q for q in queries],
|
||||
)
|
||||
return [data.embedding for data in outputs.data]
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pytest configuration for vLLM pooling MTEB tests."""
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def release_gpu_memory_between_tests():
|
||||
"""Reclaim cached GPU memory between tests in this shard.
|
||||
|
||||
These tests start many vLLM/HF engines sequentially in one pytest
|
||||
process; without explicit reclamation between tests, VRAM fragmentation
|
||||
accumulates and later tests fail at engine startup ("Engine core
|
||||
initialization failed").
|
||||
"""
|
||||
yield
|
||||
gc.collect()
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
if current_platform.is_rocm():
|
||||
return
|
||||
try:
|
||||
wait_for_memory_to_settle()
|
||||
except ValueError as e:
|
||||
# Longer-lived (class/module-scoped) engine fixtures may legitimately
|
||||
# still hold VRAM; nothing further to reclaim in that case.
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
@@ -1,279 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import torch
|
||||
from mteb.models import ModelMeta
|
||||
from mteb.types import Array
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
import tests.ci_envs as ci_envs
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
check_embeddings_close,
|
||||
get_vllm_extra_kwargs,
|
||||
)
|
||||
|
||||
# Most embedding models on the STS12 task (See #17175):
|
||||
# - Model implementation and minor changes in tensor dtype
|
||||
# results in differences less than 1e-4
|
||||
# - Different model results in differences more than 1e-3
|
||||
# 5e-4 is a good tolerance threshold
|
||||
MTEB_EMBED_TASKS = ["STS12"]
|
||||
MTEB_EMBED_TOL = 5e-4
|
||||
|
||||
|
||||
_empty_model_meta = ModelMeta(
|
||||
loader=None,
|
||||
name="vllm/model",
|
||||
revision="1",
|
||||
release_date=None,
|
||||
languages=None,
|
||||
framework=[],
|
||||
similarity_fn_name=None,
|
||||
n_parameters=None,
|
||||
memory_usage_mb=None,
|
||||
max_tokens=None,
|
||||
embed_dim=None,
|
||||
license=None,
|
||||
open_weights=None,
|
||||
public_training_code=None,
|
||||
public_training_data=None,
|
||||
use_instructions=None,
|
||||
training_datasets=None,
|
||||
modalities=["text"], # 'image' can be added to evaluate multimodal models
|
||||
)
|
||||
|
||||
|
||||
class MtebEmbedMixin(mteb.EncoderProtocol):
|
||||
mteb_model_meta = _empty_model_meta
|
||||
|
||||
def similarity(
|
||||
self,
|
||||
embeddings1: np.ndarray,
|
||||
embeddings2: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
# Cosine similarity
|
||||
norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
|
||||
norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
|
||||
sim = np.dot(embeddings1, embeddings2.T) / (norm1 * norm2.T)
|
||||
return sim
|
||||
|
||||
def similarity_pairwise(
|
||||
self,
|
||||
embeddings1: Array,
|
||||
embeddings2: Array,
|
||||
) -> Array:
|
||||
# Cosine similarity
|
||||
norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
|
||||
norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
|
||||
sim = np.sum(embeddings1 * embeddings2, axis=1) / (
|
||||
norm1.flatten() * norm2.flatten()
|
||||
)
|
||||
return sim
|
||||
|
||||
|
||||
class HfMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
sentences = [text for batch in inputs for text in batch["text"]]
|
||||
return self.model.encode(sentences)
|
||||
|
||||
|
||||
class VllmMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, vllm_model, prompt_prefix: str | None = None):
|
||||
self.llm = vllm_model
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
self.prompt_prefix = prompt_prefix
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
sentences = [
|
||||
self.prompt_prefix + text if self.prompt_prefix else text
|
||||
for batch in inputs
|
||||
for text in batch["text"]
|
||||
]
|
||||
r = self.rng.permutation(len(sentences))
|
||||
sentences = [sentences[i] for i in r]
|
||||
outputs = self.llm.embed(sentences, use_tqdm=False)
|
||||
embeds = np.array(outputs)
|
||||
embeds = embeds[np.argsort(r)]
|
||||
return embeds
|
||||
|
||||
|
||||
class OpenAIClientMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, model_name: str, client):
|
||||
self.model_name = model_name
|
||||
self.client = client
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
sentences = [text for batch in inputs for text in batch["text"]]
|
||||
r = self.rng.permutation(len(sentences))
|
||||
sentences = [sentences[i] for i in r]
|
||||
|
||||
embeddings = self.client.embeddings.create(
|
||||
model=self.model_name, input=sentences
|
||||
)
|
||||
outputs = [d.embedding for d in embeddings.data]
|
||||
embeds = np.array(outputs)
|
||||
embeds = embeds[np.argsort(r)]
|
||||
return embeds
|
||||
|
||||
|
||||
def run_mteb_embed_task(encoder: mteb.EncoderProtocol, tasks):
|
||||
tasks = mteb.get_tasks(tasks=tasks)
|
||||
results = mteb.evaluate(
|
||||
encoder,
|
||||
tasks,
|
||||
cache=None,
|
||||
show_progress_bar=False,
|
||||
)
|
||||
|
||||
main_score = results[0].scores["test"][0]["main_score"]
|
||||
return main_score
|
||||
|
||||
|
||||
def mteb_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: EmbedModelInfo,
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
atol: float | None = None,
|
||||
prompt_prefix: str | None = None,
|
||||
vllm_model_callback=None,
|
||||
):
|
||||
if atol is None:
|
||||
atol = (
|
||||
model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_EMBED_TOL
|
||||
)
|
||||
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
|
||||
|
||||
# Test embed_dims, isnan and whether to use normalize
|
||||
example_prompts = ["The chef prepared a delicious meal." * 1000]
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=model_info.max_model_len,
|
||||
**vllm_extra_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
|
||||
if vllm_model_callback is not None:
|
||||
vllm_model_callback(vllm_model)
|
||||
|
||||
# Confirm whether vllm is using the correct architecture
|
||||
if model_info.architecture:
|
||||
assert model_info.architecture in model_config.architectures
|
||||
|
||||
# Confirm whether the important configs in model_config are correct.
|
||||
pooler_config = model_config.pooler_config
|
||||
if model_info.seq_pooling_type is not None:
|
||||
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
|
||||
if model_info.tok_pooling_type is not None:
|
||||
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
|
||||
if model_info.attn_type is not None:
|
||||
assert model_config.attn_type == model_info.attn_type
|
||||
if model_info.is_prefix_caching_supported is not None:
|
||||
assert (
|
||||
model_config.is_prefix_caching_supported
|
||||
== model_info.is_prefix_caching_supported
|
||||
)
|
||||
if model_info.is_chunked_prefill_supported is not None:
|
||||
assert (
|
||||
model_config.is_chunked_prefill_supported
|
||||
== model_info.is_chunked_prefill_supported
|
||||
)
|
||||
|
||||
vllm_main_score = run_mteb_embed_task(
|
||||
VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS
|
||||
)
|
||||
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
# Test embedding_size, isnan and whether to use normalize.
|
||||
# Apply the same prompt_prefix used for scoring so this check compares
|
||||
# identical effective inputs: HF's SentenceTransformer.encode() applies
|
||||
# the model's default prompt (e.g. "Document: "), so vLLM must receive it
|
||||
# too. This mirrors VllmMtebEncoder and changes no production behavior.
|
||||
consistency_prompts = (
|
||||
[prompt_prefix + p for p in example_prompts]
|
||||
if prompt_prefix
|
||||
else example_prompts
|
||||
)
|
||||
vllm_outputs = vllm_model.embed(
|
||||
consistency_prompts,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=-1),
|
||||
)
|
||||
outputs_tensor = torch.tensor(vllm_outputs)
|
||||
assert not torch.any(torch.isnan(outputs_tensor))
|
||||
embedding_size = model_config.embedding_size
|
||||
assert torch.tensor(vllm_outputs).shape[-1] == embedding_size
|
||||
|
||||
# Accelerate mteb test by setting
|
||||
# SentenceTransformers mteb score to a constant
|
||||
if model_info.mteb_score is None:
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
is_sentence_transformer=True,
|
||||
dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype,
|
||||
) as hf_model:
|
||||
# e.g. setting default parameters for the encode method of hf_runner
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
st_main_score = run_mteb_embed_task(
|
||||
HfMtebEncoder(hf_model), MTEB_EMBED_TASKS
|
||||
)
|
||||
st_dtype = next(hf_model.model.parameters()).dtype
|
||||
|
||||
# Check embeddings close to hf outputs
|
||||
hf_outputs = hf_model.encode(example_prompts)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
else:
|
||||
st_main_score = model_info.mteb_score
|
||||
st_dtype = "Constant"
|
||||
|
||||
print("Model:", model_info.name)
|
||||
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
|
||||
print("SentenceTransformers:", st_dtype, st_main_score)
|
||||
print("Difference:", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
diff = st_main_score - vllm_main_score
|
||||
assert diff < atol, (
|
||||
f"diff={diff:.6g} tol={atol} model={model_info.name} "
|
||||
f"(st={st_main_score}, vllm={vllm_main_score})"
|
||||
)
|
||||
@@ -1,319 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import requests
|
||||
import torch
|
||||
from mteb.models import ModelMeta
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import (
|
||||
RerankModelInfo,
|
||||
get_vllm_extra_kwargs,
|
||||
)
|
||||
|
||||
# See #19344
|
||||
MTEB_RERANK_TASKS = ["NFCorpus"]
|
||||
MTEB_RERANK_LANGS = ["eng"]
|
||||
MTEB_RERANK_TOL = 2e-3
|
||||
|
||||
template_home = (
|
||||
Path(__file__).parent.parent.parent.parent.parent
|
||||
/ "examples/pooling/score/template"
|
||||
)
|
||||
|
||||
_empty_model_meta = ModelMeta(
|
||||
loader=None,
|
||||
name="vllm/model",
|
||||
revision="1",
|
||||
release_date=None,
|
||||
languages=None,
|
||||
framework=[],
|
||||
similarity_fn_name=None,
|
||||
n_parameters=None,
|
||||
memory_usage_mb=None,
|
||||
max_tokens=None,
|
||||
embed_dim=None,
|
||||
license=None,
|
||||
open_weights=None,
|
||||
public_training_code=None,
|
||||
public_training_data=None,
|
||||
use_instructions=None,
|
||||
training_datasets=None,
|
||||
modalities=["text"], # 'image' can be added to evaluate multimodal models
|
||||
)
|
||||
|
||||
|
||||
class MtebCrossEncoderMixin(mteb.CrossEncoderProtocol):
|
||||
mteb_model_meta = _empty_model_meta
|
||||
|
||||
|
||||
class VllmMtebCrossEncoder(MtebCrossEncoderMixin):
|
||||
def __init__(self, vllm_model):
|
||||
self.llm = vllm_model
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
self.chat_template: str | None = getattr(vllm_model, "chat_template", None)
|
||||
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
r = self.rng.permutation(len(queries))
|
||||
queries = [queries[i] for i in r]
|
||||
corpus = [corpus[i] for i in r]
|
||||
|
||||
outputs = self.llm.score(
|
||||
queries,
|
||||
corpus,
|
||||
use_tqdm=False,
|
||||
chat_template=self.chat_template,
|
||||
tokenization_kwargs={"truncate_prompt_tokens": -1},
|
||||
)
|
||||
scores = np.array(outputs)
|
||||
scores = scores[np.argsort(r)]
|
||||
return scores
|
||||
|
||||
|
||||
class ScoreClientMtebEncoder(MtebCrossEncoderMixin):
|
||||
mteb_model_meta = _empty_model_meta
|
||||
|
||||
def __init__(self, model_name: str, url):
|
||||
self.model_name = model_name
|
||||
self.url = url
|
||||
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
full_corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
outputs = []
|
||||
for query, corpus in zip(queries, full_corpus):
|
||||
outputs.append(self.get_score(query, corpus))
|
||||
|
||||
scores = np.array(outputs)
|
||||
return scores
|
||||
|
||||
def get_score(self, query, corpus):
|
||||
response = requests.post(
|
||||
self.url,
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"queries": query,
|
||||
"documents": corpus,
|
||||
"truncate_prompt_tokens": -1,
|
||||
},
|
||||
).json()
|
||||
return response["data"][0]["score"]
|
||||
|
||||
|
||||
class RerankClientMtebEncoder(ScoreClientMtebEncoder):
|
||||
def get_score(self, query, corpus):
|
||||
response = requests.post(
|
||||
self.url,
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"query": query,
|
||||
"documents": [corpus],
|
||||
"truncate_prompt_tokens": -1,
|
||||
},
|
||||
).json()
|
||||
return response["results"][0]["relevance_score"]
|
||||
|
||||
|
||||
class HFMtebCrossEncoder(MtebCrossEncoderMixin, HfRunner):
|
||||
chat_template: str | None = None
|
||||
|
||||
def __init__(self, model_name: str, dtype: str = "auto", **kwargs: Any) -> None:
|
||||
HfRunner.__init__(
|
||||
self, model_name=model_name, is_cross_encoder=True, dtype=dtype, **kwargs
|
||||
)
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
if self.chat_template is not None:
|
||||
tokenizer = self.model.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
outputs_list = HfRunner.classify(self, prompts)
|
||||
scores = np.array(outputs_list).squeeze(-1)
|
||||
return scores
|
||||
else:
|
||||
prompts = list(zip(queries, corpus))
|
||||
outputs_tensor = HfRunner.predict(self, prompts, show_progress_bar=False)
|
||||
return outputs_tensor.cpu().numpy()
|
||||
|
||||
|
||||
def run_mteb_rerank(cross_encoder: mteb.CrossEncoderProtocol, tasks, languages):
|
||||
with tempfile.TemporaryDirectory() as prediction_folder:
|
||||
bm25s = mteb.get_model("bm25s")
|
||||
eval_splits = ["test"]
|
||||
|
||||
mteb_tasks: list[mteb.abstasks.AbsTaskRetrieval] = mteb.get_tasks(
|
||||
tasks=tasks, languages=languages, eval_splits=eval_splits
|
||||
)
|
||||
for task in mteb_tasks:
|
||||
if not task.data_loaded:
|
||||
task.load_data()
|
||||
|
||||
mteb.evaluate(
|
||||
bm25s,
|
||||
mteb_tasks,
|
||||
prediction_folder=prediction_folder,
|
||||
show_progress_bar=False,
|
||||
# don't save results for test runs
|
||||
cache=None,
|
||||
overwrite_strategy="always",
|
||||
)
|
||||
|
||||
second_stage_tasks = []
|
||||
for task in mteb_tasks:
|
||||
second_stage_tasks.append(
|
||||
task.convert_to_reranking(
|
||||
prediction_folder,
|
||||
top_k=10,
|
||||
)
|
||||
)
|
||||
|
||||
results = mteb.evaluate(
|
||||
cross_encoder,
|
||||
second_stage_tasks,
|
||||
show_progress_bar=False,
|
||||
cache=None,
|
||||
)
|
||||
main_score = results[0].scores["test"][0]["main_score"]
|
||||
return main_score
|
||||
|
||||
|
||||
def mteb_test_rerank_models(
|
||||
vllm_runner,
|
||||
model_info: RerankModelInfo,
|
||||
hf_runner=HFMtebCrossEncoder,
|
||||
vllm_extra_kwargs=None,
|
||||
vllm_mteb_encoder=VllmMtebCrossEncoder,
|
||||
atol: float | None = None,
|
||||
):
|
||||
if atol is None:
|
||||
atol = (
|
||||
model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_RERANK_TOL
|
||||
)
|
||||
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
|
||||
|
||||
# Maybe load chat_template.
|
||||
chat_template: str | None = None
|
||||
if model_info.chat_template_name is not None:
|
||||
chat_template = (template_home / model_info.chat_template_name).read_text()
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
max_num_seqs=8,
|
||||
**vllm_extra_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
vllm_model.chat_template = chat_template
|
||||
|
||||
# Confirm whether vllm is using the correct architecture
|
||||
if model_info.architecture:
|
||||
assert model_info.architecture in model_config.architectures
|
||||
|
||||
# Score API is only enabled for num_labels == 1
|
||||
assert model_config.hf_config.num_labels == 1
|
||||
|
||||
# Confirm whether the important configs in model_config are correct.
|
||||
pooler_config = model_config.pooler_config
|
||||
if model_info.seq_pooling_type is not None:
|
||||
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
|
||||
if model_info.tok_pooling_type is not None:
|
||||
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
|
||||
if model_info.attn_type is not None:
|
||||
assert model_config.attn_type == model_info.attn_type
|
||||
if model_info.is_prefix_caching_supported is not None:
|
||||
assert (
|
||||
model_config.is_prefix_caching_supported
|
||||
== model_info.is_prefix_caching_supported
|
||||
)
|
||||
if model_info.is_chunked_prefill_supported is not None:
|
||||
assert (
|
||||
model_config.is_chunked_prefill_supported
|
||||
== model_info.is_chunked_prefill_supported
|
||||
)
|
||||
|
||||
vllm_main_score = run_mteb_rerank(
|
||||
vllm_mteb_encoder(vllm_model),
|
||||
tasks=MTEB_RERANK_TASKS,
|
||||
languages=MTEB_RERANK_LANGS,
|
||||
)
|
||||
vllm_dtype = model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
# Accelerate mteb test by setting
|
||||
# SentenceTransformers mteb score to a constant
|
||||
if model_info.mteb_score is None:
|
||||
with hf_runner(
|
||||
model_info.name, revision=model_info.revision, dtype=model_info.hf_dtype
|
||||
) as hf_model:
|
||||
hf_model.chat_template = chat_template
|
||||
st_main_score = run_mteb_rerank(
|
||||
hf_model,
|
||||
tasks=MTEB_RERANK_TASKS,
|
||||
languages=MTEB_RERANK_LANGS,
|
||||
)
|
||||
st_dtype = next(hf_model.model.model.parameters()).dtype
|
||||
else:
|
||||
st_main_score = model_info.mteb_score
|
||||
st_dtype = "Constant"
|
||||
|
||||
print("Model:", model_info.name)
|
||||
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
|
||||
print("SentenceTransformers:", st_dtype, st_main_score)
|
||||
print("Difference:", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
diff = st_main_score - vllm_main_score
|
||||
assert diff < atol, (
|
||||
f"diff={diff:.6g} tol={atol} model={model_info.name} "
|
||||
f"(st={st_main_score}, vllm={vllm_main_score})"
|
||||
)
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
RerankModelInfo,
|
||||
)
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
from .mteb_score_utils import mteb_test_rerank_models
|
||||
|
||||
MODELS = [
|
||||
########## BertModel
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-base-en",
|
||||
architecture="BertModel",
|
||||
mteb_score=0.779336792,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo("BAAI/bge-base-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-small-en", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-small-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-large-en", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-large-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-large-zh-noinstruct", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-base-en-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-base-zh-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-small-en-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-small-zh-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-large-en-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-large-zh-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
########## XLMRobertaModel
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-m3",
|
||||
architecture="XLMRobertaModel",
|
||||
mteb_score=0.787343078,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
########## Qwen2Model
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-code-v1",
|
||||
architecture="Qwen2Model",
|
||||
mteb_score=0.75724465,
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
# Skip: model's custom tokenizer on HF hub is incompatible with
|
||||
# transformers v5 (sets attrs before super().__init__, triggering
|
||||
# AttributeError on 'verbose' in __getattr__).
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
########## XLMRobertaForSequenceClassification
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-base",
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
mteb_score=0.32398,
|
||||
mteb_tol=1e-2,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-large",
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
enable_test=False,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-v2-m3",
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
|
||||
from .mteb_score_utils import (
|
||||
MtebCrossEncoderMixin,
|
||||
mteb_test_rerank_models,
|
||||
)
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-v2-gemma",
|
||||
architecture="GemmaForSequenceClassification",
|
||||
hf_overrides={
|
||||
"architectures": ["GemmaForSequenceClassification"],
|
||||
"classifier_from_token": ["Yes"],
|
||||
"method": "no_post_processing",
|
||||
},
|
||||
mteb_score=0.33757,
|
||||
mteb_tol=1e-2,
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
chat_template_name="bge-reranker-v2-gemma.jinja",
|
||||
),
|
||||
]
|
||||
|
||||
PROMPT = "Given a query A and a passage B, determine whether the passage contains an answer to the query by providing a prediction of either 'Yes' or 'No'." # noqa: E501
|
||||
|
||||
|
||||
class GemmaRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=model_name,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
|
||||
self.yes_loc = self.tokenizer.convert_tokens_to_ids("Yes")
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
def get_inputs(pairs, tokenizer, prompt=None):
|
||||
if prompt is None:
|
||||
prompt = PROMPT
|
||||
|
||||
sep = "\n"
|
||||
prompt_inputs = tokenizer(
|
||||
prompt, return_tensors=None, add_special_tokens=False
|
||||
)["input_ids"]
|
||||
sep_inputs = tokenizer(sep, return_tensors=None, add_special_tokens=False)[
|
||||
"input_ids"
|
||||
]
|
||||
inputs = []
|
||||
for query, passage in pairs:
|
||||
query_inputs = tokenizer(
|
||||
f"A: {query}",
|
||||
return_tensors=None,
|
||||
add_special_tokens=False,
|
||||
truncation=True,
|
||||
)
|
||||
passage_inputs = tokenizer(
|
||||
f"B: {passage}",
|
||||
return_tensors=None,
|
||||
add_special_tokens=False,
|
||||
truncation=True,
|
||||
)
|
||||
item = tokenizer.prepare_for_model(
|
||||
[tokenizer.bos_token_id] + query_inputs["input_ids"],
|
||||
sep_inputs + passage_inputs["input_ids"],
|
||||
truncation="only_second",
|
||||
padding=False,
|
||||
return_attention_mask=False,
|
||||
return_token_type_ids=False,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
item["input_ids"] = item["input_ids"] + sep_inputs + prompt_inputs
|
||||
item["attention_mask"] = [1] * len(item["input_ids"])
|
||||
inputs.append(item)
|
||||
return tokenizer.pad(
|
||||
inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
scores = []
|
||||
for query, document in zip(queries, corpus):
|
||||
pairs = [(query, document)]
|
||||
inputs = get_inputs(pairs, self.tokenizer)
|
||||
inputs = inputs.to(self.model.device)
|
||||
_n_tokens = inputs["input_ids"].shape[1]
|
||||
logits = self.model(**inputs, return_dict=True).logits
|
||||
_scores = (
|
||||
logits[:, -1, self.yes_loc]
|
||||
.view(
|
||||
-1,
|
||||
)
|
||||
.float()
|
||||
.sigmoid()
|
||||
)
|
||||
scores.append(_scores[0].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(
|
||||
vllm_runner,
|
||||
model_info,
|
||||
hf_runner=GemmaRerankerHfRunner,
|
||||
)
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import (
|
||||
RerankModelInfo,
|
||||
)
|
||||
|
||||
from .mteb_score_utils import mteb_test_rerank_models
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"cross-encoder/ms-marco-TinyBERT-L-2-v2",
|
||||
architecture="BertForSequenceClassification",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
mteb_score=0.32898,
|
||||
mteb_tol=1e-2,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
chat_template_name="qwen3_reranker.jinja",
|
||||
mteb_score=0.33459,
|
||||
mteb_tol=1e-2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
########## BertModel
|
||||
EmbedModelInfo(
|
||||
"intfloat/e5-small",
|
||||
architecture="BertModel",
|
||||
mteb_score=0.742285423,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo("intfloat/e5-base", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("intfloat/e5-large", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-small", architecture="BertModel", enable_test=False
|
||||
),
|
||||
########## XLMRobertaModel
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-base",
|
||||
architecture="XLMRobertaModel",
|
||||
mteb_score=0.779325955,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-large",
|
||||
architecture="XLMRobertaModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-large-instruct",
|
||||
architecture="XLMRobertaModel",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
@@ -1,111 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
|
||||
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
|
||||
|
||||
mxbai_rerank_hf_overrides = {
|
||||
"architectures": ["Qwen2ForSequenceClassification"],
|
||||
"classifier_from_token": ["0", "1"],
|
||||
"method": "from_2_way_softmax",
|
||||
}
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"mixedbread-ai/mxbai-rerank-base-v2",
|
||||
architecture="Qwen2ForSequenceClassification",
|
||||
hf_overrides=mxbai_rerank_hf_overrides,
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
chat_template_name="mxbai_rerank_v2.jinja",
|
||||
mteb_score=0.33651,
|
||||
mteb_tol=1e-2,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"mixedbread-ai/mxbai-rerank-large-v2",
|
||||
architecture="Qwen2ForSequenceClassification",
|
||||
hf_overrides=mxbai_rerank_hf_overrides,
|
||||
chat_template_name="mxbai_rerank_v2.jinja",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class MxbaiRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=model_name,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
|
||||
self.yes_loc = self.tokenizer.convert_tokens_to_ids("1")
|
||||
self.no_loc = self.tokenizer.convert_tokens_to_ids("0")
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
tokenizer = self.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
|
||||
def compute_logits(inputs):
|
||||
logits = self.model(**inputs).logits[:, -1, :]
|
||||
yes_logits = logits[:, self.yes_loc]
|
||||
no_logits = logits[:, self.no_loc]
|
||||
logits = yes_logits - no_logits
|
||||
scores = logits.float().sigmoid()
|
||||
return scores
|
||||
|
||||
scores = []
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = self.wrap_device(inputs)
|
||||
score = compute_logits(inputs)
|
||||
scores.append(score[0].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=MxbaiRerankerHfRunner)
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling_mteb_test.mteb_embed_utils import (
|
||||
mteb_test_embed_models,
|
||||
)
|
||||
from tests.models.language.pooling_mteb_test.mteb_score_utils import (
|
||||
mteb_test_rerank_models,
|
||||
)
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
RerankModelInfo,
|
||||
)
|
||||
|
||||
EMBEDDING_MODELS = [
|
||||
EmbedModelInfo(
|
||||
"nvidia/llama-nemotron-embed-1b-v2",
|
||||
architecture="LlamaBidirectionalModel",
|
||||
mteb_score=0.689164662128673,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
)
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"nvidia/llama-nemotron-rerank-1b-v2",
|
||||
architecture="LlamaBidirectionalForSequenceClassification",
|
||||
chat_template_name="nemotron-rerank.jinja",
|
||||
mteb_score=0.33994,
|
||||
mteb_tol=1e-2,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
@@ -1,59 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v1",
|
||||
architecture="NomicBertModel",
|
||||
# Fixme:
|
||||
# Update nomic-embed code to support the latest
|
||||
# HF version and remove revision set.
|
||||
revision="720244025c1a7e15661a174c63cce63c8218e52b",
|
||||
mteb_score=0.737568559,
|
||||
mteb_tol=2e-3,
|
||||
enable_test=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v1.5",
|
||||
architecture="NomicBertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/CodeRankEmbed", architecture="NomicBertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v2-moe",
|
||||
architecture="NomicBertModel",
|
||||
mteb_score=0.715488912,
|
||||
mteb_tol=2e-3,
|
||||
enable_test=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
@@ -1,133 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
from tests.utils import multi_gpu_test
|
||||
|
||||
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
|
||||
|
||||
qwen3_reranker_hf_overrides = {
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"Qwen/Qwen3-Reranker-0.6B",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
hf_overrides=qwen3_reranker_hf_overrides,
|
||||
chat_template_name="qwen3_reranker.jinja",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
mteb_score=0.33459,
|
||||
mteb_tol=1e-2,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"Qwen/Qwen3-Reranker-4B",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
chat_template_name="qwen3_reranker.jinja",
|
||||
hf_overrides=qwen3_reranker_hf_overrides,
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class Qwen3RerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=model_name,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
|
||||
self.token_false_id = self.tokenizer.convert_tokens_to_ids("no")
|
||||
self.token_true_id = self.tokenizer.convert_tokens_to_ids("yes")
|
||||
self.max_length = 40960
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
tokenizer = self.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
|
||||
def compute_logits(inputs):
|
||||
batch_scores = self.model(**inputs).logits[:, -1, :]
|
||||
true_vector = batch_scores[:, self.token_true_id]
|
||||
false_vector = batch_scores[:, self.token_false_id]
|
||||
batch_scores = torch.stack([false_vector, true_vector], dim=1)
|
||||
batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
|
||||
scores = batch_scores[:, 1].exp()
|
||||
return scores
|
||||
|
||||
scores = []
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = self.wrap_device(inputs)
|
||||
score = compute_logits(inputs)
|
||||
scores.append(score[0].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=Qwen3RerankerHfRunner)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_rerank_models_mteb_tp(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
assert model_info.architecture == "Qwen3ForSequenceClassification"
|
||||
|
||||
vllm_extra_kwargs: dict[str, Any] = {
|
||||
"tensor_parallel_size": 2,
|
||||
}
|
||||
|
||||
mteb_test_rerank_models(
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs=vllm_extra_kwargs,
|
||||
hf_runner=Qwen3RerankerHfRunner,
|
||||
)
|
||||
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-xs",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
mteb_score=0.714927797,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-s",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-long",
|
||||
is_matryoshka=False,
|
||||
architecture="NomicBertModel",
|
||||
mteb_score=0.681146831,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-l",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-v1.5",
|
||||
is_matryoshka=True,
|
||||
architecture="BertModel",
|
||||
mteb_score=0.649088363,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-l-v2.0",
|
||||
is_matryoshka=True,
|
||||
architecture="XLMRobertaModel",
|
||||
mteb_score=0.712258299,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-v2.0",
|
||||
is_matryoshka=True,
|
||||
architecture="GteModel",
|
||||
mteb_score=0.706622444,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
)
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
# ST models with projector (Dense) layers
|
||||
ST_PROJECTOR_MODELS = [
|
||||
EmbedModelInfo(
|
||||
"TencentBAC/Conan-embedding-v1",
|
||||
architecture="BertModel",
|
||||
mteb_score=0.688611955,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"google/embeddinggemma-300m",
|
||||
architecture="Gemma3TextModel",
|
||||
mteb_score=0.7473819294684156,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"voyageai/voyage-4-nano",
|
||||
architecture="VoyageQwen3BidirectionalEmbedModel",
|
||||
enable_test=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
hf_overrides={
|
||||
"architectures": ["VoyageQwen3BidirectionalEmbedModel"],
|
||||
"num_labels": 2048,
|
||||
},
|
||||
mteb_score=0.7054,
|
||||
mteb_tol=2e-3,
|
||||
# === MTEB Results ===
|
||||
# STS12: 0.6613
|
||||
# STS13: 0.6906
|
||||
# STS14: 0.6556
|
||||
# STS15: 0.7843
|
||||
# STS16: 0.7340
|
||||
# STSBenchmark: 0.7063
|
||||
# Average score: 0.7054
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
# Encoder-only attention models need enforce_eager=True to avoid
|
||||
# CUDA graph capture issues with piecewise compilation
|
||||
mteb_test_embed_models(
|
||||
hf_runner, vllm_runner, model_info, vllm_extra_kwargs={"enforce_eager": True}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
example_prompts,
|
||||
vllm_extra_kwargs={"enforce_eager": True},
|
||||
)
|
||||
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
|
||||
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"polaria-tech/zerank-2-reranker-vllm",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
chat_template_name="zerank2.jinja",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
mteb_score=0.34305,
|
||||
mteb_tol=1e-2,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class ZerankHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
# The converted checkpoint has no lm_head, so the reference is the original
|
||||
# causal LM: the logit of the "Yes" token at the last position. The converted
|
||||
# checkpoint returns sigmoid(logit / 5), which ranks documents identically.
|
||||
original_model = "zeroentropy/zerank-2-reranker"
|
||||
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=self.original_model,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.original_model, padding_side="left"
|
||||
)
|
||||
self.token_true_id = self.tokenizer.convert_tokens_to_ids("Yes")
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
tokenizer = self.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
|
||||
scores = []
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = self.wrap_device(inputs)
|
||||
logits = self.model(**inputs).logits[:, -1, :]
|
||||
scores.append(logits[0, self.token_true_id].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=ZerankHfRunner)
|
||||
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
MODELS = {
|
||||
"Gemma3nForConditionalGeneration": "google/gemma-3n-E2B-it",
|
||||
"Gemma4ForConditionalGeneration": "google/gemma-4-E2B-it",
|
||||
}
|
||||
|
||||
# Seconds. Spread wide enough that no two clips can produce the same number of
|
||||
# mel frames: equal-length clips stack normally and never reach the ragged path.
|
||||
CLIP_DURATIONS = (1.0, 3.5, 7.0, 12.0)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ragged_clips():
|
||||
"""One asset sliced to several durations, so only length varies."""
|
||||
audio, sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
clips = []
|
||||
for duration in CLIP_DURATIONS:
|
||||
num_samples = int(duration * sample_rate)
|
||||
if num_samples > len(audio):
|
||||
pytest.skip(f"asset is shorter than {duration}s")
|
||||
clips.append((audio[:num_samples], sample_rate))
|
||||
|
||||
assert len({len(clip) for clip, _ in clips}) == len(clips)
|
||||
return clips
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("arch", sorted(MODELS))
|
||||
@pytest.mark.parametrize("max_tokens", [8])
|
||||
def test_variable_length_audio_batching(
|
||||
vllm_runner,
|
||||
ragged_clips,
|
||||
arch: str,
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Test batching of requests with different audio durations.
|
||||
|
||||
Audio features are unpadded per item so a multimodal cache entry does not
|
||||
depend on the batch it was first processed in, and
|
||||
`MultiModalFieldConfig.batched` stacks items only when their shapes agree.
|
||||
A batch of differently sized clips therefore reaches the model as a list,
|
||||
which used to take EngineCore down with
|
||||
|
||||
AttributeError: 'list' object has no attribute 'squeeze'
|
||||
|
||||
failing every in-flight request with EngineDeadError.
|
||||
"""
|
||||
from transformers import AutoProcessor
|
||||
|
||||
model_name = MODELS[arch]
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(arch)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# The two architectures spell the audio placeholder differently; ask the
|
||||
# processor rather than hardcoding either.
|
||||
processor = AutoProcessor.from_pretrained(model_name)
|
||||
prompt = processor.apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "audio", "audio": None},
|
||||
{"type": "text", "text": "Transcribe this audio."},
|
||||
],
|
||||
}
|
||||
],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
) as vllm_model:
|
||||
# One call so the clips are scheduled into the same forward pass;
|
||||
# issued separately they never reach the ragged path.
|
||||
outputs = vllm_model.generate_greedy(
|
||||
[prompt] * len(ragged_clips),
|
||||
max_tokens,
|
||||
audios=[[clip] for clip in ragged_clips],
|
||||
)
|
||||
|
||||
assert len(outputs) == len(ragged_clips)
|
||||
for output in outputs:
|
||||
assert len(output[1]) > 0, "Expected non-empty output"
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
|
||||
|
||||
|
||||
def base_prompt(modalities_str: str) -> str:
|
||||
return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n" # noqa: E501
|
||||
|
||||
|
||||
INTERLEAVED_PROMPT = base_prompt("<image><video><image>\n")
|
||||
NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", ["float16"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
|
||||
"""This is a simple test to check if interleaved and non-interleaved prompts
|
||||
give the same result.
|
||||
"""
|
||||
image_cherry = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
image_stop = convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB")
|
||||
images = [image_cherry, image_stop]
|
||||
video = VideoAsset(name="baby_reading", num_frames=16).np_ndarrays
|
||||
|
||||
inputs = [
|
||||
(
|
||||
[INTERLEAVED_PROMPT],
|
||||
[images],
|
||||
[video],
|
||||
),
|
||||
(
|
||||
[NONINTERLEAVED_PROMPT],
|
||||
[images],
|
||||
[video],
|
||||
),
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
max_model_len=32768,
|
||||
max_num_seqs=2,
|
||||
tensor_parallel_size=1,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy(
|
||||
prompts, max_tokens, images=images, videos=videos
|
||||
)
|
||||
for prompts, images, videos in inputs
|
||||
]
|
||||
|
||||
all_results = [output[0][1] for output in vllm_outputs_per_case]
|
||||
outputs = [
|
||||
(total_str, total_str.find("assistant\n") + len("assistant\n"))
|
||||
for total_str in all_results
|
||||
]
|
||||
prompt_lengths = [prompt_len for _, prompt_len in outputs]
|
||||
generated_strs = [total_str[prompt_len:] for total_str, prompt_len in outputs]
|
||||
interleaved_prompt_len, noninterleaved_prompt_len = prompt_lengths
|
||||
interleaved_output_str, noninterleaved_output_str = generated_strs
|
||||
|
||||
# The two prompts are identical except for the order of modality tokens.
|
||||
assert interleaved_prompt_len == noninterleaved_prompt_len
|
||||
|
||||
# The two generated strings should be different because of the
|
||||
# interleaved modality tokens.
|
||||
assert interleaved_output_str != noninterleaved_output_str
|
||||
@@ -1,296 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.image import convert_image_mode, rescale_image_size
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
from ....conftest import (
|
||||
IMAGE_ASSETS,
|
||||
HfRunner,
|
||||
PromptAudioInput,
|
||||
PromptImageInput,
|
||||
VllmRunner,
|
||||
)
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
"cherry_blossom": "<|user|>\n<|image_1|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
}
|
||||
)
|
||||
HF_MULTIIMAGE_IMAGE_PROMPT = (
|
||||
"<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
|
||||
)
|
||||
|
||||
model_path = hf_api().snapshot_download("microsoft/Phi-4-multimodal-instruct")
|
||||
# Since the vision-lora and speech-lora co-exist with the base model,
|
||||
# we have to manually specify the path of the lora weights.
|
||||
vision_lora_path = os.path.join(model_path, "vision-lora")
|
||||
speech_question = os.path.join(
|
||||
model_path, "examples", "what_is_shown_in_this_image.wav"
|
||||
)
|
||||
models = [model_path]
|
||||
|
||||
|
||||
def vllm_to_hf_output(
|
||||
vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
|
||||
):
|
||||
"""Sanitize vllm output to be comparable with hf output."""
|
||||
_, output_str, out_logprobs = vllm_output
|
||||
|
||||
output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
|
||||
assert output_str_without_image[0] == " "
|
||||
output_str_without_image = output_str_without_image[1:]
|
||||
|
||||
hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model)
|
||||
hf_output_ids = tokenizer.encode(output_str_without_image)
|
||||
assert hf_output_ids[0] == 1
|
||||
hf_output_ids = hf_output_ids[1:]
|
||||
|
||||
return hf_output_ids, hf_output_str, out_logprobs
|
||||
|
||||
|
||||
target_dtype = "half"
|
||||
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||
(1.0,),
|
||||
(1.0, 1.0, 1.0),
|
||||
(0.25, 0.5, 1.0),
|
||||
)
|
||||
|
||||
|
||||
def run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]],
|
||||
model: str,
|
||||
*,
|
||||
max_model_len: int,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
mm_limit: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
):
|
||||
"""Inference result should be the same between hf and vllm.
|
||||
|
||||
All the image fixtures for the test are from IMAGE_ASSETS.
|
||||
For huggingface runner, we provide the PIL images as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding MultiModalConfig as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
# max_model_len should be greater than image_feature_size
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=2,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"image": mm_limit},
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enable_lora=True,
|
||||
max_lora_rank=320,
|
||||
gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
lora_request = LoRARequest("vision", 1, vision_lora_path)
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
audios=audios,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
for prompts, images, audios in inputs
|
||||
]
|
||||
|
||||
# This error occurs inside `get_peft_model`
|
||||
# FIXME: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/75
|
||||
pytest.skip("HF impl is not compatible with current transformers")
|
||||
|
||||
hf_model_kwargs = {"_attn_implementation": "sdpa"}
|
||||
with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model:
|
||||
hf_processor = hf_model.processor
|
||||
eos_token_id = hf_processor.tokenizer.eos_token_id
|
||||
|
||||
def patch_hf_processor(
|
||||
*args, text="", images=None, audio=None, sampling_rate=None, **kwargs
|
||||
):
|
||||
audios = None
|
||||
if audio is not None and sampling_rate is not None:
|
||||
audios = [(audio, sampling_rate)]
|
||||
return hf_processor(
|
||||
*args, text=text, images=images, audios=audios, **kwargs
|
||||
)
|
||||
|
||||
hf_model.processor = patch_hf_processor
|
||||
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
audios=audios,
|
||||
eos_token_id=eos_token_id,
|
||||
num_logits_to_keep=0,
|
||||
)
|
||||
for prompts, images, audios in inputs
|
||||
]
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_model_len", [12800])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_image = [
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, factor) for factor in size_factors],
|
||||
None,
|
||||
)
|
||||
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs_per_image,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
mm_limit=1,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=48)
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_model_len", [25600])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_multi_images_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_case = [
|
||||
(
|
||||
[HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
|
||||
[
|
||||
[rescale_image_size(image, factor) for image in images]
|
||||
for factor in size_factors
|
||||
],
|
||||
None,
|
||||
)
|
||||
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||
]
|
||||
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs_per_case,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
mm_limit=2,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_model_len", [12800])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_vision_speech_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
# use the example speech question so that the model outputs are reasonable
|
||||
audio = load_audio(speech_question, sr=None)
|
||||
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
|
||||
inputs_vision_speech = [
|
||||
(
|
||||
["<|user|><|image_1|><|audio_1|><|end|><|assistant|>"],
|
||||
[image],
|
||||
[audio],
|
||||
),
|
||||
]
|
||||
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs_vision_speech,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
mm_limit=1,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
@@ -1,229 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from ....conftest import AUDIO_ASSETS, AudioTestAssets, VllmRunner
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
|
||||
MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
|
||||
|
||||
AUDIO_PROMPTS = AUDIO_ASSETS.prompts(
|
||||
{
|
||||
"mary_had_lamb": "Transcribe this into English.",
|
||||
"winning_call": "What is happening in this audio clip?",
|
||||
}
|
||||
)
|
||||
|
||||
MULTI_AUDIO_PROMPT = "Describe each of the audios above."
|
||||
|
||||
AudioTuple = tuple[np.ndarray, int]
|
||||
|
||||
VLLM_PLACEHOLDER = "<|audio|>"
|
||||
HF_PLACEHOLDER = "<|audio|>"
|
||||
|
||||
CHUNKED_PREFILL_KWARGS = {
|
||||
"enable_chunked_prefill": True,
|
||||
"max_num_seqs": 2,
|
||||
# Use a very small limit to exercise chunked prefill.
|
||||
"max_num_batched_tokens": 16,
|
||||
}
|
||||
|
||||
|
||||
def params_kwargs_to_cli_args(params_kwargs: dict[str, Any]) -> list[str]:
|
||||
"""Convert kwargs to CLI args."""
|
||||
args = []
|
||||
for key, value in params_kwargs.items():
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
args.append(f"--{key.replace('_', '-')}")
|
||||
else:
|
||||
args.append(f"--{key.replace('_', '-')}={value}")
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
params=[
|
||||
pytest.param({}, marks=pytest.mark.cpu_model),
|
||||
pytest.param(CHUNKED_PREFILL_KWARGS),
|
||||
]
|
||||
)
|
||||
def server(request, audio_assets: AudioTestAssets):
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": len(audio_assets)}),
|
||||
"--trust-remote-code",
|
||||
] + params_kwargs_to_cli_args(request.param)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, args, env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"}
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
def _get_prompt(audio_count, question, placeholder):
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
placeholder = f"{placeholder}\n" * audio_count
|
||||
|
||||
return tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": f"{placeholder}{question}"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
|
||||
def run_multi_audio_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
prompts_and_audios: list[tuple[str, list[AudioTuple]]],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
**kwargs,
|
||||
):
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={
|
||||
"audio": max((len(audio) for _, audio in prompts_and_audios))
|
||||
},
|
||||
**kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[prompt for prompt, _ in prompts_and_audios],
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=[audios for _, audios in prompts_and_audios],
|
||||
)
|
||||
|
||||
# The HuggingFace model doesn't support multiple audios yet, so
|
||||
# just assert that some tokens were generated.
|
||||
assert all(tokens for tokens, *_ in vllm_outputs)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize(
|
||||
"vllm_kwargs",
|
||||
[
|
||||
pytest.param({}, marks=pytest.mark.cpu_model),
|
||||
pytest.param(CHUNKED_PREFILL_KWARGS),
|
||||
],
|
||||
)
|
||||
def test_models_with_multiple_audios(
|
||||
vllm_runner,
|
||||
audio_assets: AudioTestAssets,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
vllm_kwargs: dict,
|
||||
) -> None:
|
||||
vllm_prompt = _get_prompt(len(audio_assets), MULTI_AUDIO_PROMPT, VLLM_PLACEHOLDER)
|
||||
run_multi_audio_test(
|
||||
vllm_runner,
|
||||
[(vllm_prompt, [audio.audio_and_sample_rate for audio in audio_assets])],
|
||||
MODEL_NAME,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
**vllm_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
def test_variable_length_audio_batching(
|
||||
vllm_runner,
|
||||
audio_assets: AudioTestAssets,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Test batching of requests with different audio durations.
|
||||
|
||||
This exercises the variable-length tensor handling in
|
||||
MultiModalFlatField._reduce_data() which was buggy before
|
||||
https://github.com/vllm-project/vllm/issues/31658 was fixed.
|
||||
"""
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# Create prompts with single audio each (different durations)
|
||||
prompts_and_audios = []
|
||||
for audio, question in zip(audio_assets, AUDIO_PROMPTS):
|
||||
prompt = _get_prompt(1, question, VLLM_PLACEHOLDER)
|
||||
prompts_and_audios.append((prompt, [audio.audio_and_sample_rate]))
|
||||
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
) as vllm_model:
|
||||
# Generate for all prompts in a single batch
|
||||
# This triggers the variable-length batching code path
|
||||
outputs = vllm_model.generate_greedy(
|
||||
[prompt for prompt, _ in prompts_and_audios],
|
||||
max_tokens,
|
||||
audios=[audios for _, audios in prompts_and_audios],
|
||||
)
|
||||
|
||||
# Verify outputs were generated for each request
|
||||
assert len(outputs) == len(prompts_and_audios)
|
||||
for output in outputs:
|
||||
assert len(output[1]) > 0, "Expected non-empty output"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_serving(client, audio_assets: AudioTestAssets):
|
||||
"""Exercises online serving with/without chunked prefill enabled."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*[
|
||||
{"type": "audio_url", "audio_url": {"url": audio.url}}
|
||||
for audio in audio_assets
|
||||
],
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"What's happening in these {len(audio_assets)} audio clips?", # noqa: E501
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=MODEL_NAME, messages=messages, max_tokens=10
|
||||
)
|
||||
|
||||
assert len(chat_completion.choices) == 1
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
@@ -1,204 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
|
||||
from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.tokens.tokenizers.audio import Audio
|
||||
from transformers import VoxtralForConditionalGeneration
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
from ....conftest import AudioTestAssets
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from ...utils import check_logprobs_close
|
||||
from .test_ultravox import MULTI_AUDIO_PROMPT, run_multi_audio_test
|
||||
from .vlm_utils import model_utils
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-3B-2507"
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
"mistral",
|
||||
"--config_format",
|
||||
"mistral",
|
||||
"--load_format",
|
||||
"mistral",
|
||||
]
|
||||
|
||||
|
||||
def _get_prompt(audio_assets: AudioTestAssets, question: str) -> list[int]:
|
||||
"""Build a token-ID prompt via mistral_common for vLLM offline inference."""
|
||||
tokenizer = MistralTokenizer.from_pretrained(MODEL_NAME)
|
||||
|
||||
audios = [
|
||||
Audio.from_file(str(asset.get_local_path()), strict=False)
|
||||
for asset in audio_assets
|
||||
]
|
||||
audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
|
||||
|
||||
messages = [
|
||||
UserMessage(content=[*audio_chunks, TextChunk(text=question)]).to_openai()
|
||||
]
|
||||
return tokenizer.apply_chat_template(messages=messages)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models_with_multiple_audios(
|
||||
vllm_runner,
|
||||
audio_assets: AudioTestAssets,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
vllm_prompt = _get_prompt(audio_assets, MULTI_AUDIO_PROMPT)
|
||||
run_multi_audio_test(
|
||||
vllm_runner,
|
||||
[(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])], # type: ignore[list-item]
|
||||
MODEL_NAME,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tokenizer_mode="mistral",
|
||||
)
|
||||
|
||||
|
||||
def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
|
||||
"""Two-layer accuracy and serving validation using Mistral format.
|
||||
|
||||
1. Offline vLLM greedy output (runs first to avoid CUDA fork issues
|
||||
with multiprocessing - see vlm_utils/core.py).
|
||||
2. Online OpenAI-compatible API output must match offline — validates
|
||||
that the serving path (chat template, audio encoding, tokenization)
|
||||
does not corrupt anything.
|
||||
|
||||
Steps run sequentially so each releases the GPU before the next starts.
|
||||
"""
|
||||
question = f"What's happening in these {len(audio_assets)} audio clips?"
|
||||
max_tokens = 10
|
||||
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
|
||||
|
||||
vllm_prompt = _get_prompt(audio_assets, question)
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
tokenizer_mode="mistral",
|
||||
config_format="mistral",
|
||||
load_format="mistral",
|
||||
limit_mm_per_prompt={"audio": len(audio_assets)},
|
||||
) as vllm_model:
|
||||
offline_outputs = vllm_model.generate_greedy(
|
||||
[vllm_prompt],
|
||||
max_tokens,
|
||||
audios=[audio_data],
|
||||
)
|
||||
|
||||
offline_text = offline_outputs[0][1]
|
||||
assert offline_text, "Offline vLLM inference produced empty output"
|
||||
|
||||
def _asset_to_openai_chunk(asset):
|
||||
audio = Audio.from_file(str(asset.get_local_path()), strict=False)
|
||||
audio.format = "wav"
|
||||
return AudioChunk.from_audio(audio).to_openai()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*[_asset_to_openai_chunk(a) for a in audio_assets],
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": len(audio_assets)}),
|
||||
*MISTRAL_FORMAT_ARGS,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
server_args,
|
||||
env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"},
|
||||
) as remote_server:
|
||||
client = remote_server.get_client()
|
||||
completion = client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
assert len(completion.choices) == 1
|
||||
choice = completion.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
assert choice.message.content == offline_text, (
|
||||
f"Online serving output does not match offline inference.\n"
|
||||
f" Online: {choice.message.content!r}\n"
|
||||
f" Offline: {offline_text!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_hf_reference(hf_runner, vllm_runner, audio_assets: AudioTestAssets):
|
||||
"""Compare vLLM Mistral-format output against HF Transformers reference.
|
||||
|
||||
Instead of requiring an exact text match (which is brittle across
|
||||
attention backends), we compare per-token logprobs using the standard
|
||||
check_logprobs_close helper: when tokens diverge at a position, each
|
||||
runner's chosen token must appear in the other's top-k logprobs.
|
||||
"""
|
||||
question = f"What's happening in these {len(audio_assets)} audio clips?"
|
||||
max_tokens = 10
|
||||
num_logprobs = 5
|
||||
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
|
||||
|
||||
vllm_prompt = _get_prompt(audio_assets, question)
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
tokenizer_mode="mistral",
|
||||
config_format="mistral",
|
||||
load_format="mistral",
|
||||
limit_mm_per_prompt={"audio": len(audio_assets)},
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[vllm_prompt],
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
audios=[audio_data],
|
||||
)
|
||||
assert vllm_outputs[0][1], "vLLM inference produced empty output"
|
||||
|
||||
with hf_runner(
|
||||
MODEL_NAME,
|
||||
dtype="half",
|
||||
auto_cls=VoxtralForConditionalGeneration,
|
||||
) as hf_model:
|
||||
hf_model = model_utils.voxtral_patch_hf_runner(hf_model)
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
[question],
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
audios=[audio_data],
|
||||
)
|
||||
assert hf_outputs[0][1], "HF Transformers produced empty output"
|
||||
|
||||
print(
|
||||
f"HF Reference Comparison\n"
|
||||
f" vLLM: {vllm_outputs[0][1]!r}\n"
|
||||
f" HF: {hf_outputs[0][1]!r}"
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=vllm_outputs,
|
||||
outputs_1_lst=hf_outputs,
|
||||
name_0="vllm",
|
||||
name_1="hf",
|
||||
)
|
||||
@@ -0,0 +1,207 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Core test implementation to be shared across modalities."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .....conftest import HfRunner, VllmRunner
|
||||
from ....registry import HF_EXAMPLE_MODELS
|
||||
from .types import PromptWithMultiModalInput, RunnerOutput
|
||||
|
||||
|
||||
def run_test(
|
||||
*,
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: list[PromptWithMultiModalInput],
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
enforce_eager: bool,
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
|
||||
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
|
||||
auto_cls: type[_BaseAutoModelClass],
|
||||
use_tokenizer_eos: bool,
|
||||
comparator: Callable[..., None],
|
||||
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None,
|
||||
stop_str: list[str] | None,
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
vllm_runner_kwargs: dict[str, Any] | None,
|
||||
hf_model_kwargs: dict[str, Any] | None,
|
||||
hf_processor: Callable[[str], Any] | None,
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
|
||||
runner: RunnerOption = "auto",
|
||||
distributed_executor_backend: str | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
vllm_embeddings: torch.Tensor | None = None,
|
||||
):
|
||||
"""Modality agnostic test executor for comparing HF/vLLM outputs."""
|
||||
# In the case of embeddings, vLLM takes separate input tensors
|
||||
vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# Disable other modalities to save memory
|
||||
default_limits = {"image": 0, "video": 0, "audio": 0}
|
||||
limit_mm_per_prompt = default_limits | limit_mm_per_prompt
|
||||
|
||||
vllm_outputs_per_mm = []
|
||||
hf_outputs_per_mm = []
|
||||
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
vllm_runner_kwargs_: dict[str, Any] = {"mm_processor_cache_gb": 0}
|
||||
if model_info.tokenizer:
|
||||
vllm_runner_kwargs_["tokenizer_name"] = model_info.tokenizer
|
||||
if model_info.tokenizer_mode:
|
||||
vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
|
||||
if model_info.hf_overrides:
|
||||
vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
|
||||
if model_info.require_embed_inputs:
|
||||
for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
|
||||
vllm_runner_kwargs_[k] = model_info.require_embed_inputs
|
||||
if not model_info.enable_prefix_caching:
|
||||
vllm_runner_kwargs_["enable_prefix_caching"] = False
|
||||
|
||||
if vllm_runner_kwargs:
|
||||
vllm_runner_kwargs_.update(vllm_runner_kwargs)
|
||||
|
||||
# Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
|
||||
# already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
|
||||
if "limit_mm_per_prompt" in vllm_runner_kwargs_:
|
||||
limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=max_num_seqs,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=enforce_eager,
|
||||
runner=runner,
|
||||
**vllm_runner_kwargs_,
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
|
||||
vllm_kwargs: dict[str, Any] = {}
|
||||
if get_stop_token_ids is not None:
|
||||
vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
|
||||
if stop_str:
|
||||
vllm_kwargs["stop"] = stop_str
|
||||
|
||||
for prompts, image_data, video_data, audio_data in vllm_inputs:
|
||||
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
|
||||
vllm_kwargs_with_mm_data = vllm_kwargs | mm_data
|
||||
vllm_output = vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
**vllm_kwargs_with_mm_data,
|
||||
)
|
||||
vllm_outputs_per_mm.append(vllm_output)
|
||||
|
||||
hf_runner_kwargs: dict[str, Any] = {}
|
||||
if model_info.tokenizer:
|
||||
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
|
||||
if hf_processor is not None:
|
||||
hf_runner_kwargs["processor"] = hf_processor(model)
|
||||
|
||||
hf_model = hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=auto_cls,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
**hf_runner_kwargs,
|
||||
)
|
||||
|
||||
# Some models need to patch things like the model processor, e.g., internvl
|
||||
if patch_hf_runner is not None:
|
||||
hf_model = patch_hf_runner(hf_model)
|
||||
|
||||
with hf_model, torch.no_grad():
|
||||
tokenizer = hf_model.tokenizer
|
||||
|
||||
# Some models need to explicitly pass the eos_token_id off the tokenizer
|
||||
# or processor for a good comparison;
|
||||
# currently assume processor/tokenizer agree on the EOS, and pull it off
|
||||
# the tokenizer if requested.
|
||||
hf_kwargs = {}
|
||||
if use_tokenizer_eos:
|
||||
hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
|
||||
if stop_str:
|
||||
hf_kwargs["stop_strings"] = stop_str
|
||||
|
||||
for prompts, image_data, video_data, audio_data in inputs:
|
||||
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
|
||||
hf_kwargs_with_mm_data = hf_kwargs | mm_data
|
||||
hf_output = hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
**hf_kwargs_with_mm_data,
|
||||
)
|
||||
hf_outputs_per_mm.append(hf_output)
|
||||
|
||||
# Apply output processing / sanitation to the vLLM and HF runner results
|
||||
hf_outputs_per_mm, vllm_outputs_per_mm = process_runner_outputs(
|
||||
model,
|
||||
first_runner_outputs=hf_outputs_per_mm,
|
||||
second_runner_outputs=vllm_outputs_per_mm,
|
||||
first_runner_processor=hf_output_post_proc,
|
||||
second_runner_processor=vllm_output_post_proc,
|
||||
)
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_mm, vllm_outputs_per_mm):
|
||||
# This is usually check_logprobs_close, but it's passed through to
|
||||
# allow things like check_outputs_equal where needed
|
||||
comparator(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
def process_runner_outputs(
|
||||
model,
|
||||
first_runner_outputs,
|
||||
second_runner_outputs,
|
||||
first_runner_processor=None,
|
||||
second_runner_processor=None,
|
||||
):
|
||||
"""Applies the runner processor(s) to the runner outputs, if any."""
|
||||
if first_runner_processor is not None:
|
||||
first_runner_outputs = process_outputs(
|
||||
first_runner_processor, model, first_runner_outputs
|
||||
)
|
||||
if second_runner_processor is not None:
|
||||
second_runner_outputs = process_outputs(
|
||||
second_runner_processor, model, second_runner_outputs
|
||||
)
|
||||
return first_runner_outputs, second_runner_outputs
|
||||
|
||||
|
||||
def process_outputs(output_processor, model, outputs_per_image):
|
||||
"""Applies a model specific post-processor function to a runner's output."""
|
||||
return [
|
||||
[output_processor(res, model) for res in outputs]
|
||||
for outputs in outputs_per_image
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Types for writing multimodal model tests."""
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
from enum import Enum
|
||||
from pathlib import PosixPath
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
from pytest import MarkDecorator
|
||||
from transformers import AutoModelForCausalLM
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .....conftest import (
|
||||
AUDIO_ASSETS,
|
||||
IMAGE_ASSETS,
|
||||
HfRunner,
|
||||
ImageAsset,
|
||||
ImageTestAssets,
|
||||
PromptAudioInput,
|
||||
PromptImageInput,
|
||||
PromptVideoInput,
|
||||
)
|
||||
from ....utils import check_logprobs_close
|
||||
|
||||
# meta image tag; will be replaced by the appropriate tag for the model
|
||||
TEST_IMG_PLACEHOLDER = "<vlm_image>"
|
||||
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
|
||||
TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
|
||||
|
||||
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
|
||||
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
|
||||
}
|
||||
)
|
||||
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
|
||||
{
|
||||
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
|
||||
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
|
||||
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
|
||||
|
||||
|
||||
IMAGE_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
|
||||
EMBEDDING_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0)]
|
||||
RunnerOutput = tuple[list[int], str, SampleLogprobs | None]
|
||||
|
||||
|
||||
class PromptWithMultiModalInput(NamedTuple):
|
||||
"""Holds the multimodal input for a single test case."""
|
||||
|
||||
prompts: list[str]
|
||||
image_data: PromptImageInput | None = None
|
||||
video_data: PromptVideoInput | None = None
|
||||
audio_data: PromptAudioInput | None = None
|
||||
|
||||
|
||||
class VLMTestType(Enum):
|
||||
IMAGE = 1
|
||||
MULTI_IMAGE = 2
|
||||
EMBEDDING = 3
|
||||
VIDEO = 4
|
||||
AUDIO = 5
|
||||
CUSTOM_INPUTS = 6
|
||||
|
||||
|
||||
class SizeType(Enum):
|
||||
SIZE_FACTOR = 1
|
||||
FIXED_SIZE = 2
|
||||
|
||||
|
||||
class CustomTestOptions(NamedTuple):
|
||||
inputs: list[PromptWithMultiModalInput]
|
||||
limit_mm_per_prompt: dict[str, int]
|
||||
|
||||
|
||||
class ImageSizeWrapper(NamedTuple):
|
||||
type: SizeType
|
||||
# A size factor is a wrapper of 0+ floats,
|
||||
# while a fixed size contains an iterable of integer pairs
|
||||
data: Iterable[float] | Iterable[tuple[int, int]]
|
||||
|
||||
|
||||
class VLMTestInfo(NamedTuple):
|
||||
"""Holds the configuration for 1+ tests for one model architecture."""
|
||||
|
||||
models: list[str]
|
||||
test_type: VLMTestType | Iterable[VLMTestType]
|
||||
|
||||
# Should be None only if this is a CUSTOM_INPUTS test
|
||||
prompt_formatter: Callable[[str], str] | None = None
|
||||
img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
|
||||
video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
|
||||
audio_idx_to_prompt: Callable[[int], str] = lambda idx: "<audio>\n"
|
||||
|
||||
# Most models work on the single / multi-image prompts above, but in some
|
||||
# cases the log prob check fails, e.g., for paligemma. We allow passing
|
||||
# an override for the single image prompts / multi-image prompt for this
|
||||
# reason.
|
||||
single_image_prompts: Iterable[str] = SINGLE_IMAGE_BASE_PROMPTS
|
||||
multi_image_prompt: str = MULTI_IMAGE_BASE_PROMPT
|
||||
|
||||
# Function for converting ImageAssets to image embeddings;
|
||||
# We need to define this explicitly for embedding tests
|
||||
convert_assets_to_embeddings: (
|
||||
Callable[[ImageTestAssets], list[torch.Tensor]] | None
|
||||
) = None
|
||||
|
||||
# Exposed options for vLLM runner; we change these in a several tests,
|
||||
# but the defaults are derived from VllmRunner & the engine defaults
|
||||
# These settings are chosen to avoid OOMs when running in the CI
|
||||
enforce_eager: bool = True
|
||||
max_model_len: int = 1024
|
||||
max_num_seqs: int = 256
|
||||
runner: RunnerOption = "auto"
|
||||
tensor_parallel_size: int = 1
|
||||
vllm_runner_kwargs: dict[str, Any] | None = None
|
||||
|
||||
# Optional callable which gets a list of token IDs from the model tokenizer
|
||||
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None
|
||||
# Optional list of strings to stop generation, useful when stop tokens are
|
||||
# not special tokens in the tokenizer
|
||||
stop_str: list[str] | None = None
|
||||
|
||||
# Exposed options for HF runner
|
||||
hf_model_kwargs: dict[str, Any] | None = None
|
||||
hf_processor: Callable[[str], Any] | None = None
|
||||
# Indicates we should explicitly pass the EOS from the tokenizer
|
||||
use_tokenizer_eos: bool = False
|
||||
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None = None
|
||||
|
||||
# Post processors that if defined, will run oun the outputs of the
|
||||
# vLLM and HF runner, respectively (useful for sanitization, etc).
|
||||
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
|
||||
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
|
||||
|
||||
# Consumes the output of the callables above and checks if they're equal
|
||||
comparator: Callable[..., None] = check_logprobs_close
|
||||
|
||||
# Default expandable params per test; these defaults can be overridden in
|
||||
# instances of this object; the complete set of test cases for the model
|
||||
# is all combinations of .models + all fields below
|
||||
max_tokens: int = 128
|
||||
num_logprobs: int = 5
|
||||
dtype: str = "auto"
|
||||
distributed_executor_backend: str | None = None
|
||||
# Only expanded in video tests
|
||||
num_video_frames: int | tuple[int] = 16
|
||||
needs_video_metadata: bool = False
|
||||
|
||||
# Fixed image sizes / image size factors; most tests use image_size_factors.
|
||||
# Each inner iterable defines one request batch. All batches are run against
|
||||
# the same model instance.
|
||||
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
|
||||
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
|
||||
|
||||
# Hack for updating a prompt to take into a local path; currently only used
|
||||
# for Qwen-VL, which requires encoding the image path / url into the prompt
|
||||
# for HF runner
|
||||
prompt_path_encoder: (
|
||||
Callable[[PosixPath, str, list[ImageAsset] | ImageTestAssets], str] | None
|
||||
) = None # noqa: E501
|
||||
|
||||
# Allows configuring a test to run with custom inputs
|
||||
custom_test_opts: list[CustomTestOptions] | None = None
|
||||
|
||||
marks: list[MarkDecorator] | None = None
|
||||
|
||||
def get_non_parametrized_runner_kwargs(self):
|
||||
"""Returns a dictionary of expandable kwargs for items that are used
|
||||
in all test types, which are NOT used when creating the parametrized
|
||||
test cases.
|
||||
"""
|
||||
return {
|
||||
"enforce_eager": self.enforce_eager,
|
||||
"max_model_len": self.max_model_len,
|
||||
"max_num_seqs": self.max_num_seqs,
|
||||
"runner": self.runner,
|
||||
"tensor_parallel_size": self.tensor_parallel_size,
|
||||
"vllm_runner_kwargs": self.vllm_runner_kwargs,
|
||||
"hf_output_post_proc": self.hf_output_post_proc,
|
||||
"vllm_output_post_proc": self.vllm_output_post_proc,
|
||||
"auto_cls": self.auto_cls,
|
||||
"use_tokenizer_eos": self.use_tokenizer_eos,
|
||||
"comparator": self.comparator,
|
||||
"get_stop_token_ids": self.get_stop_token_ids,
|
||||
"hf_model_kwargs": self.hf_model_kwargs,
|
||||
"hf_processor": self.hf_processor,
|
||||
"stop_str": self.stop_str,
|
||||
"patch_hf_runner": self.patch_hf_runner,
|
||||
}
|
||||
|
||||
|
||||
class ExpandableVLMTestArgs(NamedTuple):
|
||||
"""The expanded kwargs which correspond to a single test case."""
|
||||
|
||||
model: str
|
||||
max_tokens: int
|
||||
num_logprobs: int
|
||||
dtype: str
|
||||
distributed_executor_backend: str | None
|
||||
# Sizes are used for everything except audio and custom input tests.
|
||||
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
|
||||
# Video only
|
||||
num_video_frames: int | None = None
|
||||
needs_video_metadata: bool = False
|
||||
# Custom inputs only
|
||||
custom_test_opts: CustomTestOptions | None = None
|
||||
@@ -1,40 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import vqa_ppl_test
|
||||
|
||||
MODELS = [
|
||||
(GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
|
||||
]
|
||||
|
||||
|
||||
def _pixel_size(factor: int):
|
||||
# Match max image token size with qwen's PPL test
|
||||
return {
|
||||
"size": {
|
||||
"shortest_edge": 2 * factor**2,
|
||||
"longest_edge": 2 * 1280 * factor**2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info,factor", MODELS)
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
|
||||
def test_ppl(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: GenerateModelInfo,
|
||||
factor: int,
|
||||
mm_device_do_normalize: bool,
|
||||
):
|
||||
vqa_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
|
||||
mm_processor_kwargs=_pixel_size(factor),
|
||||
)
|
||||
@@ -1,140 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
|
||||
caching.
|
||||
|
||||
Tests the use_audio_in_video feature where audio is extracted from video and
|
||||
processed together with video frames in an interleaved manner.
|
||||
|
||||
Regression test: when use_audio_in_video=True and the multimodal processor
|
||||
cache is warm, the second request goes through MultiModalProcessorSenderCache
|
||||
which sets mm_kwargs["video"] items to None on a cache hit. The processor
|
||||
must still detect use_audio_in_video=True (via token-count heuristic) and
|
||||
produce the same prompt_token_ids as the first (cache-miss) request.
|
||||
|
||||
Without the fix the cache-hit path left use_audio_in_video=False, causing
|
||||
audio placeholder tokens to be inserted separately instead of being derived
|
||||
from the interleaved video placeholders – yielding a different (wrong) token
|
||||
sequence on every subsequent request for the same video.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import LruKeyReplicatedSenderCache
|
||||
|
||||
from ....multimodal.utils import random_audio, random_video
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODELS = [
|
||||
"Qwen/Qwen2.5-Omni-3B",
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
]
|
||||
|
||||
|
||||
def create_mm_data(num_videos: int) -> dict[str, list]:
|
||||
# Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
|
||||
# stays fast even without a GPU.
|
||||
mm_data = dict[str, list](video=[], audio=[])
|
||||
for i in range(num_videos):
|
||||
rng = np.random.RandomState(i)
|
||||
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
|
||||
audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
|
||||
mm_data["video"].append(video)
|
||||
mm_data["audio"].append((audio, sr))
|
||||
return mm_data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
@pytest.mark.parametrize("num_videos", [1, 2])
|
||||
def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
|
||||
"""Regression test for https://github.com/vllm-project/vllm/pull/36800.
|
||||
|
||||
MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
|
||||
on a cache hit, so mm_kwargs["video"] items become None on the second call.
|
||||
The Qwen processor override of _maybe_apply_prompt_updates must detect
|
||||
use_audio_in_video=True via token-count heuristics and re-derive the audio
|
||||
placeholders correctly.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": num_videos, "image": 0, "video": num_videos},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Sender cache: on a cache hit returns (None, prompt_updates) for each
|
||||
# item, setting mm_kwargs["video"] = [None] – the exact condition that
|
||||
# triggered the original bug.
|
||||
sender_cache = LruKeyReplicatedSenderCache(ctx.model_config)
|
||||
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
mm_data = create_mm_data(num_videos)
|
||||
hf_processor_mm_kwargs = {"use_audio_in_video": True}
|
||||
|
||||
def run(processor, cache):
|
||||
return processor(
|
||||
[video_token_id] * num_videos,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
cache=cache,
|
||||
)["prompt_token_ids"]
|
||||
|
||||
baseline_ids = run(processor, cache=None)
|
||||
|
||||
# First call on the sender-cache processor: cache miss.
|
||||
# mm_kwargs["video"] items are real tensors; use_audio_in_video is
|
||||
# detected normally from the item data.
|
||||
first_ids = run(processor, cache=sender_cache)
|
||||
assert first_ids == baseline_ids, (
|
||||
"Cache-miss call produced different prompt_token_ids than baseline.\n"
|
||||
f" baseline : {baseline_ids}\n"
|
||||
f" cache-miss: {first_ids}"
|
||||
)
|
||||
|
||||
# Second call on the sender-cache processor: cache hit.
|
||||
# MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
|
||||
# so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
|
||||
# not detected, yielding wrong token ids.
|
||||
second_ids = run(processor, cache=sender_cache)
|
||||
assert second_ids == baseline_ids, (
|
||||
"Cache-hit call produced different prompt_token_ids than baseline.\n"
|
||||
"This is the regression introduced when use_audio_in_video detection\n"
|
||||
"fails for None mm_kwargs items on a cache hit.\n"
|
||||
f" baseline : {baseline_ids}\n"
|
||||
f" cache-hit: {second_ids}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
||||
"""A video with no audio track, combined with `use_audio_in_video=True`,
|
||||
must raise a clear `ValueError` naming the mismatch, rather than an
|
||||
opaque `StopIteration`-derived error from the underlying HF processor.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
|
||||
|
||||
# No "audio" key at all in this request's data: this is the "video has
|
||||
# no audio track" case. (The audio limit above is 1, not 0, so that
|
||||
# `mm_counts["audio"]` is still declared for this request; a 0 limit
|
||||
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
|
||||
# trips an unrelated assertion in `_apply_hf_processor_main`.)
|
||||
mm_data = {"video": [video]}
|
||||
|
||||
with pytest.raises(ValueError, match="doesn't have audio track"):
|
||||
processor(
|
||||
[video_token_id],
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={"use_audio_in_video": True},
|
||||
)
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODEL_ID = "nvidia/Cosmos3-Edge"
|
||||
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
if LOCAL_MODEL_PATH:
|
||||
model_config = ModelConfig(
|
||||
LOCAL_MODEL_PATH,
|
||||
tokenizer=LOCAL_MODEL_PATH,
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
)
|
||||
else:
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
)
|
||||
model_config = ctx.model_config
|
||||
|
||||
return MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
|
||||
def _assert_image_outputs(processor, processed, num_images: int) -> None:
|
||||
mm_data = processed["mm_kwargs"].get_data()
|
||||
grid_thw = mm_data["image_grid_thw"]
|
||||
pixel_values = mm_data["pixel_values"]
|
||||
|
||||
assert grid_thw.shape == (num_images, 3)
|
||||
assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
|
||||
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
|
||||
image_placeholders = processed["mm_placeholders"]["image"]
|
||||
|
||||
assert len(image_placeholders) == num_images
|
||||
assert [placeholder.length for placeholder in image_placeholders] == (
|
||||
expected_tokens
|
||||
)
|
||||
|
||||
image_token_id = processor.info.get_hf_processor().image_token_id
|
||||
assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
|
||||
|
||||
|
||||
def _assert_video_outputs(processor, processed) -> None:
|
||||
mm_data = processed["mm_kwargs"].get_data()
|
||||
grid_thw = mm_data["video_grid_thw"]
|
||||
pixel_values = mm_data["pixel_values_videos"]
|
||||
|
||||
assert grid_thw.shape == (1, 3)
|
||||
assert pixel_values.shape[0] == int(grid_thw.prod())
|
||||
assert len(processed["mm_placeholders"]["video"]) == 1
|
||||
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
expected_tokens = int(grid_thw.prod()) // merge_size**2
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
prompt_token_ids = processed["prompt_token_ids"]
|
||||
assert prompt_token_ids.count(video_token_id) == expected_tokens
|
||||
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
expected_frame_wrappers = int(grid_thw[:, 0].sum())
|
||||
assert (
|
||||
prompt_token_ids.count(hf_processor.vision_start_token_id)
|
||||
== expected_frame_wrappers
|
||||
)
|
||||
assert (
|
||||
prompt_token_ids.count(hf_processor.vision_end_token_id)
|
||||
== expected_frame_wrappers
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_images", [1, 2])
|
||||
def test_process_images(
|
||||
processor,
|
||||
image_assets: ImageTestAssets,
|
||||
num_images: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets[:num_images]]
|
||||
processed = processor(
|
||||
IMAGE_PLACEHOLDER * num_images,
|
||||
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_image_outputs(processor, processed, num_images)
|
||||
|
||||
|
||||
def test_process_video(processor) -> None:
|
||||
video_asset = VideoAsset(name="baby_reading", num_frames=8)
|
||||
video = (video_asset.np_ndarrays, video_asset.metadata)
|
||||
processed = processor(
|
||||
VIDEO_PLACEHOLDER,
|
||||
mm_items=processor.info.parse_mm_data({"video": [video]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_video_outputs(processor, processed)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for H2OVL's multimodal preprocessing kwargs."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _get_expected_num_patches(
|
||||
config: PreTrainedConfig,
|
||||
image: Image.Image,
|
||||
num_imgs: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
):
|
||||
from vllm.transformers_utils.processors.h2ovl import (
|
||||
calculate_h2ovl_targets,
|
||||
get_h2ovl_target_ratios,
|
||||
)
|
||||
|
||||
width, height = image.size
|
||||
|
||||
# Calculate the expected number of blocks
|
||||
if num_imgs == 1 and config.use_msac:
|
||||
# First pass
|
||||
blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_h2ovl_target_ratios(
|
||||
min_num=1,
|
||||
max_num=max_num,
|
||||
prior_aspect_ratio=None,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False, # Thumbnail is handled separately
|
||||
)
|
||||
|
||||
# Second pass
|
||||
blocks2, _, _, _ = calculate_h2ovl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_h2ovl_target_ratios(
|
||||
min_num=3,
|
||||
max_num=max_num,
|
||||
prior_aspect_ratio=aspect_ratio,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
# Add thumbnail if use_thumbnail is True and total_blocks > 1
|
||||
if config.use_thumbnail:
|
||||
blocks1 += 1 if blocks1 > 1 else 0
|
||||
blocks2 += 1 if blocks2 > 1 else 0
|
||||
|
||||
# Total blocks is the sum of blocks from both passes minus
|
||||
# overlapping
|
||||
total_blocks = blocks1 + blocks2 - 1
|
||||
|
||||
return total_blocks
|
||||
|
||||
blocks, _, _, _ = calculate_h2ovl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_h2ovl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
prior_aspect_ratio=None,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
expected_num_patches = blocks
|
||||
|
||||
if config.use_thumbnail and expected_num_patches > 1:
|
||||
expected_num_patches += 1
|
||||
|
||||
return expected_num_patches
|
||||
|
||||
|
||||
def _run_check(
|
||||
processor: BaseMultiModalProcessor,
|
||||
images: list[Image.Image],
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
config = processor.info.get_hf_config()
|
||||
|
||||
prompt = "<image>" * len(images)
|
||||
mm_data = {"image": images}
|
||||
|
||||
total_expected_num_patches = sum(
|
||||
_get_expected_num_patches(config, image, len(images), min_num, max_num)
|
||||
for image in images
|
||||
)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
|
||||
|
||||
assert img_tok_count == 256 * total_expected_num_patches
|
||||
assert pixel_shape[0] == total_expected_num_patches
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
"h2oai/h2ovl-mississippi-2b",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"size_factors",
|
||||
[
|
||||
# Single-scale
|
||||
[1.0],
|
||||
# Single-scale, batched
|
||||
[1.0, 1.0, 1.0],
|
||||
# Multi-scale
|
||||
[0.25, 0.5, 1.0],
|
||||
[4.0, 2.0, 1.0],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("min_dynamic_patch", "max_dynamic_patch"),
|
||||
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic_image_size", [True, False])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
image_assets: ImageTestAssets,
|
||||
size_factors: list[int],
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool | None,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
mm_processor_kwargs = {
|
||||
"min_dynamic_patch": min_dynamic_patch,
|
||||
"max_dynamic_patch": max_dynamic_patch,
|
||||
"dynamic_image_size": dynamic_image_size,
|
||||
}
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": len(size_factors)},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
min_num = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_num = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
_run_check(
|
||||
processor,
|
||||
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
|
||||
min_num,
|
||||
max_num,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Idefics3's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
from transformers import Idefics3Config
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
({"size": {"longest_edge": 364}}, 169),
|
||||
({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
|
||||
# Same as the previous test - don't initialize mm_processor_kwargs
|
||||
# in this test and assume that the kwargs will be correctly expanded by
|
||||
# the partial when calling the custom input processor.
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
placeholders = (
|
||||
"<image>"
|
||||
if num_imgs == 1
|
||||
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
|
||||
)
|
||||
prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
|
||||
|
||||
# Build mm_data
|
||||
image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
|
||||
dummy_image_size = (image_size * 4, image_size * 4)
|
||||
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
|
||||
mm_data = {"image": [dummy_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure the placeholders format are correct
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processed_inputs = hf_processor(
|
||||
text=prompt,
|
||||
images=mm_data["image"],
|
||||
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
|
||||
)
|
||||
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = ctx.get_hf_config().image_token_id
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for InternVL's multimodal preprocessing kwargs."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _get_expected_num_patches(
|
||||
config: PreTrainedConfig,
|
||||
image: Image.Image,
|
||||
num_imgs: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
):
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
calculate_internvl_targets,
|
||||
get_internvl_target_ratios,
|
||||
)
|
||||
|
||||
width, height = image.size
|
||||
|
||||
blocks, _, _ = calculate_internvl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_internvl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
expected_num_patches = blocks
|
||||
|
||||
if config.use_thumbnail and expected_num_patches > 1:
|
||||
expected_num_patches += 1
|
||||
|
||||
return expected_num_patches
|
||||
|
||||
|
||||
def _run_check(
|
||||
processor: BaseMultiModalProcessor,
|
||||
images: list[Image.Image],
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
config = processor.info.get_hf_config()
|
||||
|
||||
prompt = "<image>" * len(images)
|
||||
mm_data = {"image": images}
|
||||
|
||||
total_expected_num_patches = sum(
|
||||
_get_expected_num_patches(config, image, len(images), min_num, max_num)
|
||||
for image in images
|
||||
)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
|
||||
|
||||
assert img_tok_count == 256 * total_expected_num_patches
|
||||
assert pixel_shape[0] == total_expected_num_patches
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
|
||||
@pytest.mark.parametrize(
|
||||
"size_factors",
|
||||
[
|
||||
# Single-scale
|
||||
[1.0],
|
||||
# Single-scale, batched
|
||||
[1.0, 1.0, 1.0],
|
||||
# Multi-scale
|
||||
[0.25, 0.5, 1.0],
|
||||
[4.0, 2.0, 1.0],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("min_dynamic_patch", "max_dynamic_patch"),
|
||||
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic_image_size", [True, False])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
image_assets: ImageTestAssets,
|
||||
size_factors: list[int],
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool | None,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
mm_processor_kwargs = {
|
||||
"min_dynamic_patch": min_dynamic_patch,
|
||||
"max_dynamic_patch": max_dynamic_patch,
|
||||
"dynamic_image_size": dynamic_image_size,
|
||||
}
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": len(size_factors)},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
min_num = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_num = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
_run_check(
|
||||
processor,
|
||||
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
|
||||
min_num,
|
||||
max_num,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for LightOnOCR precomputed image embeddings."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("tensor_input", [False, True])
|
||||
@pytest.mark.parametrize("prompt_as_tokens", [False, True])
|
||||
def test_lightonocr_image_embeddings_preserve_lengths_and_values(
|
||||
tensor_input: bool, prompt_as_tokens: bool
|
||||
):
|
||||
"""Reserve each image's feature count and pass projected embeddings through."""
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
ctx.model_config.get_multimodal_config().enable_mm_embeds = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
hidden_size = ctx.model_config.get_hidden_size()
|
||||
feature_sizes = [3, 3] if tensor_input else [3, 7]
|
||||
embeddings = [
|
||||
torch.full((size, hidden_size), float(i))
|
||||
for i, size in enumerate(feature_sizes)
|
||||
]
|
||||
image_input = torch.stack(embeddings) if tensor_input else embeddings
|
||||
prompt = " ".join([hf_processor.image_token] * len(embeddings))
|
||||
if prompt_as_tokens:
|
||||
prompt = ctx.get_tokenizer().encode(prompt)
|
||||
|
||||
items = processor.info.parse_mm_data({"image": image_input})
|
||||
processed = processor(prompt, mm_items=items)
|
||||
|
||||
assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum(
|
||||
feature_sizes
|
||||
)
|
||||
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
|
||||
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
|
||||
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Llama4's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-4-Scout-17B-16E-Instruct"])
|
||||
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
|
||||
@pytest.mark.parametrize("num_imgs", [1, 5])
|
||||
@pytest.mark.parametrize("mm_processor_cache_gb", [0, 4])
|
||||
@pytest.mark.parametrize("tokenized_prompt", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict,
|
||||
num_imgs: int,
|
||||
mm_processor_cache_gb: int,
|
||||
tokenized_prompt: bool,
|
||||
):
|
||||
"""Ensure llama4 processor works properly."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
config = processor.info.get_hf_config()
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
prompt = (
|
||||
"<|begin_of_text|><|header_start|>user<|header_end|>"
|
||||
+ "<|image|>" * num_imgs
|
||||
+ "<|eot|><|header_start|>assistant<|header_end|>"
|
||||
)
|
||||
mm_data = {
|
||||
"image": [
|
||||
image_assets[(i % len(image_assets))].pil_image for i in range(num_imgs)
|
||||
]
|
||||
}
|
||||
if tokenized_prompt:
|
||||
prompt = tokenizer.encode(prompt)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
mm_data = processed_inputs["mm_kwargs"].get_data()
|
||||
|
||||
# place holder replacements
|
||||
prompt_token_ids = processed_inputs["prompt_token_ids"]
|
||||
assert prompt_token_ids.count(config.boi_token_index) == num_imgs
|
||||
assert prompt_token_ids.count(config.eoi_token_index) == num_imgs
|
||||
assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs
|
||||
aspect_ratios = mm_data["aspect_ratios"]
|
||||
num_x_separators = num_y_separators = 0
|
||||
for tiles_y, tiles_x in aspect_ratios:
|
||||
if tiles_x * tiles_y > 1:
|
||||
num_x_separators += (tiles_x - 1) * tiles_y
|
||||
num_y_separators += tiles_y
|
||||
assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators
|
||||
assert (
|
||||
prompt_token_ids.count(vocab[hf_processor.tile_global_token])
|
||||
== num_y_separators
|
||||
)
|
||||
|
||||
# image token offsets
|
||||
img_locs = processed_inputs["mm_placeholders"].get("image", [])
|
||||
assert len(img_locs) == num_imgs
|
||||
assert [img_loc.offset for img_loc in img_locs] == [
|
||||
i for i, v in enumerate(prompt_token_ids) if v == config.boi_token_index
|
||||
]
|
||||
|
||||
# patch sizes and masks
|
||||
num_patches_per_chunk = processor.info.get_patch_per_chunk(config.vision_config)
|
||||
assert (
|
||||
prompt_token_ids.count(config.image_token_index)
|
||||
== sum(mm_data["patches_per_image"]) * num_patches_per_chunk
|
||||
)
|
||||
assert len(mm_data["pixel_values"]) == sum(mm_data["patches_per_image"])
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pqdm.threads import pqdm
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _validate_image_max_tokens_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
max_tokens: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
info = processor.info
|
||||
feature_size = info.get_num_image_tokens(
|
||||
image_width=image_size.width, image_height=image_size.height
|
||||
)
|
||||
|
||||
try:
|
||||
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 5 minutes to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
def test_processor_max_tokens(model_id):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
info = processor.info
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 2
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(32, 4096), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_max_tokens_one,
|
||||
processor,
|
||||
info.get_max_image_tokens(), # type: ignore
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def _validate_image_prompt_replacements_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
num_imgs: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
prompt = "<image>" * num_imgs
|
||||
image = Image.new("RGB", size=image_size)
|
||||
mm_data = {"image": [image] * num_imgs}
|
||||
|
||||
try:
|
||||
# The processor will throw an error if there is a mismatch
|
||||
# in the prompt replacements
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == num_imgs
|
||||
|
||||
first_placeholder = image_placeholders[0]
|
||||
|
||||
# NOTE: There is a BOS token
|
||||
assert first_placeholder.offset == 1
|
||||
assert (
|
||||
first_placeholder.length
|
||||
== (len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
def _test_image_prompt_replacements(
|
||||
processor,
|
||||
*,
|
||||
num_imgs: int,
|
||||
image_sizes: list[ImageSize],
|
||||
) -> None:
|
||||
"""Ensure LlavaNextMultiModalProcessor
|
||||
handles prompt replacement properly for input images.
|
||||
"""
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_prompt_replacements_one,
|
||||
processor,
|
||||
num_imgs,
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
(184, 161),
|
||||
(198, 176),
|
||||
(333, 296),
|
||||
(369, 328),
|
||||
(488, 183),
|
||||
(2560, 1669),
|
||||
]
|
||||
image_sizes = [
|
||||
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
|
||||
]
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 2 hours to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1])
|
||||
def test_processor_prompt_replacements_all(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 2
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(64, 1024), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pqdm.threads import pqdm
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _validate_image_max_tokens_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
max_tokens: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
info = processor.info
|
||||
feature_size = info.get_num_image_tokens(
|
||||
image_width=image_size.width, image_height=image_size.height
|
||||
)
|
||||
|
||||
try:
|
||||
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 5 minutes to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_processor_max_tokens(model_id):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
info = processor.info
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 6
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(32, 4096), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_max_tokens_one,
|
||||
processor,
|
||||
info.get_max_image_tokens(), # type: ignore
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def _validate_image_prompt_replacements_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
num_imgs: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
prompt = "<image>" * num_imgs
|
||||
image = Image.new("RGB", size=image_size)
|
||||
mm_data = {"image": [image] * num_imgs}
|
||||
|
||||
try:
|
||||
# The processor will throw an error if there is a mismatch
|
||||
# in the prompt replacements
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == num_imgs
|
||||
|
||||
first_placeholder = image_placeholders[0]
|
||||
|
||||
assert first_placeholder.offset == 0
|
||||
assert (
|
||||
first_placeholder.length
|
||||
== len(processed_inputs["prompt_token_ids"]) // num_imgs
|
||||
)
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
def _test_image_prompt_replacements(
|
||||
processor,
|
||||
*,
|
||||
num_imgs: int,
|
||||
image_sizes: list[ImageSize],
|
||||
) -> None:
|
||||
"""Ensure LlavaOnevisionMultiModalProcessor
|
||||
handles prompt replacement properly for input images.
|
||||
"""
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_prompt_replacements_one,
|
||||
processor,
|
||||
num_imgs,
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
(184, 161),
|
||||
(198, 176),
|
||||
(333, 296),
|
||||
(369, 328),
|
||||
(488, 183),
|
||||
(2560, 1669),
|
||||
]
|
||||
image_sizes = [
|
||||
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
|
||||
]
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 2 hours to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1])
|
||||
def test_processor_prompt_replacements_all(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 6
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(64, 1024), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for MiniCPMV's multimodal preprocessing."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||
def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
|
||||
"""Calls with different kwargs must not reuse stale processor instances."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
info = processor.info
|
||||
processor_1 = info.get_hf_processor(max_slice_nums=1)
|
||||
processor_2 = info.get_hf_processor(max_slice_nums=2)
|
||||
assert processor_1.image_processor.max_slice_nums == 1
|
||||
assert processor_2.image_processor.max_slice_nums == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_ids",
|
||||
[
|
||||
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
|
||||
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
|
||||
],
|
||||
)
|
||||
def test_image_processor_for_different_models(model_ids):
|
||||
first_model_id, second_model_id = model_ids
|
||||
|
||||
first_ctx = build_model_context(
|
||||
first_model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
|
||||
first_image_processor = first_processor.info.get_image_processor()
|
||||
|
||||
second_ctx = build_model_context(
|
||||
second_model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
|
||||
second_image_processor = second_processor.info.get_image_processor()
|
||||
|
||||
second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
|
||||
|
||||
assert type(first_image_processor) is not type(second_image_processor)
|
||||
assert (
|
||||
type(first_image_processor).__module__
|
||||
!= type(second_image_processor).__module__
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||
def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = ctx.get_tokenizer()
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
|
||||
]
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
image = np.zeros((768, 1024, 3), dtype=np.uint8)
|
||||
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed["mm_placeholders"].get("image", [])
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length > 0
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
|
||||
|
||||
These exercise the vendored processor directly (no checkpoint / GPU needed), so
|
||||
they validate the long-side resize spec and the resulting prompt-token counts
|
||||
deterministically.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.transformers_utils.processors.minimax_m3 import (
|
||||
IMAGE_MAX_TOTAL_PIXELS,
|
||||
MIN_SHORT_SIDE_PIXEL,
|
||||
VIDEO_MAX_TOTAL_PIXELS,
|
||||
MiniMaxM3VLImageProcessor,
|
||||
MiniMaxM3VLVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
# Long sides are multiples of patch_size*merge_size (28) so the rounding is
|
||||
# exact and the expected token counts are unambiguous.
|
||||
LONG_SIDES = [252, 504, 1008]
|
||||
MERGE2 = 2**2 # merge_size ** 2
|
||||
|
||||
|
||||
def _image_tokens(grid_thw) -> int:
|
||||
g = list(grid_thw)
|
||||
return int(g[0] * g[1] * g[2]) // MERGE2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_smart_resize_long_side_shrink():
|
||||
# (a) long side exceeds the cap -> shrink so the long side equals the cap.
|
||||
h, w = smart_resize(
|
||||
2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
|
||||
)
|
||||
assert max(h, w) == 1008
|
||||
assert (h, w) == (1008, 504) # aspect ratio preserved
|
||||
|
||||
|
||||
def test_smart_resize_short_side_enlarge():
|
||||
# (b) long side within the cap but short side below the floor -> enlarge so
|
||||
# the short side reaches min_short_side_pixel.
|
||||
h, w = smart_resize(
|
||||
200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
|
||||
)
|
||||
assert min(h, w) == MIN_SHORT_SIDE_PIXEL # 112
|
||||
|
||||
|
||||
def test_smart_resize_total_pixels_raises():
|
||||
# (c) still over the area cap after resizing -> raise instead of inferring.
|
||||
with pytest.raises(ValueError, match="max_total_pixels"):
|
||||
smart_resize(
|
||||
5000,
|
||||
5000,
|
||||
factor=28,
|
||||
max_long_side_pixel=4000,
|
||||
max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
|
||||
)
|
||||
|
||||
|
||||
def test_smart_resize_backward_compatible_area_bound():
|
||||
# Without max_long_side_pixel the original Qwen-style area bound is used.
|
||||
assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_image_tokens_increase_with_max_long_side_pixel():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
patches = proc.get_number_of_image_patches(
|
||||
2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
|
||||
)
|
||||
counts.append(patches // MERGE2)
|
||||
|
||||
assert counts == [81, 324, 1296]
|
||||
assert counts[0] < counts[1] < counts[2]
|
||||
|
||||
|
||||
def test_image_processor_defaults_match_spec():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
assert proc.max_long_side_pixel is None # opt-in
|
||||
assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
|
||||
assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
|
||||
|
||||
|
||||
def test_image_preprocess_pipeline_monotonic():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
out = proc.preprocess(
|
||||
[image],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=long_side,
|
||||
return_tensors="pt",
|
||||
)
|
||||
counts.append(_image_tokens(out["image_grid_thw"][0]))
|
||||
assert counts == [81, 324, 1296]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_video_tokens_increase_with_max_long_side_pixel():
|
||||
proc = MiniMaxM3VLVideoProcessor()
|
||||
assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
|
||||
video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
out = proc.preprocess(
|
||||
videos=[video],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=long_side,
|
||||
return_tensors="pt",
|
||||
)
|
||||
counts.append(_image_tokens(out["video_grid_thw"][0]))
|
||||
assert counts[0] < counts[1] < counts[2]
|
||||
|
||||
|
||||
def test_video_volumetric_cap_raises():
|
||||
proc = MiniMaxM3VLVideoProcessor()
|
||||
# 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
|
||||
video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
|
||||
with pytest.raises(ValueError, match="max_total_pixels"):
|
||||
proc.preprocess(
|
||||
videos=[video],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=1008,
|
||||
return_tensors="pt",
|
||||
)
|
||||
@@ -1,53 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for mllama's multimodal preprocessing and profiling."""
|
||||
|
||||
import pytest
|
||||
from torch import prod
|
||||
from transformers import Llama4Config
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-Guard-4-12B"])
|
||||
@pytest.mark.parametrize("max_model_len", [4096, 8192, 25600, 131072])
|
||||
def test_profiling(model_id: str, max_model_len: int):
|
||||
model_config_kwargs = {
|
||||
"max_model_len": max_model_len,
|
||||
}
|
||||
mm_counts = {"image": 1}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
model_config_kwargs=model_config_kwargs,
|
||||
limit_mm_per_prompt=mm_counts,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_inputs = processor.get_dummy_mm_inputs(mm_counts)
|
||||
|
||||
hf_config = ctx.get_hf_config(Llama4Config)
|
||||
image_size = hf_config.vision_config.image_size
|
||||
patch_size = hf_config.vision_config.patch_size
|
||||
downsample_ratio = int(
|
||||
round(1.0 / (hf_config.vision_config.pixel_shuffle_ratio**2))
|
||||
)
|
||||
tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio
|
||||
|
||||
mm_data = mm_inputs["mm_kwargs"].get_data()
|
||||
chunks_per_image = prod(mm_data["patches_per_image"])
|
||||
total_num_patches = chunks_per_image * tokens_per_patch
|
||||
num_tiles = (
|
||||
mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][1]
|
||||
) # x-y separator tokens
|
||||
total_tokens = (
|
||||
total_num_patches.item() + num_tiles.item() + 3
|
||||
) # image start, image, image end
|
||||
|
||||
assert total_num_patches == sum(
|
||||
item.get_num_embeds() for item in mm_inputs["mm_placeholders"]["image"]
|
||||
)
|
||||
assert total_tokens == sum(
|
||||
placeholder.length for placeholder in mm_inputs["mm_placeholders"]["image"]
|
||||
)
|
||||
@@ -1,553 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Moondream3 multimodal processing.
|
||||
|
||||
Includes:
|
||||
- Processor creation and application tests
|
||||
- Image tokenization and placeholder expansion tests
|
||||
- Tiling and cropping logic tests (CPU-based)
|
||||
- Pixel normalization tests
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
|
||||
# Expected multimodal prefix: BOS + 729 image tokens.
|
||||
EXPECTED_IMAGE_TOKENS = 730
|
||||
# Vision encoder constants
|
||||
CROP_SIZE = 378
|
||||
PATCH_SIZE = 14
|
||||
MAX_CROPS = 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_creation(model_id: str):
|
||||
"""Test that Moondream3 processor can be created."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
assert processor is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_apply(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that Moondream3 processor can process inputs.
|
||||
|
||||
NOTE: The prompt includes the leading BOS token because Moondream3
|
||||
pre-fills BOS and image embeddings together.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert "prompt_token_ids" in processed_inputs
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_pixel_values(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that pixel values are correctly produced."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# Check mm_kwargs contains pixel_values
|
||||
mm_kwargs = processed_inputs.get("mm_kwargs")
|
||||
assert mm_kwargs is not None
|
||||
mm_data_result = mm_kwargs.get_data()
|
||||
assert "pixel_values" in mm_data_result
|
||||
|
||||
# Verify pixel_values shape
|
||||
pixel_values = mm_data_result["pixel_values"]
|
||||
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
|
||||
assert pixel_values.shape[2] == 3 # RGB channels
|
||||
assert pixel_values.shape[3] == 378 # crop height
|
||||
assert pixel_values.shape[4] == 378 # crop width
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_image_token_expansion(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that <image> placeholder is expanded to correct number of tokens."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_multi_crop_tiling(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that large images produce correct multi-crop tiling."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Create a large image that requires multiple crops
|
||||
large_image = Image.new("RGB", (1000, 1000), color="blue")
|
||||
pixel_values, tiling = processor.preprocess_image(large_image)
|
||||
|
||||
# Large images should produce more than 1x1 tiling
|
||||
assert tiling[0] >= 1 and tiling[1] >= 1
|
||||
# Check that we have global crop + local crops
|
||||
expected_crops = tiling[0] * tiling[1] + 1
|
||||
assert pixel_values.shape[0] == expected_crops
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_size",
|
||||
[
|
||||
(500, 500),
|
||||
(800, 600),
|
||||
(1920, 1080),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_tiling_various_sizes(
|
||||
image_size: tuple[int, int],
|
||||
model_id: str,
|
||||
):
|
||||
"""Test tiling with various image sizes."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
width, height = image_size
|
||||
image = Image.new("RGB", (width, height), color="red")
|
||||
pixel_values, tiling = processor.preprocess_image(image)
|
||||
|
||||
# Basic shape checks
|
||||
assert pixel_values.dim() == 4 # [num_crops, C, H, W]
|
||||
assert pixel_values.shape[1] == 3 # RGB
|
||||
assert pixel_values.shape[2] == 378 # crop height
|
||||
assert pixel_values.shape[3] == 378 # crop width
|
||||
|
||||
# Tiling should respect max_crops (12)
|
||||
assert tiling[0] * tiling[1] <= 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_pixel_normalization(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that pixel values are normalized to [-1, 1] range."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Create test image
|
||||
image = Image.new("RGB", (378, 378), color="green")
|
||||
pixel_values, _ = processor.preprocess_image(image)
|
||||
|
||||
# Normalization: (x - 0.5) / 0.5 = 2*x - 1
|
||||
# For input [0, 1], output should be [-1, 1]
|
||||
assert pixel_values.min() >= -1.0
|
||||
assert pixel_values.max() <= 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_chat_template_with_image(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that chat template correctly formats BOS + image + prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = ctx.tokenizer
|
||||
|
||||
# Use the chat template format
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
# BOS token (<|endoftext|>) should be token ID 0
|
||||
bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
|
||||
assert bos_token_id == 0
|
||||
|
||||
# First token should be BOS
|
||||
assert token_ids[0] == bos_token_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.invalid/image.png"},
|
||||
},
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
],
|
||||
id="image-first",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.invalid/image.png"},
|
||||
},
|
||||
],
|
||||
id="text-first",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_chat_template_content_list_uses_moondream_image_prefix(
|
||||
image_assets: ImageTestAssets,
|
||||
content: list[dict[str, object]],
|
||||
model_id: str,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
|
||||
prompt = hf_processor.tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": content}],
|
||||
chat_template=hf_processor.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
|
||||
expected_prompt = (
|
||||
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
|
||||
"What is in this image?<|md_reserved_2|>"
|
||||
)
|
||||
assert prompt == expected_prompt
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_bos_token_always_first(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that BOS token (ID 0) is always at position 0."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Start with BOS token explicitly
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
# Token ID 0 (<|endoftext|>) should be the first token
|
||||
assert token_ids[0] == 0, (
|
||||
f"Expected BOS token (0) at position 0, got {token_ids[0]}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_with_small_image(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test processor with image smaller than crop size."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Small image (smaller than crop size)
|
||||
small_image = Image.new("RGB", (100, 100), color="yellow")
|
||||
pixel_values, tiling = processor.preprocess_image(small_image)
|
||||
|
||||
# Small images should use 1x1 tiling
|
||||
assert tiling == (1, 1)
|
||||
# Should have 2 crops (global + 1 local)
|
||||
assert pixel_values.shape[0] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_kind",
|
||||
[
|
||||
pytest.param("numpy_hwc", id="numpy-hwc"),
|
||||
pytest.param("numpy_chw", id="numpy-chw"),
|
||||
pytest.param("torch_chw", id="torch-chw"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_preprocess_image_accepts_non_pil_inputs(
|
||||
image_assets: ImageTestAssets,
|
||||
image_kind: str,
|
||||
model_id: str,
|
||||
):
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
pil_image = image_assets[0].pil_image.convert("RGB")
|
||||
hwc_array = np.asarray(pil_image)
|
||||
expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
|
||||
|
||||
if image_kind == "numpy_hwc":
|
||||
image = hwc_array
|
||||
elif image_kind == "numpy_chw":
|
||||
image = np.transpose(hwc_array, (2, 0, 1))
|
||||
else:
|
||||
image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
|
||||
|
||||
pixel_values, tiling = processor.preprocess_image(image)
|
||||
|
||||
assert tiling == expected_tiling
|
||||
assert pixel_values.shape == expected_pixel_values.shape
|
||||
assert pixel_values.dtype == torch.bfloat16
|
||||
assert torch.equal(pixel_values, expected_pixel_values)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_apply_accepts_non_pil_image_inputs(
|
||||
image_assets: ImageTestAssets,
|
||||
image_kind: str,
|
||||
model_id: str,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
|
||||
chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
|
||||
image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"image": [image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
|
||||
assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
|
||||
|
||||
|
||||
class TestMoondream3TilingLogic:
|
||||
"""CPU-based tests for Moondream3 tiling selection logic.
|
||||
|
||||
These tests validate the select_tiling() function which determines
|
||||
how images are divided into crops for the vision encoder.
|
||||
"""
|
||||
|
||||
def test_small_image_no_tiling(self):
|
||||
"""Small images should use 1x1 tiling."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
assert tiling == (1, 1)
|
||||
|
||||
def test_exact_crop_size(self):
|
||||
"""Image exactly at crop size should use 1x1."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
assert tiling == (1, 1)
|
||||
|
||||
def test_large_square_image(self):
|
||||
"""Large square image should use multiple tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles >= 2
|
||||
assert w_tiles >= 2
|
||||
assert h_tiles * w_tiles <= MAX_CROPS
|
||||
|
||||
def test_wide_image(self):
|
||||
"""Wide image should have more width tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert w_tiles >= h_tiles
|
||||
|
||||
def test_tall_image(self):
|
||||
"""Tall image should have more height tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles >= w_tiles
|
||||
|
||||
def test_respects_max_crops(self):
|
||||
"""Tiling should not exceed max_crops."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles * w_tiles <= 4
|
||||
|
||||
|
||||
class TestMoondream3VisionShapes:
|
||||
"""CPU-based tests for vision encoder expected shapes.
|
||||
|
||||
These tests verify the mathematical relationships between
|
||||
crop size, patch size, and token counts.
|
||||
"""
|
||||
|
||||
def test_expected_patch_count(self):
|
||||
"""Test 378/14 = 27 patches per side, 729 total."""
|
||||
patches_per_side = CROP_SIZE // PATCH_SIZE
|
||||
total_patches = patches_per_side**2
|
||||
|
||||
assert patches_per_side == 27
|
||||
assert total_patches == EXPECTED_IMAGE_TOKENS - 1
|
||||
|
||||
def test_patch_embedding_input_dim(self):
|
||||
"""Test patch embedding input dimension."""
|
||||
channels = 3
|
||||
input_dim = PATCH_SIZE * PATCH_SIZE * channels
|
||||
|
||||
assert input_dim == 14 * 14 * 3
|
||||
assert input_dim == 588
|
||||
|
||||
|
||||
class TestMoondream3TauAttention:
|
||||
"""CPU-based tests for tau attention scaling components.
|
||||
|
||||
These tests validate the tau attention formula used in Moondream3:
|
||||
- Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
|
||||
- Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
|
||||
"""
|
||||
|
||||
def test_tau_position_range(self):
|
||||
"""Test tau position scaling produces values in valid range."""
|
||||
num_heads = 32
|
||||
seq_len = 100
|
||||
|
||||
tau_alpha = torch.randn(num_heads)
|
||||
positions = torch.arange(seq_len)
|
||||
|
||||
pos_float = (positions.float() + 1.0).clamp(min=1e-6)
|
||||
pos_log = pos_float.log()
|
||||
tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
|
||||
|
||||
assert tau_pos.shape == (num_heads, seq_len)
|
||||
# tau_pos should be between 0.5 and 1.5
|
||||
assert tau_pos.min() >= 0.5
|
||||
assert tau_pos.max() <= 1.5
|
||||
|
||||
def test_tau_token_output_range(self):
|
||||
"""Test tau token scaling output is bounded by tanh."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
seq_len = 100
|
||||
qkv_dim = 6144 # 2048 * 3
|
||||
num_heads = 32
|
||||
|
||||
qkv = torch.randn(seq_len, qkv_dim)
|
||||
tau_wq = torch.randn(num_heads, qkv_dim)
|
||||
|
||||
tok_feat = F.gelu(qkv)
|
||||
tok_q = torch.tanh(tok_feat @ tau_wq.t())
|
||||
|
||||
assert tok_q.shape == (seq_len, num_heads)
|
||||
# tanh output is bounded by [-1, 1]
|
||||
assert tok_q.min() >= -1.0
|
||||
assert tok_q.max() <= 1.0
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _get_expected_num_patches(
|
||||
config: PreTrainedConfig,
|
||||
image: Image.Image,
|
||||
num_imgs: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
):
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
calculate_nemotron_vl_targets,
|
||||
get_nemotron_vl_target_ratios,
|
||||
)
|
||||
|
||||
width, height = image.size
|
||||
|
||||
blocks, _, _ = calculate_nemotron_vl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_nemotron_vl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
),
|
||||
image_size=config.force_image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
expected_num_patches = blocks
|
||||
|
||||
if config.use_thumbnail and expected_num_patches > 1:
|
||||
expected_num_patches += 1
|
||||
|
||||
return expected_num_patches
|
||||
|
||||
|
||||
def _run_check(
|
||||
processor: BaseMultiModalProcessor,
|
||||
images: list[Image.Image],
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
config = processor.info.get_hf_config()
|
||||
image_processor = processor.info.get_image_processor()
|
||||
|
||||
config.use_thumbnail = image_processor.use_thumbnail
|
||||
prompt = "<image>" * len(images)
|
||||
mm_data = {"image": images}
|
||||
|
||||
total_expected_num_patches = sum(
|
||||
_get_expected_num_patches(config, image, len(images), min_num, max_num)
|
||||
for image in images
|
||||
)
|
||||
print(total_expected_num_patches)
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<image>")
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
|
||||
print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
|
||||
assert img_tok_count == 256 * total_expected_num_patches
|
||||
assert pixel_shape[0] == total_expected_num_patches
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"])
|
||||
@pytest.mark.parametrize(
|
||||
"size_factors",
|
||||
[
|
||||
# Single-scale
|
||||
[1.0],
|
||||
# Single-scale, batched
|
||||
[1.0, 1.0, 1.0],
|
||||
# Multi-scale
|
||||
[0.25, 0.5, 1.0],
|
||||
[4.0, 2.0, 1.0],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("min_dynamic_patch", "max_dynamic_patch"),
|
||||
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic_image_size", [True, False])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
image_assets: ImageTestAssets,
|
||||
size_factors: list[int],
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool | None,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
mm_processor_kwargs = {
|
||||
"min_dynamic_patch": min_dynamic_patch,
|
||||
"max_dynamic_patch": max_dynamic_patch,
|
||||
"dynamic_image_size": dynamic_image_size,
|
||||
}
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": len(size_factors)},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
min_num = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_num = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
_run_check(
|
||||
processor,
|
||||
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
|
||||
min_num,
|
||||
max_num,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.models.qwen3_asr import Qwen3ASRProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
|
||||
from .test_common import _assert_inputs_equal
|
||||
|
||||
_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
|
||||
|
||||
@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"])
|
||||
def asr_processor(request):
|
||||
config = ModelConfig(
|
||||
request.param,
|
||||
dtype="float32",
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"audio": 2},
|
||||
mm_processor_cache_gb=0,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(config)
|
||||
native = Qwen3ASRProcessor.from_pretrained(request.param)
|
||||
return processor, native
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001]
|
||||
)
|
||||
@pytest.mark.parametrize("num_audios", [1, 2])
|
||||
def test_audio_processing_matches_hf(asr_processor, samples, num_audios):
|
||||
"""Each clip must match HF independently of other clips in the request."""
|
||||
processor, native = asr_processor
|
||||
rng = np.random.RandomState(42)
|
||||
audios = [
|
||||
rng.randn(samples + index * 160).astype(np.float32)
|
||||
for index in range(num_audios)
|
||||
]
|
||||
expected = [
|
||||
native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios
|
||||
]
|
||||
actual = processor(
|
||||
_AUDIO_PROMPT * num_audios,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
expected_ids = [
|
||||
token for item in expected for token in item["input_ids"][0].tolist()
|
||||
]
|
||||
assert actual["prompt_token_ids"] == expected_ids
|
||||
masks = [item["input_features_mask"][0] for item in expected]
|
||||
expected_features = torch.cat(
|
||||
[
|
||||
item["input_features"][0, :, mask.bool()]
|
||||
for item, mask in zip(expected, masks)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
actual_features = actual["mm_kwargs"].get_data()
|
||||
torch.testing.assert_close(
|
||||
actual_features["input_audio_features"], expected_features, rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_features["audio_feature_lengths"],
|
||||
torch.stack([mask.sum() for mask in masks]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]])
|
||||
def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices):
|
||||
"""Partial cache hits must not change features, masks, or prompt expansion."""
|
||||
processor, _ = asr_processor
|
||||
config = processor.info.ctx.model_config
|
||||
config.get_multimodal_config().mm_processor_cache_gb = 1
|
||||
cache = MultiModalProcessorOnlyCache(config)
|
||||
rng = np.random.RandomState(42)
|
||||
audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)]
|
||||
for index in cached_indices:
|
||||
processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios[index]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
for batch in (audios, audios[::-1]):
|
||||
kwargs = dict(
|
||||
mm_items=processor.info.parse_mm_data({"audio": batch}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
_assert_inputs_equal(
|
||||
processor(_AUDIO_PROMPT * 2, **kwargs),
|
||||
processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache),
|
||||
)
|
||||
|
||||
|
||||
def test_text_only_processing(asr_processor):
|
||||
"""Empty media input must not invoke the audio feature extractor."""
|
||||
processor, native = asr_processor
|
||||
prompt = "Transcribe the audio."
|
||||
actual = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt)
|
||||
assert actual["mm_kwargs"].get_data() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}])
|
||||
def test_hf_processor_without_audio(asr_processor, hf_data):
|
||||
"""The media hook must not require audio or call HF for absent waveforms."""
|
||||
processor, _ = asr_processor
|
||||
assert not processor._call_hf_processor(hf_data, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}])
|
||||
def test_postprocessing_without_audio_features(asr_processor, hf_data):
|
||||
"""Non-audio output must pass through even when HF input is nonempty."""
|
||||
processor, _ = asr_processor
|
||||
processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])})
|
||||
actual = processor._postprocess_hf_mm_data(hf_data, {}, processed)
|
||||
assert actual is processed
|
||||
assert set(actual) == {"input_ids"}
|
||||
torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]]))
|
||||
|
||||
|
||||
def test_chat_template_processing(asr_processor):
|
||||
"""The generic processor loader must still render audio chat prompts."""
|
||||
processor, native = asr_processor
|
||||
chat_processor = cached_processor_from_config(processor.info.ctx.model_config)
|
||||
messages = [
|
||||
{"role": "system", "content": ""},
|
||||
{"role": "user", "content": [{"type": "audio", "audio": "unused"}]},
|
||||
]
|
||||
prompt = chat_processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
expected_prompt = native.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
assert prompt == expected_prompt
|
||||
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
|
||||
expected = native(text=expected_prompt, audio=audio, return_tensors="pt")
|
||||
actual = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
|
||||
def test_precomputed_audio_features(asr_processor):
|
||||
"""Precomputed features must bypass waveform extraction unchanged."""
|
||||
processor, _ = asr_processor
|
||||
processor.info.ctx.get_mm_config().enable_mm_embeds = True
|
||||
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
|
||||
expected = processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
features = expected["mm_kwargs"].get_data()
|
||||
actual = processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": features}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
|
||||
assert actual["mm_placeholders"] == expected["mm_placeholders"]
|
||||
assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features)
|
||||
@@ -1,140 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Qwen3 Omni audio processing and sample rate handling."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("audio_sample_rate", "audio_duration_sec"),
|
||||
[
|
||||
(16000, 1.0), # Native Whisper sample rate, 1 second
|
||||
(16000, 2.0), # Native Whisper sample rate, 2 seconds
|
||||
],
|
||||
)
|
||||
def test_processor_with_audio_sample_rate(
|
||||
model_id: str,
|
||||
audio_sample_rate: int,
|
||||
audio_duration_sec: float,
|
||||
) -> None:
|
||||
"""Test that vLLM's processor generates expected outputs with audio_sample_rate.
|
||||
|
||||
This validates that the processor correctly handles audio_sample_rate
|
||||
passed via hf_processor_mm_kwargs and generates audio tokens.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
# Create audio data at the specified sample rate
|
||||
audio_length = int(audio_sample_rate * audio_duration_sec)
|
||||
rng = np.random.RandomState(42)
|
||||
audio_data = rng.rand(audio_length).astype(np.float32)
|
||||
|
||||
# Build prompt with audio placeholder
|
||||
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
|
||||
|
||||
# Apply processor with audio_sample_rate in mm_kwargs
|
||||
hf_processor_mm_kwargs: dict[str, Any] = {
|
||||
"audio_sample_rate": audio_sample_rate,
|
||||
}
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Verify audio tokens are generated
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token)
|
||||
aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id)
|
||||
|
||||
assert aud_tok_count >= 1, (
|
||||
f"Expected at least 1 audio token but got {aud_tok_count}. "
|
||||
f"sample_rate: {audio_sample_rate}Hz, duration: {audio_duration_sec}s"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
|
||||
def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
"""Test that longer audio generates more tokens than shorter audio.
|
||||
|
||||
This validates that audio_sample_rate is being used correctly by checking
|
||||
that audio duration affects token count as expected.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
audio_sample_rate = 16000
|
||||
rng = np.random.RandomState(42)
|
||||
|
||||
def get_token_count(duration: float) -> int:
|
||||
audio_length = int(audio_sample_rate * duration)
|
||||
audio_data = rng.rand(audio_length).astype(np.float32)
|
||||
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
|
||||
hf_processor_mm_kwargs: dict[str, Any] = {
|
||||
"audio_sample_rate": audio_sample_rate,
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token)
|
||||
return processed["prompt_token_ids"].count(audio_token_id)
|
||||
|
||||
short_tokens = get_token_count(1.0)
|
||||
long_tokens = get_token_count(2.0)
|
||||
|
||||
assert long_tokens > short_tokens, (
|
||||
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
|
||||
f"Got short={short_tokens}, long={long_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
|
||||
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
|
||||
"""Audio and video tokens interleave by the fps the video was processed
|
||||
at, as in HF, not by the default of one frame a second."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
rng = np.random.RandomState(42)
|
||||
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
|
||||
audio = rng.rand(32000).astype(np.float32)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
"use_audio_in_video": True,
|
||||
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [video], "audio": [(audio, 16000)]}
|
||||
),
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
)
|
||||
expected = processor.info.get_hf_processor()(
|
||||
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
|
||||
)
|
||||
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
@@ -1,85 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for smolvlm's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
from transformers import SmolVLMConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
({"max_image_size": {"longest_edge": 384}}, 1377),
|
||||
({"max_image_size": {"longest_edge": 768}}, 405),
|
||||
({"do_image_splitting": False}, 81),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
|
||||
# Same as the previous test - don't initialize mm_processor_kwargs
|
||||
# in this test and assume that the kwargs will be correctly expanded by
|
||||
# the partial when calling the custom input processor.
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
placeholders = (
|
||||
"<image>"
|
||||
if num_imgs == 1
|
||||
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
|
||||
)
|
||||
prompt = f"<|im_start|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
|
||||
|
||||
# Build mm_data
|
||||
image_size = ctx.get_hf_config(SmolVLMConfig).vision_config.image_size
|
||||
dummy_image_size = (image_size * 4, image_size * 4)
|
||||
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
|
||||
mm_data = {"image": [dummy_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure the placeholders format are correct
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
num_patches = processor.info.get_num_patches(
|
||||
image_width=dummy_image.width,
|
||||
image_height=dummy_image.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
assert num_patches == expected_toks_per_img // hf_processor.image_seq_len
|
||||
|
||||
hf_processed_inputs = hf_processor(
|
||||
text=prompt,
|
||||
images=mm_data["image"],
|
||||
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
|
||||
)
|
||||
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = ctx.get_hf_config().image_token_id
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Whisper's multimodal preprocessing."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openai/whisper-large-v3-turbo"])
|
||||
@pytest.mark.parametrize("audio_duration_s", [5, 30, 35])
|
||||
def test_audio_features_fit_encoder_window(
|
||||
model_id: str,
|
||||
audio_duration_s: int,
|
||||
) -> None:
|
||||
"""The encoder only has positions for one 30 s window, so a longer clip
|
||||
has to be cut to that window like the HF feature extractor does by default.
|
||||
"""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
feature_extractor = processor.info.get_feature_extractor()
|
||||
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
rng = np.random.RandomState(0)
|
||||
audio = rng.rand(sampling_rate * audio_duration_s).astype(np.float32)
|
||||
|
||||
processed_inputs = processor(
|
||||
"<|startoftranscript|>",
|
||||
mm_items=processor.info.parse_mm_data({"audio": [(audio, sampling_rate)]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
input_features = processed_inputs["mm_kwargs"].get_data()["input_features"]
|
||||
assert input_features.shape[-1] == feature_extractor.nb_max_frames
|
||||
@@ -1,164 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
|
||||
from ...conftest import IMAGE_ASSETS, ImageTestAssets, VllmRunner
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
"cherry_blossom": "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||
# Single-scale
|
||||
(1.0,),
|
||||
# Single-scale, batched
|
||||
(1.0, 1.0, 1.0),
|
||||
# Multi-scale
|
||||
(0.25, 0.5, 1.0),
|
||||
)
|
||||
|
||||
|
||||
def run_awq_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
source_model: str,
|
||||
quant_model: str,
|
||||
*,
|
||||
size_factor_groups: tuple[tuple[float, ...], ...],
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
):
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_image_and_size_group = [
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, factor) for factor in size_factors],
|
||||
)
|
||||
for size_factors in size_factor_groups
|
||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
# max_model_len should be greater than image_feature_size
|
||||
with vllm_runner(
|
||||
source_model,
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
source_outputs_per_image = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs=num_logprobs, images=images
|
||||
)
|
||||
for prompts, images in inputs_per_image_and_size_group
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
quant_model,
|
||||
quantization="awq",
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
quant_outputs_per_image = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs=num_logprobs, images=images
|
||||
)
|
||||
for prompts, images in inputs_per_image_and_size_group
|
||||
]
|
||||
|
||||
for source_outputs, quant_outputs in zip(
|
||||
source_outputs_per_image, quant_outputs_per_image
|
||||
):
|
||||
# TODO: Check whether using original CLIPVisionModel can improve
|
||||
# consistency against HF
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=source_outputs,
|
||||
outputs_1_lst=quant_outputs,
|
||||
name_0="source",
|
||||
name_1="awq",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "quantization", "dtype"),
|
||||
[
|
||||
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
|
||||
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
|
||||
],
|
||||
ids=[
|
||||
"gemma4-moe-standard-awq-dot-suffix",
|
||||
"gemma4-moe-compressed-tensors-underscore-suffix",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_awq_load(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
quantization: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Regression test: AWQ weight loading must not KeyError."""
|
||||
with vllm_runner(
|
||||
model,
|
||||
quantization=quantization,
|
||||
dtype=dtype,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source_model", "quant_model"),
|
||||
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@torch.inference_mode()
|
||||
def test_awq_models(
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
source_model,
|
||||
quant_model,
|
||||
dtype,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
) -> None:
|
||||
run_awq_test(
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
source_model,
|
||||
quant_model,
|
||||
size_factor_groups=IMAGE_SIZE_FACTOR_GROUPS,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for online FP8 per-channel quantization.
|
||||
|
||||
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
|
||||
quantization) and compares log-probabilities against the same model served in
|
||||
BF16 without quantization. This exercises the full pipeline: config parsing,
|
||||
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
|
||||
loading, online quantization / shuffling, and inference.
|
||||
|
||||
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
|
||||
diverse prompts from ``tests/prompts/example.txt``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
|
||||
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
# Small dense model (no MoE) to validate the linear-only path.
|
||||
DENSE_MODEL = "Qwen/Qwen3-0.6B"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_TOKENS = 4
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_fp8_per_channel_logprobs(
|
||||
vllm_runner,
|
||||
quant_baseline_logprobs,
|
||||
example_prompts,
|
||||
model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare BF16 baseline logprobs against online per-channel-quantized
|
||||
model.
|
||||
|
||||
Runs the same model twice -- once in BF16 (baseline) and once with online
|
||||
FP8 per-channel quantization -- then checks that the top log-probabilities
|
||||
are close. Only 4 tokens are generated to keep the test fast while still
|
||||
catching numerical divergence beyond expected per-channel error.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
baseline_outputs = quant_baseline_logprobs(
|
||||
model,
|
||||
example_prompts,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_tokens=MAX_TOKENS,
|
||||
num_logprobs=NUM_LOG_PROBS,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
quantization="fp8_per_channel",
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16",
|
||||
name_1="fp8_per_channel",
|
||||
)
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
|
||||
|
||||
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
|
||||
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
|
||||
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
|
||||
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
|
||||
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
|
||||
device gate accepts the card, so the checkpoint could not be served natively at
|
||||
all.
|
||||
|
||||
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
|
||||
a correction bias), so the backend only has to ask for it. This module covers
|
||||
both halves: the gate accepts the router family, and the routing the wrapper
|
||||
requests reproduces the model's own math.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
|
||||
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
|
||||
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
|
||||
MIMO_TOP_K = 8
|
||||
|
||||
|
||||
def mimo_routing_method() -> RoutingMethodType:
|
||||
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
|
||||
return get_routing_method_type(
|
||||
scoring_func="sigmoid",
|
||||
top_k=MIMO_TOP_K,
|
||||
renormalize=True,
|
||||
num_expert_group=1,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
|
||||
|
||||
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=MIMO_TOP_K,
|
||||
hidden_dim=4096,
|
||||
intermediate_size=2048,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
routing_method=mimo_routing_method(),
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
|
||||
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
|
||||
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
|
||||
|
||||
|
||||
def test_aiter_w4a16_supports_mimo_routing() -> None:
|
||||
"""The gate accepts MiMo's router family — the reported construction abort.
|
||||
|
||||
Only the predicate the fix touches is exercised, so this runs on any host;
|
||||
the full config path is covered by the test below.
|
||||
"""
|
||||
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
|
||||
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The same gate through the real config path, device check included."""
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
mimo_moe_config(),
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
|
||||
"""The routing the wrapper requests must equal MiMo's own router math.
|
||||
|
||||
MiMo's router is ungrouped: sigmoid scores, selection on
|
||||
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
|
||||
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
|
||||
|
||||
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
|
||||
"""
|
||||
aiter_routing = pytest.importorskip(
|
||||
"aiter.ops.triton.moe.moe_routing.routing"
|
||||
).routing
|
||||
|
||||
torch.manual_seed(0)
|
||||
top_k, num_tokens = MIMO_TOP_K, 16
|
||||
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
|
||||
bias = torch.randn(num_experts, device="cuda") * 0.5
|
||||
|
||||
# Reference: select on the biased score, return the unbiased score, renorm.
|
||||
scores = torch.sigmoid(logits.float())
|
||||
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
|
||||
ref_weights = scores.gather(1, ref_ids)
|
||||
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
routing_data, topk_indx, _ = aiter_routing(
|
||||
logits,
|
||||
top_k,
|
||||
score_mode=score_mode,
|
||||
bias=bias,
|
||||
renorm=True,
|
||||
routed_scaling_factor=1.0,
|
||||
use_grouped_topk=False,
|
||||
)
|
||||
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
|
||||
# i-th entry of the expert-sorted array whose weights are gate_scal.
|
||||
hist = routing_data.expt_hist.to(torch.long).cpu()
|
||||
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
|
||||
slot = topk_indx.reshape(-1).to(torch.long).cpu()
|
||||
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
|
||||
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
|
||||
out = torch.zeros(num_tokens, top_k)
|
||||
for i in range(slot.numel()):
|
||||
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
|
||||
ids[token, position] = int(expert_of_slot[i])
|
||||
out[token, position] = float(weights[i])
|
||||
return ids, out
|
||||
|
||||
got_ids, got_weights = routed("sigmoid")
|
||||
|
||||
# aiter's triton top-k emits each token's selection sorted by *ascending
|
||||
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
|
||||
# rank order — align both sides on the expert id so the weight check
|
||||
# becomes elementwise and still validates the expert<->weight pairing.
|
||||
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
order = torch.argsort(ids, dim=-1)
|
||||
return ids.gather(1, order), w.gather(1, order)
|
||||
|
||||
got_ids, got_weights = align(got_ids, got_weights)
|
||||
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
|
||||
assert torch.equal(got_ids, ref_ids)
|
||||
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
|
||||
|
||||
# Control: the correction bias must steer selection only. Ignoring it would
|
||||
# pick from the plain sigmoid scores instead.
|
||||
plain_ids = scores.topk(top_k, dim=-1).indices
|
||||
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for AliceAI configuration."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.config import AliceAIForCausalLMConfig
|
||||
|
||||
|
||||
def _make_config(**overrides) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
**{
|
||||
"kda_allow_negative_eigenvalues": False,
|
||||
"router_score_function": "sigmoid",
|
||||
"router_bias_correction": True,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_model_config_warns_when_mtp_is_ignored(caplog) -> None:
|
||||
config = _make_config(mtp_num_hidden_layers=1)
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(
|
||||
SimpleNamespace(hf_config=config)
|
||||
)
|
||||
assert "ignores the MTP configuration and weights" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,error_type,error",
|
||||
[
|
||||
({}, None, None),
|
||||
(
|
||||
{"kda_allow_negative_eigenvalues": True},
|
||||
NotImplementedError,
|
||||
"kda_allow_negative_eigenvalues=false",
|
||||
),
|
||||
(
|
||||
{"router_score_function": "softmax"},
|
||||
ValueError,
|
||||
"router_score_function='sigmoid'",
|
||||
),
|
||||
(
|
||||
{"router_bias_correction": False},
|
||||
ValueError,
|
||||
"router_bias_correction=true",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_model_config_rejects_unsupported_alice_contract(
|
||||
overrides, error_type, error
|
||||
) -> None:
|
||||
model_config = SimpleNamespace(hf_config=_make_config(**overrides))
|
||||
|
||||
if error is None:
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
|
||||
else:
|
||||
with pytest.raises(error_type, match=error):
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pipeline_parallel_size,speculative_method,error",
|
||||
[
|
||||
(1, None, None),
|
||||
(2, None, "pipeline_parallel_size=1"),
|
||||
(1, "eagle3", "eagle3"),
|
||||
(1, "dflash", "dflash"),
|
||||
(1, "dspark", "dspark"),
|
||||
(1, "extract_hidden_states", "extract_hidden_states"),
|
||||
],
|
||||
)
|
||||
def test_runtime_config_rejects_unsupported_alice_modes(
|
||||
pipeline_parallel_size, speculative_method, error
|
||||
) -> None:
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(pipeline_parallel_size=pipeline_parallel_size),
|
||||
speculative_config=(
|
||||
None
|
||||
if speculative_method is None
|
||||
else SimpleNamespace(method=speculative_method)
|
||||
),
|
||||
)
|
||||
|
||||
if error is None:
|
||||
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
|
||||
else:
|
||||
with pytest.raises(NotImplementedError, match=error):
|
||||
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
|
||||
@@ -1,57 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
importlib.util.find_spec("terratorch") is None,
|
||||
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test() # Hangs otherwise
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
|
||||
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars",
|
||||
],
|
||||
)
|
||||
def test_inference(
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
) -> None:
|
||||
pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16)
|
||||
location_coords = torch.full((1, 2), 1.0, dtype=torch.float16)
|
||||
prompt = dict(
|
||||
prompt_token_ids=[1],
|
||||
multi_modal_data={
|
||||
"image": {
|
||||
"pixel_values": pixel_values,
|
||||
"location_coords": location_coords,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
skip_tokenizer_init=True,
|
||||
enable_mm_embeds=True,
|
||||
# Limit the maximum number of sequences to avoid the
|
||||
# test going OOM during the warmup run
|
||||
max_num_seqs=32,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
vllm_output = vllm_model.llm.encode(prompt, pooling_task="plugin")
|
||||
assert torch.equal(
|
||||
torch.isnan(vllm_output[0].outputs.data).any(), torch.tensor(False)
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,611 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import warnings
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption
|
||||
from vllm.config.pooler import SequencePoolingType, TokenPoolingType
|
||||
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
|
||||
from .. import ci_envs
|
||||
from .registry import HF_EXAMPLE_MODELS
|
||||
|
||||
TokensText = tuple[list[int], str]
|
||||
|
||||
|
||||
def check_outputs_equal(
|
||||
*,
|
||||
outputs_0_lst: Sequence[TokensText],
|
||||
outputs_1_lst: Sequence[TokensText],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
):
|
||||
"""Compare the two sequences generated by different models,
|
||||
which should be equal.
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
for prompt_idx, (outputs_0, outputs_1) in enumerate(
|
||||
zip(outputs_0_lst, outputs_1_lst)
|
||||
):
|
||||
output_ids_0, output_str_0 = outputs_0
|
||||
output_ids_1, output_str_1 = outputs_1
|
||||
|
||||
# The text and token outputs should exactly match
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\n{name_0}:\t{output_str_0!r}"
|
||||
f"\n{name_1}:\t{output_str_1!r}"
|
||||
)
|
||||
|
||||
assert output_str_0 == output_str_1, fail_msg
|
||||
assert output_ids_0 == output_ids_1, fail_msg
|
||||
|
||||
|
||||
# Representation of generated sequence as a tuple of
|
||||
# * Token ID list
|
||||
# * String
|
||||
# * List of top sample logprobs for each sampled token
|
||||
#
|
||||
# Assumes prompt logprobs were not requested.
|
||||
TokensTextLogprobs = tuple[
|
||||
list[int], str, list[dict[int, float]] | SampleLogprobs | None
|
||||
]
|
||||
|
||||
# Allow for tokens to be represented as str's rather than IDs;
|
||||
# tuple of
|
||||
# * Token string representations list
|
||||
# * String
|
||||
# * Optional list of top sample logprobs for each sampled token
|
||||
#
|
||||
# Assumes prompt logprobs were not requested.
|
||||
TextTextLogprobs = tuple[
|
||||
list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None
|
||||
]
|
||||
|
||||
# Representation of generated sequence as a tuple of
|
||||
# * Token ID list
|
||||
# * String
|
||||
# * Optional list of top sample logprobs for each sampled token
|
||||
# * Optional list of top prompt logprobs for each prompt token
|
||||
#
|
||||
# Allows prompt logprobs to be requested.
|
||||
TokensTextLogprobsPromptLogprobs = tuple[
|
||||
list[int],
|
||||
str,
|
||||
list[dict[int, float]] | SampleLogprobs | None,
|
||||
list[dict[int, float] | None] | PromptLogprobs | None,
|
||||
]
|
||||
|
||||
|
||||
def check_logprobs_close(
|
||||
*,
|
||||
outputs_0_lst: Sequence[
|
||||
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
|
||||
],
|
||||
outputs_1_lst: Sequence[
|
||||
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
|
||||
],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
num_outputs_0_skip_tokens: int = 0,
|
||||
warn_on_mismatch: bool = True,
|
||||
always_check_logprobs: bool = False,
|
||||
) -> None:
|
||||
"""Compare the logprobs of two sequences generated by different models,
|
||||
which should be similar but not necessarily equal.
|
||||
|
||||
How sample logprobs are compared:
|
||||
* `always_check_logprobs == True`: set of highest-logprob token ids
|
||||
must match between seq0 and seq1 at all sampled token offsets
|
||||
* `always_check_logprobs == False`: highest-logprob token ids are
|
||||
only compared at sampled token offsets for which generated token
|
||||
ids don't match
|
||||
|
||||
Prompt logprobs must be provided either for both input sequences, or
|
||||
for neither. If prompt logprobs are provided, then highest-logprob
|
||||
prompt token ids must match between seq0 and seq1 at all prompt token
|
||||
offsets.
|
||||
|
||||
Args:
|
||||
outputs_0_lst: First sequence to compare
|
||||
outputs_0_lst: Second sequence to compare
|
||||
name_0: sequence #0 name
|
||||
name_1: sequence #1 name
|
||||
num_outputs_0_skip_tokens: If > 0, specifies the number of initial
|
||||
sequence #0 tokens & logprobs to discard
|
||||
before comparison, i.e. all
|
||||
of sequence #1 will be compared to
|
||||
sequence #0 beginning at index
|
||||
num_outputs_0_skip_tokens
|
||||
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
|
||||
mismatch between the two sequences
|
||||
always_check_logprobs: If true, check logprobs even when tokens match
|
||||
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
# Loop through responses to each prompt.
|
||||
for prompt_idx, (outputs_0, outputs_1) in enumerate(
|
||||
zip(outputs_0_lst, outputs_1_lst)
|
||||
):
|
||||
assert len(outputs_0) == len(outputs_1)
|
||||
if len(outputs_0) == 3:
|
||||
assert len(outputs_1) == 3
|
||||
# Break out tokens, text & sample logprobs
|
||||
# (prompt logprobs were not provided)
|
||||
output_ids_0, output_str_0, logprobs_0 = outputs_0
|
||||
output_ids_1, output_str_1, logprobs_1 = outputs_1
|
||||
elif len(outputs_0) == 4:
|
||||
assert len(outputs_1) == 4
|
||||
# Break out tokens, text, sample logprobs & prompt logprobs
|
||||
(
|
||||
output_ids_0,
|
||||
output_str_0,
|
||||
logprobs_0,
|
||||
prompt_logprobs_0,
|
||||
) = outputs_0
|
||||
(
|
||||
output_ids_1,
|
||||
output_str_1,
|
||||
logprobs_1,
|
||||
prompt_logprobs_1,
|
||||
) = outputs_1
|
||||
|
||||
# Test prompt logprobs closeness
|
||||
if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:
|
||||
# Both sequences' prompt logprobs lists are not `None`
|
||||
# (although individual list elements may be `None`);
|
||||
# for each token's logprobs:
|
||||
for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
|
||||
zip(prompt_logprobs_0, prompt_logprobs_1)
|
||||
):
|
||||
fail_msg = (
|
||||
f"Prompt logprobs test:"
|
||||
f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
|
||||
f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}"
|
||||
)
|
||||
|
||||
if logprobs_elem_0 is None:
|
||||
# If the seq 0 token's logprobs are `None`,
|
||||
# the seq 1 token's logprobs must be `None`
|
||||
assert logprobs_elem_1 is None, fail_msg
|
||||
else:
|
||||
# If the seq 0 token's logprobs are not `None`,
|
||||
# the seq 1 token's logprobs must not be `None`
|
||||
assert logprobs_elem_1 is not None, fail_msg
|
||||
# Logprobs check: top-k token choices must be the same
|
||||
assert set(logprobs_elem_0.keys()) == set(
|
||||
logprobs_elem_1.keys()
|
||||
), fail_msg
|
||||
else:
|
||||
# Both sequence logprobs lists must be `None`
|
||||
fail_msg = (
|
||||
f"Prompt logprobs test:"
|
||||
f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
|
||||
f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}"
|
||||
)
|
||||
|
||||
assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Outputs tuple must have 3 or 4 elements but "
|
||||
f"{len(outputs_0)} elements were provided: "
|
||||
f"{outputs_0}"
|
||||
)
|
||||
|
||||
if logprobs_0 is None:
|
||||
logprobs_0 = [None] * len(output_ids_0)
|
||||
if logprobs_1 is None:
|
||||
logprobs_1 = [None] * len(output_ids_1)
|
||||
|
||||
# Skip specified number of initial sequence #0 tokens
|
||||
# & logprobs, leaving output text as-is for simplicity
|
||||
# (text mismatches may generate warnings but do not
|
||||
# cause the test to fail.)
|
||||
if num_outputs_0_skip_tokens < 0:
|
||||
raise ValueError("num_outputs_0_skip_tokens must be non-negative")
|
||||
output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
|
||||
logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]
|
||||
|
||||
# Loop through generated tokens.
|
||||
for idx, (output_id_0, output_id_1) in enumerate(
|
||||
zip(output_ids_0, output_ids_1)
|
||||
):
|
||||
is_tok_mismatch = output_id_0 != output_id_1
|
||||
|
||||
# If generated tokens don't match
|
||||
# or it is desired to always check logprobs,
|
||||
# then
|
||||
if is_tok_mismatch or always_check_logprobs:
|
||||
logprobs_elem_0 = logprobs_0[idx]
|
||||
logprobs_elem_1 = logprobs_1[idx]
|
||||
|
||||
# Each predicted token must be in top N logprobs of the other
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\nMatched tokens:\t{output_ids_0[:idx]}"
|
||||
f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}"
|
||||
f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}"
|
||||
)
|
||||
|
||||
assert logprobs_elem_0 is not None, fail_msg
|
||||
assert logprobs_elem_1 is not None, fail_msg
|
||||
assert output_id_0 in logprobs_elem_1, fail_msg
|
||||
assert output_id_1 in logprobs_elem_0, fail_msg
|
||||
|
||||
if warn_on_mismatch and is_tok_mismatch:
|
||||
with warnings.catch_warnings():
|
||||
# This ensures that repeated warnings are shown
|
||||
# in the output, not just the first occurrence
|
||||
warnings.simplefilter("always")
|
||||
|
||||
warnings.warn(fail_msg, stacklevel=2)
|
||||
|
||||
# Break out since sequences will now diverge.
|
||||
break
|
||||
else:
|
||||
if output_str_0 != output_str_1 and warn_on_mismatch:
|
||||
# The token outputs exactly match,
|
||||
# so the text outputs should exactly match as well
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\n{name_0}:\t{output_str_0!r}"
|
||||
f"\n{name_1}:\t{output_str_1!r}"
|
||||
)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
# This ensures that repeated warnings are shown
|
||||
# in the output, not just the first occurrence
|
||||
warnings.simplefilter("always")
|
||||
|
||||
warnings.warn(fail_msg, stacklevel=2)
|
||||
|
||||
|
||||
def build_model_context(
|
||||
model_id: str,
|
||||
runner: RunnerOption = "auto",
|
||||
dtype: ModelDType = "auto",
|
||||
model_config_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None,
|
||||
mm_processor_cache_gb: int = 0,
|
||||
):
|
||||
"""Creates an InputProcessingContext for a given model.
|
||||
|
||||
Args:
|
||||
model_id: ID of the model being considered.
|
||||
mm_processor_kwargs: optional processor kwargs for to be leveraged
|
||||
in the input processor, mapper, dummy data creation, etc.
|
||||
limit_mm_per_prompt: Multimodal limits.
|
||||
|
||||
Returns:
|
||||
InputProcessingContext for the model being considered.
|
||||
|
||||
"""
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_config_kwargs = model_config_kwargs or {}
|
||||
limit_mm_per_prompt = {
|
||||
modality: dict(limit) if isinstance(limit, Mapping) else limit
|
||||
for modality, limit in (limit_mm_per_prompt or {}).items()
|
||||
}
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
runner=runner,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
dtype=dtype,
|
||||
seed=0,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
**model_config_kwargs,
|
||||
)
|
||||
|
||||
return InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
def check_embeddings_close(
|
||||
*,
|
||||
embeddings_0_lst: Sequence[list[float]],
|
||||
embeddings_1_lst: Sequence[list[float]],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
tol: float = 1e-3,
|
||||
) -> None:
|
||||
assert len(embeddings_0_lst) == len(embeddings_1_lst)
|
||||
|
||||
for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
|
||||
zip(embeddings_0_lst, embeddings_1_lst)
|
||||
):
|
||||
assert len(embeddings_0) == len(embeddings_1), (
|
||||
f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}"
|
||||
)
|
||||
|
||||
sim = F.cosine_similarity(
|
||||
torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0
|
||||
)
|
||||
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\nCosine similarity: \t{sim:.4f}"
|
||||
f"\n{name_0}:\t{embeddings_0[:16]!r}"
|
||||
f"\n{name_1}:\t{embeddings_1[:16]!r}"
|
||||
)
|
||||
|
||||
assert sim >= 1 - tol, fail_msg
|
||||
|
||||
|
||||
def matryoshka_fy(tensor: torch.Tensor, dimensions: int):
|
||||
tensor = torch.tensor(tensor)
|
||||
tensor = tensor[..., :dimensions]
|
||||
tensor = F.normalize(tensor, p=2, dim=1)
|
||||
return tensor
|
||||
|
||||
|
||||
def softmax(data):
|
||||
if data.shape[-1] == 1:
|
||||
return F.sigmoid(data)
|
||||
else:
|
||||
return F.softmax(data, dim=-1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelInfo:
|
||||
name: str
|
||||
revision: str | None = None
|
||||
architecture: str = ""
|
||||
dtype: str = "auto"
|
||||
max_model_len: int | None = None
|
||||
hf_dtype: str = "float32"
|
||||
hf_overrides: dict[str, Any] | None = None
|
||||
seq_pooling_type: SequencePoolingType | None = None
|
||||
tok_pooling_type: TokenPoolingType | None = None
|
||||
attn_type: AttnTypeStr | None = None
|
||||
is_prefix_caching_supported: bool | None = None
|
||||
is_chunked_prefill_supported: bool | None = None
|
||||
enable_test: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbedModelInfo(ModelInfo):
|
||||
mteb_score: float | None = None
|
||||
# Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL.
|
||||
mteb_tol: float | None = None
|
||||
is_matryoshka: bool = False
|
||||
matryoshka_dimensions: list[int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RerankModelInfo(ModelInfo):
|
||||
mteb_score: float | None = None
|
||||
# Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL.
|
||||
mteb_tol: float | None = None
|
||||
chat_template_name: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerateModelInfo(ModelInfo):
|
||||
hf_dtype: str = "auto"
|
||||
hf_ppl: float | None = None
|
||||
|
||||
|
||||
def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs):
|
||||
# A model family has many models with the same architecture,
|
||||
# and we don't need to test each one.
|
||||
if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test:
|
||||
import pytest
|
||||
|
||||
pytest.skip("Skipping test.")
|
||||
|
||||
# Allow vllm to test using the given dtype, such as float32
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype
|
||||
|
||||
# Allow vllm to test using hf_overrides
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
# Allow changing the head dtype used by vllm in tests
|
||||
if ci_envs.VLLM_CI_HEAD_DTYPE is not None:
|
||||
if "hf_overrides" not in vllm_extra_kwargs:
|
||||
vllm_extra_kwargs["hf_overrides"] = {}
|
||||
vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE
|
||||
|
||||
# Allow control over whether tests use enforce_eager
|
||||
if ci_envs.VLLM_CI_ENFORCE_EAGER is not None:
|
||||
vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER
|
||||
|
||||
return vllm_extra_kwargs
|
||||
|
||||
|
||||
def dummy_hf_overrides(
|
||||
hf_config: PreTrainedConfig,
|
||||
*,
|
||||
model_arch: str = "",
|
||||
exist_overrides: dict[str, Any] | None = None,
|
||||
use_original_num_layers: bool = False,
|
||||
) -> PreTrainedConfig:
|
||||
"""Dummy HF overrides function used to create dummy model
|
||||
with only minimum nums of layer.
|
||||
"""
|
||||
# Copy because this helper is called more than once
|
||||
# while loading config, and we `.pop()`
|
||||
exist_overrides = (exist_overrides or {}).copy()
|
||||
text_config_override = exist_overrides.pop("text_config", None)
|
||||
hf_config.update(exist_overrides)
|
||||
|
||||
text_config = hf_config.get_text_config()
|
||||
if text_config_override is not None:
|
||||
# multimodal test models may override *some* text-model fields
|
||||
text_config.update(text_config_override)
|
||||
|
||||
# Ensure at least 2 expert per group
|
||||
# Since `grouped_topk` assumes top-2
|
||||
n_group = getattr(text_config, "n_group", None)
|
||||
# Kimi uses `num_expert_group` instead of `n_group`.
|
||||
if n_group is None:
|
||||
n_group = getattr(text_config, "num_expert_group", None)
|
||||
# InternS1Pro uses `router_n_groups` instead of `n_group`.
|
||||
if n_group is None:
|
||||
n_group = getattr(text_config, "router_n_groups", None)
|
||||
num_experts = n_group * 2 if n_group is not None else 2
|
||||
|
||||
# we use three layers for Gemma-3n to check
|
||||
# both normal layer and kv_shared_layer
|
||||
if use_original_num_layers:
|
||||
# Use the original number of layers from the config
|
||||
num_layers = getattr(text_config, "num_layers", 1)
|
||||
num_hidden_layers = getattr(text_config, "num_hidden_layers", 1)
|
||||
else:
|
||||
# Use minimal layers for testing
|
||||
num_layers = 1
|
||||
num_hidden_layers = (
|
||||
3
|
||||
if model_arch
|
||||
in (
|
||||
"Gemma3nForConditionalGeneration",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4MTPModel",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
update_dict = {
|
||||
"num_layers": num_layers,
|
||||
# For Gemma-3n
|
||||
"num_kv_shared_layers": 1,
|
||||
}
|
||||
|
||||
_hf_config = hf_config
|
||||
|
||||
class DummyConfig:
|
||||
hf_config = _hf_config
|
||||
hf_text_config = text_config
|
||||
|
||||
# Keep architecture conversion on the default multimodal path; this
|
||||
# helper only needs HF-derived fields, not deployment MM limits.
|
||||
@staticmethod
|
||||
def _supports_multimodal_for_mm_prefix() -> bool:
|
||||
return True
|
||||
|
||||
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
|
||||
# Only set MoE related config when the model has MoE layers.
|
||||
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
|
||||
if model_arch_config.num_experts > 0:
|
||||
num_experts_per_tok = 2
|
||||
if model_arch in (
|
||||
"Llama4ForConditionalGeneration",
|
||||
"Llama4ForCausalLM",
|
||||
"EagleLlama4ForCausalLM",
|
||||
):
|
||||
num_experts_per_tok = 1
|
||||
elif model_arch == "InternS1ProForConditionalGeneration":
|
||||
assert n_group is not None
|
||||
num_experts_per_tok = n_group
|
||||
update_dict.update(
|
||||
{
|
||||
"num_experts": num_experts,
|
||||
"num_experts_per_tok": num_experts_per_tok,
|
||||
# Kimi uses `num_experts_per_token`.
|
||||
"num_experts_per_token": num_experts_per_tok,
|
||||
"num_local_experts": num_experts,
|
||||
# Otherwise there will not be any expert layers
|
||||
"first_k_dense_replace": 0,
|
||||
# To avoid OOM on DeepSeek-V3
|
||||
"n_routed_experts": num_experts,
|
||||
}
|
||||
)
|
||||
|
||||
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
|
||||
# from num_layers for its dual-attention layers).
|
||||
if model_arch not in (
|
||||
"LongcatFlashForCausalLM",
|
||||
"LongCatFlashMTPModel",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
):
|
||||
update_dict["num_hidden_layers"] = num_hidden_layers
|
||||
|
||||
text_config.update(update_dict)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
# HY V4 uses this to decide which sparse-attention indexers are shared.
|
||||
if indexer_types := getattr(text_config, "indexer_types", None):
|
||||
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
|
||||
|
||||
# Update n_layers and moe configs for Moondream3 model
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
text_config.update(
|
||||
{
|
||||
"n_layers": num_hidden_layers,
|
||||
"moe_num_experts": num_experts,
|
||||
"moe_experts_per_token": 2,
|
||||
"moe_start_layer": num_hidden_layers,
|
||||
}
|
||||
)
|
||||
|
||||
if hasattr(hf_config, "vision_config"):
|
||||
vision_config = hf_config.vision_config
|
||||
vision_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
if layer_types := getattr(vision_config, "layer_types", None):
|
||||
vision_config.update({"layer_types": layer_types[:1]})
|
||||
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
vision_config.update({"enc_n_layers": 1})
|
||||
|
||||
# e.g.: ibm-granite/granite-speech-3.3-2b
|
||||
if hasattr(hf_config, "encoder_config"):
|
||||
hf_config.encoder_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# e.g.: Qwen/Qwen2-Audio-7B-Instruct
|
||||
# audio_config may exist but be None (e.g. audio-less Gemma4 variants).
|
||||
if getattr(hf_config, "audio_config", None) is not None:
|
||||
hf_config.audio_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
"encoder_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
return hf_config
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the unified Cohere Command parser tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
|
||||
_SPECIAL_TOKEN_RE = re.compile(r"(<\|[A-Z_]+\|>)")
|
||||
REPLACEMENT_CHAR = "�"
|
||||
|
||||
|
||||
class MockCohereTokenizer:
|
||||
"""Byte-level stand-in for the Cohere tokenizer.
|
||||
|
||||
Text round-trips through UTF-8 bytes so a multi-byte character can split
|
||||
across "tokens" (reproducing trailing U+FFFD buffering), while each
|
||||
``<|...|>`` marker is a single id so token-id reasoning-end gating behaves
|
||||
as with the real tokenizer.
|
||||
"""
|
||||
|
||||
_SPECIAL_TOKEN_IDS = {
|
||||
tok: 256 + i
|
||||
for i, tok in enumerate(
|
||||
(
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
"<|CHATBOT_TOKEN|>",
|
||||
"<|START_RESPONSE|>",
|
||||
"<|END_RESPONSE|>",
|
||||
"<|START_TEXT|>",
|
||||
"<|END_TEXT|>",
|
||||
"<|START_ACTION|>",
|
||||
"<|END_ACTION|>",
|
||||
)
|
||||
)
|
||||
}
|
||||
_ID_TO_SPECIAL_TOKEN = {v: k for k, v in _SPECIAL_TOKEN_IDS.items()}
|
||||
|
||||
def convert_tokens_to_ids(self, token: str) -> int:
|
||||
return self._SPECIAL_TOKEN_IDS.get(token, 0)
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
ids: list[int] = []
|
||||
for part in _SPECIAL_TOKEN_RE.split(text):
|
||||
if part in self._SPECIAL_TOKEN_IDS:
|
||||
ids.append(self._SPECIAL_TOKEN_IDS[part])
|
||||
else:
|
||||
ids.extend(part.encode("utf-8"))
|
||||
return ids
|
||||
|
||||
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
out: list[str] = []
|
||||
for special, run in itertools.groupby(
|
||||
ids, self._ID_TO_SPECIAL_TOKEN.__contains__
|
||||
):
|
||||
if not special:
|
||||
out.append(bytes(run).decode("utf-8", errors="replace"))
|
||||
elif not skip_special_tokens:
|
||||
out.extend(self._ID_TO_SPECIAL_TOKEN[i] for i in run)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def token_deltas(
|
||||
tokenizer: MockCohereTokenizer, text: str, chunk_size: int = 1
|
||||
) -> list[tuple[str, list[int]]]:
|
||||
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
|
||||
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
|
||||
U+FFFD) into the next step as real streaming does."""
|
||||
ids = tokenizer.encode(text)
|
||||
deltas: list[tuple[str, list[int]]] = []
|
||||
prev = ""
|
||||
pending: list[int] = []
|
||||
for start in range(0, len(ids), chunk_size):
|
||||
end = start + chunk_size
|
||||
pending.extend(ids[start:end])
|
||||
current = tokenizer.decode(ids[:end])
|
||||
if current.endswith(REPLACEMENT_CHAR) and end < len(ids):
|
||||
continue
|
||||
deltas.append((current[len(prev) :], pending))
|
||||
prev, pending = current, []
|
||||
return deltas
|
||||
|
||||
|
||||
def make_parser(
|
||||
tokenizer: MockCohereTokenizer,
|
||||
name: str,
|
||||
tools: list[dict] | None = None,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
model_config=None,
|
||||
) -> Parser:
|
||||
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
|
||||
name (``cohere_command3`` / ``cohere_command4``)."""
|
||||
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
|
||||
assert cls is not None
|
||||
return cls(
|
||||
tokenizer,
|
||||
tools,
|
||||
model_config=model_config,
|
||||
chat_template_kwargs=chat_template_kwargs or {},
|
||||
)
|
||||
|
||||
|
||||
def drive_parser(
|
||||
parser: Parser,
|
||||
request: ChatCompletionRequest,
|
||||
deltas: list[tuple[str, list[int]]],
|
||||
) -> list[DeltaMessage]:
|
||||
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
|
||||
return the non-``None`` messages in order."""
|
||||
out: list[DeltaMessage] = []
|
||||
for i, (text, ids) in enumerate(deltas):
|
||||
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
|
||||
if delta is not None:
|
||||
out.append(delta)
|
||||
return out
|
||||
|
||||
|
||||
def stream_parser(parser, request, tokenizer, text: str, chunk_size: int = 1):
|
||||
"""Stream ``text`` through ``parser`` in ``chunk_size``-token deltas."""
|
||||
return drive_parser(parser, request, token_deltas(tokenizer, text, chunk_size))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.quantization import (
|
||||
QuantizationConfigArgs,
|
||||
QuantSpec,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import get_quantization_config
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def quant_spec(**kwargs: Any) -> QuantSpec:
|
||||
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
|
||||
return TypeAdapter(QuantSpec).validate_python(kwargs)
|
||||
|
||||
|
||||
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
|
||||
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
|
||||
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
|
||||
|
||||
|
||||
def _limit_num_hidden_layers(
|
||||
model: torch.nn.Module, num_hidden_layers: int | None
|
||||
) -> None:
|
||||
if num_hidden_layers is None:
|
||||
return
|
||||
|
||||
original_load_weights = model.load_weights
|
||||
|
||||
def should_load_weight(name: str) -> bool:
|
||||
layers_prefix = "layers."
|
||||
if name.startswith(layers_prefix):
|
||||
layer_name = name.removeprefix(layers_prefix)
|
||||
elif ".layers." in name:
|
||||
layer_name = name.split(".layers.", maxsplit=1)[1]
|
||||
else:
|
||||
return True
|
||||
|
||||
layer_idx_str = layer_name.split(".", maxsplit=1)[0]
|
||||
if layer_idx_str.isdigit():
|
||||
return int(layer_idx_str) < num_hidden_layers
|
||||
return True
|
||||
|
||||
def load_weights(weights):
|
||||
weights = (
|
||||
(name, weight) for name, weight in weights if should_load_weight(name)
|
||||
)
|
||||
return original_load_weights(weights)
|
||||
|
||||
model.load_weights = load_weights
|
||||
|
||||
|
||||
def load_model_without_vllm_runner(
|
||||
model_path: str,
|
||||
*,
|
||||
dtype: str | torch.dtype = "bfloat16",
|
||||
quantization: str | None = None,
|
||||
model_config_kwargs: dict[str, Any] | None = None,
|
||||
vllm_config_kwargs: dict[str, Any] | None = None,
|
||||
model_loader_cls: type = DefaultModelLoader,
|
||||
) -> tuple[torch.nn.Module, VllmConfig]:
|
||||
"""Instantiate a model, load weights, and process them for inference."""
|
||||
model_config = ModelConfig(
|
||||
model=model_path,
|
||||
dtype=dtype,
|
||||
quantization=quantization,
|
||||
**(model_config_kwargs or {}),
|
||||
)
|
||||
model_config.quantization_config = resolve_quantization_config(
|
||||
model_config.quantization, model_config.quantization_config
|
||||
)
|
||||
vllm_config_args = dict(vllm_config_kwargs or {})
|
||||
vllm_config_args.setdefault(
|
||||
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
|
||||
)
|
||||
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
|
||||
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
|
||||
text_config_overrides = hf_overrides.get("text_config") or {}
|
||||
num_hidden_layers = text_config_overrides.get(
|
||||
"num_hidden_layers", hf_overrides.get("num_hidden_layers")
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
model_loader = model_loader_cls(vllm_config.load_config)
|
||||
if num_hidden_layers is not None:
|
||||
original_load_weights = model_loader.load_weights
|
||||
|
||||
def load_weights(model, model_config):
|
||||
_limit_num_hidden_layers(model, num_hidden_layers)
|
||||
original_load_weights(model, model_config)
|
||||
|
||||
model_loader.load_weights = load_weights
|
||||
model = model_loader.load_model(vllm_config, model_config)
|
||||
|
||||
return model, vllm_config
|
||||
|
||||
|
||||
def is_quant_method_supported(quant_method: str) -> bool:
|
||||
# Currently, quantization tests only run GPUs
|
||||
if current_platform.is_cpu():
|
||||
return False
|
||||
try:
|
||||
current_platform.verify_quantization(quant_method)
|
||||
except ValueError:
|
||||
return False
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
capability = current_platform.get_device_capability()
|
||||
assert capability is not None
|
||||
|
||||
min_capability = get_quantization_config(quant_method).get_min_capability()
|
||||
|
||||
return capability.to_int() >= min_capability
|
||||
|
||||
|
||||
def _test_online_quant_peak_mem_impl(
|
||||
quantization_arg_value,
|
||||
vllm_runner,
|
||||
caplog_mp_spawn,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
# Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
|
||||
# 1. it covers both Linear and MoE paths
|
||||
# 2. it is already used by other tests in CI, so adding it here
|
||||
# does not increase disk space for CI runners
|
||||
# I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
|
||||
# which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
|
||||
# 1.3 GiB fp8), but could not as adding one more model makes CI
|
||||
# run out of disk space.
|
||||
model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
|
||||
# Force spawn to ensure caplog_mp_spawn works consistently
|
||||
# (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
with (
|
||||
caplog_mp_spawn(logging.DEBUG) as log_holder,
|
||||
vllm_runner(
|
||||
model_name,
|
||||
quantization=quantization_arg_value,
|
||||
enforce_eager=True,
|
||||
) as llm,
|
||||
):
|
||||
outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
|
||||
log_text = log_holder.text
|
||||
|
||||
# Parse memory usage from captured logs
|
||||
model_memory_gib = None
|
||||
peak_memory_gib = None
|
||||
for line in log_text.splitlines():
|
||||
if model_memory_gib is None:
|
||||
match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
|
||||
if match:
|
||||
model_memory_gib = float(match.group(1))
|
||||
if peak_memory_gib is None:
|
||||
match = re.search(
|
||||
r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
|
||||
)
|
||||
if match:
|
||||
peak_memory_gib = float(match.group(1))
|
||||
|
||||
assert model_memory_gib is not None, "Could not find model loading memory log"
|
||||
assert peak_memory_gib is not None, "Could not find peak memory log"
|
||||
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
|
||||
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
|
||||
|
||||
expected_model_memory_gib = 6.7
|
||||
|
||||
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
|
||||
# GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
|
||||
# number is expected as when we load and quantize weights in a streaming
|
||||
# fashion we need to have individual weights in bf16 + fp8 alive at the
|
||||
# same time.
|
||||
expected_peak_memory_gib = expected_model_memory_gib * 1.4
|
||||
|
||||
assert model_memory_gib < expected_model_memory_gib, (
|
||||
f"{model_memory_gib=} higher than {expected_model_memory_gib}"
|
||||
)
|
||||
assert peak_memory_gib < expected_peak_memory_gib, (
|
||||
f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
|
||||
class StreamingReasoningReconstructor:
|
||||
def __init__(self):
|
||||
self.reasoning = None
|
||||
self.other_content = None
|
||||
|
||||
def append_delta(self, delta: DeltaMessage):
|
||||
# content and the reasoning content should not be present
|
||||
# at the same time
|
||||
assert delta.content is None or delta.reasoning is None, (
|
||||
"Both content and reasoning content are present in the delta message"
|
||||
)
|
||||
if delta.content is not None:
|
||||
if self.other_content is None:
|
||||
self.other_content = delta.content
|
||||
else:
|
||||
self.other_content += delta.content
|
||||
else:
|
||||
if self.reasoning is None:
|
||||
self.reasoning = delta.reasoning
|
||||
else:
|
||||
self.reasoning += delta.reasoning
|
||||
|
||||
|
||||
def run_reasoning_extraction(
|
||||
reasoning_parser: ReasoningParser,
|
||||
model_output: list[str],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
streaming: bool = False,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if streaming:
|
||||
reconstructor = run_reasoning_extraction_streaming(
|
||||
reasoning_parser,
|
||||
model_output,
|
||||
request,
|
||||
)
|
||||
return (
|
||||
reconstructor.reasoning,
|
||||
reconstructor.other_content or None,
|
||||
)
|
||||
else:
|
||||
reasoning, content = run_reasoning_extraction_nonstreaming(
|
||||
reasoning_parser, model_output, request
|
||||
)
|
||||
return reasoning, content
|
||||
|
||||
|
||||
def run_reasoning_extraction_mistral(
|
||||
reasoning_parser: ReasoningParser,
|
||||
model_output: list[int],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
streaming: bool = False,
|
||||
) -> tuple[str | None, str | None]:
|
||||
assert is_mistral_tokenizer(reasoning_parser.model_tokenizer), type(
|
||||
reasoning_parser.model_tokenizer
|
||||
)
|
||||
if streaming:
|
||||
reconstructor = run_reasoning_extraction_streaming_mistral(
|
||||
reasoning_parser,
|
||||
model_output,
|
||||
request,
|
||||
)
|
||||
return (
|
||||
reconstructor.reasoning,
|
||||
reconstructor.other_content or None,
|
||||
)
|
||||
else:
|
||||
str_output = reasoning_parser.model_tokenizer.convert_ids_to_tokens(
|
||||
model_output
|
||||
)
|
||||
reasoning, content = run_reasoning_extraction_nonstreaming(
|
||||
reasoning_parser, str_output, request
|
||||
)
|
||||
return reasoning, content
|
||||
|
||||
|
||||
def run_reasoning_extraction_nonstreaming(
|
||||
reasoning_parser: ReasoningParser,
|
||||
model_output: list[str],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
return reasoning_parser.extract_reasoning(
|
||||
model_output="".join(model_output), request=request
|
||||
)
|
||||
|
||||
|
||||
def run_reasoning_extraction_streaming(
|
||||
reasoning_parser: ReasoningParser,
|
||||
model_deltas: list[str],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> StreamingReasoningReconstructor:
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
reconstructor = StreamingReasoningReconstructor()
|
||||
previous_text = ""
|
||||
previous_tokens: list[int] = []
|
||||
tokenizer = cast(PreTrainedTokenizerBase, reasoning_parser.model_tokenizer)
|
||||
for delta in model_deltas:
|
||||
token_delta = [
|
||||
reasoning_parser.vocab[token]
|
||||
for token in tokenizer.tokenize(delta)
|
||||
if token in reasoning_parser.vocab
|
||||
]
|
||||
current_text = previous_text + delta
|
||||
current_tokens = previous_tokens + token_delta
|
||||
delta_message = reasoning_parser.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta,
|
||||
previous_tokens,
|
||||
current_tokens,
|
||||
token_delta,
|
||||
)
|
||||
if delta_message is not None:
|
||||
reconstructor.append_delta(delta_message)
|
||||
previous_text = current_text
|
||||
previous_tokens = current_tokens
|
||||
return reconstructor
|
||||
|
||||
|
||||
def run_reasoning_extraction_streaming_mistral(
|
||||
reasoning_parser: ReasoningParser,
|
||||
model_deltas: list[int],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> StreamingReasoningReconstructor:
|
||||
assert is_mistral_tokenizer(reasoning_parser.model_tokenizer), type(
|
||||
reasoning_parser.model_tokenizer
|
||||
)
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
reconstructor = StreamingReasoningReconstructor()
|
||||
previous_text = ""
|
||||
previous_tokens: list[int] = []
|
||||
for model_delta in model_deltas:
|
||||
token_delta = [model_delta]
|
||||
delta = reasoning_parser.model_tokenizer.convert_ids_to_tokens([model_delta])[0]
|
||||
current_text = previous_text + delta
|
||||
current_tokens = previous_tokens + token_delta
|
||||
delta_message = reasoning_parser.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta,
|
||||
previous_tokens,
|
||||
current_tokens,
|
||||
token_delta,
|
||||
)
|
||||
if delta_message is not None:
|
||||
reconstructor.append_delta(delta_message)
|
||||
previous_text = current_text
|
||||
previous_tokens = current_tokens
|
||||
return reconstructor
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,380 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from types import NoneType
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import run_tool_extraction
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolParserTestConfig:
|
||||
"""Configuration for a tool parser's common tests.
|
||||
|
||||
This dataclass contains all the test data and expected results needed
|
||||
to run the common test suite for a parser. Each parser test file
|
||||
creates one instance of this config with parser-specific values.
|
||||
|
||||
Attributes:
|
||||
parser_name: Name used with ToolParserManager (e.g., "mistral")
|
||||
|
||||
Test data (model outputs):
|
||||
no_tool_calls_output: Plain text without any tool syntax
|
||||
single_tool_call_output: One tool call with simple arguments
|
||||
parallel_tool_calls_output: Multiple tool calls in one response
|
||||
various_data_types_output: Tool with various data types
|
||||
empty_arguments_output: Tool call with no parameters
|
||||
surrounding_text_output: Tool call mixed with regular text
|
||||
escaped_strings_output: Tool call with escaped chars
|
||||
malformed_input_outputs: List of invalid inputs
|
||||
|
||||
Expected results:
|
||||
single_tool_call_expected_name: Expected function name
|
||||
single_tool_call_expected_args: Expected arguments dict
|
||||
parallel_tool_calls_count: Number of tools in parallel test
|
||||
parallel_tool_calls_names: Function names in order
|
||||
single_tool_call_expected_content: Content field when tool called
|
||||
parallel_tool_calls_expected_content: Content for parallel test
|
||||
|
||||
xfail markers:
|
||||
xfail_streaming: Mapping test name to xfail reason (streaming only)
|
||||
xfail_nonstreaming: Mapping test name to xfail reason (non-streaming)
|
||||
|
||||
Special flags:
|
||||
allow_empty_or_json_empty_args: True if "" or "{}" both valid for empty args
|
||||
supports_typed_arguments: True if the parser supports typed function arguments
|
||||
|
||||
"""
|
||||
|
||||
# Parser identification
|
||||
parser_name: str
|
||||
|
||||
# Test data - model outputs for each common test
|
||||
no_tool_calls_output: str
|
||||
single_tool_call_output: str
|
||||
parallel_tool_calls_output: str
|
||||
various_data_types_output: str
|
||||
empty_arguments_output: str
|
||||
surrounding_text_output: str
|
||||
escaped_strings_output: str
|
||||
malformed_input_outputs: list[str]
|
||||
|
||||
# Expected results for specific tests (optional overrides)
|
||||
single_tool_call_expected_name: str = "get_weather"
|
||||
single_tool_call_expected_args: dict[str, Any] = field(
|
||||
default_factory=lambda: {"city": "Tokyo"}
|
||||
)
|
||||
parallel_tool_calls_count: int = 2
|
||||
parallel_tool_calls_names: list[str] = field(
|
||||
default_factory=lambda: ["get_weather", "get_time"]
|
||||
)
|
||||
|
||||
# xfail configuration - maps test name to xfail reason
|
||||
xfail_streaming: dict[str, str] = field(default_factory=dict)
|
||||
xfail_nonstreaming: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Content expectations (some parsers strip content, others don't)
|
||||
single_tool_call_expected_content: str | None = None
|
||||
parallel_tool_calls_expected_content: str | None = None
|
||||
|
||||
# Special assertions for edge cases
|
||||
allow_empty_or_json_empty_args: bool = True # "{}" or "" for empty args
|
||||
supports_typed_arguments: bool = True
|
||||
|
||||
|
||||
class ToolParserTests:
|
||||
"""Mixin class providing common test suite for tool parsers.
|
||||
|
||||
To use this mixin in a parser test file:
|
||||
|
||||
1. Create a test_config fixture that returns a ToolParserTestConfig instance
|
||||
2. Inherit from this class
|
||||
3. Add parser-specific tests as additional methods
|
||||
|
||||
Example:
|
||||
class TestMistralToolParser(ToolParserTests):
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="mistral",
|
||||
no_tool_calls_output="Plain text...",
|
||||
# ... other config ...
|
||||
)
|
||||
|
||||
# Parser-specific tests
|
||||
def test_mistral_specific_feature(self, tool_parser):
|
||||
# Custom test logic
|
||||
pass
|
||||
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
"""Override this to provide parser-specific configuration."""
|
||||
raise NotImplementedError(
|
||||
"Subclass must provide test_config fixture returning ToolParserTestConfig"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""Override this to provide parser-specific tokenizer."""
|
||||
return default_tokenizer
|
||||
|
||||
@pytest.fixture
|
||||
def tool_parser(self, test_config: ToolParserTestConfig, tokenizer: TokenizerLike):
|
||||
return ToolParserManager.get_tool_parser(test_config.parser_name)(tokenizer)
|
||||
|
||||
@pytest.fixture(params=[True, False])
|
||||
def streaming(self, request: pytest.FixtureRequest) -> bool:
|
||||
return request.param
|
||||
|
||||
def test_no_tool_calls(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles plain text without tool syntax."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_no_tool_calls"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.no_tool_calls_output, streaming=streaming
|
||||
)
|
||||
assert content == test_config.no_tool_calls_output, (
|
||||
f"Expected content to match input, got {content}"
|
||||
)
|
||||
assert len(tool_calls) == 0, f"Expected no tool calls, got {len(tool_calls)}"
|
||||
|
||||
def test_single_tool_call_simple_args(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser extracts one tool with simple arguments."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_single_tool_call_simple_args"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.single_tool_call_output, streaming=streaming
|
||||
)
|
||||
|
||||
# Content check (some parsers strip it)
|
||||
if test_config.single_tool_call_expected_content is not None:
|
||||
assert content == test_config.single_tool_call_expected_content
|
||||
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
assert tool_calls[0].type == "function"
|
||||
assert tool_calls[0].function.name == test_config.single_tool_call_expected_name
|
||||
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
for key, value in test_config.single_tool_call_expected_args.items():
|
||||
assert args.get(key) == value, (
|
||||
f"Expected {key}={value}, got {args.get(key)}"
|
||||
)
|
||||
|
||||
def test_parallel_tool_calls(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles multiple tools in one response."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_parallel_tool_calls"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser,
|
||||
test_config.parallel_tool_calls_output,
|
||||
streaming=streaming,
|
||||
)
|
||||
|
||||
assert len(tool_calls) == test_config.parallel_tool_calls_count, (
|
||||
f"Expected {test_config.parallel_tool_calls_count} "
|
||||
f"tool calls, got {len(tool_calls)}"
|
||||
)
|
||||
|
||||
# Verify tool names match expected
|
||||
for i, expected_name in enumerate(test_config.parallel_tool_calls_names):
|
||||
assert tool_calls[i].type == "function"
|
||||
assert tool_calls[i].function.name == expected_name
|
||||
|
||||
# Verify unique IDs
|
||||
ids = [tc.id for tc in tool_calls]
|
||||
assert len(ids) == len(set(ids)), "Tool call IDs should be unique"
|
||||
|
||||
def test_various_data_types(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles all JSON types in arguments."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_various_data_types"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser,
|
||||
test_config.various_data_types_output,
|
||||
streaming=streaming,
|
||||
)
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
# Verify all expected fields present
|
||||
required_fields_types = {
|
||||
"string_field": str,
|
||||
"int_field": int,
|
||||
"float_field": float,
|
||||
"bool_field": bool,
|
||||
"null_field": NoneType,
|
||||
"array_field": list,
|
||||
"object_field": dict,
|
||||
}
|
||||
for required_field, expected_type in required_fields_types.items():
|
||||
assert required_field in args, (
|
||||
f"Expected field '{required_field}' in arguments"
|
||||
)
|
||||
if test_config.supports_typed_arguments:
|
||||
found_type = type(args[required_field])
|
||||
assert found_type is expected_type, (
|
||||
f"Expected field '{required_field}' to have type {expected_type}, "
|
||||
f"got {found_type}"
|
||||
)
|
||||
|
||||
def test_empty_arguments(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles parameterless tool calls."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_empty_arguments"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.empty_arguments_output, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
|
||||
args = tool_calls[0].function.arguments
|
||||
if test_config.allow_empty_or_json_empty_args:
|
||||
assert args in ["{}", ""], f"Expected empty args, got {args}"
|
||||
else:
|
||||
assert args == "{}", f"Expected {{}}, got {args}"
|
||||
|
||||
def test_surrounding_text(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser extracts tools from mixed content."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_surrounding_text"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.surrounding_text_output, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) >= 1, (
|
||||
f"Expected at least 1 tool call, got {len(tool_calls)}"
|
||||
)
|
||||
|
||||
def test_escaped_strings(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles escaped characters in arguments."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_escaped_strings"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.escaped_strings_output, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
# At minimum, verify we can parse and have expected fields
|
||||
# Exact escaping behavior varies by parser
|
||||
assert len(args) > 0, "Expected some arguments with escaped strings"
|
||||
|
||||
def test_malformed_input(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser gracefully handles invalid syntax."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_malformed_input"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
for malformed_input in test_config.malformed_input_outputs:
|
||||
# Should not raise exception
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, malformed_input, streaming=streaming
|
||||
)
|
||||
# Parser should handle gracefully (exact behavior varies)
|
||||
|
||||
def test_streaming_reconstruction(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
):
|
||||
"""Verify streaming produces same result as non-streaming."""
|
||||
test_name = "test_streaming_reconstruction"
|
||||
self.apply_xfail_mark(request, test_config, test_name, True)
|
||||
|
||||
test_output = test_config.single_tool_call_output
|
||||
|
||||
# Non-streaming result
|
||||
content_non, tools_non = run_tool_extraction(
|
||||
tool_parser, test_output, streaming=False
|
||||
)
|
||||
|
||||
# Streaming result
|
||||
content_stream, tools_stream = run_tool_extraction(
|
||||
tool_parser, test_output, streaming=True
|
||||
)
|
||||
|
||||
# Compare results
|
||||
assert content_non == content_stream, "Content should match between modes"
|
||||
assert len(tools_non) == len(tools_stream), "Tool count should match"
|
||||
if len(tools_non) > 0:
|
||||
assert tools_non[0].function.name == tools_stream[0].function.name
|
||||
assert tools_non[0].function.arguments == tools_stream[0].function.arguments
|
||||
|
||||
def apply_xfail_mark(self, request, test_config, test_name, streaming):
|
||||
reason = None
|
||||
if streaming and test_name in test_config.xfail_streaming:
|
||||
reason = test_config.xfail_streaming[test_name]
|
||||
elif not streaming and test_name in test_config.xfail_nonstreaming:
|
||||
reason = test_config.xfail_nonstreaming[test_name]
|
||||
if reason is not None:
|
||||
mark = pytest.mark.xfail(reason=reason, strict=True)
|
||||
request.node.add_marker(mark)
|
||||
@@ -0,0 +1,167 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser
|
||||
|
||||
|
||||
class StreamingToolReconstructor:
|
||||
def __init__(self, assert_one_tool_per_delta: bool = True):
|
||||
self.tool_calls: list[ToolCall] = []
|
||||
self.other_content: str = ""
|
||||
self._assert_one_tool_per_delta = assert_one_tool_per_delta
|
||||
|
||||
def append_delta(self, delta: DeltaMessage):
|
||||
if delta.content is not None:
|
||||
self.other_content += delta.content
|
||||
else:
|
||||
assert delta.tool_calls, (
|
||||
"Streaming results should have either content or tool calls (or both)"
|
||||
)
|
||||
if self._assert_one_tool_per_delta:
|
||||
# Note: This isn't strictly required by the API and may not be
|
||||
# possible to adhere to depending on the token space and number of
|
||||
# tokens per streamed response from the model, but it is required
|
||||
# by tool_use tests, so we enforce it here by default also.
|
||||
assert len(delta.tool_calls) < 2, (
|
||||
"Streaming should include only one tool call per update."
|
||||
)
|
||||
for call_delta in delta.tool_calls:
|
||||
assert call_delta.type is None or call_delta.type == "function", (
|
||||
"Streaming tool calls should only emit function calls. Got "
|
||||
f"{call_delta.type}"
|
||||
)
|
||||
current_tool_call = (
|
||||
self.tool_calls[call_delta.index]
|
||||
if call_delta.index < len(self.tool_calls)
|
||||
else None
|
||||
)
|
||||
if current_tool_call:
|
||||
assert not call_delta.function.name, (
|
||||
"Streaming tool calls should emit the full function name "
|
||||
f"exactly once. Got {call_delta.function.name}"
|
||||
)
|
||||
assert not call_delta.id, (
|
||||
"Streaming tool calls must emit function id only once. Got "
|
||||
f"{call_delta.id}"
|
||||
)
|
||||
assert call_delta.index == len(self.tool_calls) - 1, (
|
||||
f"Incorrect index for tool delta. Got {call_delta.index}, "
|
||||
f"expected {len(self.tool_calls) - 1}"
|
||||
)
|
||||
current_tool_call.function.arguments += call_delta.function.arguments
|
||||
else:
|
||||
assert call_delta.id is not None, (
|
||||
"Streaming tool calls must have an id on first appearance"
|
||||
)
|
||||
assert call_delta.function.name is not None, (
|
||||
"Streaming tool calls must have a function name on first appearance"
|
||||
)
|
||||
assert call_delta.index == len(self.tool_calls), (
|
||||
f"Incorrect index for tool delta. Got {call_delta.index}, "
|
||||
f"expected {len(self.tool_calls)}"
|
||||
)
|
||||
self.tool_calls.append(
|
||||
ToolCall(
|
||||
id=call_delta.id,
|
||||
function=FunctionCall(
|
||||
name=call_delta.function.name,
|
||||
arguments=call_delta.function.arguments or "",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_tool_extraction(
|
||||
tool_parser: ToolParser,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | None = None,
|
||||
streaming: bool = False,
|
||||
assert_one_tool_per_delta: bool = True,
|
||||
) -> tuple[str | None, list[ToolCall]]:
|
||||
if streaming:
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser,
|
||||
model_output,
|
||||
request,
|
||||
assert_one_tool_per_delta=assert_one_tool_per_delta,
|
||||
)
|
||||
return reconstructor.other_content or None, reconstructor.tool_calls
|
||||
else:
|
||||
extracted = run_tool_extraction_nonstreaming(tool_parser, model_output, request)
|
||||
assert extracted.tools_called == bool(extracted.tool_calls)
|
||||
return extracted.content, extracted.tool_calls
|
||||
|
||||
|
||||
def run_tool_extraction_nonstreaming(
|
||||
tool_parser: ToolParser,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> ExtractedToolCallInformation:
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
return tool_parser.extract_tool_calls(model_output, request)
|
||||
|
||||
|
||||
def split_string_into_token_deltas(tokenizer: TokenizerLike, text: str) -> list[str]:
|
||||
# Split a string into a series of deltas using the provided tokenizer. Each
|
||||
# delta will be the string equivalent of a single token.
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
previously_decoded_text = ""
|
||||
deltas = []
|
||||
for i in range(1, len(token_ids) + 1):
|
||||
current_tokens = token_ids[:i]
|
||||
current_text = tokenizer.decode(current_tokens)
|
||||
new_text = current_text[len(previously_decoded_text) :]
|
||||
previously_decoded_text = current_text
|
||||
deltas.append(new_text)
|
||||
return deltas
|
||||
|
||||
|
||||
def run_tool_extraction_streaming(
|
||||
tool_parser: ToolParser,
|
||||
model_deltas: Iterable[str],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
assert_one_tool_per_delta: bool = True,
|
||||
) -> StreamingToolReconstructor:
|
||||
if isinstance(model_deltas, str):
|
||||
model_deltas = split_string_into_token_deltas(
|
||||
tool_parser.model_tokenizer, model_deltas
|
||||
)
|
||||
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
reconstructor = StreamingToolReconstructor(
|
||||
assert_one_tool_per_delta=assert_one_tool_per_delta
|
||||
)
|
||||
previous_text = ""
|
||||
previous_tokens: list[int] = []
|
||||
for delta in model_deltas:
|
||||
token_delta = [
|
||||
tool_parser.vocab.get(token)
|
||||
for token in tool_parser.model_tokenizer.tokenize(delta)
|
||||
if token in tool_parser.vocab
|
||||
]
|
||||
current_text = previous_text + delta
|
||||
current_tokens = previous_tokens + token_delta
|
||||
delta_message = tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta,
|
||||
previous_tokens,
|
||||
current_tokens,
|
||||
token_delta,
|
||||
request,
|
||||
)
|
||||
if delta_message is not None:
|
||||
reconstructor.append_delta(delta_message)
|
||||
previous_text = current_text
|
||||
previous_tokens = current_tokens
|
||||
return reconstructor
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the agent-file pre-commit check.
|
||||
|
||||
The hook passes silently on a healthy tree, so CI would not notice a rule that
|
||||
stopped firing. Each case breaks one convention in a minimal repo and asserts
|
||||
the check reports it.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.pre_commit.check_agent_files import check
|
||||
|
||||
SKILL_MD = ".agents/skills/demo/SKILL.md"
|
||||
SKILL = "---\nname: demo\ndescription: Demo things. Use when testing.\n---\n"
|
||||
|
||||
|
||||
def write(path: str, text: str) -> None:
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(path).write_text(text)
|
||||
|
||||
|
||||
def tracked() -> dict[str, str]:
|
||||
"""Mimic `git ls-files -s` for every file and symlink under the cwd."""
|
||||
files = {}
|
||||
for root, dirs, names in os.walk("."):
|
||||
for name in dirs + names:
|
||||
path = os.path.normpath(os.path.join(root, name))
|
||||
if os.path.islink(path):
|
||||
files[path] = "120000"
|
||||
elif os.path.isfile(path):
|
||||
files[path] = "100644"
|
||||
return files
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
write("AGENTS.md", "See the [guide](docs/guide.md#some-heading).\n")
|
||||
write("docs/guide.md", "## Some Heading\n")
|
||||
write(SKILL_MD, SKILL)
|
||||
os.makedirs(".claude/skills")
|
||||
os.symlink("../../.agents/skills/demo", ".claude/skills/demo")
|
||||
|
||||
|
||||
def test_valid_repo_passes():
|
||||
assert check(tracked()) == ([], [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("break_repo", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
lambda: os.unlink(".claude/skills/demo"),
|
||||
"must be a symlink",
|
||||
id="skill-not-linked",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(".claude/skills/other/SKILL.md", SKILL),
|
||||
"only symlinks",
|
||||
id="real-file-in-claude-skills",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: os.symlink("../../.agents/skills/gone", ".claude/skills/gone"),
|
||||
"only symlinks",
|
||||
id="dangling-link",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL.replace("demo", "other")),
|
||||
"must be 'demo'",
|
||||
id="name-differs-from-dir",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL.replace("Demo things", "Demo: things")),
|
||||
"not valid YAML",
|
||||
id="unquoted-colon-in-description",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, "# Demo\n"),
|
||||
"missing YAML frontmatter",
|
||||
id="no-frontmatter",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("vllm/CLAUDE.md", "@AGENTS.md\n"),
|
||||
"remove it",
|
||||
id="claude-md",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("AGENTS.md", "line\n" * 201),
|
||||
"200-line budget",
|
||||
id="root-guide-over-budget",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("vllm/AGENTS.md", "line\n" * 301),
|
||||
"300-line budget",
|
||||
id="domain-guide-over-budget",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL + "line\n" * 500),
|
||||
"500-line budget",
|
||||
id="skill-over-budget",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write("AGENTS.md", "[gone](docs/gone.md)\n"),
|
||||
"does not exist",
|
||||
id="dead-link",
|
||||
),
|
||||
pytest.param(
|
||||
lambda: write(SKILL_MD, SKILL + "[guide](../../../docs/guide.md#gone)\n"),
|
||||
"no heading",
|
||||
id="dead-anchor",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_broken_convention_is_reported(break_repo, expected):
|
||||
break_repo()
|
||||
errors, _ = check(tracked())
|
||||
assert len(errors) == 1
|
||||
assert expected in errors[0]
|
||||
|
||||
|
||||
def test_description_without_trigger_is_advisory():
|
||||
write(SKILL_MD, SKILL.replace(" Use when testing.", ""))
|
||||
errors, notes = check(tracked())
|
||||
assert not errors
|
||||
assert "when to use" in notes[0]
|
||||
@@ -1,879 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the test-tethering pre-commit check.
|
||||
|
||||
The check answers one question - "does any Buildkite job (in
|
||||
``.buildkite/test_areas/`` or the legacy ``.buildkite/test-amd.yaml``) collect
|
||||
this test file?" - by parsing job ``commands`` as shell. The risk that
|
||||
matters is a *false tether*: a misparse that invents coverage makes the gate
|
||||
pass while the test still never runs, which is the exact failure the check
|
||||
exists to catch. So the parser cases below pin both directions, and
|
||||
:func:`test_no_selection_matches_everything` guards the catastrophic version of
|
||||
it.
|
||||
"""
|
||||
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.pre_commit.check_test_tethering as checker
|
||||
from tools.pre_commit.check_test_tethering import (
|
||||
FindSelection,
|
||||
PytestSelection,
|
||||
_parse_command,
|
||||
_to_repo_relative,
|
||||
all_test_modules,
|
||||
allowlist_entries_missing_reason,
|
||||
is_test_module,
|
||||
is_tethered,
|
||||
load_allowlist,
|
||||
load_selections,
|
||||
main,
|
||||
normalize_test_path,
|
||||
run_changed_files_check,
|
||||
run_full_scan,
|
||||
)
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def runs(command: str, test_file: str) -> bool:
|
||||
"""True if `command` collects `test_file` (path relative to ``tests/``)."""
|
||||
return any(sel.runs(test_file) for sel in _parse_command(command))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# pytest path arguments
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("command", "test_file", "expected"),
|
||||
[
|
||||
# Directory sweeps cover everything nested underneath.
|
||||
("pytest -v -s kernels", "kernels/test_attention.py", True),
|
||||
("pytest -v -s kernels/", "kernels/moe/test_moe.py", True),
|
||||
("pytest -v -s kernels", "lora/test_lora.py", False),
|
||||
# An exact file argument covers that file and nothing beside it.
|
||||
("pytest -v -s kernels/test_a.py", "kernels/test_a.py", True),
|
||||
("pytest -v -s kernels/test_a.py", "kernels/test_b.py", False),
|
||||
# Commands run from the repo root and from tests/ alike.
|
||||
("pytest -v -s tests/kernels/test_a.py", "kernels/test_a.py", True),
|
||||
("pytest -v -s ./kernels/test_a.py", "kernels/test_a.py", True),
|
||||
(
|
||||
"pytest -v -s /vllm-workspace/tests/kernels/test_a.py",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
# A ::nodeid selector still collects the file.
|
||||
("pytest -v -s kernels/test_a.py::test_one", "kernels/test_a.py", True),
|
||||
("pytest -v -s kernels/test_a.py::Cls::test_one", "kernels/test_a.py", True),
|
||||
# Several positional paths, including a bare root-level file.
|
||||
("pytest -v -s test_envs.py test_outputs.py", "test_outputs.py", True),
|
||||
("pytest -v -s test_envs.py test_outputs.py", "test_regression.py", False),
|
||||
# A bare word is a path only when it is really a directory under tests/.
|
||||
("pytest -v -s samplers", "samplers/test_sampler.py", True),
|
||||
# Glob path args do not cross a directory separator, matching the shell.
|
||||
("pytest -v -s kernels/test_*.py", "kernels/test_a.py", True),
|
||||
("pytest -v -s kernels/test_*.py", "kernels/moe/test_a.py", False),
|
||||
("pytest -v -s kernels/test_*.py", "kernels/helper.py", False),
|
||||
],
|
||||
)
|
||||
def test_pytest_path_arguments(command, test_file, expected):
|
||||
assert runs(command, test_file) is expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# --ignore / --deselect
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("command", "test_file", "expected"),
|
||||
[
|
||||
# --ignore removes a file from an otherwise covering sweep, in both the
|
||||
# `=` and space-separated spellings.
|
||||
("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_a.py", False),
|
||||
("pytest -v -s kernels --ignore kernels/test_a.py", "kernels/test_a.py", False),
|
||||
# ...and only that file.
|
||||
("pytest -v -s kernels --ignore=kernels/test_a.py", "kernels/test_b.py", True),
|
||||
# --ignore of a directory removes the whole subtree.
|
||||
("pytest -v -s kernels --ignore=kernels/moe", "kernels/moe/test_moe.py", False),
|
||||
(
|
||||
"pytest -v -s lora --ignore-glob=lora/*_tp.py",
|
||||
"lora/test_llama_tp.py",
|
||||
False,
|
||||
),
|
||||
# --ignore-glob is a whole-path fnmatch (pytest semantics): its `*`
|
||||
# crosses `/`, so a bare `*_tp.py` drops a nested match too. A positional
|
||||
# path glob would not - that is the distinction the parser has to keep.
|
||||
("pytest -v -s kernels --ignore-glob=*_tp.py", "kernels/test_a_tp.py", False),
|
||||
(
|
||||
"pytest -v -s kernels --ignore-glob=*_tp.py",
|
||||
"kernels/nested/test_a_tp.py",
|
||||
False,
|
||||
),
|
||||
("pytest -v -s kernels --ignore-glob '*_tp.py'", "kernels/test_a.py", True),
|
||||
# A node-id deselect leaves the file collected; a bare-path one does not.
|
||||
(
|
||||
"pytest -v -s kernels --deselect kernels/test_a.py::test_one",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"pytest -v -s kernels --deselect kernels/test_a.py",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_ignore_and_deselect(command, test_file, expected):
|
||||
assert runs(command, test_file) is expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Options, env prefixes and other token noise
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("command", "test_file", "expected"),
|
||||
[
|
||||
# A -k / -m filter narrows which tests run but does not un-run the file.
|
||||
("pytest -v -s kernels -k 'not slow'", "kernels/test_a.py", True),
|
||||
("pytest -v -s kernels -m core_model", "kernels/test_a.py", True),
|
||||
# A value-taking option's value is never mistaken for a path.
|
||||
("pytest -m core_model", "core_model", False),
|
||||
("pytest -k kernels", "kernels/test_a.py", False),
|
||||
("pytest -n 4 kernels/test_a.py", "kernels/test_a.py", True),
|
||||
# Buildkite shard flags carry a $$VAR and must not swallow the path.
|
||||
(
|
||||
(
|
||||
"pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB "
|
||||
"--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT"
|
||||
),
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
# A leading env-var assignment is not a path.
|
||||
(
|
||||
"VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s kernels",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
# `python -m pytest` is the same invocation.
|
||||
("python3 -m pytest -v -s kernels/test_a.py", "kernels/test_a.py", True),
|
||||
# Quoted paths, wrapper commands and redirects all survive tokenizing.
|
||||
("pytest -v -s 'kernels/test_a.py'", "kernels/test_a.py", True),
|
||||
("timeout 600 pytest kernels/test_a.py", "kernels/test_a.py", True),
|
||||
("pytest kernels/test_a.py > out.log 2>&1", "kernels/test_a.py", True),
|
||||
# Each sub-command of a compound line is classified on its own.
|
||||
("pytest kernels/test_a.py && pytest lora/test_b.py", "lora/test_b.py", True),
|
||||
("pytest kernels/test_a.py ; pytest lora/test_b.py", "lora/test_b.py", True),
|
||||
# An unresolvable $VAR path yields no coverage rather than a guess.
|
||||
("pytest -v -s $TEST_TARGET", "kernels/test_a.py", False),
|
||||
],
|
||||
)
|
||||
def test_option_and_token_handling(command, test_file, expected):
|
||||
assert runs(command, test_file) is expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# find | xargs pytest pipelines
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("command", "test_file", "expected"),
|
||||
[
|
||||
("find kernels -name 'test_*.py' | xargs pytest -v", "kernels/test_a.py", True),
|
||||
(
|
||||
"find kernels -name 'test_*.py' | xargs pytest -v",
|
||||
"kernels/moe/test_b.py",
|
||||
True,
|
||||
),
|
||||
# A non-matching filename is not collected by the find expression.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | xargs pytest -v",
|
||||
"kernels/helper.py",
|
||||
False,
|
||||
),
|
||||
# -maxdepth 1 keeps direct children and drops nested ones.
|
||||
(
|
||||
"find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"find kernels -maxdepth 1 -name 'test_*.py' | xargs pytest",
|
||||
"kernels/moe/test_b.py",
|
||||
False,
|
||||
),
|
||||
# -not -name excludes its target and leaves siblings alone.
|
||||
(
|
||||
"find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest",
|
||||
"kernels/test_skip.py",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"find kernels -name 'test_*.py' -not -name 'test_skip.py' | xargs pytest",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
# The root confines the sweep.
|
||||
("find kernels -name 'test_*.py' | xargs pytest", "lora/test_a.py", False),
|
||||
# -maxdepth is honored even when the find root is the tests/ root itself
|
||||
# ("." or "tests/"), not just a subdir.
|
||||
(
|
||||
"find . -maxdepth 1 -name 'test_*.py' | xargs pytest",
|
||||
"test_regression.py",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"find . -maxdepth 1 -name 'test_*.py' | xargs pytest",
|
||||
"v1/test_scheduler.py",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"find tests/ -maxdepth 1 -name 'test_*.py' | xargs pytest",
|
||||
"v1/test_scheduler.py",
|
||||
False,
|
||||
),
|
||||
# -exec pytest wires the sweep just as `| xargs pytest` does.
|
||||
(
|
||||
"find compile -name 'test_*.py' -exec pytest -s -v {} \\;",
|
||||
"compile/test_a.py",
|
||||
True,
|
||||
),
|
||||
# A pytest in a *separate* command on the line (not fed by the find) does
|
||||
# not arm the sweep - the find here is piped to `wc`.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | wc -l ; pytest test_regression.py",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"find kernels -name 'test_*.py' -delete && pytest test_regression.py",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
# ...but only a *non-filtering* intermediate stage. `sort` reorders,
|
||||
# every match still reaches pytest.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | sort | xargs pytest",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
# `head` drops all but one match - the sweep is not fully tethered.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | head -n 1 | xargs pytest",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
(
|
||||
"find kernels -name 'test_*.py' | grep attention | xargs pytest",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
# `xargs` runs `echo` here, not pytest - the trailing `pytest` is echo's
|
||||
# argument.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | xargs echo pytest",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
# An `-exec` with no `\;` / `+` terminator is a `find` syntax error - it
|
||||
# never runs pytest.
|
||||
(
|
||||
"find kernels -name 'test_*.py' -exec pytest {}",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
# An `-exec pytest` with no `{}` runs only the named file for every
|
||||
# match - the matched files never reach pytest.
|
||||
(
|
||||
"find kernels -name 'test_*.py' -exec pytest test_regression.py \\;",
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
# `{}` as an option *value* (`--ignore {}`) doesn't reach pytest as a
|
||||
# test path either - only `test_regression.py` runs.
|
||||
(
|
||||
(
|
||||
"find kernels -name 'test_*.py' -exec pytest test_regression.py "
|
||||
"--ignore {} \\;"
|
||||
),
|
||||
"kernels/test_a.py",
|
||||
False,
|
||||
),
|
||||
# xargs value-flags with a space don't swallow the command.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | xargs -0 -I {} pytest {}",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
# `xargs env FOO=1 pytest` - xargs runs `env`, which runs pytest.
|
||||
(
|
||||
"find kernels -name 'test_*.py' | xargs env FOO=1 pytest",
|
||||
"kernels/test_a.py",
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_find_pipelines(command, test_file, expected):
|
||||
assert runs(command, test_file) is expected
|
||||
|
||||
|
||||
def test_find_pipeline_parses_as_find_selection():
|
||||
(selection,) = [
|
||||
s
|
||||
for s in _parse_command("find kernels -name 'test_*.py' | xargs pytest -v")
|
||||
if isinstance(s, FindSelection)
|
||||
]
|
||||
assert selection.root == "kernels"
|
||||
assert selection.name_globs == ["test_*.py"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Direct file runners
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("command", "test_file", "expected"),
|
||||
[
|
||||
(
|
||||
"python3 standalone_tests/lazy_imports.py",
|
||||
"standalone_tests/lazy_imports.py",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"python standalone_tests/lazy_imports.py",
|
||||
"standalone_tests/lazy_imports.py",
|
||||
True,
|
||||
),
|
||||
# `timeout -k <grace> <dur>` wrapping a runner: the grace value must not
|
||||
# be consumed as the duration (which would hide `python`).
|
||||
(
|
||||
"timeout -k 5s 30s python standalone_tests/lazy_imports.py",
|
||||
"standalone_tests/lazy_imports.py",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"torchrun --nproc-per-node 2 distributed/test_a.py",
|
||||
"distributed/test_a.py",
|
||||
True,
|
||||
),
|
||||
# The option's value is a count, not a path.
|
||||
("torchrun --nproc-per-node 2 distributed/test_a.py", "2", False),
|
||||
(
|
||||
"VLLM_X=1 python3 standalone_tests/lazy_imports.py | grep -q ok",
|
||||
"standalone_tests/lazy_imports.py",
|
||||
True,
|
||||
),
|
||||
# Only the executed script counts; the tokens after it are that script's
|
||||
# own argv. `--suffix v1` must not tether the whole tests/v1 subtree.
|
||||
(
|
||||
(
|
||||
"python3 features/tensorize_vllm_model.py --model facebook/opt-125m "
|
||||
"serialize --serialized-directory /tmp/ --suffix v1"
|
||||
),
|
||||
"v1/test_scheduler.py",
|
||||
False,
|
||||
),
|
||||
(
|
||||
(
|
||||
"python3 features/tensorize_vllm_model.py --model facebook/opt-125m "
|
||||
"serialize --serialized-directory /tmp/ --suffix v1"
|
||||
),
|
||||
"features/tensorize_vllm_model.py",
|
||||
True,
|
||||
),
|
||||
# A script named by an absolute `$VAR` path still resolves.
|
||||
(
|
||||
'python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/e2e/test_x.py"',
|
||||
"v1/e2e/test_x.py",
|
||||
True,
|
||||
),
|
||||
# `python -m <module>` runs the module, not a script - only `-m pytest`
|
||||
# collects tests. `compileall kernels` byte-compiles, it does not run.
|
||||
("python -m compileall kernels", "kernels/test_a.py", False),
|
||||
("python -m mypy kernels/test_a.py", "kernels/test_a.py", False),
|
||||
# `coverage run -m pytest` still resolves through to pytest.
|
||||
("coverage run -m pytest kernels/test_a.py", "kernels/test_a.py", True),
|
||||
# `python -c '<code>' file.py` runs the code string; `file.py` is argv.
|
||||
("python -c 'pass' kernels/test_a.py", "kernels/test_a.py", False),
|
||||
("python -c 'import sys' kernels/test_a.py", "kernels/test_a.py", False),
|
||||
# a `-m pytest` *after* the `-c` code operand is that program's argv too.
|
||||
("python -c 'pass' -m pytest kernels/test_a.py", "kernels/test_a.py", False),
|
||||
# ...but `-c` as the *script's* own arg still tethers the script.
|
||||
("python kernels/test_a.py -c config.py", "kernels/test_a.py", True),
|
||||
# ...and pytest's own `-c <cfg>` (after `-m pytest`) is fine.
|
||||
("python -m pytest kernels/test_a.py -c setup.cfg", "kernels/test_a.py", True),
|
||||
],
|
||||
)
|
||||
def test_direct_runners(command, test_file, expected):
|
||||
assert runs(command, test_file) is expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Commands that must contribute no coverage
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command",
|
||||
[
|
||||
"export VLLM_WORKER_MULTIPROC_METHOD=spawn",
|
||||
"pip install pytest-timeout pytest-forked",
|
||||
"uv pip install -r requirements/test/cuda.in",
|
||||
"bash tests/does_not_exist.sh",
|
||||
"echo 'pytest kernels/test_a.py'",
|
||||
# `pytest` as an argument to another command - `echo` runs, not pytest.
|
||||
"echo pytest kernels/test_a.py",
|
||||
'pytest -v -s "unbalanced',
|
||||
# A `.sh` that is only an *argument* (Buildkite printing / copying a
|
||||
# filename) is not a script invocation.
|
||||
"echo tests/foo.sh",
|
||||
"cp scripts/run_tests.sh /tmp/",
|
||||
# A `find` whose output never reaches pytest is not a test selection.
|
||||
"find kernels -name 'test_*.py'",
|
||||
"find kernels -name 'test_*.py' | wc -l",
|
||||
"find kernels -name 'test_*.py' -delete",
|
||||
"find kernels -name 'test_*.py' | xargs echo pytest",
|
||||
"find kernels -name 'test_*.py' -exec pytest {}",
|
||||
],
|
||||
)
|
||||
def test_non_test_commands_contribute_nothing(command):
|
||||
assert _parse_command(command) == [] or not any(
|
||||
sel.runs("kernels/test_a.py") for sel in _parse_command(command)
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Job-level yaml parsing
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _write_area(tmp_path, monkeypatch, body: str):
|
||||
mod = checker
|
||||
area_dir = tmp_path / "test_areas"
|
||||
area_dir.mkdir(exist_ok=True)
|
||||
(area_dir / "synthetic.yaml").write_text(textwrap.dedent(body))
|
||||
monkeypatch.setattr(mod, "TEST_AREAS_DIR", area_dir)
|
||||
# Keep the real test-amd.yaml out of these isolated parsing tests.
|
||||
monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-such-test-amd.yaml")
|
||||
# The yaml-error path reports a repo-relative name, so REPO_ROOT has to
|
||||
# contain the synthetic area too.
|
||||
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
|
||||
return load_selections()
|
||||
|
||||
|
||||
def test_plain_step_commands_are_collected(tmp_path, monkeypatch):
|
||||
selections = _write_area(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
steps:
|
||||
- label: ":nvidia: (H200) Kernels"
|
||||
commands:
|
||||
- pytest -v -s kernels
|
||||
""",
|
||||
)
|
||||
assert is_tethered("kernels/test_a.py", selections)
|
||||
|
||||
|
||||
def test_singular_command_key_is_collected(tmp_path, monkeypatch):
|
||||
selections = _write_area(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
steps:
|
||||
- label: "Singular"
|
||||
command: pytest -v -s kernels/test_a.py
|
||||
""",
|
||||
)
|
||||
assert is_tethered("kernels/test_a.py", selections)
|
||||
|
||||
|
||||
def test_test_amd_yaml_coverage_counts_as_tethered(tmp_path, monkeypatch):
|
||||
"""A test wired only into the legacy test-amd.yaml is still run by CI."""
|
||||
mod = checker
|
||||
(tmp_path / "test_areas").mkdir()
|
||||
amd_yaml = tmp_path / "test-amd.yaml"
|
||||
amd_yaml.write_text(
|
||||
textwrap.dedent(
|
||||
"""
|
||||
steps:
|
||||
- label: ":amd: (MI300) Quantization"
|
||||
commands:
|
||||
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/
|
||||
- pytest -v -s rocm/test_moe_weight_replay.py
|
||||
"""
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas")
|
||||
monkeypatch.setattr(mod, "TEST_AMD_YAML", amd_yaml)
|
||||
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
|
||||
selections = load_selections()
|
||||
assert is_tethered("rocm/test_moe_weight_replay.py", selections)
|
||||
assert is_tethered("quantization/test_foo.py", selections)
|
||||
|
||||
|
||||
def test_missing_test_amd_yaml_is_not_fatal(tmp_path, monkeypatch):
|
||||
"""When the migration finally deletes test-amd.yaml, the checker still runs."""
|
||||
mod = checker
|
||||
(tmp_path / "test_areas").mkdir()
|
||||
monkeypatch.setattr(mod, "TEST_AREAS_DIR", tmp_path / "test_areas")
|
||||
monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "gone.yaml")
|
||||
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
|
||||
assert load_selections() == []
|
||||
|
||||
|
||||
def test_job_dirs_come_from_ci_config(tmp_path, monkeypatch):
|
||||
"""The pipeline directories are whatever `.buildkite/ci_config*.yaml` lists
|
||||
under `job_dirs:` - so a test wired only into e.g. `hardware_tests/` counts,
|
||||
and an unreadable config falls back to `test_areas/` alone."""
|
||||
mod = checker
|
||||
bk = tmp_path / ".buildkite"
|
||||
(bk / "test_areas").mkdir(parents=True)
|
||||
(bk / "hardware_tests").mkdir()
|
||||
(bk / "ci_config.yaml").write_text("job_dirs:\n - .buildkite/hardware_tests\n")
|
||||
(bk / "ci_config_broken.yaml").write_text("job_dirs: [oops\n") # ignored
|
||||
(bk / "hardware_tests" / "cpu.yaml").write_text(
|
||||
textwrap.dedent(
|
||||
"""
|
||||
steps:
|
||||
- label: "CPU"
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/test_cpu_thing.py
|
||||
"""
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
|
||||
monkeypatch.setattr(mod, "TEST_AREAS_DIR", bk / "test_areas")
|
||||
monkeypatch.setattr(mod, "TEST_AMD_YAML", tmp_path / "no-amd.yaml")
|
||||
|
||||
assert (bk / "hardware_tests") in checker._pipeline_job_dirs()
|
||||
assert is_tethered("v1/e2e/test_cpu_thing.py", load_selections())
|
||||
|
||||
|
||||
def test_step_level_ignore_is_honored(tmp_path, monkeypatch):
|
||||
selections = _write_area(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
steps:
|
||||
- label: "Kernels"
|
||||
commands:
|
||||
- pytest -v -s kernels --ignore=kernels/test_a.py
|
||||
""",
|
||||
)
|
||||
assert not is_tethered("kernels/test_a.py", selections)
|
||||
assert is_tethered("kernels/test_b.py", selections)
|
||||
|
||||
|
||||
def test_amd_mirror_only_command_is_collected(tmp_path, monkeypatch):
|
||||
"""A mirror that overrides `commands` runs tests the base step never does."""
|
||||
selections = _write_area(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
"""
|
||||
steps:
|
||||
- label: ":nvidia: (H200) Base"
|
||||
commands:
|
||||
- pytest -v -s kernels/test_base.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Base"
|
||||
commands:
|
||||
- pytest -v -s kernels/test_amd_only.py
|
||||
""",
|
||||
)
|
||||
assert is_tethered("kernels/test_base.py", selections)
|
||||
assert is_tethered("kernels/test_amd_only.py", selections)
|
||||
|
||||
|
||||
def test_shell_script_continuation_vars_and_nesting(tmp_path, monkeypatch):
|
||||
"""The v1 P/D integration scripts name their test through a `\\`-continued
|
||||
line, a literal shell var, and a nested `bash "$SCRIPT"`. Each indirection
|
||||
has to be followed or the check falsely reports the test untethered."""
|
||||
scripts = tmp_path / "tests" / "v1" / "kv_connector" / "nixl_integration"
|
||||
scripts.mkdir(parents=True)
|
||||
(scripts / "inner.sh").write_text(
|
||||
"python3 -m pytest -s -v \\\n"
|
||||
' "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_inner.py"\n'
|
||||
)
|
||||
(scripts / "sweep.sh").write_text(
|
||||
'SCRIPT="v1/kv_connector/nixl_integration/inner.sh"\n'
|
||||
'IMPORT_CANARY="v1/kv_connector/nixl_integration/test_canary.py"\n'
|
||||
'LITERAL="v1/kv_connector/nixl_integration/test_not_run.py"\n'
|
||||
'python3 -m pytest -s -x "${IMPORT_CANARY}"\n'
|
||||
# single-quoted: the shell passes a literal $LITERAL, collecting nothing
|
||||
"python3 -m pytest -s -x '${LITERAL}'\n"
|
||||
'if ! env FOO=1 bash "${SCRIPT}"; then exit 1; fi\n'
|
||||
)
|
||||
monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
|
||||
monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests")
|
||||
|
||||
selections = _parse_command("bash v1/kv_connector/nixl_integration/sweep.sh")
|
||||
|
||||
def tethered(path):
|
||||
return any(s.runs(path) for s in selections)
|
||||
|
||||
# `\`-continuation + $VAR-prefixed absolute path, resolved through a nested
|
||||
# `env FOO=1 bash "$SCRIPT"`:
|
||||
assert tethered("v1/kv_connector/nixl_integration/test_inner.py")
|
||||
# literal `VAR=path` + double-quoted `"$VAR"` (the import-canary pattern):
|
||||
assert tethered("v1/kv_connector/nixl_integration/test_canary.py")
|
||||
# single-quoted `'$VAR'` is NOT expanded by the shell, so not tethered:
|
||||
assert not tethered("v1/kv_connector/nixl_integration/test_not_run.py")
|
||||
|
||||
|
||||
def test_bash_dash_c_does_not_follow_a_script_operand(tmp_path, monkeypatch):
|
||||
"""`bash -c '<code>' foo.sh` runs the `-c` string; `foo.sh` is `$0` and is
|
||||
never executed, so its contents must not count as coverage."""
|
||||
(tmp_path / "tests").mkdir()
|
||||
(tmp_path / "tests" / "decoy.sh").write_text("pytest -v -s lora/test_decoy.py\n")
|
||||
monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
|
||||
monkeypatch.setattr(checker, "TESTS_DIR", tmp_path / "tests")
|
||||
|
||||
with_c = _parse_command("bash -c 'echo hi' tests/decoy.sh")
|
||||
assert not any(s.runs("lora/test_decoy.py") for s in with_c)
|
||||
# ...but a plain `bash tests/decoy.sh` still follows it.
|
||||
plain = _parse_command("bash tests/decoy.sh")
|
||||
assert any(s.runs("lora/test_decoy.py") for s in plain)
|
||||
# ...and a `-c` that is the *script's* own flag (not bash's) is not `bash -c`.
|
||||
script_flag = _parse_command("bash tests/decoy.sh -c models.txt")
|
||||
assert any(s.runs("lora/test_decoy.py") for s in script_flag)
|
||||
# A `bash -c` trailing arg that looks like a command string is `$0`, not run.
|
||||
cmd_string_arg = _parse_command("bash -c 'echo ok' 'pytest lora/test_x.py'")
|
||||
assert not any(s.runs("lora/test_x.py") for s in cmd_string_arg)
|
||||
# ...but the `-c` code string *itself* is parsed - it is what runs.
|
||||
code_runs = _parse_command("bash -c 'pytest lora/test_x.py'")
|
||||
assert any(s.runs("lora/test_x.py") for s in code_runs)
|
||||
# a combined short-option group resolves the `-c` operand with `c` in any
|
||||
# position (`-ec`, `-ce`), and its trailing args are still `$0`, not run.
|
||||
for grp in ("-ec", "-ce", "-euxc"):
|
||||
assert any(
|
||||
s.runs("lora/test_x.py")
|
||||
for s in _parse_command(f"bash {grp} 'pytest lora/test_x.py'")
|
||||
)
|
||||
assert not any(
|
||||
s.runs("lora/test_x.py")
|
||||
for s in _parse_command("bash -ce 'echo ok' 'pytest lora/test_x.py'")
|
||||
)
|
||||
|
||||
|
||||
def test_unparsable_yaml_is_fatal(tmp_path, monkeypatch):
|
||||
"""Silently skipping a bad yaml would drop its coverage and report false
|
||||
untethered files, so it must fail loudly instead."""
|
||||
with pytest.raises(SystemExit):
|
||||
_write_area(tmp_path, monkeypatch, "steps: [oops\n")
|
||||
|
||||
|
||||
def test_non_mapping_yaml_is_fatal(tmp_path, monkeypatch):
|
||||
"""A yaml that parses but isn't a pipeline mapping (a bare list or scalar)
|
||||
must raise the actionable error, not crash on ``.get``."""
|
||||
with pytest.raises(SystemExit):
|
||||
_write_area(tmp_path, monkeypatch, "- just\n- a\n- list\n")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# is_test_module / path normalization
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "expected"),
|
||||
[
|
||||
("tests/kernels/test_a.py", True),
|
||||
("tests/kernels/a_test.py", True),
|
||||
("tests/kernels/conftest.py", False),
|
||||
("tests/kernels/__init__.py", False),
|
||||
("tests/kernels/utils.py", False),
|
||||
("tests/kernels/test_a.txt", False),
|
||||
("vllm/kernels/test_a.py", False),
|
||||
],
|
||||
)
|
||||
def test_is_test_module(path, expected):
|
||||
assert is_test_module(path) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("tests/kernels/test_a.py", "kernels/test_a.py"),
|
||||
("./kernels/test_a.py", "kernels/test_a.py"),
|
||||
("/vllm-workspace/tests/kernels/test_a.py", "kernels/test_a.py"),
|
||||
("'kernels/test_a.py'", "kernels/test_a.py"),
|
||||
("kernels/test_a.py::test_one", "kernels/test_a.py"),
|
||||
("kernels/", "kernels"),
|
||||
("${GIT_ROOT}/tests/kernels/test_a.py", "kernels/test_a.py"),
|
||||
("$GIT_ROOT/tests/kernels/test_a.py", "kernels/test_a.py"),
|
||||
],
|
||||
)
|
||||
def test_normalize_test_path(raw, expected):
|
||||
assert normalize_test_path(raw) == expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Whole-tree invariants
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_real_tree_parses_into_selections():
|
||||
assert len(load_selections()) > 200
|
||||
assert len(all_test_modules()) > 1000
|
||||
|
||||
|
||||
def test_no_selection_matches_everything():
|
||||
"""A path arg that normalizes to '' or '.' would tether every test in the
|
||||
repo and silently disarm the gate - the worst possible misparse."""
|
||||
for selection in load_selections():
|
||||
if isinstance(selection, PytestSelection):
|
||||
offenders = [
|
||||
p
|
||||
for p in selection.included_paths
|
||||
if normalize_test_path(p) in ("", ".")
|
||||
]
|
||||
assert not offenders, f"match-everything pytest arg: {offenders}"
|
||||
else:
|
||||
assert normalize_test_path(selection.root) not in ("", ".")
|
||||
|
||||
|
||||
def test_tethering_is_deterministic():
|
||||
selections = load_selections()
|
||||
modules = all_test_modules()[:200]
|
||||
first = [is_tethered(m, selections) for m in modules]
|
||||
assert first == [is_tethered(m, load_selections()) for m in modules]
|
||||
|
||||
|
||||
def test_allowlist_entries_are_well_formed():
|
||||
allowlist = load_allowlist()
|
||||
assert allowlist, "allowlist should not be empty while gaps remain"
|
||||
assert all(is_test_module(path) for path in allowlist)
|
||||
assert all(not path.startswith("/") for path in allowlist)
|
||||
|
||||
|
||||
def test_real_allowlist_entries_all_have_a_reason():
|
||||
assert allowlist_entries_missing_reason() == []
|
||||
|
||||
|
||||
def test_bare_allowlist_entry_is_flagged(tmp_path, monkeypatch):
|
||||
allowlist = tmp_path / "allowlist.txt"
|
||||
allowlist.write_text(
|
||||
"tests/a/test_ok.py # torchrun multi-GPU, no job\n"
|
||||
"tests/b/test_bare.py\n"
|
||||
" # a standalone comment line is fine\n"
|
||||
)
|
||||
monkeypatch.setattr(checker, "ALLOWLIST_PATH", allowlist)
|
||||
assert allowlist_entries_missing_reason() == ["tests/b/test_bare.py"]
|
||||
|
||||
|
||||
def test_real_tree_has_no_unallowlisted_gaps():
|
||||
"""Every test module is either collected by a job or allowlisted.
|
||||
|
||||
Deliberately non-strict: a stale allowlist entry is advisory by design, so
|
||||
asserting `strict=True` here would make unrelated allowlist cleanup block
|
||||
CI - the exact behaviour the two-mode design avoids.
|
||||
"""
|
||||
assert run_full_scan(load_selections(), load_allowlist(), strict=False) == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CLI behaviour
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_repo(tmp_path, monkeypatch):
|
||||
"""Point the module at a throwaway tree so the changed-files path can be
|
||||
exercised without writing test files into the real repo."""
|
||||
mod = checker
|
||||
(tmp_path / "tests" / "kernels").mkdir(parents=True)
|
||||
(tmp_path / "tests" / "kernels" / "test_new.py").write_text("def test_x(): pass\n")
|
||||
monkeypatch.setattr(mod, "REPO_ROOT", tmp_path)
|
||||
monkeypatch.setattr(mod, "TESTS_DIR", tmp_path / "tests")
|
||||
monkeypatch.setattr(mod, "ALLOWLIST_PATH", tmp_path / "allowlist.txt")
|
||||
# The throwaway tree is not a git checkout, so the tracked-file inventory
|
||||
# has to be stubbed rather than shelled out.
|
||||
monkeypatch.setattr(mod, "all_test_modules", lambda: ["tests/kernels/test_new.py"])
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_untethered_changed_file_fails(fake_repo, capsys):
|
||||
rc = run_changed_files_check(["tests/kernels/test_new.py"], [], set())
|
||||
assert rc == 1
|
||||
assert "is not run by any Buildkite job" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_tethered_changed_file_passes(fake_repo):
|
||||
selections = _parse_command("pytest -v -s kernels")
|
||||
assert (
|
||||
run_changed_files_check(["tests/kernels/test_new.py"], selections, set()) == 0
|
||||
)
|
||||
|
||||
|
||||
def test_allowlisted_changed_file_passes(fake_repo):
|
||||
allowlist = {"tests/kernels/test_new.py"}
|
||||
assert run_changed_files_check(["tests/kernels/test_new.py"], [], allowlist) == 0
|
||||
|
||||
|
||||
def test_non_test_and_missing_paths_are_ignored(fake_repo):
|
||||
paths = ["tests/kernels/conftest.py", "vllm/config.py", "tests/kernels/gone.py"]
|
||||
assert run_changed_files_check(paths, [], set()) == 0
|
||||
|
||||
|
||||
def test_stale_allowlist_entries_are_advisory_but_reported(fake_repo, capsys):
|
||||
"""Dangling entries must not block an unrelated PR, but they have to be
|
||||
printed - the hook sets `verbose: true` so a passing run still shows them."""
|
||||
allowlist = {"tests/kernels/gone.py"}
|
||||
rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=False)
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "no longer exists" in out
|
||||
|
||||
|
||||
def test_stale_allowlist_entries_are_fatal_under_all(fake_repo, capsys):
|
||||
allowlist = {"tests/kernels/gone.py"}
|
||||
rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True)
|
||||
assert rc == 1
|
||||
assert "no longer exists" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_now_tethered_allowlist_entry_is_reported(fake_repo, capsys):
|
||||
allowlist = {"tests/kernels/test_new.py"}
|
||||
rc = run_full_scan(_parse_command("pytest -v -s kernels"), allowlist, strict=True)
|
||||
assert rc == 1
|
||||
assert "is now tethered" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_passes_a_tethered_file(monkeypatch):
|
||||
"""The CLI's normal pre-commit path: a tethered file exits 0.
|
||||
|
||||
`--all` is not asserted here on purpose - it implies `strict=True`, so a
|
||||
stale allowlist entry would turn unrelated cleanup into a CI failure.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"sys.argv",
|
||||
["check_test_tethering.py", "tests/tools/test_check_test_tethering.py"],
|
||||
)
|
||||
assert main() == 0
|
||||
|
||||
|
||||
def test_main_flags_an_untethered_file(monkeypatch):
|
||||
# Must be named like a test module, or the check correctly skips it.
|
||||
orphan = REPO_ROOT / "tests" / "test_tethering_probe_orphan.py"
|
||||
orphan.write_text("def test_x(): pass\n")
|
||||
monkeypatch.setattr("sys.argv", ["check_test_tethering.py", str(orphan)])
|
||||
try:
|
||||
assert main() == 1
|
||||
finally:
|
||||
orphan.unlink()
|
||||
|
||||
|
||||
def test_to_repo_relative_accepts_absolute_paths():
|
||||
absolute = str(REPO_ROOT / "tests" / "kernels" / "test_a.py")
|
||||
assert _to_repo_relative(absolute) == "tests/kernels/test_a.py"
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import ast
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.pre_commit.validate_config import validate_ast
|
||||
|
||||
_TestConfig1 = '''
|
||||
@config
|
||||
class _TestConfig1:
|
||||
a: int
|
||||
"""docstring"""
|
||||
'''
|
||||
|
||||
_TestConfig2 = """
|
||||
@config
|
||||
class _TestConfig2:
|
||||
a: int = 1
|
||||
"""
|
||||
|
||||
_TestConfig3 = '''
|
||||
@config
|
||||
class _TestConfig3:
|
||||
a: Union[Literal[1], Literal[2]] = 1
|
||||
"""docstring"""
|
||||
'''
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("test_config", "expected_error"),
|
||||
[
|
||||
(_TestConfig1, "must have a default"),
|
||||
(_TestConfig2, "must have a docstring"),
|
||||
(_TestConfig3, "must use a single Literal"),
|
||||
],
|
||||
)
|
||||
def test_config(test_config, expected_error):
|
||||
tree = ast.parse(test_config)
|
||||
with pytest.raises(Exception, match=expected_error):
|
||||
validate_ast(tree)
|
||||
+2290
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,153 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
import random
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||
|
||||
|
||||
class DeviceConfig(NamedTuple):
|
||||
available: bool
|
||||
backends: list[str]
|
||||
|
||||
|
||||
# Maps each device to its availability and supported backends.
|
||||
DEVICE_BACKENDS: dict[str, DeviceConfig] = {
|
||||
"cuda": DeviceConfig(
|
||||
available=current_platform.is_cuda()
|
||||
and current_platform.has_device_capability(80),
|
||||
# FlashInfer backend temporarily disabled due to invariant CTA sizes.
|
||||
# See FlashInfer issue #2424
|
||||
backends=["FLASH_ATTN", "TRITON_ATTN", "FLEX_ATTENTION"],
|
||||
),
|
||||
"xpu": DeviceConfig(
|
||||
available=current_platform.is_xpu() and HAS_TRITON,
|
||||
backends=["TRITON_ATTN"],
|
||||
),
|
||||
"rocm": DeviceConfig(
|
||||
available=current_platform.is_rocm() and HAS_TRITON,
|
||||
backends=["TRITON_ATTN"],
|
||||
),
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
|
||||
TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
|
||||
|
||||
# Override backends for MLA models.
|
||||
if os.getenv("VLLM_TEST_MODEL"):
|
||||
config = get_config(TEST_MODEL, trust_remote_code=False)
|
||||
if ModelArchConfigConvertorBase(config, config.get_text_config()).is_deepseek_mla():
|
||||
DEVICE_BACKENDS["cuda"] = DeviceConfig(
|
||||
available=DEVICE_BACKENDS["cuda"].available,
|
||||
backends=["TRITON_MLA"]
|
||||
+ (["FLASH_ATTN_MLA"] if flash_attn_supports_mla() else []),
|
||||
)
|
||||
DEVICE_BACKENDS["xpu"] = DeviceConfig(
|
||||
available=DEVICE_BACKENDS["xpu"].available,
|
||||
backends=[],
|
||||
)
|
||||
DEVICE_BACKENDS["rocm"] = DeviceConfig(
|
||||
available=DEVICE_BACKENDS["rocm"].available,
|
||||
backends=["TRITON_MLA"],
|
||||
)
|
||||
|
||||
# Only include backends for devices that are actually available.
|
||||
BACKENDS: list[str] = sorted(
|
||||
{b for cfg in DEVICE_BACKENDS.values() if cfg.available for b in cfg.backends}
|
||||
)
|
||||
|
||||
skip_unsupported = pytest.mark.skipif(
|
||||
not any(cfg.available for cfg in DEVICE_BACKENDS.values()),
|
||||
reason="Requires CUDA >= Ampere (SM80), ROCm, or Intel XPU with Triton",
|
||||
)
|
||||
|
||||
skip_if_not_cuda = pytest.mark.skipif(
|
||||
not DEVICE_BACKENDS["cuda"].available,
|
||||
reason="Requires CUDA >= Ampere (SM80)",
|
||||
)
|
||||
|
||||
skip_if_not_cuda_alike = pytest.mark.skipif(
|
||||
not (DEVICE_BACKENDS["cuda"].available or DEVICE_BACKENDS["rocm"].available),
|
||||
reason="Requires CUDA >= Ampere (SM80) or ROCm",
|
||||
)
|
||||
|
||||
requires_mx = pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and current_platform.supports_mx()),
|
||||
reason="requires a ROCm device with native MX support (gfx95x)",
|
||||
)
|
||||
|
||||
|
||||
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
|
||||
# Generate more realistic prompts that will actually produce varied tokens
|
||||
# Use a mix of common English text patterns
|
||||
|
||||
prompt_templates = [
|
||||
# Question-answer style
|
||||
"Question: What is the capital of France?\nAnswer: The capital of France is",
|
||||
"Q: How does photosynthesis work?\nA: Photosynthesis is the process by which",
|
||||
"User: Can you explain quantum mechanics?\nAssistant: Quantum mechanics is",
|
||||
# Story/narrative style
|
||||
"Once upon a time in a distant galaxy, there lived",
|
||||
"The old man walked slowly down the street, remembering",
|
||||
"In the year 2157, humanity finally discovered",
|
||||
# Technical/code style
|
||||
"To implement a binary search tree in Python, first we need to",
|
||||
"The algorithm works by iterating through the array and",
|
||||
"Here's how to optimize database queries using indexing:",
|
||||
# Factual/informative style
|
||||
"The Renaissance was a period in European history that",
|
||||
"Climate change is caused by several factors including",
|
||||
"The human brain contains approximately 86 billion neurons which",
|
||||
# Conversational style
|
||||
"I've been thinking about getting a new laptop because",
|
||||
"Yesterday I went to the store and bought",
|
||||
"My favorite thing about summer is definitely",
|
||||
]
|
||||
|
||||
# Pick a random template
|
||||
base_prompt = random.choice(prompt_templates)
|
||||
|
||||
if max_words < min_words:
|
||||
max_words = min_words
|
||||
target_words = random.randint(min_words, max_words)
|
||||
|
||||
if target_words > 50:
|
||||
# For longer prompts, repeat context
|
||||
padding_text = (
|
||||
" This is an interesting topic that deserves more explanation. "
|
||||
# TODO: Update to * (target_words // 10) to better align with word ratio
|
||||
* (target_words // 50)
|
||||
)
|
||||
base_prompt = padding_text + base_prompt
|
||||
|
||||
return base_prompt
|
||||
|
||||
|
||||
def _extract_step_logprobs(request_output):
|
||||
if getattr(request_output, "outputs", None):
|
||||
inner = request_output.outputs[0]
|
||||
if hasattr(inner, "logprobs") and inner.logprobs is not None:
|
||||
t = torch.tensor(
|
||||
[
|
||||
inner.logprobs[i][tid].logprob
|
||||
for i, tid in enumerate(inner.token_ids)
|
||||
],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return t, inner.token_ids
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def is_device_capability_below_90() -> bool:
|
||||
return not current_platform.has_device_capability(90)
|
||||
@@ -0,0 +1,601 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AsyncExitStack
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import msgspec
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleCoordinator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine.core_client import DPAsyncMPClient
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
|
||||
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
# Number of data parallel ranks for external LB testing
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
# Default tensor parallel size to use
|
||||
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
|
||||
class DictStore:
|
||||
def __init__(self, values: dict[str, bytes]):
|
||||
self.values = values
|
||||
|
||||
def check(self, keys: list[str]) -> bool:
|
||||
return all(key in self.values for key in keys)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self.values[key]
|
||||
|
||||
def set(self, key: str, value: bytes) -> None:
|
||||
self.values[key] = value
|
||||
|
||||
|
||||
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_master_port=12345,
|
||||
_data_parallel_master_port_list=[12346, 12347],
|
||||
)
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
control_store, reconfig_store = DictStore({}), DictStore({})
|
||||
coordinator.reconfig_store_ref = reconfig_store
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
|
||||
)
|
||||
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
|
||||
|
||||
for store in (control_store, reconfig_store):
|
||||
# DictStore preserves insertion order: metadata must precede the epoch.
|
||||
assert list(store.values)[-1] == coordinator.key("current_epoch")
|
||||
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
|
||||
bootstrap
|
||||
)
|
||||
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
|
||||
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_get_reconfig_store", lambda store=store: store
|
||||
)
|
||||
assert coordinator.get_status()["requested_data_parallel_size"] == 3
|
||||
|
||||
|
||||
def test_external_elastic_ep_calculates_target_expert_redundancy():
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0)
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
|
||||
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("dp_rank", range(4))
|
||||
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
|
||||
"""Validate expert capacity on every rank before waiting for an epoch."""
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_rank=dp_rank,
|
||||
elastic_ep_max_dp_size=4,
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
_coord_store_port=12345,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
store = DictStore({})
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot retain all model experts"):
|
||||
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
|
||||
|
||||
assert not store.values
|
||||
assert coordinator.prepared_scale is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_elastic_ep_late_rank_observes_epoch_error():
|
||||
coordinator = ExternalElasticEPScaleCoordinator(
|
||||
cast("DPAsyncMPClient", SimpleNamespace())
|
||||
)
|
||||
epoch = "failed-epoch"
|
||||
store = DictStore(
|
||||
{
|
||||
coordinator.key("current_epoch"): epoch.encode(),
|
||||
coordinator.key(epoch, "error"): b"scale failed",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="scale failed"):
|
||||
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
|
||||
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
dp_size: int,
|
||||
api_server_count: int,
|
||||
base_server_args: list,
|
||||
tp_size: int = TP_SIZE,
|
||||
):
|
||||
self.model_name = model_name
|
||||
self.dp_size = dp_size
|
||||
self.tp_size = tp_size
|
||||
self.api_server_count = api_server_count
|
||||
self.base_server_args = base_server_args
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
|
||||
def start_rank(
|
||||
self,
|
||||
rank: int,
|
||||
dp_size: int | None = None,
|
||||
elastic_ep_scale_up_launch: bool = False,
|
||||
) -> tuple[RemoteOpenAIServer, list[str]]:
|
||||
"""Start one externally managed DP rank."""
|
||||
dp_size = self.dp_size if dp_size is None else dp_size
|
||||
server_args = self.base_server_args.copy()
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank),
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
|
||||
),
|
||||
}
|
||||
if elastic_ep_scale_up_launch:
|
||||
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
|
||||
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
server_args,
|
||||
auto_port=False,
|
||||
env_dict=env_dict,
|
||||
)
|
||||
print(
|
||||
f"Server rank {rank} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
server_and_args = (server, server_args)
|
||||
self.servers.append(server_and_args)
|
||||
return server_and_args
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
"""Start all initial server instances for external LB mode."""
|
||||
start_errors: list[Exception] = []
|
||||
for rank in range(self.dp_size):
|
||||
# Use a thread to start each server to allow parallel initialization
|
||||
def start_server(r: int):
|
||||
try:
|
||||
self.start_rank(r)
|
||||
except Exception as e:
|
||||
print(f"Failed to start server rank {r}: {e}")
|
||||
start_errors.append(e)
|
||||
|
||||
thread = threading.Thread(target=start_server, args=(rank,))
|
||||
thread.start()
|
||||
|
||||
self.server_threads.append(thread)
|
||||
|
||||
# Wait for all servers to start
|
||||
for thread in self.server_threads:
|
||||
thread.join()
|
||||
|
||||
if start_errors:
|
||||
start_error = start_errors[0]
|
||||
raise RuntimeError("Failed to start external LB servers") from start_error
|
||||
|
||||
# Give servers additional time to fully initialize and coordinate
|
||||
time.sleep(2)
|
||||
|
||||
if len(self.servers) != self.dp_size:
|
||||
raise Exception("Servers failed to start")
|
||||
|
||||
return self.servers
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Stop all server instances."""
|
||||
servers = [s for s, _ in self.servers]
|
||||
self.servers.clear()
|
||||
try:
|
||||
RemoteOpenAIServer.shutdown_many(servers)
|
||||
except Exception as e:
|
||||
print(f"Error stopping servers: {e}")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def default_server_args():
|
||||
return [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"128",
|
||||
"--enforce-eager",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
scope="module",
|
||||
params=[(1, False), (4, False), (1, True)],
|
||||
ids=["api-1-standard", "api-4-standard", "api-1-ep-enabled"],
|
||||
)
|
||||
def server_manager(request, default_server_args):
|
||||
api_server_count, enable_expert_parallel = request.param
|
||||
server_args = default_server_args.copy()
|
||||
if enable_expert_parallel:
|
||||
server_args.append("--enable-expert-parallel")
|
||||
|
||||
server_manager = ExternalLBServerManager(
|
||||
MODEL_NAME, DP_SIZE, api_server_count, server_args
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
yield server_manager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def servers(server_manager):
|
||||
return server_manager.servers
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def clients(servers: list[tuple[RemoteOpenAIServer, list[str]]]):
|
||||
# Create a client for each server
|
||||
async with AsyncExitStack() as stack:
|
||||
yield [
|
||||
await stack.enter_async_context(server.get_async_client())
|
||||
for server, _ in servers
|
||||
]
|
||||
|
||||
|
||||
def _get_parallel_config(server: RemoteOpenAIServer):
|
||||
response = requests.get(server.url_for("server_info?config_format=json"))
|
||||
response.raise_for_status()
|
||||
|
||||
vllm_config = response.json()["vllm_config"]
|
||||
return vllm_config["parallel_config"]
|
||||
|
||||
|
||||
def _assert_expert_parallel_config(
|
||||
servers: list[tuple[RemoteOpenAIServer, list[str]]],
|
||||
):
|
||||
for server, server_args in servers:
|
||||
expected = "--enable-expert-parallel" in server_args
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["enable_expert_parallel"] is expected
|
||||
|
||||
|
||||
def _send_scale_command(
|
||||
server: RemoteOpenAIServer, new_dp_size: int
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
server.url_for("scale_elastic_ep"),
|
||||
json={"new_data_parallel_size": new_dp_size},
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_scale_requests_to_start(
|
||||
scale_requests: list[Future[requests.Response]],
|
||||
timeout: float = 30,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for request in scale_requests:
|
||||
if request.done():
|
||||
response = request.result()
|
||||
pytest.fail(
|
||||
"Scale request finished before the new rank was launched: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
if all(request.running() for request in scale_requests):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
pytest.fail("Timed out waiting for all scale requests to start")
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=3)
|
||||
@pytest.mark.skipif(
|
||||
TP_SIZE != 1 or current_platform.device_count() < 3,
|
||||
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
|
||||
reason="Elastic EP scaling is not supported by the Rust frontend",
|
||||
)
|
||||
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
|
||||
server_args = [
|
||||
*default_server_args,
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--attention-backend",
|
||||
"TRITON_MLA",
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
"3",
|
||||
"--enable-eplb",
|
||||
"--eplb-config",
|
||||
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
|
||||
]
|
||||
server_manager = ExternalLBServerManager(
|
||||
ELASTIC_EP_MODEL_NAME,
|
||||
dp_size=2,
|
||||
api_server_count=1,
|
||||
base_server_args=server_args,
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
existing_servers = [server for server, _ in server_manager.servers]
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
scale_requests = [
|
||||
executor.submit(_send_scale_command, server, 3)
|
||||
for server in existing_servers
|
||||
]
|
||||
_wait_for_scale_requests_to_start(scale_requests)
|
||||
server_manager.start_rank(
|
||||
rank=2,
|
||||
dp_size=3,
|
||||
elastic_ep_scale_up_launch=True,
|
||||
)
|
||||
|
||||
for request in scale_requests:
|
||||
response = request.result(timeout=600)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
for server, _ in server_manager.servers:
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["data_parallel_size"] == 3
|
||||
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
|
||||
with server.get_client() as client:
|
||||
completion = client.completions.create(
|
||||
model=ELASTIC_EP_MODEL_NAME,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].finish_reason in ("length", "stop")
|
||||
|
||||
|
||||
def test_external_lb_server_info(server_manager):
|
||||
servers = server_manager.servers
|
||||
api_server_count = server_manager.api_server_count
|
||||
|
||||
for i, (server, _) in enumerate(servers):
|
||||
print(f"Testing {i=}")
|
||||
|
||||
# Each request will hit one of the API servers
|
||||
# `n_reqs` is set so that there is a good chance each server
|
||||
# receives at least one request
|
||||
n_reqs = 2 * api_server_count * api_server_count
|
||||
parallel_configs = [_get_parallel_config(server) for _ in range(n_reqs)]
|
||||
api_process_counts = [c["_api_process_count"] for c in parallel_configs]
|
||||
api_process_ranks = [c["_api_process_rank"] for c in parallel_configs]
|
||||
|
||||
assert all(c == api_server_count for c in api_process_counts), (
|
||||
api_process_counts
|
||||
)
|
||||
assert all(0 <= r < api_server_count for r in api_process_ranks), (
|
||||
api_process_ranks
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_external_lb_single_completion(
|
||||
clients: list[openai.AsyncOpenAI],
|
||||
servers: list[tuple[RemoteOpenAIServer, list[str]]],
|
||||
model_name: str,
|
||||
) -> None:
|
||||
async def make_request(client: openai.AsyncOpenAI):
|
||||
completion = await client.completions.create(
|
||||
model=model_name, prompt="Hello, my name is", max_tokens=10, temperature=1.0
|
||||
)
|
||||
|
||||
assert completion.id is not None
|
||||
assert completion.choices is not None and len(completion.choices) == 1
|
||||
|
||||
choice = completion.choices[0]
|
||||
# The exact number of tokens can vary slightly with temperature=1.0,
|
||||
# so we check for a reasonable minimum length.
|
||||
assert len(choice.text) >= 1
|
||||
# Finish reason might not always be 'length' if the model finishes early
|
||||
# or due to other reasons, especially with high temperature.
|
||||
# So, we'll accept 'length' or 'stop'.
|
||||
assert choice.finish_reason in ("length", "stop")
|
||||
|
||||
# Token counts can also vary, so we check they are positive.
|
||||
assert completion.usage.completion_tokens > 0
|
||||
assert completion.usage.prompt_tokens > 0
|
||||
assert completion.usage.total_tokens > 0
|
||||
return completion
|
||||
|
||||
# Test single request to each server
|
||||
for i, client in enumerate(clients):
|
||||
result = await make_request(client)
|
||||
assert result is not None
|
||||
print(f"Server {i} handled single completion request successfully")
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send requests to all servers in round-robin fashion
|
||||
num_requests_per_server = 25 # Total 50 requests across 2 servers
|
||||
all_tasks = []
|
||||
|
||||
for i, client in enumerate(clients):
|
||||
tasks = [make_request(client) for _ in range(num_requests_per_server)]
|
||||
all_tasks.extend(tasks)
|
||||
|
||||
results = await asyncio.gather(*all_tasks)
|
||||
assert len(results) == num_requests_per_server * len(clients)
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Second burst of requests
|
||||
all_tasks = []
|
||||
for i, client in enumerate(clients):
|
||||
tasks = [make_request(client) for _ in range(num_requests_per_server)]
|
||||
all_tasks.extend(tasks)
|
||||
|
||||
results = await asyncio.gather(*all_tasks)
|
||||
assert len(results) == num_requests_per_server * len(clients)
|
||||
assert all(completion is not None for completion in results)
|
||||
|
||||
_, server_args = servers[0]
|
||||
api_server_count = (
|
||||
server_args.count("--api-server-count")
|
||||
and server_args[server_args.index("--api-server-count") + 1]
|
||||
or 1
|
||||
)
|
||||
print(
|
||||
f"Successfully completed external LB test with {len(clients)} servers "
|
||||
f"(API server count: {api_server_count})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[MODEL_NAME],
|
||||
)
|
||||
async def test_external_lb_completion_streaming(
|
||||
clients: list[openai.AsyncOpenAI],
|
||||
servers: list[tuple[RemoteOpenAIServer, list[str]]],
|
||||
model_name: str,
|
||||
) -> None:
|
||||
prompt = "What is an LLM?"
|
||||
|
||||
async def make_streaming_request(client: openai.AsyncOpenAI):
|
||||
# Perform a non-streaming request to get the expected full output
|
||||
single_completion = await client.completions.create(
|
||||
model=model_name,
|
||||
prompt=prompt,
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
single_output = single_completion.choices[0].text
|
||||
|
||||
# Perform the streaming request
|
||||
stream = await client.completions.create(
|
||||
model=model_name, prompt=prompt, max_tokens=5, temperature=0.0, stream=True
|
||||
)
|
||||
chunks: list[str] = []
|
||||
finish_reason_count = 0
|
||||
last_chunk = None
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk.choices[0].text)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
finish_reason_count += 1
|
||||
last_chunk = chunk # Keep track of the last chunk
|
||||
|
||||
# finish reason should only return in the last block for OpenAI API
|
||||
assert finish_reason_count == 1, "Finish reason should appear exactly once."
|
||||
assert last_chunk is not None, "Stream should have yielded at least one chunk."
|
||||
assert last_chunk.choices[0].finish_reason == "length", (
|
||||
"Finish reason should be 'length'."
|
||||
)
|
||||
# Check that the combined text matches the non-streamed version.
|
||||
assert "".join(chunks) == single_output, (
|
||||
"Streamed output should match non-streamed output."
|
||||
)
|
||||
return True # Indicate success for this request
|
||||
|
||||
# Test single request to each server
|
||||
for i, client in enumerate(clients):
|
||||
result = await make_streaming_request(client)
|
||||
assert result is not None
|
||||
print(f"Server {i} handled single streaming request successfully")
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Send streaming requests to all servers in round-robin fashion
|
||||
num_requests_per_server = 25 # Total 50 requests across 2 servers
|
||||
all_tasks = []
|
||||
|
||||
for i, client in enumerate(clients):
|
||||
tasks = [make_streaming_request(client) for _ in range(num_requests_per_server)]
|
||||
all_tasks.extend(tasks)
|
||||
|
||||
results = await asyncio.gather(*all_tasks)
|
||||
assert len(results) == num_requests_per_server * len(clients)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Second burst of streaming requests
|
||||
all_tasks = []
|
||||
for i, client in enumerate(clients):
|
||||
tasks = [make_streaming_request(client) for _ in range(num_requests_per_server)]
|
||||
all_tasks.extend(tasks)
|
||||
|
||||
results = await asyncio.gather(*all_tasks)
|
||||
assert len(results) == num_requests_per_server * len(clients)
|
||||
assert all(results), "Not all streaming requests completed successfully."
|
||||
|
||||
_, server_args = servers[0]
|
||||
api_server_count = (
|
||||
server_args.count("--api-server-count")
|
||||
and server_args[server_args.index("--api-server-count") + 1]
|
||||
or 1
|
||||
)
|
||||
print(
|
||||
f"Successfully completed external LB streaming test with "
|
||||
f"{len(clients)} servers (API server count: {api_server_count})"
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
|
||||
from ..utils import compute_acceptance_len
|
||||
|
||||
|
||||
def load_and_process_dataset(data_name: str):
|
||||
from datasets import load_dataset
|
||||
|
||||
if data_name == "gsm8k":
|
||||
dataset = load_dataset("openai/gsm8k", "main", split="test")
|
||||
prompt_fmt = (
|
||||
"{question}\nPlease reason step by step,"
|
||||
" and put your final answer within \\boxed{{}}."
|
||||
)
|
||||
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
|
||||
elif data_name == "mt-bench":
|
||||
dataset = load_dataset("HuggingFaceH4/mt_bench_prompts", split="train")
|
||||
dataset = dataset.map(lambda x: {"turns": x["prompt"]})
|
||||
elif data_name == "humaneval":
|
||||
dataset = load_dataset("openai/openai_humaneval", split="test")
|
||||
prompt_fmt = (
|
||||
"Write a solution to the following problem and make sure"
|
||||
" that it passes the tests:\n```python\n{prompt}\n```"
|
||||
)
|
||||
dataset = dataset.map(lambda x: {"turns": [prompt_fmt.format(**x)]})
|
||||
|
||||
return dataset
|
||||
|
||||
|
||||
def run_acceptance_length_eval(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
vllm_runner,
|
||||
spec_config: dict[str, Any],
|
||||
expected_acceptance_lengths: dict[str, float],
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
use_mrv2: bool,
|
||||
):
|
||||
"""E2E acceptance-rate validation for speculative decoding.
|
||||
|
||||
Drives one or more datasets (keyed in ``expected_acceptance_lengths``)
|
||||
through the spec decode engine and asserts the mean acceptance length
|
||||
stays within tolerance of the reference figure for each dataset.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_mrv2 else "0")
|
||||
runner_config = spec_config.copy()
|
||||
model = runner_config.pop("model")
|
||||
runner_config.setdefault("block_size", None)
|
||||
runner_config.setdefault("enable_chunked_prefill", None)
|
||||
runner_config.setdefault("compilation_config", CompilationConfig())
|
||||
with vllm_runner(model, **runner_config) as spec_runner:
|
||||
spec_llm = spec_runner.llm
|
||||
# MT-Bench has 80 prompts and HumanEval has 164; truncate GSM8K.
|
||||
max_prompts_per_dataset = 200
|
||||
|
||||
tokenizer = spec_llm.get_tokenizer()
|
||||
for dataset_name, expected_len in expected_acceptance_lengths.items():
|
||||
dataset = load_and_process_dataset(dataset_name)
|
||||
prev_metrics = None
|
||||
acceptance_lengths = []
|
||||
for i in range(min(max_prompts_per_dataset, len(dataset))):
|
||||
user_content = dataset[i]["turns"][0]
|
||||
prompt_text = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": user_content}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# Greedy (temp=0) so acceptance length is deterministic and comparable
|
||||
# across runs.
|
||||
spec_llm.generate(
|
||||
[prompt_text],
|
||||
SamplingParams(temperature=0, max_tokens=2048),
|
||||
use_tqdm=False,
|
||||
)
|
||||
current_metrics = spec_llm.get_metrics()
|
||||
acceptance_len = compute_acceptance_len(current_metrics, prev_metrics)
|
||||
prev_metrics = current_metrics
|
||||
acceptance_lengths.append(acceptance_len)
|
||||
|
||||
mean_acceptance_length = sum(acceptance_lengths) / len(acceptance_lengths)
|
||||
# Fairly tight tolerance of 95% against the reference figures,
|
||||
# watching for regressions. Can be relaxed if test is flaky but be sure to
|
||||
# check for genuine issues such as #40727.
|
||||
expected_len = expected_len * 0.95
|
||||
print(
|
||||
f"acceptance_len for {dataset_name}: {mean_acceptance_length:.2f}"
|
||||
f" (expected at least {expected_len:.2f})"
|
||||
)
|
||||
|
||||
assert mean_acceptance_length >= expected_len, (
|
||||
f"acceptance_len for {dataset_name} is below expected threshold: "
|
||||
f"{mean_acceptance_length:.2f} < {expected_len:.2f}"
|
||||
)
|
||||
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts, evaluate_gsm8k_offline
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
from vllm.assets.image import VLM_IMAGES_DIR
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.metrics.reader import Metric
|
||||
|
||||
|
||||
def _skip_if_insufficient_gpus_for_tp(tp_size: int):
|
||||
"""Skip if fewer than ``tp_size`` accelerators are available."""
|
||||
available_gpus = torch.accelerator.device_count()
|
||||
if available_gpus < tp_size:
|
||||
pytest.skip(
|
||||
f"Test requires {tp_size} GPUs, but only {available_gpus} available"
|
||||
)
|
||||
|
||||
|
||||
Messages = list[dict[str, Any]]
|
||||
|
||||
|
||||
def get_test_prompts(mm_enabled: bool, num_prompts: int = 100) -> list[Messages]:
|
||||
prompt_types = ["repeat", "gsm8k"]
|
||||
if mm_enabled:
|
||||
prompt_types.append("mm")
|
||||
prompts: list[Messages] = []
|
||||
|
||||
num_repeat_prompts = num_prompts // len(prompt_types)
|
||||
if mm_enabled:
|
||||
num_gsm8k_prompts = num_prompts // len(prompt_types)
|
||||
num_mm_prompts = num_prompts - num_repeat_prompts - num_gsm8k_prompts
|
||||
else:
|
||||
num_mm_prompts = 0
|
||||
num_gsm8k_prompts = num_prompts - num_repeat_prompts
|
||||
|
||||
# Generate a mixed batch of prompts, some of which can be easily
|
||||
# predicted by n-gram matching and some which likely cannot.
|
||||
set_random_seed(0)
|
||||
for _ in range(num_repeat_prompts):
|
||||
word_choices = ["test", "temp", "hello", "where"]
|
||||
word = random.choice(word_choices)
|
||||
prompts.append(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"""
|
||||
please repeat the word '{word}' 10 times.
|
||||
give no other output than the word at least ten times in a row,
|
||||
in lowercase with spaces between each word and without quotes.
|
||||
""",
|
||||
}
|
||||
]
|
||||
)
|
||||
prompts.extend(
|
||||
[{"role": "user", "content": prompt}]
|
||||
for prompt in _build_gsm8k_prompts(
|
||||
num_questions=num_gsm8k_prompts, num_shots=5
|
||||
)[0]
|
||||
)
|
||||
for _ in range(num_mm_prompts):
|
||||
placeholders = [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"{VLLM_S3_BUCKET_URL}/{VLM_IMAGES_DIR}/stop_sign.jpg"
|
||||
},
|
||||
}
|
||||
]
|
||||
prompt = [
|
||||
*placeholders,
|
||||
{"type": "text", "text": "The meaning of the image is"},
|
||||
]
|
||||
prompts.append([{"role": "user", "content": prompt}])
|
||||
|
||||
return prompts
|
||||
|
||||
|
||||
def get_instruct_coder_messages(n: int) -> list[Messages]:
|
||||
from vllm.benchmarks.datasets import InstructCoderDataset
|
||||
|
||||
dataset = InstructCoderDataset(
|
||||
dataset_path="likaixin/InstructCoder", dataset_split="train"
|
||||
)
|
||||
prompts: Iterable[str] = dataset.sample_prompts(n=n)
|
||||
return [[{"role": "user", "content": prompt}] for prompt in prompts]
|
||||
|
||||
|
||||
def greedy_sampling() -> SamplingParams:
|
||||
return SamplingParams(temperature=0, max_tokens=10, ignore_eos=False)
|
||||
|
||||
|
||||
def stochastic_sampling() -> SamplingParams:
|
||||
return SamplingParams(temperature=1.0, max_tokens=10, ignore_eos=False)
|
||||
|
||||
|
||||
def evaluate_llm_for_gsm8k(llm: LLM, expected_accuracy_threshold: float = 0.70) -> None:
|
||||
"""Evaluate the LLM on GSM8K and check that accuracy is above a sanity threshold.
|
||||
|
||||
The default threshold assumes the LLM uses the same target model as the "model_name"
|
||||
fixture, with max model len == 4096. Precomputed reference value is 75% to 80%
|
||||
on GSM8K with greedy decoding, so we check that it's above a sanity threshold of 70%
|
||||
to verify that the model is correct.
|
||||
"""
|
||||
if expected_accuracy_threshold <= 0.0:
|
||||
print("Skipping GSM8K evaluation")
|
||||
return
|
||||
results = evaluate_gsm8k_offline(llm)
|
||||
accuracy = results["accuracy"]
|
||||
print(f"GSM8K accuracy: {accuracy:.3f}")
|
||||
assert accuracy >= expected_accuracy_threshold, (
|
||||
f"Expected GSM8K accuracy >= {expected_accuracy_threshold}, got {accuracy:.3f}"
|
||||
)
|
||||
|
||||
|
||||
def assert_request_outputs_match(
|
||||
ref_outputs: Sequence[RequestOutput],
|
||||
spec_outputs: Sequence[RequestOutput],
|
||||
*,
|
||||
required_matches: int,
|
||||
context: str,
|
||||
max_mismatches: int = 3,
|
||||
) -> None:
|
||||
"""Assert a thresholded exact-text match with bounded failure diagnostics."""
|
||||
assert ref_outputs, f"{context}: no reference outputs"
|
||||
assert len(ref_outputs) == len(spec_outputs), (
|
||||
f"{context}: output count differs: "
|
||||
f"reference={len(ref_outputs)}, speculative={len(spec_outputs)}"
|
||||
)
|
||||
assert 0 <= required_matches <= len(ref_outputs), (
|
||||
f"{context}: invalid required_matches={required_matches} for "
|
||||
f"{len(ref_outputs)} outputs"
|
||||
)
|
||||
|
||||
mismatches: list[str] = []
|
||||
matches = 0
|
||||
for index, (ref_output, spec_output) in enumerate(zip(ref_outputs, spec_outputs)):
|
||||
assert ref_output.outputs, (
|
||||
f"{context}: reference output {index} has no candidate"
|
||||
)
|
||||
assert spec_output.outputs, (
|
||||
f"{context}: speculative output {index} has no candidate"
|
||||
)
|
||||
ref_candidate = ref_output.outputs[0]
|
||||
spec_candidate = spec_output.outputs[0]
|
||||
if ref_candidate.text == spec_candidate.text:
|
||||
matches += 1
|
||||
elif len(mismatches) < max_mismatches:
|
||||
mismatches.append(
|
||||
f"[{index}] ref_text={ref_candidate.text[:240]!r}, "
|
||||
f"spec_text={spec_candidate.text[:240]!r}\n"
|
||||
f" ref_token_ids={list(ref_candidate.token_ids)[:64]}\n"
|
||||
f" spec_token_ids={list(spec_candidate.token_ids)[:64]}"
|
||||
)
|
||||
|
||||
print(
|
||||
f"{context}: exact text matches={matches}/{len(ref_outputs)} "
|
||||
f"(required={required_matches})"
|
||||
)
|
||||
mismatch_summary = "\n".join(mismatches) or "no mismatches captured"
|
||||
assert matches >= required_matches, (
|
||||
f"{context}: only {matches}/{len(ref_outputs)} outputs matched; "
|
||||
f"required at least {required_matches}. First mismatches:\n"
|
||||
f"{mismatch_summary}"
|
||||
)
|
||||
|
||||
|
||||
def get_spec_decode_metric_value(metrics: Sequence[Metric], metric_name: str) -> float:
|
||||
"""Get a spec-decode metric with an actionable error when stats are absent."""
|
||||
name2metric = {metric.name: metric for metric in metrics}
|
||||
metric = name2metric.get(metric_name)
|
||||
assert metric is not None, (
|
||||
f"Missing metric {metric_name!r}. Ensure disable_log_stats=False. "
|
||||
"Available spec-decode metrics: "
|
||||
f"{sorted(name for name in name2metric if 'spec_decode' in name) or ['<none>']}"
|
||||
)
|
||||
return float(metric.value)
|
||||
|
||||
|
||||
def compute_acceptance_rate(
|
||||
metrics: list[Metric], prev_metrics: list[Metric] | None = None
|
||||
) -> float:
|
||||
n_draft_toks = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_draft_tokens"
|
||||
)
|
||||
if n_draft_toks == 0:
|
||||
return float("nan")
|
||||
n_accepted_toks = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
if prev_metrics is not None:
|
||||
n_draft_toks -= get_spec_decode_metric_value(
|
||||
prev_metrics, "vllm:spec_decode_num_draft_tokens"
|
||||
)
|
||||
n_accepted_toks -= get_spec_decode_metric_value(
|
||||
prev_metrics, "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
if n_draft_toks <= 0:
|
||||
return float("nan")
|
||||
return n_accepted_toks / n_draft_toks
|
||||
|
||||
|
||||
def compute_acceptance_len(
|
||||
metrics: list[Metric], prev_metrics: list[Metric] | None = None
|
||||
) -> float:
|
||||
n_drafts = get_spec_decode_metric_value(metrics, "vllm:spec_decode_num_drafts")
|
||||
n_accepted_toks = get_spec_decode_metric_value(
|
||||
metrics, "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
if n_drafts == 0:
|
||||
return 1
|
||||
if prev_metrics is not None:
|
||||
n_drafts -= get_spec_decode_metric_value(
|
||||
prev_metrics, "vllm:spec_decode_num_drafts"
|
||||
)
|
||||
n_accepted_toks -= get_spec_decode_metric_value(
|
||||
prev_metrics, "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
if n_drafts <= 0:
|
||||
return 1
|
||||
return 1 + (n_accepted_toks / n_drafts)
|
||||
@@ -1,279 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing and registration behaviour of the EPD proxy."""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from examples.disaggregated.disaggregated_encoder import (
|
||||
disagg_epd_proxy as proxy_module,
|
||||
)
|
||||
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
|
||||
EPDProxy,
|
||||
EPDProxyConfig,
|
||||
InstanceRecord,
|
||||
InstanceRegistry,
|
||||
InstanceRole,
|
||||
build_app,
|
||||
extract_mm_items,
|
||||
)
|
||||
|
||||
ENCODE = InstanceRole.ENCODE
|
||||
PREFILL = InstanceRole.PREFILL
|
||||
DECODE = InstanceRole.DECODE
|
||||
PD = InstanceRole.PREFILL_DECODE
|
||||
|
||||
IMAGE_ITEM = {"type": "image_url", "image_url": {"url": "http://img/0.png"}}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def proxy():
|
||||
registry = InstanceRegistry(probe_interval=0)
|
||||
return EPDProxy(registry)
|
||||
|
||||
|
||||
class TestRouting:
|
||||
def test_no_decode_instance_is_service_unavailable(self, proxy):
|
||||
"""The proxy comes up before anything registers, so this is normal."""
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
proxy.route(num_items=0)
|
||||
assert excinfo.value.status_code == 503
|
||||
|
||||
def test_media_without_an_encoder_is_service_unavailable(self, proxy):
|
||||
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
|
||||
assert proxy.route(num_items=0).decode.url == "http://d0:8000"
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
proxy.route(num_items=1)
|
||||
assert excinfo.value.status_code == 503
|
||||
|
||||
def test_prefill_is_optional(self, proxy):
|
||||
"""An E+PD deployment registers no prefill instance at all."""
|
||||
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
|
||||
assert proxy.route(num_items=0).prefill is None
|
||||
|
||||
def test_encoder_roster_tracks_new_instances(self, proxy):
|
||||
proxy.registry.register(InstanceRecord(PD, "http://d0:8000"))
|
||||
for index in range(2):
|
||||
proxy.registry.register(InstanceRecord(ENCODE, f"http://e{index}:8000"))
|
||||
assert proxy.route(num_items=3).encoder_urls == [
|
||||
"http://e0:8000",
|
||||
"http://e1:8000",
|
||||
]
|
||||
proxy.registry.unregister("http://e0:8000")
|
||||
assert proxy.route(num_items=3).encoder_urls == ["http://e1:8000"]
|
||||
|
||||
@pytest.mark.parametrize("unregister", [False, True])
|
||||
def test_standalone_decode_always_requires_healthy_prefill(self, unregister):
|
||||
app = build_app()
|
||||
proxy = app.state.proxy
|
||||
proxy.registry.register(InstanceRecord(DECODE, "http://d0:8000"))
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
proxy.route(num_items=0)
|
||||
assert excinfo.value.status_code == 503
|
||||
proxy.registry.register(InstanceRecord(PREFILL, "http://p0:8000"))
|
||||
route = proxy.route(num_items=0)
|
||||
assert route.decode.url == "http://d0:8000"
|
||||
assert route.consumer.url == "http://p0:8000"
|
||||
if unregister:
|
||||
proxy.registry.unregister("http://p0:8000")
|
||||
else:
|
||||
for _ in range(proxy.registry._fail_threshold):
|
||||
proxy.registry._on_probe_failure(route.prefill, 0)
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
proxy.route(num_items=0)
|
||||
assert excinfo.value.status_code == 503
|
||||
|
||||
|
||||
class TestConsumerAddress:
|
||||
"""Which stage receives the embedding depends on the topology."""
|
||||
|
||||
def test_shared_storage_connectors_name_no_target(self, proxy):
|
||||
"""Rotate replicas without requiring a push endpoint."""
|
||||
proxy.registry.register(InstanceRecord(PD, "http://d0:8000", dp_size=2))
|
||||
route = proxy.route(num_items=0)
|
||||
assert route.consumer_zmq is None
|
||||
assert route.dp_rank == 0
|
||||
assert proxy.route(num_items=0).dp_rank == 1
|
||||
|
||||
def test_decode_is_the_consumer_when_prefill_is_not_split_out(self, proxy):
|
||||
proxy.registry.register(
|
||||
InstanceRecord(PD, "http://d0:8000", ec_zmq_addrs=["tcp://d0:20001"])
|
||||
)
|
||||
assert proxy.route(num_items=0).consumer_zmq == "tcp://d0:20001"
|
||||
|
||||
def test_prefill_is_the_consumer_when_it_is_split_out(self, proxy):
|
||||
"""In E+P+D the prefill instance consumes the embedding, not decode.
|
||||
|
||||
Reading the address out of whichever list it came from -- rather than
|
||||
from the record that reported one -- sends the push to the wrong
|
||||
instance in this topology.
|
||||
"""
|
||||
proxy.registry.register(
|
||||
InstanceRecord(PREFILL, "http://p0:8000", ec_zmq_addrs=["tcp://p0:20001"])
|
||||
)
|
||||
proxy.registry.register(InstanceRecord(DECODE, "http://d0:8000"))
|
||||
assert proxy.route(num_items=0).consumer_zmq == "tcp://p0:20001"
|
||||
|
||||
|
||||
class TestProxyApi:
|
||||
@pytest.fixture
|
||||
def client(self, monkeypatch):
|
||||
monkeypatch.setenv("ADMIN_API_KEY", "test-key")
|
||||
config = EPDProxyConfig(probe_interval=0)
|
||||
with TestClient(build_app(config)) as client:
|
||||
client.headers["X-API-Key"] = "test-key"
|
||||
yield client
|
||||
|
||||
def test_proxy_serves_before_anything_registers(self, client):
|
||||
assert client.get("/health").status_code == 200
|
||||
assert client.get("/instances").json()["encode"]["live"] == []
|
||||
|
||||
def test_requests_are_refused_until_a_decoder_registers(self, client):
|
||||
response = client.post(
|
||||
"/v1/chat/completions", json={"model": "m", "messages": []}
|
||||
)
|
||||
assert response.status_code == 503
|
||||
assert client.get("/v1/models").status_code == 503
|
||||
|
||||
def test_http_registration_keeps_consumer_replica_and_address_together(
|
||||
self, client
|
||||
):
|
||||
payload: dict[str, Any] = {
|
||||
"role": "prefill_decode",
|
||||
"url": "http://d0:8000/",
|
||||
"dp_size": 2,
|
||||
"ec_zmq_addrs": ["tcp://d0:20001", "tcp://d0:20003"],
|
||||
}
|
||||
assert client.post("/instances", json=payload).json() == {"registered": True}
|
||||
assert client.post("/instances", json=payload).json() == {"registered": False}
|
||||
proxy = client.app.state.proxy
|
||||
for rank in [0, 1, 0]:
|
||||
route = proxy.route(0)
|
||||
assert route.dp_rank == rank
|
||||
assert route.consumer_zmq == payload["ec_zmq_addrs"][rank]
|
||||
assert client.delete("/instances", params={"url": payload["url"]}).json() == {
|
||||
"removed": True
|
||||
}
|
||||
assert proxy.registry.pick(PD) is None
|
||||
|
||||
def test_registration_rejects_invalid_topology(self, client):
|
||||
cases: list[tuple[dict[str, Any], int]] = [
|
||||
({"dp_size": 0}, 422),
|
||||
({"dp_size": 2, "ec_zmq_addrs": ["tcp://d0:20001"]}, 422),
|
||||
({"role": "decode", "ec_zmq_addrs": ["tcp://d0:20001"]}, 400),
|
||||
({"role": "encode", "ec_zmq_addrs": ["tcp://d0:20001"]}, 400),
|
||||
]
|
||||
for fields, status in cases:
|
||||
response = client.post(
|
||||
"/instances",
|
||||
json={"url": "http://d0:8000", "role": "prefill_decode", **fields},
|
||||
)
|
||||
assert response.status_code == status
|
||||
|
||||
def test_registration_requires_admin_key(self, client):
|
||||
client.headers.pop("X-API-Key")
|
||||
assert (
|
||||
client.post(
|
||||
"/instances", json={"role": "encode", "url": "http://e0:8000"}
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
assert (
|
||||
client.delete("/instances", params={"url": "http://e0:8000"}).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("split_role", ["prefill", "decode"])
|
||||
@pytest.mark.parametrize("combined_first", [False, True])
|
||||
def test_registration_rejects_mixed_topologies(
|
||||
self, client, split_role, combined_first
|
||||
):
|
||||
roles = ["prefill_decode", split_role]
|
||||
if not combined_first:
|
||||
roles.reverse()
|
||||
first = {"role": roles[0], "url": "http://first:8000"}
|
||||
second = {"role": roles[1], "url": "http://second:8000"}
|
||||
assert client.post("/instances", json=first).status_code == 200
|
||||
assert client.post("/instances", json=second).status_code == 409
|
||||
# Health eviction must not silently permit a topology switch.
|
||||
registry = client.app.state.registry
|
||||
record = registry._live[first["url"]]
|
||||
for _ in range(registry._fail_threshold):
|
||||
registry._on_probe_failure(record, 0)
|
||||
assert client.post("/instances", json=second).status_code == 409
|
||||
assert client.delete("/instances", params={"url": first["url"]}).json() == {
|
||||
"removed": True
|
||||
}
|
||||
assert client.post("/instances", json=second).status_code == 200
|
||||
|
||||
|
||||
def test_extract_mm_items_finds_media_across_messages():
|
||||
req = {
|
||||
"messages": [
|
||||
{"role": "user", "content": "plain text"},
|
||||
{"role": "user", "content": [IMAGE_ITEM, {"type": "text", "text": "hi"}]},
|
||||
{"role": "user", "content": [IMAGE_ITEM]},
|
||||
]
|
||||
}
|
||||
assert extract_mm_items(req) == [IMAGE_ITEM, IMAGE_ITEM]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("push", [False, True])
|
||||
def test_encoder_handles_and_json_metadata_survive_rewrite(proxy, push, monkeypatch):
|
||||
"""Keep main's hash-keyed handles and per-item push IDs after the proxy move."""
|
||||
proxy.registry.register(InstanceRecord(ENCODE, "http://encoder"))
|
||||
proxy.registry.register(
|
||||
InstanceRecord(
|
||||
PD,
|
||||
"http://decode",
|
||||
ec_zmq_addrs=["tcp://decode:14579"] if push else [],
|
||||
)
|
||||
)
|
||||
handle = {
|
||||
"metadata": {"image_grid_thw": [[1, 2, 3]]},
|
||||
"item_indices": [0, 1],
|
||||
"peer_port": 5601,
|
||||
}
|
||||
response = Mock(
|
||||
status=200,
|
||||
read=AsyncMock(
|
||||
return_value=msgspec.json.encode(
|
||||
{"ec_transfer_params": {"engine-hash": handle}}
|
||||
)
|
||||
),
|
||||
)
|
||||
session = Mock(post=AsyncMock(return_value=response))
|
||||
monkeypatch.setattr(proxy_module, "encode_session", session)
|
||||
original = {"messages": [{"role": "user", "content": [IMAGE_ITEM, IMAGE_ITEM]}]}
|
||||
route = proxy.route(2)
|
||||
prepared, _, _ = asyncio.run(
|
||||
proxy_module.prepare_for_decode(
|
||||
original, "request", route.encoder_urls, None, route.consumer_zmq
|
||||
)
|
||||
)
|
||||
assert "ec_transfer_params" not in original
|
||||
assert prepared["ec_transfer_params"]["engine-hash"] == handle
|
||||
items = prepared["ec_transfer_params"]["ec_items"]
|
||||
assert [item["mm_hash"] for item in items] == ["engine-hash", "engine-hash"]
|
||||
assert items[0]["transfer_id"] != items[1]["transfer_id"]
|
||||
assert session.post.call_count == 1
|
||||
sent = msgspec.json.decode(session.post.call_args.kwargs["data"]).get(
|
||||
"ec_transfer_params"
|
||||
)
|
||||
if push:
|
||||
assert sent == {
|
||||
"consumer_zmq": "tcp://decode:14579",
|
||||
"ec_items": [{"transfer_id": item["transfer_id"]} for item in items],
|
||||
}
|
||||
else:
|
||||
assert sent is None
|
||||
for index in range(2):
|
||||
assert prepared["messages"][0]["content"][index]["image_embeds"] == {
|
||||
"image_grid_thw": [1, 2, 3]
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Dynamic registration and liveness for the EPD proxy's instance registry."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
|
||||
InstanceRecord,
|
||||
InstanceRegistry,
|
||||
InstanceRole,
|
||||
)
|
||||
|
||||
ENCODE = InstanceRole.ENCODE
|
||||
DECODE = InstanceRole.DECODE
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry():
|
||||
return InstanceRegistry(probe_interval=0, fail_threshold=3, evicted_ttl=60.0)
|
||||
|
||||
|
||||
def _fill(registry, role, count, prefix="http://h"):
|
||||
for index in range(count):
|
||||
registry.register(InstanceRecord(role, f"{prefix}{index}:8000"))
|
||||
|
||||
|
||||
async def _probe_round(registry, healthy: set[str], now: float = 0.0):
|
||||
"""Run one probe pass where only `healthy` URLs answer."""
|
||||
|
||||
async def fake_probe(self, session, url):
|
||||
return url in healthy
|
||||
|
||||
with (
|
||||
patch.object(InstanceRegistry, "_probe", fake_probe),
|
||||
patch(
|
||||
"examples.disaggregated.disaggregated_encoder.disagg_epd_proxy.time.monotonic",
|
||||
return_value=now,
|
||||
),
|
||||
):
|
||||
await registry._probe_once(session=None)
|
||||
|
||||
|
||||
class TestRegistration:
|
||||
def test_registering_is_idempotent_and_reports_novelty(self, registry):
|
||||
record = InstanceRecord(ENCODE, "http://e0:8000")
|
||||
assert registry.register(record) is True
|
||||
assert registry.register(record) is False
|
||||
assert registry.urls(ENCODE) == ["http://e0:8000"]
|
||||
|
||||
def test_transfer_addresses_stay_with_the_registered_consumer(self, registry):
|
||||
"""Roster updates must not shift a consumer's transfer addresses."""
|
||||
registry.register(InstanceRecord(DECODE, "http://d0:8000"))
|
||||
registry.register(
|
||||
InstanceRecord(
|
||||
DECODE,
|
||||
"http://d0:8000",
|
||||
ec_zmq_addrs=["tcp://d0:20001", "tcp://d0:20002"],
|
||||
dp_size=2,
|
||||
)
|
||||
)
|
||||
picked = registry.pick(DECODE)
|
||||
assert picked.ec_zmq_addrs == ["tcp://d0:20001", "tcp://d0:20002"]
|
||||
assert picked.dp_size == 2
|
||||
|
||||
def test_unregister_is_terminal(self, registry):
|
||||
registry.register(InstanceRecord(ENCODE, "http://e0:8000"))
|
||||
assert registry.unregister("http://e0:8000") is True
|
||||
assert registry.unregister("http://e0:8000") is False
|
||||
assert registry.pick(ENCODE) is None
|
||||
|
||||
def test_round_robin_survives_registration(self, registry):
|
||||
_fill(registry, ENCODE, 2)
|
||||
assert [record.url for record in registry.pick_many(ENCODE, 2)] == [
|
||||
"http://h0:8000",
|
||||
"http://h1:8000",
|
||||
]
|
||||
registry.register(InstanceRecord(ENCODE, "http://h2:8000"))
|
||||
assert registry.pick(ENCODE).url == "http://h2:8000"
|
||||
|
||||
|
||||
class TestLiveness:
|
||||
@pytest.mark.asyncio
|
||||
async def test_identical_reregistration_preserves_inflight_probe_results(
|
||||
self, registry
|
||||
):
|
||||
"""Repeated registrations must not suppress eviction or recovery."""
|
||||
url = "http://e0:8000"
|
||||
registry.register(InstanceRecord(ENCODE, url))
|
||||
healthy = False
|
||||
|
||||
async def probe(self, session, url):
|
||||
registry.register(InstanceRecord(ENCODE, url))
|
||||
return healthy
|
||||
|
||||
with patch.object(InstanceRegistry, "_probe", probe):
|
||||
for _ in range(registry._fail_threshold):
|
||||
await registry._probe_once(None)
|
||||
assert registry.status()["encode"] == {"live": [], "evicted": [url]}
|
||||
|
||||
healthy = True
|
||||
await registry._probe_once(None)
|
||||
assert registry.status()["encode"] == {"live": [url], "evicted": []}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_consecutive_failures_stop_routing(self, registry):
|
||||
registry.register(InstanceRecord(ENCODE, "http://e0:8000"))
|
||||
registry.register(InstanceRecord(ENCODE, "http://e1:8000"))
|
||||
for _ in range(registry._fail_threshold):
|
||||
await _probe_round(registry, healthy={"http://e1:8000"})
|
||||
assert registry.urls(ENCODE) == ["http://e1:8000"]
|
||||
assert registry.status()["encode"]["evicted"] == ["http://e0:8000"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("role", [ENCODE, DECODE])
|
||||
async def test_a_recovered_instance_rejoins_without_re_registering(
|
||||
self, registry, role
|
||||
):
|
||||
"""Restarting every encoder to recover from a blip is not acceptable."""
|
||||
registry.register(InstanceRecord(role, "http://e0:8000"))
|
||||
for _ in range(registry._fail_threshold):
|
||||
await _probe_round(registry, healthy=set())
|
||||
assert registry.urls(role) == []
|
||||
|
||||
await _probe_round(registry, healthy={"http://e0:8000"}, now=1)
|
||||
assert registry.urls(role) == ["http://e0:8000"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_re_registration_does_not_override_failed_health_checks(
|
||||
self, registry
|
||||
):
|
||||
record = InstanceRecord(ENCODE, "http://e0:8000")
|
||||
registry.register(record)
|
||||
for _ in range(registry._fail_threshold):
|
||||
await _probe_round(registry, healthy=set())
|
||||
registry.register(record)
|
||||
assert registry.urls(ENCODE) == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("healthy", [False, True])
|
||||
async def test_inflight_probe_cannot_restore_unregistered_instance(
|
||||
self, registry, healthy
|
||||
):
|
||||
record = InstanceRecord(ENCODE, "http://e0:8000")
|
||||
registry.register(record)
|
||||
for _ in range(registry._fail_threshold):
|
||||
await _probe_round(registry, healthy=set())
|
||||
|
||||
async def probe(self, session, url):
|
||||
registry.unregister(url)
|
||||
return healthy
|
||||
|
||||
with patch.object(InstanceRegistry, "_probe", probe):
|
||||
await registry._probe_once(None)
|
||||
assert registry.status()["encode"] == {"live": [], "evicted": []}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,712 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import (
|
||||
EOS_TOKEN_ID,
|
||||
create_model_runner_output,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import (
|
||||
KVEventsConfig,
|
||||
KVTransferConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
|
||||
OffloadingConnector,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadingWorker,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
RequestOffloadingContext,
|
||||
TransferResult,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
|
||||
def to_key(int_hash: int) -> OffloadKey:
|
||||
return make_offload_key(str(int_hash).encode(), 0)
|
||||
|
||||
|
||||
def to_keys(int_hashes: Iterable[int]) -> list[OffloadKey]:
|
||||
return [to_key(i) for i in int_hashes]
|
||||
|
||||
|
||||
class MockLoadStoreSpec(LoadStoreSpec):
|
||||
def __init__(self, offload_keys: Iterable[OffloadKey]):
|
||||
self.offload_keys: list[OffloadKey] = list(offload_keys)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return repr(self.offload_keys)
|
||||
|
||||
|
||||
class MockOffloadingWorker(OffloadingWorker):
|
||||
def __init__(self):
|
||||
self.transfer_specs: dict[int, tuple[LoadStoreSpec, LoadStoreSpec]] = {}
|
||||
self.completed_transfers: list[TransferResult] = []
|
||||
self.waiting_jobs: set[int] = set()
|
||||
self.completed_jobs: list[int] = []
|
||||
self.flushed_jobs: set[int] = set()
|
||||
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
finished = self.completed_transfers
|
||||
self.completed_transfers = []
|
||||
return finished
|
||||
|
||||
def submit_store(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool: # type: ignore[override]
|
||||
self.transfer_specs[job_id] = (src_spec, dst_spec)
|
||||
self.waiting_jobs.add(job_id)
|
||||
return True
|
||||
|
||||
def submit_load(
|
||||
self, job_id: int, src_spec: LoadStoreSpec, dst_spec: LoadStoreSpec
|
||||
) -> bool: # type: ignore[override]
|
||||
self.transfer_specs[job_id] = (src_spec, dst_spec)
|
||||
self.waiting_jobs.add(job_id)
|
||||
return True
|
||||
|
||||
def complete_jobs(self, job_ids: set[int]) -> None:
|
||||
for job_id in job_ids:
|
||||
if job_id in self.waiting_jobs:
|
||||
self.waiting_jobs.remove(job_id)
|
||||
self.completed_jobs.append(job_id)
|
||||
result = TransferResult(
|
||||
job_id=job_id,
|
||||
success=True,
|
||||
transfer_size=None,
|
||||
transfer_time=None,
|
||||
)
|
||||
self.completed_transfers.append(result)
|
||||
|
||||
def wait(self, job_ids: set[int]) -> None:
|
||||
self.flushed_jobs |= job_ids
|
||||
self.complete_jobs(job_ids)
|
||||
|
||||
|
||||
class MockOffloadingSpec(OffloadingSpec):
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
super().__init__(config)
|
||||
|
||||
self.manager = MagicMock(spec=OffloadingManager)
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = LookupResult.MISS
|
||||
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
self.manager.get_stats.return_value = None
|
||||
self.manager.config_info.return_value = [{}]
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
return self.manager
|
||||
|
||||
def get_worker(self, _: CanonicalKVCaches) -> OffloadingWorker:
|
||||
return self.handler
|
||||
|
||||
def complete_transfers(self):
|
||||
self.handler.complete_jobs(self.handler.waiting_jobs.copy())
|
||||
|
||||
def get_completed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
|
||||
specs = [
|
||||
self.handler.transfer_specs[job_id]
|
||||
for job_id in self.handler.completed_jobs
|
||||
]
|
||||
self.handler.completed_jobs.clear()
|
||||
return specs
|
||||
|
||||
def get_flushed_transfers(self) -> list[tuple[LoadStoreSpec, LoadStoreSpec]]:
|
||||
specs = [
|
||||
self.handler.transfer_specs[job_id] for job_id in self.handler.flushed_jobs
|
||||
]
|
||||
self.handler.flushed_jobs.clear()
|
||||
return specs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GPUBlock:
|
||||
group_idx: int
|
||||
request_block_offset: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferSummary:
|
||||
gpu_blocks: list[GPUBlock]
|
||||
offload_addresses: list[Any]
|
||||
|
||||
|
||||
class RequestRunner:
|
||||
def __init__(
|
||||
self,
|
||||
block_size: int,
|
||||
num_gpu_blocks: int,
|
||||
blocks_per_chunk: int = 1,
|
||||
async_scheduling: bool = True,
|
||||
kv_cache_groups: list[KVCacheGroupSpec] | None = None,
|
||||
extra_config_overrides: dict[str, Any] | None = None,
|
||||
worker_count: int = 1,
|
||||
retention_interval: int | None = None,
|
||||
speculative_config: Any | None = None,
|
||||
):
|
||||
assert blocks_per_chunk == 1 or kv_cache_groups is None, (
|
||||
"blocks_per_chunk > 1 requires all groups to have the same "
|
||||
"block size, so kv_cache_groups must be None (use default group)"
|
||||
)
|
||||
|
||||
self.blocks_per_chunk: int = blocks_per_chunk
|
||||
self.block_size: int = block_size
|
||||
self.num_gpu_blocks: int = num_gpu_blocks
|
||||
self.async_scheduling: bool = async_scheduling
|
||||
|
||||
self.req_id: int = -1
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
max_num_batched_tokens=1000,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
vllm_config.scheduler_config.async_scheduling = async_scheduling
|
||||
vllm_config.parallel_config.world_size = worker_count
|
||||
vllm_config.cache_config.prefix_cache_retention_interval = retention_interval
|
||||
if speculative_config is not None:
|
||||
vllm_config.speculative_config = speculative_config
|
||||
|
||||
extra_config: dict[str, Any] = {
|
||||
"spec_name": "MockOffloadingSpec",
|
||||
"spec_module_path": "tests.v1.kv_connector.unit.offloading_connector.utils", # noqa: E501
|
||||
# Preserve legacy behavior for tests; new opt-in tests override.
|
||||
"offload_prompt_only": False,
|
||||
# Exercise the self-describing KV events path by default;
|
||||
# opt-out tests override this to cover the legacy placeholders.
|
||||
"self_describing_kv_events": True,
|
||||
}
|
||||
if blocks_per_chunk > 1:
|
||||
extra_config["block_size"] = block_size * blocks_per_chunk
|
||||
if extra_config_overrides:
|
||||
extra_config.update(extra_config_overrides)
|
||||
|
||||
vllm_config.kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=extra_config,
|
||||
)
|
||||
vllm_config.kv_events_config = KVEventsConfig(
|
||||
# Enable so the offloading events tracker is active, but use the
|
||||
# null publisher: these tests drain take_events directly and a
|
||||
# real ZMQ publisher would bind a port per test.
|
||||
enable_kv_cache_events=True,
|
||||
publisher="null",
|
||||
)
|
||||
|
||||
if kv_cache_groups is None:
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
# Groups overlay one allocation, sized by the largest group.
|
||||
block_stride = max(
|
||||
group.kv_cache_spec.page_size_bytes * len(group.layer_names)
|
||||
for group in kv_cache_groups
|
||||
)
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(
|
||||
size=block_stride * num_gpu_blocks,
|
||||
layers=list(group.layer_names),
|
||||
layer_stride=group.kv_cache_spec.page_size_bytes * num_gpu_blocks,
|
||||
block_stride=group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
for group in kv_cache_groups
|
||||
]
|
||||
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_gpu_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
self.num_kv_groups = len(kv_cache_config.kv_cache_groups)
|
||||
|
||||
scheduler_block_size, hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
|
||||
scheduler_cls = AsyncScheduler if async_scheduling else Scheduler
|
||||
self.scheduler = scheduler_cls(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
self.worker_connector = OffloadingConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
# register worker kv_caches to enable OffloadingWorker creations
|
||||
kv_caches: dict[str, torch.Tensor] = {}
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
for layer_name in group.layer_names:
|
||||
# Shape follows FlashAttention layout:
|
||||
# Shape: (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
kv_caches[layer_name] = torch.empty(
|
||||
num_gpu_blocks,
|
||||
2,
|
||||
spec.block_size,
|
||||
spec.num_kv_heads,
|
||||
spec.head_size,
|
||||
dtype=spec.dtype,
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
self.worker_connector.register_kv_caches(kv_caches)
|
||||
|
||||
# extract connector of scheduler
|
||||
scheduler_connector = self.scheduler.connector
|
||||
assert scheduler_connector is not None
|
||||
assert isinstance(scheduler_connector, OffloadingConnector)
|
||||
self.scheduler_connector: OffloadingConnector = scheduler_connector
|
||||
|
||||
# extract mocked OffloadingManager of scheduler connector
|
||||
self.connector_scheduler = scheduler_connector.connector_scheduler
|
||||
assert self.connector_scheduler is not None
|
||||
manager = self.connector_scheduler.manager
|
||||
assert isinstance(manager, MagicMock)
|
||||
self.manager: MagicMock = manager
|
||||
|
||||
num_kv_groups = len(kv_cache_config.kv_cache_groups)
|
||||
assert len(self.connector_scheduler.config.kv_group_configs) == num_kv_groups
|
||||
for group_config, kv_cache_group in zip(
|
||||
self.connector_scheduler.config.kv_group_configs,
|
||||
kv_cache_config.kv_cache_groups,
|
||||
):
|
||||
tokens_per_block = kv_cache_group.kv_cache_spec.block_size
|
||||
assert group_config.tokens_per_block == tokens_per_block
|
||||
assert group_config.tokens_per_chunk == tokens_per_block * blocks_per_chunk
|
||||
|
||||
# extract OffloadingSpec of worker_connector
|
||||
connector_worker = self.worker_connector.connector_worker
|
||||
assert connector_worker is not None
|
||||
offloading_spec = connector_worker.spec
|
||||
assert isinstance(offloading_spec, MockOffloadingSpec)
|
||||
self.offloading_spec: MockOffloadingSpec = offloading_spec
|
||||
|
||||
# mapping (offloading address) -> GPUBlock
|
||||
self.offloaded: dict[Any, GPUBlock] = {}
|
||||
|
||||
self.completed_loads: list[TransferSummary] = []
|
||||
self.completed_stores: list[TransferSummary] = []
|
||||
self.flushed_gpu_blocks: set[GPUBlock] = set()
|
||||
self.kv_connector_stats: list[Any] = []
|
||||
|
||||
# block_id -> GPUBlock
|
||||
self.gpu_blocks: dict[int, GPUBlock] = {}
|
||||
|
||||
init_none_hash(sha256)
|
||||
self._block_hasher = get_request_block_hasher(block_size, sha256)
|
||||
|
||||
self._dummy_ctx: ForwardContext = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={},
|
||||
slot_mapping={},
|
||||
)
|
||||
|
||||
def _record_kv_connector_stats(self, engine_outputs: dict[int, Any]) -> None:
|
||||
for output in engine_outputs.values():
|
||||
scheduler_stats = output.scheduler_stats
|
||||
if scheduler_stats is not None and scheduler_stats.kv_connector_stats:
|
||||
self.kv_connector_stats.append(scheduler_stats.kv_connector_stats)
|
||||
|
||||
def new_request(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
kv_transfer_params: dict | None = None,
|
||||
skip_reading_prefix_cache: bool = False,
|
||||
):
|
||||
self.req_id += 1
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=1000,
|
||||
skip_reading_prefix_cache=skip_reading_prefix_cache or None,
|
||||
)
|
||||
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
|
||||
req = Request(
|
||||
request_id=str(self.req_id),
|
||||
prompt_token_ids=token_ids,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
block_hasher=self._block_hasher,
|
||||
)
|
||||
if kv_transfer_params is not None:
|
||||
req.kv_transfer_params = kv_transfer_params
|
||||
|
||||
self.scheduler.add_request(req)
|
||||
|
||||
def _parse_transfers(self):
|
||||
for src_spec, dst_spec in self.offloading_spec.get_flushed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
# store flush
|
||||
for block_id in src_spec.block_ids:
|
||||
self.flushed_gpu_blocks.add(self.gpu_blocks[block_id.item()])
|
||||
else:
|
||||
# load flush
|
||||
for block_id in dst_spec.block_ids:
|
||||
self.flushed_gpu_blocks.add(self.gpu_blocks[block_id.item()])
|
||||
|
||||
blocks_per_chunk = self.blocks_per_chunk
|
||||
|
||||
for src_spec, dst_spec in self.offloading_spec.get_completed_transfers():
|
||||
if isinstance(src_spec, GPULoadStoreSpec):
|
||||
store = True
|
||||
gpu_spec = src_spec
|
||||
offload_spec = dst_spec
|
||||
else:
|
||||
store = False
|
||||
gpu_spec = dst_spec
|
||||
offload_spec = src_spec
|
||||
|
||||
assert isinstance(offload_spec, MockLoadStoreSpec)
|
||||
assert isinstance(gpu_spec, GPULoadStoreSpec)
|
||||
assert len(gpu_spec.group_sizes) == self.num_kv_groups
|
||||
|
||||
gpu_blocks: list[GPUBlock] = []
|
||||
for block_id in gpu_spec.block_ids:
|
||||
gpu_blocks.append(self.gpu_blocks[block_id.item()])
|
||||
|
||||
# list of (offload_key, sub_block_offset)
|
||||
offload_addresses: list[Any] = []
|
||||
for offload_key in offload_spec.offload_keys:
|
||||
for sub_block_idx in range(blocks_per_chunk):
|
||||
offload_addresses.append((offload_key, sub_block_idx))
|
||||
|
||||
assert gpu_spec.block_indices is not None
|
||||
assert len(gpu_spec.block_indices) == self.num_kv_groups
|
||||
|
||||
gpu_block_offset = 0
|
||||
offload_address_offset = 0
|
||||
for group_size, logical_offset in zip(
|
||||
gpu_spec.group_sizes, gpu_spec.block_indices
|
||||
):
|
||||
gpu_block_end_offset = gpu_block_offset + group_size
|
||||
assert gpu_block_end_offset <= len(gpu_blocks)
|
||||
|
||||
offload_addresses_to_skip = logical_offset % blocks_per_chunk
|
||||
offload_addresses_end_offset = (
|
||||
offload_address_offset + offload_addresses_to_skip + group_size
|
||||
)
|
||||
assert offload_addresses_end_offset <= len(offload_addresses)
|
||||
|
||||
offload_addresses = (
|
||||
offload_addresses[:offload_address_offset]
|
||||
+ offload_addresses[
|
||||
offload_address_offset + offload_addresses_to_skip :
|
||||
]
|
||||
)
|
||||
|
||||
gpu_block_offset += group_size
|
||||
offload_address_offset += group_size
|
||||
|
||||
assert gpu_block_offset == len(gpu_blocks)
|
||||
assert offload_address_offset == len(offload_addresses)
|
||||
|
||||
transfer_summary = TransferSummary(gpu_blocks, offload_addresses)
|
||||
if store:
|
||||
self.completed_stores.append(transfer_summary)
|
||||
else:
|
||||
self.completed_loads.append(transfer_summary)
|
||||
|
||||
def _update_gpu_blocks(self):
|
||||
for group_idx, manager in enumerate(
|
||||
self.scheduler.kv_cache_manager.coordinator.single_type_managers
|
||||
):
|
||||
for blocks in manager.req_to_blocks.values():
|
||||
for block_idx, block in enumerate(blocks):
|
||||
self.gpu_blocks[block.block_id] = GPUBlock(group_idx, block_idx)
|
||||
|
||||
def _run(
|
||||
self,
|
||||
decoded_tokens: list[int],
|
||||
complete_transfers: bool,
|
||||
post_step_fn: Callable[[], None] | None = None,
|
||||
):
|
||||
"""Runs multiple engine (scheduler + worker) steps.
|
||||
Assumes a single request is running.
|
||||
|
||||
Args:
|
||||
decoded_tokens: the tokens to yield at each step.
|
||||
complete_transfers: complete transfers immediately
|
||||
post_step_fn: optional callback invoked after each step's
|
||||
update_from_output(), before the next schedule().
|
||||
|
||||
"""
|
||||
tokens_iter = iter(decoded_tokens)
|
||||
token_id = next(tokens_iter, None)
|
||||
prev_scheduler_output = None
|
||||
prev_model_runner_output = None
|
||||
while True:
|
||||
# Strict-always-False frees the request immediately on EOS, but
|
||||
# the worker may still have a deferred store queued. In production
|
||||
# the next request's step drains it; in single-request tests we
|
||||
# must keep stepping until the scheduler sees no in-flight jobs
|
||||
# and no pending finished_req_ids awaiting build_connector_meta.
|
||||
if (
|
||||
not self.scheduler.requests
|
||||
and not self.connector_scheduler._jobs
|
||||
and not self.scheduler.finished_req_ids
|
||||
):
|
||||
break
|
||||
|
||||
scheduler_output = self.scheduler.schedule()
|
||||
self._update_gpu_blocks()
|
||||
|
||||
kv_connector_metadata = scheduler_output.kv_connector_metadata
|
||||
assert kv_connector_metadata is not None
|
||||
assert isinstance(kv_connector_metadata, OffloadingConnectorMetadata)
|
||||
|
||||
self.worker_connector.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
self.worker_connector.bind_connector_metadata(kv_connector_metadata)
|
||||
self.worker_connector.start_load_kv(self._dummy_ctx)
|
||||
|
||||
if complete_transfers:
|
||||
self.offloading_spec.complete_transfers()
|
||||
|
||||
self.worker_connector.wait_for_save()
|
||||
finished_sending, finished_recving = self.worker_connector.get_finished(
|
||||
scheduler_output.finished_req_ids
|
||||
)
|
||||
worker_meta = (
|
||||
self.worker_connector.build_connector_worker_meta()
|
||||
or OffloadingWorkerMetadata()
|
||||
)
|
||||
|
||||
self.worker_connector.clear_connector_metadata()
|
||||
|
||||
model_runner_output = create_model_runner_output(
|
||||
reqs=self.scheduler.running,
|
||||
finished_sending=finished_sending,
|
||||
finished_recving=finished_recving,
|
||||
token_id=token_id or 0,
|
||||
kv_connector_worker_meta=worker_meta,
|
||||
)
|
||||
|
||||
prev_token_id = token_id
|
||||
if self.scheduler.running:
|
||||
token_id = next(tokens_iter, None)
|
||||
|
||||
if self.async_scheduling:
|
||||
# in async scheduling we update the output of the previous step
|
||||
if prev_model_runner_output is not None:
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
prev_scheduler_output, prev_model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
prev_scheduler_output = scheduler_output
|
||||
prev_model_runner_output = model_runner_output
|
||||
else:
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
scheduler_output, model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
|
||||
if post_step_fn is not None:
|
||||
post_step_fn()
|
||||
|
||||
if (
|
||||
prev_token_id == EOS_TOKEN_ID
|
||||
and prev_token_id != token_id
|
||||
and (
|
||||
self.scheduler.requests
|
||||
or self.connector_scheduler._jobs
|
||||
or self.scheduler.finished_req_ids
|
||||
)
|
||||
):
|
||||
# continue for one more step to allow offloading to kick off
|
||||
continue
|
||||
|
||||
if token_id is None:
|
||||
if self.async_scheduling:
|
||||
# Flush the previous step's output.
|
||||
engine_outputs = self.scheduler.update_from_output(
|
||||
prev_scheduler_output, prev_model_runner_output
|
||||
)
|
||||
self._record_kv_connector_stats(engine_outputs)
|
||||
prev_model_runner_output = None
|
||||
if self.scheduler.requests:
|
||||
# Request still running, just exhausted decoded_tokens.
|
||||
break
|
||||
if not self.scheduler.finished_req_ids and (
|
||||
not complete_transfers or not self.connector_scheduler._jobs
|
||||
):
|
||||
break
|
||||
|
||||
self._parse_transfers()
|
||||
|
||||
if EOS_TOKEN_ID in decoded_tokens:
|
||||
assert not self.scheduler.running
|
||||
|
||||
def _to_gpu_blocks(
|
||||
self, blocks: tuple[int | tuple[int, int], ...]
|
||||
) -> list[GPUBlock]:
|
||||
gpu_blocks: list[GPUBlock] = []
|
||||
for block in blocks:
|
||||
if isinstance(block, int):
|
||||
for group_idx in range(self.num_kv_groups):
|
||||
gpu_blocks.append(
|
||||
GPUBlock(group_idx=group_idx, request_block_offset=block)
|
||||
)
|
||||
else:
|
||||
group_idx, offset = block
|
||||
gpu_blocks.append(
|
||||
GPUBlock(group_idx=group_idx, request_block_offset=offset)
|
||||
)
|
||||
return gpu_blocks
|
||||
|
||||
def run(
|
||||
self,
|
||||
decoded_tokens: list[int],
|
||||
complete_transfers: bool = True,
|
||||
expected_stored: tuple[int | tuple[int, int], ...] = (),
|
||||
expected_loaded: tuple[int | tuple[int, int], ...] = (),
|
||||
expected_flushed: tuple[int | tuple[int, int], ...] = (),
|
||||
post_step_fn: Callable[[], None] | None = None,
|
||||
):
|
||||
"""Runs multiple engine (scheduler + worker) steps.
|
||||
Assumes a single request is running.
|
||||
|
||||
Args:
|
||||
decoded_tokens: the tokens to yield at each step.
|
||||
complete_transfers: complete transfers immediately
|
||||
expected_stored: GPU blocks
|
||||
that are expected to be written during the run.
|
||||
expected_loaded: GPU blocks
|
||||
that are expected to be loaded during the run.
|
||||
expected_flushed: GPU blocks
|
||||
that are expected to be flushed during the run.
|
||||
|
||||
A GPU block is either a (group_idx: int, request_block_offset: int)
|
||||
or just request_block_offset: int.
|
||||
The latter case is a convenience for representing all groups.
|
||||
|
||||
"""
|
||||
expected_stored_gpu_blocks = self._to_gpu_blocks(expected_stored)
|
||||
expected_loaded_gpu_blocks = self._to_gpu_blocks(expected_loaded)
|
||||
expected_flushed_gpu_blocks = self._to_gpu_blocks(expected_flushed)
|
||||
|
||||
self.manager.reset_mock()
|
||||
self._run(decoded_tokens, complete_transfers, post_step_fn=post_step_fn)
|
||||
|
||||
loaded_gpu_blocks: set[GPUBlock] = set()
|
||||
for transfer in self.completed_loads:
|
||||
for gpu_block, offloaded_address in zip(
|
||||
transfer.gpu_blocks, transfer.offload_addresses
|
||||
):
|
||||
loaded_gpu_blocks.add(gpu_block)
|
||||
assert gpu_block == self.offloaded[offloaded_address]
|
||||
|
||||
assert set(expected_loaded_gpu_blocks) == loaded_gpu_blocks
|
||||
self.completed_loads.clear()
|
||||
|
||||
stored_gpu_blocks: set[GPUBlock] = set()
|
||||
for transfer in self.completed_stores:
|
||||
for gpu_block, offloaded_address in zip(
|
||||
transfer.gpu_blocks, transfer.offload_addresses
|
||||
):
|
||||
stored_gpu_blocks.add(gpu_block)
|
||||
self.offloaded[offloaded_address] = gpu_block
|
||||
|
||||
assert set(expected_stored_gpu_blocks) == stored_gpu_blocks
|
||||
self.completed_stores.clear()
|
||||
|
||||
assert set(expected_flushed_gpu_blocks) == self.flushed_gpu_blocks
|
||||
self.flushed_gpu_blocks.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_runner():
|
||||
runners = []
|
||||
|
||||
def runner_factory(
|
||||
block_size,
|
||||
num_gpu_blocks,
|
||||
async_scheduling,
|
||||
blocks_per_chunk=1,
|
||||
kv_cache_groups=None,
|
||||
extra_config_overrides=None,
|
||||
worker_count=1,
|
||||
retention_interval=None,
|
||||
speculative_config=None,
|
||||
):
|
||||
runner = RequestRunner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
extra_config_overrides=extra_config_overrides,
|
||||
worker_count=worker_count,
|
||||
retention_interval=retention_interval,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
runners.append(runner)
|
||||
return runner
|
||||
|
||||
yield runner_factory # pass factory to the test
|
||||
|
||||
|
||||
def generate_store_output(keys: Iterable[OffloadKey]):
|
||||
keys = list(keys)
|
||||
return PrepareStoreOutput(
|
||||
keys_to_store=list(keys),
|
||||
store_spec=MockLoadStoreSpec(keys),
|
||||
evicted_keys=[],
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,700 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, count
|
||||
from typing import Any, Literal
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
DeviceConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
|
||||
ExampleConnector,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.async_scheduler import AsyncScheduler
|
||||
from vllm.v1.core.sched.scheduler import Scheduler, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
EOS_TOKEN_ID = 50256
|
||||
|
||||
|
||||
def create_mock_connector() -> Mock:
|
||||
connector = Mock()
|
||||
connector.get_external_cache_hit_sources.side_effect = lambda request, num_tokens: (
|
||||
{CacheHitSource.EXTERNAL_UNSPECIFIED: num_tokens} if num_tokens else {}
|
||||
)
|
||||
return connector
|
||||
|
||||
|
||||
def maybe_update_block_size(block_size: int) -> int:
|
||||
return 64 if current_platform.is_xpu() else block_size
|
||||
|
||||
|
||||
def assert_scheduler_empty(scheduler: Scheduler):
|
||||
"""Confirm the scheduler is "empty" - i.e. no leaks."""
|
||||
# Scheduler Metadata.
|
||||
assert len(scheduler.requests) == 0
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.kv_holding_waiting) == 0
|
||||
assert not scheduler.deferred_waiting
|
||||
assert len(scheduler.running) == 0
|
||||
assert len(scheduler.finished_req_ids) == 0
|
||||
assert len(scheduler.finished_recving_kv_req_ids) == 0
|
||||
assert len(scheduler._inflight_prefills) == 0
|
||||
assert not scheduler._kv_fetch_stages
|
||||
assert not any(scheduler._kv_fetch_counts.values())
|
||||
|
||||
# EncoderCacheManager.
|
||||
assert len(scheduler.encoder_cache_manager.freed) == 0
|
||||
assert len(scheduler.encoder_cache_manager.cached) == 0
|
||||
|
||||
# KVCache Manager.
|
||||
assert (
|
||||
len(
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
len(
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].num_cached_block
|
||||
)
|
||||
== 0
|
||||
)
|
||||
num_free_blocks = (
|
||||
scheduler.kv_cache_manager.block_pool.free_block_queue.num_free_blocks
|
||||
)
|
||||
assert num_free_blocks == (scheduler.kv_cache_manager.block_pool.num_gpu_blocks - 1)
|
||||
|
||||
# NOTE(rob): just the ref count on blocks will be 0. The hash
|
||||
# value, etc will remain since we lazily evict for prefix cache.
|
||||
for block in scheduler.kv_cache_manager.block_pool.blocks:
|
||||
assert block.ref_cnt == 0
|
||||
|
||||
|
||||
def create_vllm_config(
|
||||
model: str = "facebook/opt-125m",
|
||||
max_num_seqs: int = 16,
|
||||
max_num_batched_tokens: int = 64,
|
||||
block_size: int = 16,
|
||||
max_model_len: int = 10000,
|
||||
enable_chunked_prefill: bool = True,
|
||||
enable_permute_local_kv: bool = False,
|
||||
kv_connector_extra_config: dict[str, Any] | None = None,
|
||||
dtype: str = "float16",
|
||||
cache_dtype: str = "auto",
|
||||
hf_overrides: dict[str, Any] | None = None,
|
||||
attention_backend: str | None = None,
|
||||
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
|
||||
kv_connector: str = "NixlConnector",
|
||||
kv_connector_module_path: str | None = None,
|
||||
kv_role: str = "kv_consumer",
|
||||
disable_hybrid_kv_cache_manager: bool | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
) -> VllmConfig:
|
||||
"""Initialize VllmConfig For Testing."""
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
trust_remote_code=True,
|
||||
dtype=dtype,
|
||||
seed=42,
|
||||
hf_overrides=hf_overrides or {},
|
||||
)
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager,
|
||||
)
|
||||
# Cache config, optionally force APC
|
||||
cache_config = CacheConfig(
|
||||
block_size=block_size,
|
||||
gpu_memory_utilization=0.9,
|
||||
cache_dtype=cache_dtype,
|
||||
enable_prefix_caching=True,
|
||||
)
|
||||
# Connectors are constructed after layout resolution; mirror that here.
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector=kv_connector,
|
||||
kv_connector_module_path=kv_connector_module_path,
|
||||
kv_role=kv_role,
|
||||
enable_permute_local_kv=enable_permute_local_kv,
|
||||
kv_connector_extra_config=kv_connector_extra_config or {},
|
||||
kv_load_failure_policy=kv_load_failure_policy,
|
||||
)
|
||||
attention_config = AttentionConfig(backend=attention_backend)
|
||||
speculative_config = (
|
||||
SpeculativeConfig(model="ngram", num_speculative_tokens=num_speculative_tokens)
|
||||
if num_speculative_tokens is not None
|
||||
else None
|
||||
)
|
||||
return VllmConfig(
|
||||
scheduler_config=scheduler_config,
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
device_config=DeviceConfig("cpu"),
|
||||
attention_config=attention_config,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
def create_scheduler(
|
||||
vllm_config: VllmConfig,
|
||||
num_blocks: int = 10000,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
hash_block_size: int | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Initialize Scheduler For Testing."""
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
if kv_cache_config is None:
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks, # A large number of blocks to hold all requests
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
vllm_config.cache_config.num_gpu_blocks = num_blocks
|
||||
|
||||
scheduler_cls = (
|
||||
AsyncScheduler if vllm_config.scheduler_config.async_scheduling else Scheduler
|
||||
)
|
||||
return scheduler_cls(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
|
||||
_request_count = count(1)
|
||||
_none_hash_initialized = False
|
||||
|
||||
|
||||
def create_request(
|
||||
request_id: int | None = None,
|
||||
num_tokens: int = 10,
|
||||
common_prefix_len=0,
|
||||
max_tokens: int = 16,
|
||||
do_remote_decode: bool = False,
|
||||
do_remote_prefill: bool = False,
|
||||
num_remote_blocks: int = 3,
|
||||
block_size: int = 16,
|
||||
hash_fn: Callable = sha256,
|
||||
) -> Request:
|
||||
"""Make dummy request for testing."""
|
||||
assert num_tokens >= common_prefix_len >= 0
|
||||
|
||||
if request_id is None:
|
||||
request_id = next(_request_count)
|
||||
|
||||
global _none_hash_initialized
|
||||
if not _none_hash_initialized:
|
||||
init_none_hash(hash_fn)
|
||||
_none_hash_initialized = True
|
||||
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
if do_remote_decode:
|
||||
assert not do_remote_prefill
|
||||
kv_transfer_params = dict(do_remote_prefill=False, do_remote_decode=True)
|
||||
elif do_remote_prefill:
|
||||
kv_transfer_params = dict(
|
||||
do_remote_prefill=True,
|
||||
do_remote_decode=False,
|
||||
remote_engine_id="my-engine-id",
|
||||
remote_request_id=f"prefill-{request_id}",
|
||||
remote_block_ids=list(range(num_remote_blocks)),
|
||||
remote_host="my-host",
|
||||
remote_port=1234,
|
||||
tp_size=1,
|
||||
)
|
||||
|
||||
max_tokens = 1 if do_remote_decode else max_tokens
|
||||
sampling_params = SamplingParams(max_tokens=max_tokens)
|
||||
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
|
||||
common_prefix = [1] * common_prefix_len if common_prefix_len > 0 else []
|
||||
suffix = [i * request_id for i in range(num_tokens - common_prefix_len)]
|
||||
prompt_token_ids = common_prefix + suffix
|
||||
|
||||
req = Request(
|
||||
request_id=f"id-{request_id}",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
mm_features=None,
|
||||
block_hasher=get_request_block_hasher(block_size, hash_fn),
|
||||
)
|
||||
req.kv_transfer_params = kv_transfer_params
|
||||
return req
|
||||
|
||||
|
||||
def create_model_runner_output(
|
||||
reqs: list[Request],
|
||||
finished_sending: set[str] | None = None,
|
||||
finished_recving: set[str] | None = None,
|
||||
invalid_block_ids: set[int] | None = None,
|
||||
use_eos: bool = False,
|
||||
token_id: int = 0,
|
||||
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
|
||||
) -> ModelRunnerOutput:
|
||||
"""Make dummy model runner output for testing."""
|
||||
# Make request data.
|
||||
req_ids = [req.request_id for req in reqs]
|
||||
req_id_to_index = {req_id: idx for idx, req_id in enumerate(req_ids)}
|
||||
|
||||
# Make sampled tokens.
|
||||
sampled_token = EOS_TOKEN_ID if use_eos else token_id
|
||||
sampled_token_ids = [[sampled_token] for _ in req_ids]
|
||||
|
||||
kv_connector_output = (
|
||||
None
|
||||
if (
|
||||
finished_sending is None
|
||||
and finished_recving is None
|
||||
and invalid_block_ids is None
|
||||
and kv_connector_worker_meta is None
|
||||
)
|
||||
else KVConnectorOutput(
|
||||
finished_sending=finished_sending,
|
||||
finished_recving=finished_recving,
|
||||
invalid_block_ids=invalid_block_ids or set(),
|
||||
kv_connector_worker_meta=kv_connector_worker_meta,
|
||||
)
|
||||
)
|
||||
|
||||
# Make output data structure.
|
||||
return ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index=req_id_to_index,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=None,
|
||||
kv_connector_output=kv_connector_output,
|
||||
)
|
||||
|
||||
|
||||
class TestExampleConnector(ExampleConnector):
|
||||
def __init__(
|
||||
self,
|
||||
config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
self.name = config.kv_transfer_config.kv_connector_extra_config["name"]
|
||||
self._connector = ExampleConnector(config, role, kv_cache_config)
|
||||
self.call_record: dict[str, int] = defaultdict(int)
|
||||
# Use a unique temp file per connector
|
||||
self._event_file = (
|
||||
tempfile.gettempdir()
|
||||
+ f"/connector_{self.name}-{self.role.name}_events.log"
|
||||
)
|
||||
# Start with an empty file
|
||||
with open(self._event_file, "w") as _:
|
||||
pass
|
||||
|
||||
def __getattribute__(self, name):
|
||||
if name in (
|
||||
"_connector",
|
||||
"call_record",
|
||||
"name",
|
||||
"_event_file",
|
||||
"__class__",
|
||||
"__dict__",
|
||||
"__getattribute__",
|
||||
"__init__",
|
||||
): # avoid recursion
|
||||
return object.__getattribute__(self, name)
|
||||
if not hasattr(self._connector, name):
|
||||
return object.__getattribute__(self, name)
|
||||
attr = getattr(self._connector, name)
|
||||
|
||||
# Intercept calls to the connector interface and write an event
|
||||
# for each one to a file, which can be read back in the main test proc.
|
||||
if callable(attr):
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
self.call_record[name] += 1
|
||||
|
||||
# Include args that we're interested in
|
||||
to_log = [name]
|
||||
for arg in args:
|
||||
if isinstance(arg, int):
|
||||
to_log.append(str(arg))
|
||||
elif isinstance(arg, KVCacheBlocks):
|
||||
to_log.append(f"num_blocks={[len(b) for b in arg.blocks]}")
|
||||
|
||||
# Log the event as a line to the file
|
||||
try:
|
||||
with open(self._event_file, "a") as f:
|
||||
f.write(" ".join(to_log) + "\n")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Could not log event {name} for {self.name}: {e}")
|
||||
return attr(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
return attr
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MockKVConfig:
|
||||
matched_tokens: int = 0
|
||||
is_async: bool = False
|
||||
num_defers_before_matching: int = 0
|
||||
supports_divergent_local_hybrid_hits: bool = False
|
||||
|
||||
|
||||
class MockKVConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self):
|
||||
# Scheduler tests check metadata.requests
|
||||
self.requests: list = []
|
||||
|
||||
|
||||
class MockKVConnector(KVConnectorBase_V1):
|
||||
"""Mock KV connector for scheduler tests, supporting both sync and async mode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
extra_config = self._kv_transfer_config.kv_connector_extra_config
|
||||
self.config = MockKVConfig(
|
||||
matched_tokens=extra_config["matched_tokens"],
|
||||
is_async=extra_config["is_async"],
|
||||
num_defers_before_matching=extra_config.get(
|
||||
"num_defers_before_matching", 0
|
||||
),
|
||||
supports_divergent_local_hybrid_hits=extra_config.get(
|
||||
"supports_divergent_local_hybrid_hits", False
|
||||
),
|
||||
)
|
||||
self._defers_left: defaultdict[str, int] = defaultdict(
|
||||
lambda: self.config.num_defers_before_matching
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return self.config.supports_divergent_local_hybrid_hits
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
if self._defers_left[request.request_id] > 0:
|
||||
self._defers_left[request.request_id] -= 1
|
||||
return (None, False)
|
||||
return (self.config.matched_tokens, self.config.is_async)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
):
|
||||
pass
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
metadata = MockKVConnectorMetadata()
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
for req_id in chain(
|
||||
(req.req_id for req in scheduler_output.scheduled_new_reqs),
|
||||
(
|
||||
req_id
|
||||
for req_id in cached_reqs.req_ids
|
||||
if req_id in cached_reqs.resumed_req_ids
|
||||
),
|
||||
):
|
||||
metadata.requests.append({"req_id": req_id})
|
||||
return metadata
|
||||
|
||||
def start_load_kv(self, kv_caches, finished_req_ids):
|
||||
pass
|
||||
|
||||
def wait_for_layer_load(self, layer_name):
|
||||
pass
|
||||
|
||||
def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs):
|
||||
pass
|
||||
|
||||
def wait_for_save(self):
|
||||
pass
|
||||
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"TestExampleConnector", __name__, TestExampleConnector.__name__
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MockKVConnector", __name__, MockKVConnector.__name__
|
||||
)
|
||||
|
||||
|
||||
def make_kv_cache_config(
|
||||
block_size: int,
|
||||
swa_enabled: bool = False,
|
||||
mamba_enabled: bool = False,
|
||||
sw_size: int = 128,
|
||||
num_blocks: int = 100,
|
||||
mamba_cache_mode: Literal["align", "none"] = "none",
|
||||
) -> KVCacheConfig:
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0", "layer2"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
)
|
||||
]
|
||||
if swa_enabled:
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
["layer1", "layer3"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
sliding_window=sw_size,
|
||||
),
|
||||
)
|
||||
)
|
||||
if mamba_enabled:
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
["mamba0", "mamba1"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((16,), (16,)),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
),
|
||||
)
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
def make_nixl_scheduler(
|
||||
has_mamba: bool = False,
|
||||
is_hma_required: bool = False,
|
||||
bounded_replay: bool = False,
|
||||
heartbeat: bool = False,
|
||||
kv_lease_duration: int = 30,
|
||||
):
|
||||
"""Create a NixlConnectorScheduler via __new__ (skipping __init__).
|
||||
|
||||
Only sets the flags needed by the tests. When *heartbeat=True* the
|
||||
scheduler-side heartbeat bookkeeping fields are also initialised.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import (
|
||||
NixlConnectorScheduler,
|
||||
)
|
||||
|
||||
sched = object.__new__(NixlConnectorScheduler)
|
||||
sched._has_mamba = has_mamba
|
||||
sched._bounded_replay = bounded_replay
|
||||
sched._is_hma_required = is_hma_required
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
sched.vllm_config = SimpleNamespace(num_prefill_lookahead_tokens=0)
|
||||
|
||||
if heartbeat:
|
||||
sched._heartbeat_by_engine = {}
|
||||
sched._heartbeat_req_engine = {}
|
||||
sched._last_heartbeat_time = 0.0
|
||||
sched._kv_lease_duration = kv_lease_duration
|
||||
sched._heartbeat_interval = kv_lease_duration // 6
|
||||
# Fields touched by build_connector_meta / request_finished:
|
||||
sched._reqs_need_recv = {}
|
||||
sched._hisparse_host_blocks_to_recv = {}
|
||||
sched._reqs_need_send = {}
|
||||
sched._reqs_in_batch = set()
|
||||
sched._reqs_not_processed = set()
|
||||
sched._reqs_need_save = {}
|
||||
sched.use_host_buffer = False
|
||||
sched.engine_id = "test-engine"
|
||||
sched.transfer_tp_size = 1
|
||||
sched.side_channel_host = "localhost"
|
||||
sched.side_channel_port = 5555
|
||||
sched.blocks_per_sw = []
|
||||
sched.is_bidirectional_kv_xfer_enabled = False
|
||||
return sched
|
||||
|
||||
|
||||
def make_nixl_push_scheduler(
|
||||
*,
|
||||
decoder_kv_blocks_ttl: float = 30.0,
|
||||
push_registration_timeout: float | None = None,
|
||||
is_bidirectional_kv_xfer_enabled: bool = False,
|
||||
has_mamba: bool = False,
|
||||
):
|
||||
"""Create a NixlPushConnectorScheduler via __new__ (skipping __init__).
|
||||
|
||||
The push scheduler can't reuse :func:`make_nixl_scheduler` because it
|
||||
is a different class (``NixlPushConnectorScheduler`` vs
|
||||
``NixlConnectorScheduler``) and carries push-specific state. Only the
|
||||
fields touched by the unit tests are populated.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import (
|
||||
NixlPushConnectorScheduler,
|
||||
)
|
||||
|
||||
sched = object.__new__(NixlPushConnectorScheduler)
|
||||
|
||||
# Base scheduler fields (shared with pull / heartbeat path).
|
||||
sched._reqs_need_recv = {}
|
||||
sched._reqs_need_send = {}
|
||||
sched._reqs_in_batch = set()
|
||||
sched._reqs_not_processed = set()
|
||||
sched._reqs_need_save = {}
|
||||
sched._kv_lease_duration = 30
|
||||
sched.decoder_kv_blocks_ttl = decoder_kv_blocks_ttl
|
||||
sched.use_host_buffer = False
|
||||
sched.engine_id = "decode-engine"
|
||||
sched.transfer_tp_size = 1
|
||||
sched.side_channel_host = "127.0.0.1"
|
||||
sched.side_channel_port = 5600
|
||||
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
||||
sched._has_mamba = has_mamba
|
||||
sched._bounded_replay = False
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
# vllm_config is consulted for parallel_config.tensor_parallel_size, and by
|
||||
# `_prefill_backoff` on both the P and D prefill paths.
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.parallel_config.tensor_parallel_size = 1
|
||||
vllm_config.num_prefill_lookahead_tokens = 0
|
||||
sched.vllm_config = vllm_config
|
||||
|
||||
# Push-specific state.
|
||||
sched._push_pending_registrations = {}
|
||||
sched._push_registration_deadlines = {}
|
||||
sched._finished_request_blocks = {}
|
||||
sched._newly_finished_push_blocks = {}
|
||||
sched._push_registration_timeout = (
|
||||
push_registration_timeout
|
||||
if push_registration_timeout is not None
|
||||
else decoder_kv_blocks_ttl
|
||||
)
|
||||
|
||||
# Heartbeat fields touched by base request_finished /
|
||||
# update_connector_output.
|
||||
sched._heartbeat_by_engine = {}
|
||||
sched._heartbeat_req_engine = {}
|
||||
sched._last_heartbeat_time = 0.0
|
||||
sched.blocks_per_sw = []
|
||||
|
||||
return sched
|
||||
|
||||
|
||||
def make_moriio_writer(fake_worker: Any) -> Any:
|
||||
"""Build a MoRIIOWriter with internals stubbed for unit tests.
|
||||
|
||||
Bypasses ``__init__`` and wires only the write/finalize state the tests
|
||||
touch, including the deferred-task fields used by the routing suite.
|
||||
"""
|
||||
import threading
|
||||
from queue import Queue
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWriter,
|
||||
)
|
||||
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
writer._write_task_q = Queue()
|
||||
writer._write_state_lock = threading.Lock()
|
||||
writer._scheduled_writes = defaultdict(int)
|
||||
writer._scheduled_layers = defaultdict(set)
|
||||
writer._sealed_writes = {}
|
||||
writer._deferred_tasks = []
|
||||
writer._defer_timeout = 60.0
|
||||
writer.ensure_worker_started = lambda: None
|
||||
return writer
|
||||
|
||||
|
||||
def expand_strided_descs(strided_descs: np.ndarray) -> np.ndarray:
|
||||
"""Expand Nx5 (addr, len, dev, stride, count) rows to one row per block.
|
||||
|
||||
Mirrors how NIXL indexes a strided descriptor list, so desc ids computed
|
||||
by the connector can be resolved to (addr, len, dev) byte ranges.
|
||||
"""
|
||||
strided_descs = np.asarray(strided_descs, dtype=np.uint64).reshape(-1, 5)
|
||||
counts = strided_descs[:, 4].astype(np.intp)
|
||||
starts = np.cumsum(strided_descs[:, 4]) - strided_descs[:, 4]
|
||||
out = np.repeat(strided_descs[:, :3], counts, axis=0)
|
||||
offsets = np.arange(len(out), dtype=np.uint64) - np.repeat(starts, counts)
|
||||
out[:, 0] += offsets * np.repeat(strided_descs[:, 3], counts)
|
||||
return out
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterator
|
||||
from enum import Enum
|
||||
from typing import NamedTuple
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm import CompletionOutput
|
||||
from vllm.config import ReasoningConfig
|
||||
from vllm.utils.torch_utils import make_tensor_with_pad
|
||||
from vllm.v1.sample.logits_processor import BatchUpdate, LogitsProcessor
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
|
||||
def create_mock_reasoning_config(
|
||||
start_token_ids: list[int], end_token_ids: list[int]
|
||||
) -> ReasoningConfig:
|
||||
return MagicMock(
|
||||
spec=ReasoningConfig,
|
||||
reasoning_start_token_ids=start_token_ids,
|
||||
reasoning_end_token_ids=end_token_ids,
|
||||
)
|
||||
|
||||
|
||||
class BatchLogprobsComposition(Enum):
|
||||
"""Types of logprobs configs to include in test batch."""
|
||||
|
||||
NONE = 0
|
||||
SAMPLE = 1
|
||||
PROMPT = 2
|
||||
SAMPLE_PROMPT = 3
|
||||
|
||||
|
||||
BatchLogprobsSpecType = list[tuple[int | None, int | None]]
|
||||
|
||||
|
||||
def get_test_batch(
|
||||
batch_logprobs_composition: BatchLogprobsComposition,
|
||||
) -> BatchLogprobsSpecType:
|
||||
"""Generate logprobs configs for a batch of requests.
|
||||
|
||||
A given request's logprobs configuration is (1) num_sample_logprobs and (2)
|
||||
num_prompt_logprobs. The batch logprobs configuration is the list of request
|
||||
logprobs configs.
|
||||
|
||||
batch_logprobs_composition == NONE yields a batch with no sample or prompt
|
||||
logprobs
|
||||
|
||||
batch_logprobs_composition == SAMPLE yields a batch with some requests
|
||||
configured for sample logprobs only, and others configured for no logprobs
|
||||
|
||||
batch_logprobs_composition == PROMPT yields a batch with some requests
|
||||
configured for prompt logprobs only, and others configured for no logprobs
|
||||
|
||||
batch_logprobs_composition == SAMPLE_PROMPT yields a batch with some
|
||||
requests configured for sample logprobs and prompt logprobs, some configured
|
||||
for only sample logprobs or only prompt logprobs, and some configured for
|
||||
no logprobs
|
||||
|
||||
Args:
|
||||
batch_logprobs_composition: types of logprobs configs to include in batch
|
||||
|
||||
Returns:
|
||||
list of (Optional[num_sample_logprobs], Optional[num_prompt_logprobs])
|
||||
tuples
|
||||
|
||||
"""
|
||||
if batch_logprobs_composition == BatchLogprobsComposition.NONE:
|
||||
# No requests with sample or prompt logprobs
|
||||
return [(None, None)]
|
||||
elif batch_logprobs_composition == BatchLogprobsComposition.SAMPLE:
|
||||
# Requests requiring sample logprobs or no logprobs
|
||||
return [
|
||||
(None, None),
|
||||
(0, None),
|
||||
(5, None),
|
||||
(3, None),
|
||||
]
|
||||
elif batch_logprobs_composition == BatchLogprobsComposition.PROMPT:
|
||||
# Requests requiring prompt logprobs or no logprobs
|
||||
return [
|
||||
(None, None),
|
||||
(None, 0),
|
||||
(None, 6),
|
||||
(None, 5),
|
||||
]
|
||||
elif batch_logprobs_composition == BatchLogprobsComposition.SAMPLE_PROMPT:
|
||||
# Requests requiring either no logprobs, just
|
||||
# sample logprobs, just prompt logprobs, or
|
||||
# both sample and prompt logprobs
|
||||
return [
|
||||
(None, None),
|
||||
(0, None),
|
||||
(5, None),
|
||||
(3, None),
|
||||
(0, 3),
|
||||
(6, 0),
|
||||
(6, 3),
|
||||
(None, 6),
|
||||
(None, 5),
|
||||
(None, 0),
|
||||
]
|
||||
else:
|
||||
raise ValueError("Invalid logprobs batch configuration for test.")
|
||||
|
||||
|
||||
def assert_incr_detok_str_matches_non_incr_detok_str(
|
||||
incremental_detokenization_str: str,
|
||||
non_incremental_detokenization_str: str,
|
||||
msg: str,
|
||||
) -> None:
|
||||
"""Compare incrementally detok. text to non-incrementally detok. text.
|
||||
|
||||
Fail if the strings mismatch after non-alphanumeric characters are stripped
|
||||
out.
|
||||
|
||||
Rationale: incremental detokenization in the text generation process allows
|
||||
the tokenizer to adjust the next token text output based on the token's
|
||||
context in the string. However, logprobs detokenization detokenizes each
|
||||
token individually, and the resultant strings may include some
|
||||
non-alphanumeric placeholder characters where there could be i.e.
|
||||
whitespace. So, this function compares only the alphanumeric text
|
||||
between two strings and fails if there is a mismatch, which helps
|
||||
with validating logprobs detokenization.
|
||||
|
||||
Args:
|
||||
incremental_detokenization_str: incrementally-detokenized generated text
|
||||
non_incremental_detokenization_str: non-incrementally-detokenized logprob
|
||||
tokens
|
||||
msg: error message if `assert` fails
|
||||
|
||||
"""
|
||||
rgx = r"[^a-zA-Z0-9]+"
|
||||
assert re.sub(rgx, "", incremental_detokenization_str) == re.sub(
|
||||
rgx, "", non_incremental_detokenization_str
|
||||
), msg
|
||||
|
||||
|
||||
def compute_correct_cumulative_logprob(completion_output: CompletionOutput) -> float:
|
||||
"""Compute known-good value for evaluating cumulative logprob.
|
||||
|
||||
Args:
|
||||
completion_output: completion output from engine
|
||||
|
||||
Returns:
|
||||
Known-good cumulative logprob value
|
||||
|
||||
"""
|
||||
token_ids = completion_output.token_ids
|
||||
logprobs = completion_output.logprobs
|
||||
assert logprobs is not None
|
||||
values: list[float] = []
|
||||
for tok_id, lp in zip(token_ids, logprobs):
|
||||
assert lp is not None
|
||||
values.append(lp[tok_id].logprob)
|
||||
return sum(values)
|
||||
|
||||
|
||||
def create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor:
|
||||
fake_logits = torch.full((batch_size, vocab_size), 1e-2, dtype=torch.float)
|
||||
return fake_logits
|
||||
|
||||
|
||||
def create_penalty_tensor(
|
||||
batch_size: int, penalty_value: float, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
return torch.full(
|
||||
(batch_size,), fill_value=penalty_value, dtype=torch.float, device=device
|
||||
)
|
||||
|
||||
|
||||
def create_prompt_tokens_tensor(
|
||||
prompt_token_ids: list[list[int]],
|
||||
vocab_size: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
return make_tensor_with_pad(
|
||||
prompt_token_ids,
|
||||
pad=vocab_size,
|
||||
device=device,
|
||||
dtype=torch.int64,
|
||||
pin_memory=False,
|
||||
)
|
||||
|
||||
|
||||
class LogitsprocsTestFakes(NamedTuple):
|
||||
"""Wraps fake data structures to support testing."""
|
||||
|
||||
logits: torch.Tensor
|
||||
sampling_metadata: SamplingMetadata
|
||||
|
||||
def get_logitsprocs_by_cls(
|
||||
self,
|
||||
cls: type[LogitsProcessor],
|
||||
) -> Iterator[LogitsProcessor]:
|
||||
"""Yield logits processors of a specific class.
|
||||
|
||||
Args:
|
||||
cls: :class:`LogitsProcessor` subclass
|
||||
|
||||
Returns:
|
||||
Iterator over logits processors
|
||||
|
||||
"""
|
||||
return (
|
||||
lp for lp in self.sampling_metadata.logitsprocs.all if isinstance(lp, cls)
|
||||
)
|
||||
|
||||
def get_logitsprocs(self) -> Iterator[LogitsProcessor]:
|
||||
"""Iterator over all logits processors."""
|
||||
return self.sampling_metadata.logitsprocs.all
|
||||
|
||||
|
||||
def fake_update_logitsprocs_state(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
batch_update: BatchUpdate | None,
|
||||
) -> None:
|
||||
"""Imitate logits processors persistent batch state update
|
||||
in engine core"""
|
||||
for logitproc in test_fakes.get_logitsprocs():
|
||||
logitproc.update_state(batch_update)
|
||||
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
|
||||
if holder is not None:
|
||||
holder.sync_batch(batch_update)
|
||||
|
||||
|
||||
def fake_apply_logitsprocs(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
slice_indices: list[int],
|
||||
slot_output_token_ids: list[list[int]] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Imitate application of logits processors in engine core.
|
||||
|
||||
When ``thinking_budget_state_holder`` has tracked requests, this mirrors
|
||||
:meth:`Sampler.apply_logits_processors` by refreshing per-slot
|
||||
``output_token_ids`` (if ``slot_output_token_ids`` is provided), then
|
||||
``update_state`` + ``apply_to_logits`` on the holder after built-in logits
|
||||
processors.
|
||||
"""
|
||||
logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone()
|
||||
for processor in test_fakes.get_logitsprocs():
|
||||
logits = processor.apply(logits)
|
||||
|
||||
md = test_fakes.sampling_metadata
|
||||
holder = md.thinking_budget_state_holder
|
||||
if holder is not None and holder.has_tracked_requests():
|
||||
if slot_output_token_ids is not None:
|
||||
for i, toks in enumerate(slot_output_token_ids):
|
||||
if i < len(md.output_token_ids):
|
||||
md.output_token_ids[i] = list(toks)
|
||||
holder.update_state(md.output_token_ids, md.spec_token_ids, None)
|
||||
logits = holder.apply_to_logits(logits, False, md.spec_token_ids)
|
||||
return logits
|
||||
|
||||
|
||||
def create_allowed_token_ids(
|
||||
batch_size: int,
|
||||
vocab_size: int,
|
||||
num_allowed_token_ids: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
mask: torch.Tensor | None = None
|
||||
for i in range(batch_size):
|
||||
if i % 2 == 1:
|
||||
continue
|
||||
if mask is None:
|
||||
mask = torch.zeros(
|
||||
(batch_size, vocab_size), dtype=torch.bool, device=device
|
||||
)
|
||||
start = min(i, vocab_size - 1)
|
||||
end = min(i + num_allowed_token_ids, vocab_size - 1)
|
||||
mask[i, start:end] = True
|
||||
return mask
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,461 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
try:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
except ImportError:
|
||||
MLACommonMetadata = type("AbsentMLACommonMetadata", (), {})
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
# Request tokens
|
||||
token_ids: torch.Tensor
|
||||
# Slot mappings, should have the same length as token_ids
|
||||
slot_mapping: torch.Tensor
|
||||
# Is store or load
|
||||
is_store: bool
|
||||
mm_hashes: list[str]
|
||||
|
||||
@staticmethod
|
||||
def make_meta(
|
||||
token_ids: list[int],
|
||||
block_ids: list[int],
|
||||
block_size: int,
|
||||
is_store: bool,
|
||||
mm_hashes: list[str],
|
||||
) -> "ReqMeta":
|
||||
valid_num_tokens = align_to_block_size(len(token_ids), block_size)
|
||||
token_ids_tensor = torch.tensor(token_ids)[:valid_num_tokens]
|
||||
block_ids_tensor = torch.tensor(block_ids)
|
||||
num_blocks = block_ids_tensor.shape[0]
|
||||
block_offsets = torch.arange(0, block_size)
|
||||
slot_mapping = (
|
||||
block_offsets.reshape((1, block_size))
|
||||
+ block_ids_tensor.reshape((num_blocks, 1)) * block_size
|
||||
)
|
||||
slot_mapping = slot_mapping.flatten()[:valid_num_tokens]
|
||||
return ReqMeta(
|
||||
token_ids=token_ids_tensor,
|
||||
slot_mapping=slot_mapping,
|
||||
is_store=is_store,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExampleConnectorMetadata(KVConnectorMetadata):
|
||||
requests: list[ReqMeta] = field(default_factory=list)
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
block_ids: list[int],
|
||||
block_size: int,
|
||||
is_store: bool,
|
||||
mm_hashes: list[str],
|
||||
) -> None:
|
||||
self.requests.append(
|
||||
ReqMeta.make_meta(token_ids, block_ids, block_size, is_store, mm_hashes)
|
||||
)
|
||||
|
||||
|
||||
class ExampleConnector(KVConnectorBase_V1):
|
||||
# NOTE: This is Simple debug implementation of the KV connector.
|
||||
# It save / load the KV cache to / from the disk.
|
||||
# It does extra work which will overwrite the existing prefix-cache in GPU
|
||||
# - to remove the overhead, need to add some "mask" in the ReqMeta class
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
role=role,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
self._block_size = vllm_config.cache_config.block_size
|
||||
self._requests_need_load: dict[str, Request] = {}
|
||||
self._storage_path = self._kv_transfer_config.get_from_extra_config(
|
||||
"shared_storage_path", "/tmp"
|
||||
)
|
||||
logger.info(self._kv_transfer_config)
|
||||
logger.info("Shared storage path is %s", self._storage_path)
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""Start loading the KV cache from the connector buffer to vLLM's
|
||||
paged KV buffer.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs: additional arguments for the load operation
|
||||
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
|
||||
def inject_kv_into_layer(
|
||||
dst_kv_cache_layer: torch.Tensor,
|
||||
src_kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
) -> None:
|
||||
"""Inject the KV cache into the layer.
|
||||
|
||||
Args:
|
||||
dst_kv_cache_layer (torch.Tensor): the destination KV cache layer,
|
||||
a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
src_kv_cache (torch.Tensor): the source KV cache.
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
attn_metadata (AttentionMetadata): attention metadata for the
|
||||
current forward pass.
|
||||
|
||||
"""
|
||||
# Deliberate bulk transfers in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping = slot_mapping.to(
|
||||
dst_kv_cache_layer.device, non_blocking=True
|
||||
)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
|
||||
-1, dst_kv_cache_layer.shape[-1]
|
||||
)
|
||||
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
|
||||
else:
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
dst_kv_cache_layer[block_idxs, :, offsets] = src_kv_cache
|
||||
|
||||
# Get the metadata
|
||||
metadata: KVConnectorMetadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ExampleConnectorMetadata)
|
||||
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is None:
|
||||
logger.warning("In connector.start_load_kv, but the attn_metadata is None")
|
||||
return
|
||||
|
||||
# Load the KV for each request each layer
|
||||
for request in metadata.requests:
|
||||
if request.is_store:
|
||||
continue
|
||||
logger.info(
|
||||
"Inject KV cache of %d tokens to the paged memory",
|
||||
len(request.slot_mapping),
|
||||
)
|
||||
for layer_name in forward_context.no_compile_layers:
|
||||
layer = forward_context.no_compile_layers[layer_name]
|
||||
|
||||
# Only process layers that have kv_cache
|
||||
# attribute (attention layers) Skip non-attention
|
||||
# layers like FusedMoEFactory/MLP etc.
|
||||
kv_cache_layer = getattr(layer, "kv_cache", None)
|
||||
if kv_cache_layer is None:
|
||||
continue
|
||||
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
# Deliberate bulk load in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
|
||||
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, dict):
|
||||
inject_kv_into_layer(
|
||||
kv_cache_layer,
|
||||
kv_cache,
|
||||
request.slot_mapping,
|
||||
attn_metadata[layer_name],
|
||||
)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""Blocking until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer.
|
||||
|
||||
This interface will be useful for layer-by-layer pipelining.
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Start saving the KV cache of the layer from vLLM's paged buffer
|
||||
to the connector.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer.
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
|
||||
def extract_kv_from_layer(
|
||||
layer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Extract the KV cache from the layer.
|
||||
|
||||
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
"""
|
||||
# Deliberate bulk transfer in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
return layer[block_idxs, :, offsets]
|
||||
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, ExampleConnectorMetadata)
|
||||
for request in connector_metadata.requests:
|
||||
if request.is_store:
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
kv_cache = extract_kv_from_layer(kv_layer, request.slot_mapping)
|
||||
with gpu_sync_allowed():
|
||||
tensors = {"kv_cache": kv_cache.detach().cpu()}
|
||||
safetensors.torch.save_file(tensors, filename)
|
||||
|
||||
def wait_for_save(self):
|
||||
return
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# NOTE: in this debug implementation, we assume that the prompt is
|
||||
# cached_prompt + newly_generated_single_token
|
||||
# Therefore, we use prompt_token_ids[:-1] to determine the folder name
|
||||
|
||||
# NOTE: in current v1 scheduler, the num_computed_tokens is aligned
|
||||
# with the block granularity. And it expects the returned blocks and
|
||||
# num_computed_tokens to also be aligned with the block granularity.
|
||||
if not self._found_match_for_request(request):
|
||||
return 0, False
|
||||
|
||||
logger.info("External Cache Hit!")
|
||||
|
||||
# Now, first num_tokens_to_check tokens are hit, we need to prepare
|
||||
# the metadata for the worker connector to correctly load the KV
|
||||
token_ids = request.prompt_token_ids or []
|
||||
num_tokens_to_check = align_to_block_size(len(token_ids) - 1, self._block_size)
|
||||
|
||||
return num_tokens_to_check - num_computed_tokens, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If blocks were allocated, add to _requests_need_load,
|
||||
such that we load the KVs in the next forward pass.
|
||||
"""
|
||||
if num_external_tokens > 0:
|
||||
self._requests_need_load[request.request_id] = request
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify any fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleConnectorMetadata()
|
||||
|
||||
total_need_load = 0
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
token_ids = new_req.prompt_token_ids or []
|
||||
mm_hashes = [f.identifier for f in new_req.mm_features]
|
||||
if new_req.req_id in self._requests_need_load:
|
||||
meta.add_request(
|
||||
token_ids=token_ids,
|
||||
block_ids=new_req.block_ids[0],
|
||||
block_size=self._block_size,
|
||||
is_store=False,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
total_need_load += 1
|
||||
else:
|
||||
# NOTE: here, we set the store and load being exclusive,
|
||||
# but a single request can have both store and load.
|
||||
# NOTE(rob): for this debug implementation, we only cache
|
||||
# the original prompt tokens.
|
||||
if not self._found_match_for_prompt(token_ids, mm_hashes):
|
||||
meta.add_request(
|
||||
token_ids=token_ids,
|
||||
block_ids=new_req.block_ids[0],
|
||||
block_size=self._block_size,
|
||||
is_store=True,
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
resumed_from_preemption = req_id in cached_reqs.resumed_req_ids
|
||||
if not resumed_from_preemption or req_id not in self._requests_need_load:
|
||||
continue
|
||||
|
||||
num_computed_tokens = cached_reqs.num_computed_tokens[i]
|
||||
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
|
||||
# NOTE(rob): cached_req_data does not have the full
|
||||
# list of token ids (only new tokens). So we look it
|
||||
# up in the actual request object.
|
||||
request = self._requests_need_load[req_id]
|
||||
total_tokens = num_computed_tokens + num_new_tokens
|
||||
token_ids = request.all_token_ids[:total_tokens]
|
||||
|
||||
# NOTE(rob): For resumed req, new_block_ids is all
|
||||
# of the block_ids for the request.
|
||||
assert new_block_ids is not None
|
||||
block_ids = new_block_ids[0]
|
||||
|
||||
meta.add_request(
|
||||
token_ids=token_ids,
|
||||
block_ids=block_ids,
|
||||
block_size=self._block_size,
|
||||
is_store=False,
|
||||
mm_hashes=[f.identifier for f in request.mm_features],
|
||||
)
|
||||
total_need_load += 1
|
||||
|
||||
assert total_need_load == len(self._requests_need_load)
|
||||
self._requests_need_load.clear()
|
||||
return meta
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
# ==============================
|
||||
|
||||
def _found_match_for_request(
|
||||
self,
|
||||
request: "Request",
|
||||
) -> bool:
|
||||
"""Check if the cache is hit for the request."""
|
||||
return self._found_match_for_prompt(
|
||||
list(request.prompt_token_ids or []),
|
||||
[f.identifier for f in request.mm_features],
|
||||
)
|
||||
|
||||
def _found_match_for_prompt(
|
||||
self,
|
||||
prompt_token_ids: list[int],
|
||||
mm_hashes: list[str],
|
||||
) -> bool:
|
||||
num_tokens_to_check = align_to_block_size(
|
||||
len(prompt_token_ids) - 1, self._block_size
|
||||
)
|
||||
foldername = self._generate_foldername_debug(
|
||||
torch.tensor(prompt_token_ids)[:num_tokens_to_check],
|
||||
mm_hashes,
|
||||
create_folder=False,
|
||||
)
|
||||
return os.path.exists(foldername)
|
||||
|
||||
def _generate_foldername_debug(
|
||||
self,
|
||||
token_ids: torch.Tensor,
|
||||
mm_hashes: list[str],
|
||||
create_folder=False,
|
||||
) -> str:
|
||||
"""Generate a folder name based on the hash of the bytes of the input
|
||||
ids.
|
||||
"""
|
||||
token_bytes = token_ids.numpy().tobytes()
|
||||
# Add mm_hashes to the bytes being hashed to avoid path traversal and
|
||||
# to create a canonical key.
|
||||
if mm_hashes:
|
||||
mm_str = "-".join(mm_hashes)
|
||||
token_bytes += mm_str.encode("utf-8")
|
||||
input_ids_hash = safe_hash(token_bytes, usedforsecurity=False).hexdigest()
|
||||
|
||||
foldername = os.path.join(self._storage_path, input_ids_hash)
|
||||
if create_folder:
|
||||
os.makedirs(foldername, exist_ok=True)
|
||||
return foldername
|
||||
|
||||
def _generate_filename_debug(
|
||||
self,
|
||||
layer_name: str,
|
||||
token_ids: torch.Tensor,
|
||||
mm_hashes: list[str],
|
||||
) -> str:
|
||||
"""Generate a file name based on the layer name and the hash
|
||||
of the bytes of the input ids.
|
||||
"""
|
||||
foldername = self._generate_foldername_debug(
|
||||
token_ids, mm_hashes=mm_hashes, create_folder=True
|
||||
)
|
||||
return os.path.join(foldername, f"{layer_name}.safetensors")
|
||||
|
||||
|
||||
def align_to_block_size(num_tokens: int, block_size) -> int:
|
||||
"""Align the number of tokens to the block size."""
|
||||
return (num_tokens - 1) // block_size * block_size
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HyperConnection (Gated Residual) utilities for the AMD model variant.
|
||||
|
||||
Implements the HyperConnection residual scheme proposed in
|
||||
"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant
|
||||
delays each HC combine to the following HC mix boundary. HC glue kernels,
|
||||
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
|
||||
standard vLLM Linear modules.
|
||||
|
||||
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
|
||||
(HC outer, HS inner — checkpoint-native layout).
|
||||
|
||||
Typical usage inside a transformer decoder layer::
|
||||
|
||||
self.attn_hc = GatedResidual(hc_config)
|
||||
|
||||
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
|
||||
attention_output = attention(block_input)
|
||||
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
|
||||
hidden_states, attention_output, injection
|
||||
)
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from ..common.hyperconnection import (
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from .ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
hc_silu,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gated-residual variant
|
||||
# ---------------------------------------------------------------------------
|
||||
class GatedResidual(nn.Module):
|
||||
"""Gated HyperConnection with learnable low-rank mixing and injection.
|
||||
|
||||
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
|
||||
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
|
||||
over the HC streams). When passed a pending block output and an injection,
|
||||
it fuses their residual combine with the RMSNorm. Final mixers use
|
||||
``use_combine=False`` and do not produce a new injection.
|
||||
|
||||
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
|
||||
are vLLM Linear modules (merged replicated linear for down+inject), so
|
||||
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
|
||||
standard quant_method mechanism.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: HyperConnectionConfig,
|
||||
use_combine: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.lora_rank = config.hc_lowrank
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_combine = use_combine
|
||||
|
||||
norm_size = (
|
||||
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
|
||||
)
|
||||
group_size = config.hidden_size if config.hc_per_branch_norm else None
|
||||
# Normalize each H-sized HC stream independently while retaining a
|
||||
# separate affine weight for every element of the HC*H layout.
|
||||
self.hc_norm = GroupedGemmaRMSNorm(
|
||||
norm_size,
|
||||
eps=config.rms_norm_eps,
|
||||
group_size=group_size,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
|
||||
# -- vLLM Linear weights --------------------------------------------
|
||||
# The merged skinny-GEMM shape is physically padded to 16 rows for
|
||||
# alignment and efficient backend dispatch.
|
||||
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
|
||||
if use_combine:
|
||||
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
|
||||
self.hyper_hidden_size,
|
||||
[self.lora_rank, self.hc_count]
|
||||
+ ([self.pad_size] if self.pad_size else []),
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
self.lora_rank,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
|
||||
return_bias=False,
|
||||
)
|
||||
self.input_mix_weight_up = ReplicatedLinear(
|
||||
self.lora_rank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def mix(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
xn = grouped_gemma_rmsnorm(
|
||||
hidden_states,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine_and_mix(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prev_block_output: torch.Tensor,
|
||||
prev_injection: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Consume a pending combine, then prepare the next block input.
|
||||
|
||||
``hidden_states`` is the multi-stream state from before the pending
|
||||
block's mix. Its combine with ``block_output`` is fused with this
|
||||
module's input RMSNorm.
|
||||
"""
|
||||
hidden_states, xn = hc_combine_norm(
|
||||
hidden_states,
|
||||
prev_block_output,
|
||||
prev_injection,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return hc_combine(hidden_states, block_output, injection, self.hc_count)
|
||||
|
||||
@property
|
||||
def hyper_hidden_size(self) -> int:
|
||||
return self.hc_count * self.hidden_size
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionConfig",
|
||||
]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user