From 54a396586e27d5ec363d80b1feb0b48c1e4504f2 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Mon, 19 Oct 2026 01:06:30 +0700 Subject: [PATCH] [Mini-VLLM] Repair test suite after SM86 cut - Restore example connector, test helpers, and dropped patch tests - Guard MLA metadata import in the example connector - Point the PLE test at common.ops.ple after upstream #60021 move - Cut tests for removed features: tools, DRY, epd proxy, STT batch, diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer Co-authored-by: Hinata Signed-off-by: MOVZX --- examples/template_chatml.jinja | 2 + tests/compile/fusions_e2e/common.py | 144 + tests/distributed/test_events.py | 408 + tests/distributed/test_weight_transfer.py | 2399 +++++ tests/entrypoints/launchers/test_run_batch.py | 166 - .../token_in_token_out/test_serving_tokens.py | 23 +- tests/evals/gsm8k/gsm8k_eval.py | 507 ++ tests/kernels/helion/test_benchmark_script.py | 90 - tests/kernels/mamba/test_mamba_mixer2.py | 140 - tests/kernels/mamba/test_recoverssm_ple.py | 2 +- tests/kernels/moe/test_deepgemm.py | 634 ++ tests/kernels/moe/utils.py | 658 ++ tests/kernels/quant_utils.py | 290 + .../test_runai_model_streamer_s3.py | 123 - .../generation_ppl_test/test_gemma.py | 18 - .../language/generation_ppl_test/test_gpt.py | 24 - tests/models/language/pooling/embed_utils.py | 90 + .../language/pooling_mteb_test/__init__.py | 0 .../language/pooling_mteb_test/conftest.py | 37 - .../pooling_mteb_test/mteb_embed_utils.py | 279 - .../pooling_mteb_test/mteb_score_utils.py | 319 - .../language/pooling_mteb_test/test_baai.py | 123 - .../test_bge_reranker_v2_gemma.py | 138 - .../pooling_mteb_test/test_cross_encoder.py | 39 - .../pooling_mteb_test/test_intfloat.py | 63 - .../pooling_mteb_test/test_mxbai_rerank.py | 111 - .../pooling_mteb_test/test_nemotron.py | 54 - .../language/pooling_mteb_test/test_nomic.py | 59 - .../pooling_mteb_test/test_qwen3_reranker.py | 133 - .../test_snowflake_arctic_embed.py | 103 - .../pooling_mteb_test/test_st_projector.py | 41 - .../language/pooling_mteb_test/test_voyage.py | 58 - .../language/pooling_mteb_test/test_zerank.py | 95 - .../generation/test_gemma_ragged_audio.py | 96 - .../multimodal/generation/test_interleaved.py | 79 - .../multimodal/generation/test_phi4mm.py | 296 - .../multimodal/generation/test_ultravox.py | 229 - .../multimodal/generation/test_voxtral.py | 204 - .../multimodal/generation/vlm_utils/core.py | 207 + .../generation/vlm_utils/model_utils.py | 1429 +++ .../multimodal/generation/vlm_utils/types.py | 218 + .../generation_ppl_test/test_glm.py | 40 - .../processing/test_audio_in_video.py | 140 - .../processing/test_cosmos3_edge.py | 113 - .../multimodal/processing/test_h2ovl.py | 181 - .../multimodal/processing/test_idefics3.py | 76 - .../multimodal/processing/test_internvl.py | 135 - .../multimodal/processing/test_lightonocr.py | 50 - .../multimodal/processing/test_llama4.py | 89 - .../multimodal/processing/test_llava_next.py | 212 - .../processing/test_llava_onevision.py | 211 - .../multimodal/processing/test_minicpmv.py | 90 - .../multimodal/processing/test_minimax_m3.py | 138 - .../multimodal/processing/test_mllama4.py | 53 - .../multimodal/processing/test_moondream3.py | 553 -- .../multimodal/processing/test_nemotron_vl.py | 137 - .../multimodal/processing/test_qwen3_asr.py | 181 - .../multimodal/processing/test_qwen3_omni.py | 140 - .../multimodal/processing/test_smolvlm.py | 85 - .../multimodal/processing/test_whisper.py | 37 - tests/models/quantization/test_awq.py | 164 - .../quantization/test_fp8_per_channel.py | 79 - .../test_mimo_v2_w4a16_routing.py | 176 - tests/models/qwen4_exp/test_ple.py | 2781 ++++++ tests/models/registry.py | 1492 ---- tests/models/test_alice_ai_config.py | 93 - tests/models/test_terratorch.py | 57 - .../models/transformers/fusers/test_linear.py | 1439 +++ tests/models/utils.py | 611 ++ tests/parser/cohere/utils.py | 131 + tests/parser/engine/trace_builder.py | 1358 +++ tests/quantization/utils.py | 199 + tests/reasoning/utils.py | 162 + tests/test_config.py | 4268 +++++++++ tests/tool_parsers/common_tests.py | 380 + tests/tool_parsers/utils.py | 167 + tests/tools/__init__.py | 0 tests/tools/test_check_agent_files.py | 129 - tests/tools/test_check_test_tethering.py | 879 -- tests/tools/test_config_validator.py | 42 - tests/utils.py | 2290 +++++ tests/v1/attention/test_attention_backends.py | 2043 +++++ tests/v1/core/test_prefix_caching.py | 6280 ++++++++++++++ tests/v1/core/test_scheduler.py | 7688 +++++++++++++++++ tests/v1/determinism/utils.py | 153 + tests/v1/distributed/test_external_lb_dp.py | 601 ++ .../e2e/spec_decode/acceptance_rates/utils.py | 102 + tests/v1/e2e/spec_decode/utils.py | 230 + tests/v1/ec_connector/unit/test_epd_proxy.py | 279 - .../v1/ec_connector/unit/test_epd_registry.py | 156 - .../unit/offloading_connector/test_config.py | 1315 +++ .../unit/offloading_connector/utils.py | 712 ++ .../kv_connector/unit/test_nixl_connector.py | 4820 +++++++++++ tests/v1/kv_connector/unit/utils.py | 700 ++ tests/v1/sample/test_dry.py | 1034 --- tests/v1/sample/utils.py | 277 + tests/v1/simple_kv_offload/test_scheduler.py | 3395 ++++++++ .../kv_connector/v1/example_connector.py | 461 + vllm/models/qwen4_exp/amd/__init__.py | 2 + vllm/models/qwen4_exp/amd/hyperconnection.py | 205 + vllm/models/qwen4_exp/amd/indexer_qsa.py | 420 + vllm/models/qwen4_exp/amd/low_latency_gemm.py | 14 + vllm/models/qwen4_exp/amd/model.py | 1097 +++ vllm/models/qwen4_exp/amd/model_state.py | 143 + vllm/models/qwen4_exp/amd/mtp.py | 495 ++ vllm/models/qwen4_exp/amd/ops/__init__.py | 3 + vllm/models/qwen4_exp/amd/ops/hc.py | 487 ++ vllm/models/qwen4_exp/amd/ops/qsa.py | 1122 +++ vllm/models/qwen4_exp/amd/ops/qsa_prepare.py | 703 ++ .../qwen4_exp/amd/ops/sequence_parallel.py | 47 + vllm/models/qwen4_exp/amd/ple_layer.py | 883 ++ vllm/models/qwen4_exp/amd/qsa.py | 544 ++ 112 files changed, 55726 insertions(+), 10371 deletions(-) create mode 100644 examples/template_chatml.jinja create mode 100644 tests/compile/fusions_e2e/common.py create mode 100644 tests/distributed/test_events.py create mode 100644 tests/distributed/test_weight_transfer.py create mode 100644 tests/evals/gsm8k/gsm8k_eval.py delete mode 100644 tests/kernels/helion/test_benchmark_script.py delete mode 100644 tests/kernels/mamba/test_mamba_mixer2.py create mode 100644 tests/kernels/moe/test_deepgemm.py create mode 100644 tests/kernels/moe/utils.py create mode 100644 tests/kernels/quant_utils.py delete mode 100644 tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py delete mode 100644 tests/models/language/generation_ppl_test/test_gemma.py delete mode 100644 tests/models/language/generation_ppl_test/test_gpt.py create mode 100644 tests/models/language/pooling/embed_utils.py delete mode 100644 tests/models/language/pooling_mteb_test/__init__.py delete mode 100644 tests/models/language/pooling_mteb_test/conftest.py delete mode 100644 tests/models/language/pooling_mteb_test/mteb_embed_utils.py delete mode 100644 tests/models/language/pooling_mteb_test/mteb_score_utils.py delete mode 100644 tests/models/language/pooling_mteb_test/test_baai.py delete mode 100644 tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py delete mode 100644 tests/models/language/pooling_mteb_test/test_cross_encoder.py delete mode 100644 tests/models/language/pooling_mteb_test/test_intfloat.py delete mode 100644 tests/models/language/pooling_mteb_test/test_mxbai_rerank.py delete mode 100644 tests/models/language/pooling_mteb_test/test_nemotron.py delete mode 100644 tests/models/language/pooling_mteb_test/test_nomic.py delete mode 100644 tests/models/language/pooling_mteb_test/test_qwen3_reranker.py delete mode 100644 tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py delete mode 100644 tests/models/language/pooling_mteb_test/test_st_projector.py delete mode 100644 tests/models/language/pooling_mteb_test/test_voyage.py delete mode 100644 tests/models/language/pooling_mteb_test/test_zerank.py delete mode 100644 tests/models/multimodal/generation/test_gemma_ragged_audio.py delete mode 100644 tests/models/multimodal/generation/test_interleaved.py delete mode 100644 tests/models/multimodal/generation/test_phi4mm.py delete mode 100644 tests/models/multimodal/generation/test_ultravox.py delete mode 100644 tests/models/multimodal/generation/test_voxtral.py create mode 100644 tests/models/multimodal/generation/vlm_utils/core.py create mode 100644 tests/models/multimodal/generation/vlm_utils/model_utils.py create mode 100644 tests/models/multimodal/generation/vlm_utils/types.py delete mode 100644 tests/models/multimodal/generation_ppl_test/test_glm.py delete mode 100644 tests/models/multimodal/processing/test_audio_in_video.py delete mode 100644 tests/models/multimodal/processing/test_cosmos3_edge.py delete mode 100644 tests/models/multimodal/processing/test_h2ovl.py delete mode 100644 tests/models/multimodal/processing/test_idefics3.py delete mode 100644 tests/models/multimodal/processing/test_internvl.py delete mode 100644 tests/models/multimodal/processing/test_lightonocr.py delete mode 100644 tests/models/multimodal/processing/test_llama4.py delete mode 100644 tests/models/multimodal/processing/test_llava_next.py delete mode 100644 tests/models/multimodal/processing/test_llava_onevision.py delete mode 100644 tests/models/multimodal/processing/test_minicpmv.py delete mode 100644 tests/models/multimodal/processing/test_minimax_m3.py delete mode 100644 tests/models/multimodal/processing/test_mllama4.py delete mode 100644 tests/models/multimodal/processing/test_moondream3.py delete mode 100644 tests/models/multimodal/processing/test_nemotron_vl.py delete mode 100644 tests/models/multimodal/processing/test_qwen3_asr.py delete mode 100644 tests/models/multimodal/processing/test_qwen3_omni.py delete mode 100644 tests/models/multimodal/processing/test_smolvlm.py delete mode 100644 tests/models/multimodal/processing/test_whisper.py delete mode 100644 tests/models/quantization/test_awq.py delete mode 100644 tests/models/quantization/test_fp8_per_channel.py delete mode 100644 tests/models/quantization/test_mimo_v2_w4a16_routing.py create mode 100644 tests/models/qwen4_exp/test_ple.py delete mode 100644 tests/models/test_alice_ai_config.py delete mode 100644 tests/models/test_terratorch.py create mode 100644 tests/models/transformers/fusers/test_linear.py create mode 100644 tests/models/utils.py create mode 100644 tests/parser/cohere/utils.py create mode 100644 tests/parser/engine/trace_builder.py create mode 100644 tests/quantization/utils.py create mode 100644 tests/reasoning/utils.py create mode 100644 tests/test_config.py create mode 100644 tests/tool_parsers/common_tests.py create mode 100644 tests/tool_parsers/utils.py delete mode 100644 tests/tools/__init__.py delete mode 100644 tests/tools/test_check_agent_files.py delete mode 100644 tests/tools/test_check_test_tethering.py delete mode 100644 tests/tools/test_config_validator.py create mode 100644 tests/utils.py create mode 100644 tests/v1/attention/test_attention_backends.py create mode 100644 tests/v1/core/test_prefix_caching.py create mode 100644 tests/v1/core/test_scheduler.py create mode 100644 tests/v1/determinism/utils.py create mode 100644 tests/v1/distributed/test_external_lb_dp.py create mode 100644 tests/v1/e2e/spec_decode/acceptance_rates/utils.py create mode 100644 tests/v1/e2e/spec_decode/utils.py delete mode 100644 tests/v1/ec_connector/unit/test_epd_proxy.py delete mode 100644 tests/v1/ec_connector/unit/test_epd_registry.py create mode 100644 tests/v1/kv_connector/unit/offloading_connector/test_config.py create mode 100644 tests/v1/kv_connector/unit/offloading_connector/utils.py create mode 100644 tests/v1/kv_connector/unit/test_nixl_connector.py create mode 100644 tests/v1/kv_connector/unit/utils.py delete mode 100644 tests/v1/sample/test_dry.py create mode 100644 tests/v1/sample/utils.py create mode 100644 tests/v1/simple_kv_offload/test_scheduler.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py create mode 100644 vllm/models/qwen4_exp/amd/__init__.py create mode 100644 vllm/models/qwen4_exp/amd/hyperconnection.py create mode 100644 vllm/models/qwen4_exp/amd/indexer_qsa.py create mode 100644 vllm/models/qwen4_exp/amd/low_latency_gemm.py create mode 100644 vllm/models/qwen4_exp/amd/model.py create mode 100644 vllm/models/qwen4_exp/amd/model_state.py create mode 100644 vllm/models/qwen4_exp/amd/mtp.py create mode 100644 vllm/models/qwen4_exp/amd/ops/__init__.py create mode 100644 vllm/models/qwen4_exp/amd/ops/hc.py create mode 100644 vllm/models/qwen4_exp/amd/ops/qsa.py create mode 100644 vllm/models/qwen4_exp/amd/ops/qsa_prepare.py create mode 100644 vllm/models/qwen4_exp/amd/ops/sequence_parallel.py create mode 100644 vllm/models/qwen4_exp/amd/ple_layer.py create mode 100644 vllm/models/qwen4_exp/amd/qsa.py diff --git a/examples/template_chatml.jinja b/examples/template_chatml.jinja new file mode 100644 index 0000000000..4844e681e1 --- /dev/null +++ b/examples/template_chatml.jinja @@ -0,0 +1,2 @@ +{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\n'}}{% endif %}{% endfor %} +{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\n' }}{% endif %} \ No newline at end of file diff --git a/tests/compile/fusions_e2e/common.py b/tests/compile/fusions_e2e/common.py new file mode 100644 index 0000000000..a4d996b48d --- /dev/null +++ b/tests/compile/fusions_e2e/common.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import itertools +from collections.abc import Callable, Iterable +from typing import Any, NamedTuple + +import pytest +import regex as re + +from vllm.platforms import current_platform +from vllm.v1.attention.backends.registry import AttentionBackendEnum + + +class Matches(NamedTuple): + # simple pointwise + aiter_rms_quant_fusion: int = 0 + rms_quant_fusion: int = 0 + act_quant_fusion: int = 0 + norm_rope_fusion: int = 0 + attn_quant_fusion: int = 0 + # distributed + ar_rms_fusion: int = 0 + aiter_ar_rms_fusion: int = 0 + + +class ModelFusionInfo(NamedTuple): + model_name: str + matches: Callable[[int], Matches] + """Given number of hidden layers, produces the matches object""" + model_kwargs: dict[str, Any] = {} + hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n} + + +class AttentionBackendCase(NamedTuple): + backend: AttentionBackendEnum + model_kwargs: dict[str, Any] = {} + """Additional args required for attn+quant fusion""" + + +is_blackwell = lambda: current_platform.is_device_capability_family(100) +"""Are we running on Blackwell, a lot of tests depend on it""" + + +def nvfp4_kernel_exposes_input_quant_key() -> bool: + """Check if the NVFP4 kernel selected on this platform exposes input_quant_key. + + FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to + layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant) + output format and the cutedsl backend's expected input format. + + FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports + manual fusion. + """ + if not current_platform.is_cuda(): + return False + + try: + from vllm.model_executor.kernels.linear.nvfp4.flashinfer import ( + FlashInferCuteDslNvFp4LinearKernel, + ) + + is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported() + if is_supported: + return False + except ImportError: + pass + + try: + from vllm.model_executor.kernels.linear.nvfp4.flashinfer import ( + FlashInferCutlassNvFp4LinearKernel, + ) + + is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported() + if is_supported: + return True + except ImportError: + pass + + # Fallback: assume no manual fusion support + return False + + +def custom_ops_combos(*custom_ops: str) -> Iterable[str]: + """Generate all combinations of custom ops for parametrization.""" + custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops] + for op_list in itertools.product(*custom_ops_lists): + yield ",".join(op_list) + + +# Quick inline validation +assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"] +assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [ + "-quant_fp8,-rms_norm", + "-quant_fp8,+rms_norm", + "+quant_fp8,-rms_norm", + "+quant_fp8,+rms_norm", +] + + +def has_cuda_graph_wrapper_metadata() -> bool: + from importlib import import_module + + try: + module = import_module("torch._inductor.utils") + module.CUDAGraphWrapperMetadata # noqa B018 + except AttributeError: + return False + return True + + +INDUCTOR_GRAPH_PARTITION = [ + pytest.param( + True, + marks=pytest.mark.skipif( + not has_cuda_graph_wrapper_metadata(), + reason="torch version does not support Inductor partition", + ), + id="inductor_partition", + ), + pytest.param(False, id="dynamo_partition"), +] + +FUSION_LOG_PATTERNS: dict[str, re.Pattern] = { + "aiter_rms_quant_fusion": re.compile( + r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns" + ), + "rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"), + "act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"), + # The combined pass can consume these sites before the standalone pass. + "norm_rope_fusion": re.compile( + r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|" + r"qk_norm_rope_kvcache_fusion.py:\d+] " + r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)" + ), + "attn_quant_fusion": re.compile( + r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes" + ), + "ar_rms_fusion": re.compile( + r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns" + ), + "aiter_ar_rms_fusion": re.compile( + r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns" + ), +} diff --git a/tests/distributed/test_events.py b/tests/distributed/test_events.py new file mode 100644 index 0000000000..dee0b99bb0 --- /dev/null +++ b/tests/distributed/test_events.py @@ -0,0 +1,408 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import threading +import time + +import msgspec +import pytest + +from vllm.distributed.kv_events import ( + EventBatch, + EventPublisherFactory, + NullEventPublisher, +) + +DP_RANK = 0 + + +class EventSample( + msgspec.Struct, + tag=True, # type: ignore + array_like=True, # type: ignore +): + """Test event for publisher testing.""" + + id: int + value: str + + +class SampleBatch(EventBatch): + """Test event batch for publisher testing.""" + + events: list[EventSample] + + +def create_test_events(count: int) -> SampleBatch: + """Create a batch of test events.""" + events = [EventSample(id=i, value=f"test-{i}") for i in range(count)] + return SampleBatch(ts=time.time(), events=events) + + +def test_basic_publishing(publisher, subscriber): + """Test basic event publishing works.""" + test_batch = create_test_events(5) + publisher.publish(test_batch) + + result = subscriber.receive_one(timeout=1000) + assert result is not None, "No message received" + + seq, received = result + assert seq == 0, "Sequence number mismatch" + assert received.ts == pytest.approx(test_batch.ts, abs=0.1), "Timestamp mismatch" + assert len(received.events) == len(test_batch.events), "Number of events mismatch" + + for i, event in enumerate(received.events): + assert event.id == i, "Event id mismatch" + assert event.value == f"test-{i}", "Event value mismatch" + + +def test_multiple_events(publisher, subscriber): + """Test publishing and receiving multiple event batches.""" + for _ in range(10): + batch = create_test_events(2) + publisher.publish(batch) + + received = [] + for _ in range(10): + data = subscriber.receive_one(timeout=100) + if data: + received.append(data) + + assert len(received) == 10, "Number of messages mismatch" + seqs = [seq for seq, _ in received] + assert seqs == list(range(10)), "Sequence numbers mismatch" + + +def test_replay_mechanism(publisher, subscriber): + """Test the replay mechanism works correctly.""" + for _ in range(19): + batch = create_test_events(1) + publisher.publish(batch) + + # Drain live events to ensure publisher has buffered them. + for _ in range(19): + assert subscriber.receive_one(timeout=1000) is not None + + subscriber.request_replay(10) + + replayed = subscriber.receive_replay() + + assert len(replayed) == 9, ( + f"Expected 9 replayed messages (seq 10-18), got {len(replayed)}" + ) + seqs = [seq for seq, _ in replayed] + assert seqs == list(range(10, 19)), "Replayed sequences should be 10-18" + + +def test_replay_includes_topic(publisher, subscriber, publisher_config): + """Test that replay responses include the topic, matching PUB format.""" + for _ in range(5): + publisher.publish(create_test_events(1)) + + # Drain live events to ensure publisher has processed them. + for _ in range(5): + assert subscriber.receive_one(timeout=1000) is not None + + subscriber.request_replay(0) + + # receive_replay unpacks (topic, seq, payload) and asserts + # topic == publisher topic for each message. + replayed = subscriber.receive_replay() + assert len(replayed) == 5, f"Expected 5 replayed messages, got {len(replayed)}" + seqs = [seq for seq, _ in replayed] + assert seqs == list(range(5)), "Replayed sequences should be 0-4" + + +def test_buffer_limit(publisher, subscriber, publisher_config): + """Test buffer limit behavior.""" + buffer_size = publisher_config.buffer_steps + + # Publish more events than the buffer can hold + for i in range(buffer_size + 10): + batch = create_test_events(1) + publisher.publish(batch) + + time.sleep(0.5) # Need publisher to process above requests + subscriber.request_replay(0) + + replayed = subscriber.receive_replay() + + assert len(replayed) == buffer_size, ( + f"Expected {buffer_size} replayed messages, got {len(replayed)}" + ) + + seqs = [seq for seq, _ in replayed] + assert seqs == list(range(10, buffer_size + 10)), ( + "Should replay seq 11 through buffer_size+10" + ) + + +def test_topic_filtering(publisher_config): + """Test that a subscriber only receives messages matching its topic filter.""" + publisher_config.replay_endpoint = None + + publisher_config.topic = "foo" + pub = EventPublisherFactory.create(publisher_config, DP_RANK) + + from .conftest import MockSubscriber + + sub_foo = MockSubscriber(publisher_config.endpoint, None, "foo") + sub_bar = MockSubscriber(publisher_config.endpoint, None, "bar") + + try: + time.sleep(0.1) + + for _ in range(3): + pub.publish(create_test_events(1)) + + foo_received = [sub_foo.receive_one(timeout=200) for _ in range(3)] + assert all(msg is not None for msg in foo_received), ( + "Subscriber with matching topic should receive messages" + ) + + bar_received = [sub_bar.receive_one(timeout=200) for _ in range(3)] + assert all(msg is None for msg in bar_received), ( + "Subscriber with non-matching topic should receive no messages" + ) + finally: + pub.shutdown() + sub_foo.close() + sub_bar.close() + + +def test_high_volume(publisher, subscriber): + """Test publishing and receiving a high volume of events.""" + num_batches = 10_000 + events_per_batch = 100 + + # Publish events in a separate thread to not block + def publish_events(): + for i in range(num_batches): + batch = create_test_events(events_per_batch) + publisher.publish(batch) + # Small delay to avoid overwhelming + if i % 100 == 0: + time.sleep(0.01) + + received: list[tuple[int, SampleBatch]] = [] + + publisher_thread = threading.Thread(target=publish_events) + publisher_thread.start() + + start_time = time.time() + while len(received) < num_batches: + if time.time() - start_time > 10: # Timeout after 10 seconds + break + + result = subscriber.receive_one(timeout=100) + if result: + received.append(result) + + publisher_thread.join() + + assert len(received) >= num_batches * 0.9, "We should have received most messages" + + seqs = [seq for seq, _ in received] + assert sorted(seqs) == seqs, "Sequence numbers should be in order" + + +def test_null_publisher(): + """Test that NullEventPublisher can be used without errors.""" + publisher = NullEventPublisher(DP_RANK) + + # This should not raise any errors + batch = create_test_events(5) + publisher.publish(batch) + publisher.shutdown() + + +def test_data_parallel_rank_tagging(publisher_config): + """Test that events are properly tagged with their data parallel rank.""" + publisher_config.topic = "foo" + pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK) + pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1) + + # Hardcode the expected endpoints based on port offsetting behavior + # Both ranks get offsets according to _offset_endpoint_port function + base_endpoint = publisher_config.endpoint + if "tcp://" in base_endpoint: + # For TCP endpoints: tcp://localhost:5557 -> tcp://localhost:5557, tcp://localhost:5558 + expected_endpoint_0 = base_endpoint # rank 0 gets port + 0 = same port + expected_endpoint_1 = base_endpoint.replace( + ":5557", ":5558" + ) # rank 1 gets port + 1 + else: + # For inproc endpoints: inproc://test -> inproc://test_dp0, inproc://test_dp1 + expected_endpoint_0 = base_endpoint # rank 0 gets base + expected_endpoint_1 = base_endpoint + "_dp1" # rank 1 gets _dp1 + + from .conftest import MockSubscriber + + sub_0 = MockSubscriber(expected_endpoint_0, None, publisher_config.topic) + sub_1 = MockSubscriber(expected_endpoint_1, None, publisher_config.topic) + + try: + time.sleep(0.1) # Let publishers start up + + # Publish events from different ranks + batch_0 = create_test_events(2) + batch_1 = create_test_events(3) + + pub_0.publish(batch_0) + pub_1.publish(batch_1) + + # Receive events from rank 0 + result_0 = sub_0.receive_one(timeout=200) + assert result_0 is not None, "No message received from rank 0" + seq_0, received_0 = result_0 + + # Receive events from rank 1 + result_1 = sub_1.receive_one(timeout=200) + assert result_1 is not None, "No message received from rank 1" + seq_1, received_1 = result_1 + + # Verify DP rank tagging + assert received_0.data_parallel_rank == 0, ( + f"Expected DP rank 0, got {received_0.data_parallel_rank}" + ) + assert received_1.data_parallel_rank == 1, ( + f"Expected DP rank 1, got {received_1.data_parallel_rank}" + ) + + # Verify event content is correct + assert len(received_0.events) == 2, "Wrong number of events from rank 0" + assert len(received_1.events) == 3, "Wrong number of events from rank 1" + + finally: + pub_0.shutdown() + pub_1.shutdown() + sub_0.close() + sub_1.close() + + +def test_event_publisher_factory(random_port): + """Test event publisher factory creation behavior under different configurations.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed.kv_events import ZmqEventPublisher + + # test config is None + publisher = EventPublisherFactory.create(None, DP_RANK) + assert isinstance(publisher, NullEventPublisher) + publisher.shutdown() + + # test disable kv cache events + config = KVEventsConfig( + enable_kv_cache_events=False, + publisher="zmq", # Even if zmq is specified, should return NullEventPublisher + endpoint="tcp://localhost:5557", + ) + publisher = EventPublisherFactory.create(config, DP_RANK) + assert isinstance(publisher, NullEventPublisher) + publisher.shutdown() + + # test zmq publisher + config = KVEventsConfig( + enable_kv_cache_events=True, + publisher="zmq", + endpoint=f"tcp://*:{random_port}", + replay_endpoint=f"tcp://*:{random_port + 100}", + ) + publisher = EventPublisherFactory.create(config, DP_RANK + 1) + assert isinstance(publisher, ZmqEventPublisher) + resolved_config = publisher.get_publisher_config() + assert resolved_config.endpoint == f"tcp://*:{random_port + 1}" + assert resolved_config.replay_endpoint == f"tcp://*:{random_port + 101}" + assert config.endpoint == f"tcp://*:{random_port}" + publisher.shutdown() + + # test unknown publisher + with pytest.raises(ValueError, match="Input should be"): + KVEventsConfig( + enable_kv_cache_events=True, + publisher="unknown_publisher", + endpoint="tcp://localhost:5557", + ) + + # test publisher not specified + config = KVEventsConfig( + enable_kv_cache_events=True, + # publisher not specified, should default to "zmq" + endpoint="tcp://localhost:5557", + ) + publisher = EventPublisherFactory.create(config, DP_RANK) + assert isinstance(publisher, ZmqEventPublisher) + publisher.shutdown() + + +@pytest.mark.parametrize( + ("endpoint", "expected"), + [("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")], +) +def test_offset_endpoint_port_keeps_port_zero(endpoint, expected): + """Port 0 is not offset: every rank asks the OS for its own port.""" + from vllm.distributed.kv_events import ZmqEventPublisher + + assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected + + +@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"]) +def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip): + """Two ranks binding tcp://*:0 get distinct OS-assigned ports on this + node's IP and report endpoints a subscriber can use for events and replay.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed import kv_events + from vllm.utils.network_utils import split_zmq_path + + from .conftest import MockSubscriber + + monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip) + config = KVEventsConfig( + enable_kv_cache_events=True, + publisher="zmq", + endpoint="tcp://*:0", + replay_endpoint="tcp://*:0", + topic="kv", + ) + pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)] + try: + resolved = [pub.get_publisher_config() for pub in pubs] + endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)] + hosts_ports = {split_zmq_path(e)[1:] for e in endpoints} + assert {h for h, _ in hosts_ports} == {node_ip} + assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports} + assert config.endpoint == "tcp://*:0" + + sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv") + try: + # PUB/SUB drops messages sent before the subscription lands, so + # keep publishing until both ranks are heard. + ranks: set[int] = set() + deadline = time.time() + 10 + while ranks != {0, 1} and time.time() < deadline: + for pub in pubs: + pub.publish(create_test_events(1)) + while (result := sub.receive_one(timeout=100)) is not None: + ranks.add(result[1].data_parallel_rank) + assert ranks == {0, 1} + sub.request_replay(0, socket_idx=1) + assert sub.receive_replay(socket_idx=1) + finally: + sub.close() + finally: + for pub in pubs: + pub.shutdown() + + +@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"]) +def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip): + """tcp://*:0 is advertised as this node's IP; with no usable IP the + publisher refuses instead of reporting a wildcard.""" + from vllm.config.kv_events import KVEventsConfig + from vllm.distributed import kv_events + + monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip) + config = KVEventsConfig( + enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0" + ) + with pytest.raises(ValueError, match="VLLM_HOST_IP"): + EventPublisherFactory.create(config, DP_RANK) diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py new file mode 100644 index 0000000000..642193ce88 --- /dev/null +++ b/tests/distributed/test_weight_transfer.py @@ -0,0 +1,2399 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for weight transfer engine backends. + +Unit tests for engine classes (parsing, validation, registry). +Integration tests for NCCL and IPC weight transfer between processes using Ray. +""" + +import builtins +import importlib.util +import pickle +import runpy +import threading +import time +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pybase64 as base64 +import pytest +import ray +import torch +from torch.multiprocessing.reductions import reduce_tensor + +from vllm.config.parallel import ParallelConfig +from vllm.config.weight_transfer import WeightTransferConfig +from vllm.distributed.weight_transfer import ( + HTTPVLLMWeightSyncClient, + ModuleSource, + ParamMeta, + RayVLLMWeightSyncClient, + TrainerWeightTransferEngine, + VLLMWeightSyncClient, + WeightSource, + WeightTransferEngineFactory, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.base import ( + TrainerInitInfo, + WeightTransferEngine, + WeightTransferInitRequest, + WeightTransferUpdateRequest, + layerwise_groups, +) +from vllm.distributed.weight_transfer.ipc_engine import ( + IPCTrainerInitInfo, + IPCTrainerWeightTransferEngine, + IPCWeightTransferEngine, + IPCWeightTransferInitInfo, + IPCWeightTransferUpdateInfo, +) +from vllm.distributed.weight_transfer.nccl_engine import ( + NCCLTrainerInitInfo, + NCCLTrainerWeightTransferEngine, + NCCLWeightTransferEngine, + NCCLWeightTransferInitInfo, + NCCLWeightTransferUpdateInfo, +) +from vllm.distributed.weight_transfer.packed_tensor import ( + DEFAULT_PACKED_BUFFER_SIZE_BYTES, + DEFAULT_PACKED_NUM_BUFFERS, +) +from vllm.distributed.weight_transfer.sparse_nccl_engine import ( + SparseNCCLTrainerInitInfo, + SparseNCCLTrainerWeightTransferEngine, + SparseNCCLWeightTransferEngine, + SparseNCCLWeightTransferUpdateInfo, + SparseWeightPatch, +) +from vllm.platforms import current_platform +from vllm.utils.nccl import _nccl_has_no_cache +from vllm.utils.network_utils import get_open_port + + +def _init_ray_for_weight_transfer() -> None: + if ray.is_initialized(): + return + ray.init( + ignore_reinit_error=True, + runtime_env={ + "env_vars": { + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", + "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES": "1", + "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES": "1", + } + }, + ) + + +def _get_ray_assigned_device() -> torch.device: + gpu_ids = ray.get_gpu_ids() + if not gpu_ids: + return torch.device("cuda:0") + return torch.device(f"cuda:{int(gpu_ids[0])}") + + +def _set_ray_assigned_device() -> torch.device: + device = _get_ray_assigned_device() + current_platform.set_device(device) + return device + + +def create_mock_parallel_config( + rank: int = 0, + world_size: int = 1, + dp_rank: int = 0, +) -> ParallelConfig: + """Create a mock ParallelConfig for testing.""" + config = MagicMock(spec=ParallelConfig) + config.rank = rank + config.world_size = world_size + config.data_parallel_rank = dp_rank + config.data_parallel_index = dp_rank + return config + + +def create_mock_vllm_config( + rank: int = 0, + world_size: int = 1, + dp_rank: int = 0, +) -> MagicMock: + """Create a mock VllmConfig exposing parallel_config and model_config.""" + vllm_config = MagicMock() + vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank) + vllm_config.model_config = MagicMock() + return vllm_config + + +# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation --- + + +class TestNCCLWeightTransferUpdateInfoValidation: + """Test NCCLWeightTransferUpdateInfo dataclass validation.""" + + def test_valid_update_info(self): + info = NCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32", "float32"], + shapes=[[10, 10], [10]], + ) + assert info.names == ["layer.weight", "layer.bias"] + assert info.dtype_names == ["float32", "float32"] + assert info.shapes == [[10, 10], [10]] + + def test_mismatched_dtype_names_raises(self): + with pytest.raises(ValueError, match="dtype_names"): + NCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32"], # Only one dtype + shapes=[[10, 10], [10]], + ) + + def test_mismatched_shapes_raises(self): + with pytest.raises(ValueError, match="shapes"): + NCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32", "float32"], + shapes=[[10, 10]], # Only one shape + ) + + def test_empty_lists_valid(self): + info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[]) + assert len(info.names) == 0 + + +# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation --- + + +class TestSparseNCCLWeightTransferUpdateInfoValidation: + """Test SparseNCCLWeightTransferUpdateInfo dataclass validation.""" + + def test_valid_sparse_update_info(self): + info = SparseNCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32", "bfloat16"], + shapes=[[10, 10], [10]], + num_updates_list=[4, 2], + ) + assert info.num_updates_list == [4, 2] + + def test_mismatched_dtype_names_raises(self): + with pytest.raises(ValueError, match="dtype_names"): + SparseNCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32"], + shapes=[[10, 10], [10]], + num_updates_list=[4, 2], + ) + + def test_rejects_empty_num_updates_list(self): + with pytest.raises(ValueError, match="cannot be empty"): + SparseNCCLWeightTransferUpdateInfo( + names=[], + dtype_names=[], + shapes=[], + num_updates_list=[], + ) + + def test_rejects_mismatched_num_updates(self): + with pytest.raises(ValueError, match="`num_updates_list`"): + SparseNCCLWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32", "float32"], + shapes=[[10, 10], [10]], + num_updates_list=[3], + ) + + def test_rejects_negative_num_updates(self): + with pytest.raises(ValueError, match="non-negative"): + SparseNCCLWeightTransferUpdateInfo( + names=["layer.weight"], + dtype_names=["float32"], + shapes=[[10, 10]], + num_updates_list=[-1], + ) + + +# --- Unit Tests: Engine Parsing --- + + +@pytest.mark.parametrize( + "engine_type", + [ + NCCLWeightTransferEngine, + NCCLTrainerWeightTransferEngine, + SparseNCCLWeightTransferEngine, + SparseNCCLTrainerWeightTransferEngine, + ], +) +def test_nccl_shutdown_destroys_communicator_once(engine_type): + engine = object.__new__(engine_type) + group = MagicMock() + engine.model_update_group = group + + engine.shutdown() + engine.shutdown() + + group.destroy.assert_called_once_with() + assert engine.model_update_group is None + + +class TestNCCLEngineParsing: + """Test NCCLWeightTransferEngine parsing methods.""" + + def _make_engine(self): + config = WeightTransferConfig(backend="nccl") + return NCCLWeightTransferEngine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + + def test_parse_init_info_valid(self): + engine = self._make_engine() + init_info = engine.parse_init_info( + { + "master_address": "127.0.0.1", + "master_port": 12345, + "rank_offset": 1, + "world_size": 3, + } + ) + assert isinstance(init_info, NCCLWeightTransferInitInfo) + assert init_info.master_address == "127.0.0.1" + assert init_info.master_port == 12345 + assert init_info.rank_offset == 1 + assert init_info.world_size == 3 + + def test_parse_init_info_missing_field_raises(self): + engine = self._make_engine() + with pytest.raises(ValueError, match="Invalid init_info"): + engine.parse_init_info({"master_address": "127.0.0.1"}) + + def test_parse_update_info_valid(self): + engine = self._make_engine() + update_info = engine.parse_update_info( + { + "names": ["w1", "w2"], + "dtype_names": ["float32", "bfloat16"], + "shapes": [[100, 100], [50]], + } + ) + assert isinstance(update_info, NCCLWeightTransferUpdateInfo) + assert update_info.names == ["w1", "w2"] + assert update_info.dtype_names == ["float32", "bfloat16"] + assert update_info.shapes == [[100, 100], [50]] + + +# --- Unit Tests: Engine Registry --- + + +class TestEngineRegistry: + """Test weight transfer engine registry.""" + + def test_create_engine_nccl(self): + config = WeightTransferConfig(backend="nccl") + engine = WeightTransferEngineFactory.create_engine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + assert isinstance(engine, NCCLWeightTransferEngine) + + def test_create_engine_ipc(self): + config = WeightTransferConfig(backend="ipc") + engine = WeightTransferEngineFactory.create_engine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + assert isinstance(engine, IPCWeightTransferEngine) + + def test_create_engine_sparse_nccl(self): + config = WeightTransferConfig(backend="sparse_nccl") + engine = WeightTransferEngineFactory.create_engine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + assert isinstance(engine, SparseNCCLWeightTransferEngine) + + def test_modelexpress_registration_is_native_and_lazy(self, monkeypatch): + from vllm.distributed.weight_transfer import factory + + imported = [] + module = MagicMock() + module.ModelExpressWeightTransferEngine.__name__ = ( + "ModelExpressWeightTransferEngine" + ) + + def import_module(name): + imported.append(name) + return module + + monkeypatch.setattr(factory.importlib, "import_module", import_module) + config = WeightTransferConfig(backend="modelexpress") + vllm_config = create_mock_vllm_config() + device = torch.device("cpu") + model = MagicMock(spec=torch.nn.Module) + + assert imported == [] + engine = WeightTransferEngineFactory.create_engine( + config, vllm_config, device, model + ) + + assert imported == ["vllm.distributed.weight_transfer.modelexpress_engine"] + module.ModelExpressWeightTransferEngine.assert_called_once_with( + config, vllm_config, device, model + ) + assert engine is module.ModelExpressWeightTransferEngine.return_value + + def test_create_engine_invalid_backend(self): + config = WeightTransferConfig(backend="invalid") + with pytest.raises(ValueError, match="Invalid weight transfer backend"): + WeightTransferEngineFactory.create_engine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + + def test_register_duplicate_raises(self): + with pytest.raises(ValueError, match="already registered"): + WeightTransferEngineFactory.register_engine( + "nccl", NCCLWeightTransferEngine + ) + + +# --- Test receive_weights without init raises --- + + +def test_nccl_receive_weights_without_init_raises(): + """Test that receive_weights raises if init_transfer_engine wasn't called.""" + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + config = WeightTransferConfig(backend="nccl") + engine = NCCLWeightTransferEngine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + + update_info = NCCLWeightTransferUpdateInfo( + names=["w"], dtype_names=["float32"], shapes=[[10]] + ) + + with pytest.raises(RuntimeError, match="not initialized"): + engine.receive_weights(update_info) + + +def test_sparse_nccl_receive_weights_without_init_raises(): + """Test that sparse receive raises if init_transfer_engine wasn't called.""" + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + config = WeightTransferConfig(backend="sparse_nccl") + engine = SparseNCCLWeightTransferEngine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + + update_info = SparseNCCLWeightTransferUpdateInfo( + names=["w"], + dtype_names=["float32"], + shapes=[[10]], + num_updates_list=[2], + ) + + with pytest.raises(RuntimeError, match="not initialized"): + engine.receive_weights(update_info) + + +# --- Integration Test: NCCL Weight Transfer Between Ray Tasks --- + + +@ray.remote(num_gpus=1) +def trainer_broadcast_tensor( + master_address: str, + master_port: int, + world_size: int, + tensor_shape: list[int], + tensor_dtype: str, +) -> bool: + """Trainer task that broadcasts a tensor via NCCL.""" + import torch + + device = _set_ray_assigned_device() + + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + from vllm.distributed.utils import StatelessProcessGroup + + # Create process group as rank 0 (trainer) + pg = StatelessProcessGroup.create( + host=master_address, + port=master_port, + rank=0, + world_size=world_size, + ) + comm = PyNcclCommunicator(pg, device=device.index) + + # Create and broadcast the tensor + dtype = getattr(torch, tensor_dtype) + tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device=device) + comm.broadcast(tensor_to_send, src=0, stream=torch.cuda.current_stream()) + torch.accelerator.synchronize() + + return True + + +# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process. +@ray.remote(num_gpus=1, max_calls=1) +def inference_receive_tensor( + master_address: str, + master_port: int, + world_size: int, + tensor_shape: list[int], + tensor_dtype: str, + batch_invariant: bool = False, +) -> dict: + """Inference task that receives tensor via NCCLWeightTransferEngine.""" + import contextlib + from unittest.mock import MagicMock + + import torch + + _set_ray_assigned_device() + if batch_invariant: + from vllm.model_executor.determinism.batch_invariant import ( + override_envs_for_invariance, + ) + + override_envs_for_invariance() + # Like vLLM's own groups, a pinned communicator makes NCCL read the + # pins before the transfer group exists. + torch.distributed.init_process_group( + "nccl", + init_method=f"tcp://127.0.0.1:{get_open_port()}", + rank=0, + world_size=1, + ) + torch.distributed.all_reduce(torch.ones(1, device="cuda")) + + from vllm.config.parallel import ParallelConfig + from vllm.config.weight_transfer import WeightTransferConfig + from vllm.distributed.weight_transfer.nccl_engine import ( + NCCLWeightTransferEngine, + NCCLWeightTransferInitInfo, + NCCLWeightTransferUpdateInfo, + ) + + class Recorder(torch.nn.Module): + def __init__(self): + super().__init__() + self.received = [] + + def load_weights(self, weights): + for name, tensor in weights: + self.received.append((name, tensor.clone())) + + config = WeightTransferConfig(backend="nccl") + vllm_config = MagicMock() + parallel_config = MagicMock(spec=ParallelConfig) + parallel_config.rank = 0 + parallel_config.world_size = 1 + parallel_config.data_parallel_rank = 0 + parallel_config.data_parallel_index = 0 + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() + + recorder = Recorder() + engine = NCCLWeightTransferEngine( + config, vllm_config, torch.device("cuda"), recorder + ) + # Transport-only test: bypass the set_current_vllm_config context that + # receive_weights enters, since vllm_config here is a mock. + import vllm.config as _vllm_config_mod + + _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() + + # Initialize the engine (joins as rank 1) + # Trainer broadcasts a single tensor unpacked, so the worker must not + # expect the packed wire format (packed is a must-agree wire param shipped + # on the init info). + init_info = NCCLWeightTransferInitInfo( + master_address=master_address, + master_port=master_port, + rank_offset=1, # Trainer is rank 0, we become rank 1 + world_size=world_size, + packed=False, + ) + engine.init_transfer_engine(init_info) + + update_info = NCCLWeightTransferUpdateInfo( + names=["test.weight"], + dtype_names=[tensor_dtype], + shapes=[tensor_shape], + ) + engine.receive_weights(update_info) + torch.accelerator.synchronize() + + # Verify we received the tensor + success = False + received_shape = None + received_sum = None + + if len(recorder.received) == 1: + name, tensor = recorder.received[0] + received_shape = list(tensor.shape) + received_sum = tensor.sum().item() + if received_shape == tensor_shape: + expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item() + if abs(received_sum - expected_sum) < 0.01: + success = True + + engine.shutdown() + + return { + "success": success, + "received_shape": received_shape, + "received_sum": received_sum, + } + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 2, + reason="Need at least 2 GPUs to run NCCL weight transfer test.", +) +@pytest.mark.parametrize( + "batch_invariant", + [ + pytest.param(False, id="default"), + pytest.param( + True, + id="batch-invariant-worker", + marks=pytest.mark.skipif( + not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7." + ), + ), + ], +) +def test_nccl_weight_transfer_between_processes(batch_invariant): + """Test NCCL weight transfer from trainer to inference process using Ray. + + This test verifies that the NCCLWeightTransferEngine can receive + tensors broadcast by a trainer process via NCCL, including when the + worker runs with batch-invariance NCCL pins the trainer does not have. + """ + _init_ray_for_weight_transfer() + + master_address = "127.0.0.1" + master_port = get_open_port() + world_size = 2 # 1 trainer + 1 inference worker + + tensor_shape = [100, 100] + tensor_dtype = "float32" + + inference_future = inference_receive_tensor.remote( + master_address, + master_port, + world_size, + tensor_shape, + tensor_dtype, + batch_invariant, + ) + trainer_future = trainer_broadcast_tensor.remote( + master_address, master_port, world_size, tensor_shape, tensor_dtype + ) + + # A mismatched NCCL config deadlocks instead of failing. + trainer_result, result = ray.get([trainer_future, inference_future], timeout=300) + + assert trainer_result, "Trainer should complete successfully" + assert result["success"], ( + f"Weight transfer failed. " + f"Received shape: {result['received_shape']}, " + f"Received sum: {result['received_sum']}" + ) + + +def test_sparse_nccl_checkpoint_chunks_to_ep_local_experts_cpu(monkeypatch): + """Replay global expert patches through two EP-local loaders on CPU.""" + + class CaptureClient: + def __init__(self): + self.order: list[str] = [] + self.update_infos: list[dict] = [] + + def start_weight_update(self): + self.order.append("start") + + def update_weights(self, update_info): + self.order.append("update") + self.update_infos.append(update_info) + + def finish_weight_update(self): + self.order.append("finish") + + monkeypatch.setattr(torch.cuda, "current_stream", lambda *_, **__: None) + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + monkeypatch.setattr(torch.accelerator, "device_index", lambda *_: nullcontext()) + + client = CaptureClient() + sender = SparseNCCLTrainerWeightTransferEngine(client=client) + sender.model_update_group = MagicMock() + sender.model_update_group.device = torch.device("cpu") + wire_payloads = [] + sender.model_update_group.broadcast.side_effect = lambda tensor, **_: ( + wire_payloads.append(tensor.clone()) + ) + + expert_names = [ + "model.layers.0.mlp.experts.0.gate_proj.weight", + "model.layers.0.mlp.experts.1.gate_proj.weight", + ] + client.start_weight_update() + for name, index, value in zip( + expert_names, + (0, 3), + (5.0, 7.0), + strict=True, + ): + sender.send_weight_chunk( + [ + SparseWeightPatch( + name=name, + indices=torch.tensor([index], dtype=torch.int32), + values=torch.tensor([value]), + full_shape=(2, 2), + ) + ] + ) + client.finish_weight_update() + + assert client.order == ["start", "update", "update", "finish"] + assert [info["names"] for info in client.update_infos] == [ + [expert_names[0]], + [expert_names[1]], + ] + assert [info["shapes"] for info in client.update_infos] == [ + [[2, 2]], + [[2, 2]], + ] + assert [info["num_updates_list"] for info in client.update_infos] == [ + [1], + [1], + ] + + expected_gates = ( + [[5.0, -1.0], [-1.0, -1.0]], + [[-1.0, -1.0], [-1.0, 7.0]], + ) + for ep_rank, expected_gate in enumerate(expected_gates): + model = torch.nn.Module() + model.register_parameter( + "w13_weight", + torch.nn.Parameter(torch.full((1, 4, 2), -1.0), requires_grad=False), + ) + local_name = expert_names[ep_rank] + seen_names: list[str] = [] + loaded_names: set[str] = set() + copy_count = [0] + + def load_weights( + weights, + local_name=local_name, + target=model, + seen=seen_names, + loaded=loaded_names, + count=copy_count, + ): + for name, checkpoint_weight in weights: + seen.append(name) + if name != local_name: + continue + target.w13_weight.data[0, :2].copy_(checkpoint_weight) + loaded.add(name) + count[0] += 1 + return loaded + + model.load_weights = load_weights + receiver = SparseNCCLWeightTransferEngine( + WeightTransferConfig(backend="sparse_nccl"), + create_mock_vllm_config(rank=ep_rank, world_size=2), + torch.device("cpu"), + model, + ) + receiver.model_update_group = MagicMock() + receiver.model_update_group.device = torch.device("cpu") + payloads = iter(wire_payloads) + receiver.model_update_group.broadcast.side_effect = ( + lambda tensor, payloads=payloads, **_: tensor.copy_(next(payloads)) + ) + + receiver.start_weight_update() + for update_info in client.update_infos: + receiver.receive_weights(SparseNCCLWeightTransferUpdateInfo(**update_info)) + receiver.finish_weight_update() + + assert seen_names == expert_names + assert loaded_names == {local_name} + assert copy_count == [1] + assert model.w13_weight[0, :2].tolist() == expected_gate + assert model.w13_weight[0, 2:].eq(-1).all() + foreign_value = 7.0 if ep_rank == 0 else 5.0 + assert not model.w13_weight.eq(foreign_value).any() + receiver.shutdown() + + sender.shutdown() + + +# --- Unit Tests: IPCWeightTransferUpdateInfo Validation --- + + +class TestIPCWeightTransferUpdateInfoValidation: + """Test IPCWeightTransferUpdateInfo dataclass validation.""" + + def test_valid_update_info(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + _, ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}] + + info = IPCWeightTransferUpdateInfo( + names=["layer.weight"], + dtype_names=["float32"], + shapes=[[10, 10]], + ipc_handles=ipc_handles, + ) + assert info.names == ["layer.weight"] + assert info.dtype_names == ["float32"] + assert info.shapes == [[10, 10]] + assert len(info.ipc_handles) == 1 + + def test_mismatched_dtype_names_raises(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + _, ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}, {gpu_uuid: ipc_handle}] + + with pytest.raises(ValueError, match="dtype_names"): + IPCWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32"], # Only one dtype + shapes=[[10, 10], [10]], + ipc_handles=ipc_handles, + ) + + def test_mismatched_shapes_raises(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + _, ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}, {gpu_uuid: ipc_handle}] + + with pytest.raises(ValueError, match="shapes"): + IPCWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32", "float32"], + shapes=[[10, 10]], # Only one shape + ipc_handles=ipc_handles, + ) + + def test_mismatched_ipc_handles_raises(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + _, ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}] # Only one handle + + with pytest.raises(ValueError, match="ipc_handles"): + IPCWeightTransferUpdateInfo( + names=["layer.weight", "layer.bias"], + dtype_names=["float32", "float32"], + shapes=[[10, 10], [10]], + ipc_handles=ipc_handles, + ) + + def test_valid_update_info_from_pickled(self, monkeypatch): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}] + + pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") + + info = IPCWeightTransferUpdateInfo( + names=["layer.weight"], + dtype_names=["float32"], + shapes=[[10, 10]], + ipc_handles_pickled=pickled, + ) + assert info.ipc_handles == ipc_handles + assert info.ipc_handles_pickled is None + + def test_pickled_requires_insecure_serialization_flag(self, monkeypatch): + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0") + + with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"): + IPCWeightTransferUpdateInfo( + names=[], + dtype_names=[], + shapes=[], + ipc_handles_pickled=base64.b64encode(pickle.dumps([])).decode("utf-8"), + ) + + def test_both_handles_and_pickled_raises(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}] + + pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") + + with pytest.raises(ValueError, match="Cannot specify both"): + IPCWeightTransferUpdateInfo( + names=["layer.weight"], + dtype_names=["float32"], + shapes=[[10, 10]], + ipc_handles=ipc_handles, + ipc_handles_pickled=pickled, + ) + + def test_neither_handles_nor_pickled_raises(self): + with pytest.raises(ValueError, match="must be provided"): + IPCWeightTransferUpdateInfo( + names=["layer.weight"], + dtype_names=["float32"], + shapes=[[10, 10]], + ) + + def test_empty_lists_valid(self): + info = IPCWeightTransferUpdateInfo( + names=[], + dtype_names=[], + shapes=[], + ipc_handles=[], + ) + assert len(info.names) == 0 + + +# --- Unit Tests: IPC Engine Parsing --- + + +class TestIPCEngineParsing: + """Test IPCWeightTransferEngine parsing methods.""" + + def _make_engine(self): + config = WeightTransferConfig(backend="ipc") + return IPCWeightTransferEngine( + config, + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + + def test_parse_update_info_valid(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + engine = self._make_engine() + + dummy_tensor1 = torch.ones(100, 100, device="cuda:0") + dummy_tensor2 = torch.ones(50, device="cuda:0") + _, ipc_args1 = reduce_tensor(dummy_tensor1) + _, ipc_args2 = reduce_tensor(dummy_tensor2) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_args1}, {gpu_uuid: ipc_args2}] + + update_info = engine.parse_update_info( + { + "names": ["w1", "w2"], + "dtype_names": ["float32", "bfloat16"], + "shapes": [[100, 100], [50]], + "ipc_handles": ipc_handles, + } + ) + + assert isinstance(update_info, IPCWeightTransferUpdateInfo) + assert update_info.names == ["w1", "w2"] + assert update_info.dtype_names == ["float32", "bfloat16"] + assert update_info.shapes == [[100, 100], [50]] + assert len(update_info.ipc_handles) == 2 + + def test_parse_update_info_pickled(self, monkeypatch): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + engine = self._make_engine() + + dummy_tensor1 = torch.ones(100, 100, device="cuda:0") + dummy_tensor2 = torch.ones(50, device="cuda:0") + _, ipc_args1 = reduce_tensor(dummy_tensor1) + _, ipc_args2 = reduce_tensor(dummy_tensor2) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_args1}, {gpu_uuid: ipc_args2}] + + pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") + + update_info = engine.parse_update_info( + { + "names": ["w1", "w2"], + "dtype_names": ["float32", "bfloat16"], + "shapes": [[100, 100], [50]], + "ipc_handles_pickled": pickled, + } + ) + + assert isinstance(update_info, IPCWeightTransferUpdateInfo) + assert update_info.names == ["w1", "w2"] + assert len(update_info.ipc_handles) == 2 + assert gpu_uuid in update_info.ipc_handles[0] + assert gpu_uuid in update_info.ipc_handles[1] + + def test_parse_update_info_ignores_none_pickled_handles(self): + engine = self._make_engine() + ipc_handles = [{"gpu-uuid": ("ipc-args",)}] + + update_info = engine.parse_update_info( + { + "names": ["w1"], + "dtype_names": ["float32"], + "shapes": [[1]], + "ipc_handles": ipc_handles, + "ipc_handles_pickled": None, + } + ) + + assert isinstance(update_info, IPCWeightTransferUpdateInfo) + assert update_info.ipc_handles == ipc_handles + + def test_parse_update_info_both_handles_and_pickled_raises(self): + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + engine = self._make_engine() + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + _, ipc_handle = reduce_tensor(dummy_tensor) + gpu_uuid = str(torch.cuda.get_device_properties(0).uuid) + ipc_handles = [{gpu_uuid: ipc_handle}] + + pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") + + with pytest.raises(ValueError, match="Cannot specify both"): + engine.parse_update_info( + { + "names": ["layer.weight"], + "dtype_names": ["float32"], + "shapes": [[10, 10]], + "ipc_handles": ipc_handles, + "ipc_handles_pickled": pickled, + } + ) + + +# --- Integration Test: IPC Weight Transfer Between Ray Tasks --- + + +def get_physical_gpu_id(device_index: int = 0) -> str: + """Get physical GPU UUID for a device.""" + props = torch.cuda.get_device_properties(device_index) + return str(props.uuid) + + +@ray.remote(num_gpus=0.5) +class TrainerActor: + """Trainer actor that creates and holds CUDA IPC handles.""" + + def __init__(self, tensor_shape: list[int], tensor_dtype: str): + device = _set_ray_assigned_device() + + # Create tensor on GPU and keep it alive + dtype = getattr(torch, tensor_dtype) + self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device) + self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer + + _, ipc_args = reduce_tensor(self.tensor) + gpu_uuid = get_physical_gpu_id(device.index) + + torch.accelerator.synchronize() + + self.ipc_handle_dict = { + "ipc_handle": ipc_args, + "gpu_uuid": gpu_uuid, + "shape": tensor_shape, + "dtype": tensor_dtype, + } + + def get_ipc_handle_dict(self) -> dict: + """Return IPC handle dict. Tensor stays alive in this actor.""" + return self.ipc_handle_dict + + +@ray.remote(num_gpus=0.5) +def inference_receive_ipc_tensor( + ipc_handle_dict: dict, + mode: str = "ray", +) -> dict: + """Inference task that receives tensor via IPCWeightTransferEngine.""" + import contextlib + import os + + # Worker-side: ipc_handles_pickled is deserialized via pickle. + if mode == "http": + os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" + + from unittest.mock import MagicMock + + import torch + + device = _set_ray_assigned_device() + + from vllm.config.parallel import ParallelConfig + from vllm.config.weight_transfer import WeightTransferConfig + from vllm.distributed.weight_transfer.ipc_engine import ( + IPCWeightTransferEngine, + ) + + class Recorder(torch.nn.Module): + def __init__(self): + super().__init__() + self.received = [] + + def load_weights(self, weights): + for name, tensor in weights: + self.received.append((name, tensor.clone())) + + # Trainer sends unpacked IPC handles; the worker learns packed=False from + # the init handshake below (IPCWeightTransferInitInfo defaults to False). + config = WeightTransferConfig(backend="ipc") + vllm_config = MagicMock() + parallel_config = MagicMock(spec=ParallelConfig) + parallel_config.rank = 0 + parallel_config.world_size = 1 + parallel_config.data_parallel_rank = 0 + parallel_config.data_parallel_index = 0 + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() + + recorder = Recorder() + engine = IPCWeightTransferEngine(config, vllm_config, device, recorder) + # Transport-only test: bypass the set_current_vllm_config context that + # receive_weights enters, since vllm_config here is a mock. + import vllm.config as _vllm_config_mod + + _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() + + init_info = IPCWeightTransferInitInfo() + engine.init_transfer_engine(init_info) + + ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}] + + if mode == "ray": + update_dict: dict = { + "names": ["test.weight"], + "dtype_names": [ipc_handle_dict["dtype"]], + "shapes": [ipc_handle_dict["shape"]], + "ipc_handles": ipc_handles, + } + elif mode == "http": + pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8") + update_dict = { + "names": ["test.weight"], + "dtype_names": [ipc_handle_dict["dtype"]], + "shapes": [ipc_handle_dict["shape"]], + "ipc_handles_pickled": pickled, + } + else: + raise ValueError(f"Unknown mode: {mode}") + + update_info = engine.parse_update_info(update_dict) + engine.receive_weights(update_info) + torch.accelerator.synchronize() + + success = False + received_shape = None + received_sum = None + + if len(recorder.received) == 1: + name, tensor = recorder.received[0] + received_shape = list(tensor.shape) + received_sum = tensor.sum().item() + if received_shape == ipc_handle_dict["shape"]: + expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item() + if abs(received_sum - expected_sum) < 0.01: + success = True + + engine.shutdown() + + return { + "success": success, + "received_shape": received_shape, + "received_sum": received_sum, + } + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 1, + reason="Need at least 1 GPU to run IPC weight transfer test.", +) +@pytest.mark.parametrize("mode", ["ray", "http"]) +def test_ipc_weight_transfer_between_processes(mode: str): + """Test IPC weight transfer from trainer to inference process using Ray.""" + from ray.util.placement_group import placement_group + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + + _init_ray_for_weight_transfer() + + pg = placement_group([{"GPU": 1, "CPU": 2}]) + ray.get(pg.ready()) + + scheduling_strategy = PlacementGroupSchedulingStrategy( + placement_group=pg, + placement_group_capture_child_tasks=True, + ) + + tensor_shape = [100, 100] + tensor_dtype = "float32" + + trainer_actor = TrainerActor.options( # type: ignore[attr-defined] + scheduling_strategy=scheduling_strategy + ).remote(tensor_shape, tensor_dtype) + + ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote()) + + inference_result = ray.get( + inference_receive_ipc_tensor.options( + scheduling_strategy=scheduling_strategy + ).remote(ipc_handle_dict, mode=mode) + ) + + assert inference_result["success"], ( + f"IPC weight transfer failed (mode={mode}). " + f"Received shape: {inference_result['received_shape']}, " + f"Received sum: {inference_result['received_sum']}" + ) + + +def test_ipc_receive_weights_missing_gpu_uuid_raises(): + """Test that receive_weights raises if GPU UUID not found in IPC handles.""" + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU for this test") + + config = WeightTransferConfig(backend="ipc") + engine = IPCWeightTransferEngine( + config, + create_mock_vllm_config(), + torch.device("cuda:0"), + MagicMock(spec=torch.nn.Module), + ) + # No init handshake here, so the engine keeps its default packed=False. + + dummy_tensor = torch.ones(10, 10, device="cuda:0") + _, ipc_handle = reduce_tensor(dummy_tensor) + wrong_uuid = "wrong-uuid-12345" + ipc_handles = [{wrong_uuid: ipc_handle}] + + update_info = IPCWeightTransferUpdateInfo( + names=["w"], + dtype_names=["float32"], + shapes=[[10, 10]], + ipc_handles=ipc_handles, + ) + + with pytest.raises(ValueError, match="IPC handle not found"): + engine.receive_weights(update_info) + + +class RecordingClient: + """A fake VLLMWeightSyncClient that records the order of calls.""" + + def __init__(self): + self.order: list[str] = [] + self.last_init_info: dict | None = None + self.last_update_info: dict | None = None + + def init_weight_transfer_engine(self, init_info: dict) -> None: + self.order.append("init") + self.last_init_info = init_info + + def start_weight_update(self) -> None: + self.order.append("start") + + def update_weights(self, update_info: dict) -> None: + self.order.append("update") + self.last_update_info = update_info + + def finish_weight_update(self, weight_version: str | None = None) -> None: + self.order.append("finish") + + +def _module_with(*pairs): + """A tiny nn.Module exposing the given (name, tensor) pairs as parameters, + so trainer tests can build a ModuleSource without a real model.""" + module = torch.nn.Module() + for name, tensor in pairs: + module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False)) + return module + + +class _DummyTrainerEngine(TrainerWeightTransferEngine): + """Minimal concrete trainer engine to exercise base-class + factory.""" + + @classmethod + def trainer_init(cls, init_info, *, client, source): + return cls(client=client, source=source) + + def send_weights(self): + pass + + +class TestTrainerClients: + """Structural protocol conformance for the built-in clients.""" + + def test_recording_client_is_protocol(self): + assert isinstance(RecordingClient(), VLLMWeightSyncClient) + + def test_http_client_is_protocol(self): + assert isinstance( + HTTPVLLMWeightSyncClient("http://localhost:8000"), VLLMWeightSyncClient + ) + + def test_ray_client_is_protocol(self): + assert isinstance(RayVLLMWeightSyncClient(MagicMock()), VLLMWeightSyncClient) + + def test_ray_client_sends_typed_requests(self, monkeypatch): + """Ray client must hand the actor typed Request objects, not raw dicts.""" + import ray + + monkeypatch.setattr(ray, "get", lambda refs: None) + handle = MagicMock() + client = RayVLLMWeightSyncClient(handle) + + client.init_weight_transfer_engine({"master_addr": "x"}) + (init_req,), _ = handle.init_weight_transfer_engine.remote.call_args + assert isinstance(init_req, WeightTransferInitRequest) + assert init_req.init_info == {"master_addr": "x"} + + client.update_weights({"names": ["w"]}) + (update_req,), _ = handle.update_weights.remote.call_args + assert isinstance(update_req, WeightTransferUpdateRequest) + assert update_req.update_info == {"names": ["w"]} + + client.finish_weight_update("step-42") + handle.finish_weight_update.remote.assert_called_once_with() + handle.update_weight_version.remote.assert_called_once_with("step-42") + + client.shutdown_weight_transfer_engine() + handle.shutdown_weight_transfer_engine.remote.assert_called_once_with() + + def test_http_client_pickles_ipc_handles_for_json(self, monkeypatch): + """HTTP update_weights must encode raw ipc_handles as a base64 pickle.""" + captured = {} + + def fake_post(self, path, json=None): + captured["path"] = path + captured["json"] = json + + monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post) + client = HTTPVLLMWeightSyncClient("http://localhost:8000") + client.update_weights({"names": ["w"], "ipc_handles": [{"gpu": ("args",)}]}) + sent = captured["json"]["update_info"] + assert "ipc_handles" not in sent + assert "ipc_handles_pickled" in sent + assert pickle.loads(base64.b64decode(sent["ipc_handles_pickled"])) == [ + {"gpu": ("args",)} + ] + + client.update_weights([{"ipc_handles": {"gpu": ("args",)}}, {"names": []}]) + sent = captured["json"]["update_info"] + assert sent[1] == {"names": []} + assert pickle.loads(base64.b64decode(sent[0]["ipc_handles_pickled"])) == { + "gpu": ("args",) + } + + def test_http_client_passes_through_nccl_update_info(self, monkeypatch): + """NCCL update_info has only JSON-native fields and passes unchanged.""" + captured = {} + + def fake_post(self, path, json=None): + captured["path"] = path + captured["json"] = json + + monkeypatch.setattr(HTTPVLLMWeightSyncClient, "_post", fake_post) + client = HTTPVLLMWeightSyncClient("http://localhost:8000") + update_info = {"names": ["w"], "dtype_names": ["float32"], "shapes": [[4]]} + client.update_weights(update_info) + assert captured["json"]["update_info"] == update_info + + client.finish_weight_update("step-42") + assert captured["json"] == {"weight_version": "step-42"} + + client.shutdown_weight_transfer_engine() + assert captured == {"path": "shutdown_weight_transfer_engine", "json": None} + + +class TestModuleSource: + """`ModuleSource` metadata vs. materialized iteration (dense, no GPU).""" + + def test_metadata_reads_shape_and_dtype(self): + source = ModuleSource( + _module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3))) + ) + meta = source.metadata() + assert [m.name for m in meta] == ["w", "b"] + assert [m.shape for m in meta] == [(2, 3), (3,)] + assert all(m.dtype == torch.float32 for m in meta) + + def test_iteration_yields_materialized_tensors(self): + w = torch.arange(6, dtype=torch.float32).reshape(2, 3) + source = ModuleSource(_module_with(("w", w))) + pairs = list(source) + assert [name for name, _ in pairs] == ["w"] + assert torch.equal(pairs[0][1], w) + + def test_source_is_reiterable(self): + source = ModuleSource(_module_with(("w", torch.zeros(2)))) + assert [n for n, _ in source] == [n for n, _ in source] == ["w"] + + def test_metadata_agrees_with_iteration(self): + """The two channels must line up element-for-element: engines declare + the round from `metadata()` and then send what iteration yields.""" + source = ModuleSource( + _module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3))) + ) + meta = source.metadata() + pairs = list(source) + assert [m.name for m in meta] == [name for name, _ in pairs] + assert [m.dtype for m in meta] == [t.dtype for _, t in pairs] + assert [m.shape for m in meta] == [tuple(t.shape) for _, t in pairs] + + +class TestWeightSourceGroupContract: + """`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what + backends gather and free by, so the default must agree with + `layerwise_groups` over `metadata()`, restricted to what this rank holds.""" + + class _Source(WeightSource): + """Minimal source over an ordered (name, tensor) list, optionally holding + only some names (in which case it iterates only their groups).""" + + def __init__(self, names, held=None, reverse=False): + self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)] + self._held = held + self._reverse = reverse + + def metadata(self): + return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs] + + def held_names(self): + return self._held + + def __iter__(self): + pairs = self._pairs + if self._held is not None: + keep = set(self._held) + pairs = [(n, t) for n, t in pairs if n in keep] + return iter(list(reversed(pairs)) if self._reverse else pairs) + + def _source(self, names, held=None, reverse=False): + return self._Source(names, held, reverse) + + def test_groups_defaults_to_the_layerwise_partition(self): + names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] + assert self._source(names).groups() == layerwise_groups(names) + + def test_groups_keeps_only_groups_holding_a_held_name(self): + names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] + held = ["model.layers.0.a", "model.layers.1.a"] + assert self._source(names, held=held).groups() == [ + ["model.layers.0.a"], + ["model.layers.1.a"], + ] + + def test_groups_keeps_a_partially_held_group_whole(self): + """One held name selects the WHOLE group, unheld members included. + + A group is the collective unit, so its membership has to be + rank-uniform: a rank that narrowed it to what it holds would disagree + with its peers about which names group *g* covers, and the gather would + desynchronize. Selection is per group, but only whole groups; the + per-name split is the backend's, via owner sets. This is the + foreign-expert shape under expert parallelism — a rank holds some + experts of a layer, not all of them. + """ + names = ["model.layers.0.a", "model.layers.0.b", "model.layers.1.a"] + held = ["model.layers.0.a"] + assert self._source(names, held=held).groups() == [ + ["model.layers.0.a", "model.layers.0.b"], + ] + + def test_groups_order_follows_metadata_not_the_declaration(self): + """The declaration is a SET of names; the groups it selects still come + out in metadata order, so each pairs with the right ``iter_groups()`` + batch however the source listed them.""" + names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] + source = self._source( + names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"] + ) + assert source.groups() == [["model.layers.0.a"], ["model.layers.1.a"]] + assert [ns for ns, _ in source.iter_groups()] == [ + ["model.layers.0.a"], + ["model.layers.1.a"], + ] + + def test_iter_groups_batches_the_stream_per_group(self): + names = ["embed.w", "model.layers.0.a", "model.layers.0.b", "norm.w"] + batches = list(self._source(names).iter_groups()) + assert [ns for ns, _ in batches] == [ + ["embed.w"], + ["model.layers.0.a", "model.layers.0.b"], + ["norm.w"], + ] + assert all(len(ns) == len(ts) for ns, ts in batches) + + def test_iter_groups_yields_the_tensors_iteration_produced(self): + names = ["model.layers.0.a", "model.layers.0.b"] + (batch,) = list(self._source(names).iter_groups()) + _names, tensors = batch + assert [float(t[0]) for t in tensors] == [0.0, 1.0] + + def test_iter_groups_yields_only_held_groups(self): + names = ["embed.w", "model.layers.0.a", "model.layers.1.a"] + batches = list(self._source(names, held=["model.layers.1.a"]).iter_groups()) + assert [ns for ns, _ in batches] == [["model.layers.1.a"]] + + def test_out_of_order_iteration_raises(self): + """Materializing is usually a collective, so a rank that iterates out of + order deadlocks its peers -- fail loudly instead.""" + source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True) + with pytest.raises(RuntimeError, match="iteration order must match"): + list(source.iter_groups()) + + def test_a_source_may_override_iter_groups(self): + """The extension point: materialize a whole group in one step instead of + one generator resume per tensor.""" + calls = [] + + class _Batched(TestWeightSourceGroupContract._Source): + def iter_groups(self): + for group in self.groups(): + calls.append(len(group)) + yield group, [torch.zeros(2) for _ in group] + + source = _Batched(["model.layers.0.a", "model.layers.0.b"]) + assert [ns for ns, _ in source.iter_groups()] == [ + ["model.layers.0.a", "model.layers.0.b"] + ] + assert calls == [2] + + +class TestDeferredProcessingContract: + """`defers_processing` and `drain_pending` are two halves of one contract: a + caller that takes over the update tail (running its own + `finalize_layerwise_reload` instead of going through `finish_weight_update`) + reads the flag and calls the method. Both must be answerable on any engine, or + that caller ends up reaching through a getattr.""" + + def _engines(self): + registry = dict(WeightTransferEngineFactory._registry) + # ModelExpress is an optional, separately installed package, but its + # backend is always registered. Skip it when the package is missing. + if importlib.util.find_spec("modelexpress") is None: + registry.pop("modelexpress") + return {name: loader() for name, loader in registry.items()} + + def test_every_engine_declares_whether_it_defers(self): + for name, cls in self._engines().items(): + assert isinstance(cls.defers_processing, bool), name + + def test_every_engine_can_be_drained(self): + """The default is a no-op, so a caller never has to check whether the + method exists before calling it.""" + for name, cls in self._engines().items(): + assert callable(cls.drain_pending), name + + def test_the_default_is_not_to_defer(self): + assert WeightTransferEngine.defers_processing is False + + def test_a_synchronous_engine_drains_as_a_no_op(self): + engine = object.__new__(WeightTransferEngineFactory._registry["nccl"]()) + engine.drain_pending() # must not raise, and must not need any state + + def test_the_rdt_engine_defers_and_overrides_the_drain(self): + """The one engine the contract exists for.""" + cls = WeightTransferEngineFactory._registry["sharded_rdt"]() + assert cls.defers_processing is True + assert cls.drain_pending is not WeightTransferEngine.drain_pending + + +class TestTrainerFactory: + """WeightTransferTrainerFactory registry mechanics.""" + + def test_registry_has_all_backends(self): + assert "nccl" in WeightTransferTrainerFactory._registry + assert "ipc" in WeightTransferTrainerFactory._registry + assert "sparse_nccl" in WeightTransferTrainerFactory._registry + + def test_register_and_dispatch(self): + saved = dict(WeightTransferTrainerFactory._registry) + try: + WeightTransferTrainerFactory.register_engine("dummy", _DummyTrainerEngine) + engine = WeightTransferTrainerFactory.trainer_init( + MagicMock(backend="dummy"), # backend read from the init info + client=RecordingClient(), + source=ModuleSource(_module_with(("w", torch.zeros(2)))), + ) + assert isinstance(engine, _DummyTrainerEngine) + with pytest.raises(ValueError, match="already registered"): + WeightTransferTrainerFactory.register_engine( + "dummy", _DummyTrainerEngine + ) + finally: + WeightTransferTrainerFactory._registry = saved + + def test_unknown_backend_raises(self): + with pytest.raises(ValueError, match="Invalid weight transfer backend"): + WeightTransferTrainerFactory.trainer_init( + MagicMock(backend="nope"), + client=RecordingClient(), + source=ModuleSource(_module_with(("w", torch.zeros(2)))), + ) + + def test_ipc_init_info_declares_backend(self): + assert IPCTrainerInitInfo.backend == "ipc" + + def test_nccl_init_info_declares_backend(self): + assert NCCLTrainerInitInfo.backend == "nccl" + + def test_sparse_nccl_init_info_declares_backend(self): + assert SparseNCCLTrainerInitInfo.backend == "sparse_nccl" + + def test_trainer_init_info_subclass_must_set_backend(self): + with pytest.raises(TypeError, match="class-level `backend`"): + + class _NoBackend(TrainerInitInfo): + pass + + +class TestTrainerEngineBase: + """Base-class construction (no GPU).""" + + def test_source_stored_and_sender_by_default(self): + engine = _DummyTrainerEngine( + client=RecordingClient(), + source=ModuleSource(_module_with(("w", torch.zeros(2)))), + ) + assert engine.is_sender is True + assert [name for name, _ in engine.source] == ["w"] + + def test_shutdown_default_is_noop(self): + engine = _DummyTrainerEngine( + client=RecordingClient(), + source=ModuleSource(_module_with(("w", torch.zeros(2)))), + is_sender=False, + ) + assert engine.is_sender is False + engine.shutdown() # must not raise + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 1, + reason="Need at least 1 GPU (CUDA IPC handles).", +) +def test_ipc_trainer_send_weights_drives_client_in_order(): + """send_weights issues start -> update -> finish and ships per-round metadata; + the packed wire param rides the init info, not the per-round update_info.""" + client = RecordingClient() + engine = IPCTrainerWeightTransferEngine( + client=client, + source=ModuleSource(_module_with(("w", torch.ones(4, device="cuda")))), + packed=False, + ) + + engine.send_weights() + + assert client.order == ["start", "update", "finish"] + assert client.last_update_info is not None + assert client.last_update_info["names"] == ["w"] + assert client.last_update_info["shapes"] == [[4]] + assert "packed" not in client.last_update_info + + +def test_ipc_trainer_init_ships_packed_to_worker(): + """trainer_init drives the inference-side init handshake and propagates the + must-agree `packed` flag to the worker.""" + if torch.accelerator.device_count() < 1: + pytest.skip("Need at least 1 GPU (CUDA IPC handles).") + + client = RecordingClient() + engine = WeightTransferTrainerFactory.trainer_init( + init_info=IPCTrainerInitInfo(rank=0, packed=True), # backend from init info + client=client, + source=ModuleSource(_module_with(("w", torch.ones(4, device="cuda")))), + ) + + assert isinstance(engine, IPCTrainerWeightTransferEngine) + assert engine.is_sender is True + assert engine.packed is True + assert client.order == ["init"] + assert client.last_init_info == {"packed": True} + + +def test_nccl_trainer_init_ships_worker_init_info(monkeypatch): + """The sender's trainer_init drives the inference-side init handshake with + the worker-shaped init info (rank_offset=1) while opening its own endpoint, + and propagates the must-agree wire params to the worker.""" + import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod + + # Bypass the real NCCL rendezvous. + monkeypatch.setattr( + nccl_engine_mod, "open_trainer_endpoint", lambda info: MagicMock() + ) + + client = RecordingClient() + engine = WeightTransferTrainerFactory.trainer_init( + init_info=NCCLTrainerInitInfo( + master_address="127.0.0.1", + master_port=29500, + world_size=3, + rank=0, + packed=True, + packed_buffer_size_bytes=1024, + packed_num_buffers=3, + ), + client=client, + source=ModuleSource(_module_with(("w", torch.zeros(4)))), + ) + + assert isinstance(engine, NCCLTrainerWeightTransferEngine) + assert engine.is_sender is True + assert engine.packed is True + assert client.order == ["init"] + assert client.last_init_info == { + "master_address": "127.0.0.1", + "master_port": 29500, + "rank_offset": 1, + "world_size": 3, + "packed": True, + "packed_buffer_size_bytes": 1024, + "packed_num_buffers": 3, + } + + +def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch): + """The worker engine reads packed + buffer geometry from the + trainer-supplied init info at the handshake, not from the config or the + per-round update info.""" + import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod + + monkeypatch.setattr( + nccl_engine_mod, "worker_init_process_group", lambda info, pc: MagicMock() + ) + + engine = NCCLWeightTransferEngine( + WeightTransferConfig(backend="nccl"), + create_mock_vllm_config(), + torch.device("cuda:0"), + MagicMock(spec=torch.nn.Module), + ) + assert engine.packed is False # pre-handshake default (legacy unpacked) + engine.init_transfer_engine( + NCCLWeightTransferInitInfo( + master_address="127.0.0.1", + master_port=29500, + rank_offset=1, + world_size=2, + packed=True, + packed_buffer_size_bytes=2048, + packed_num_buffers=4, + ) + ) + + assert engine.packed is True + assert engine.packed_buffer_size_bytes == 2048 + assert engine.packed_num_buffers == 4 + + +def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client(): + """Non-sender trainer ranks build an engine without opening an endpoint or + touching the client; they only join the collectives in send_weights.""" + client = RecordingClient() + engine = WeightTransferTrainerFactory.trainer_init( + init_info=NCCLTrainerInitInfo( + master_address="127.0.0.1", + master_port=29500, + world_size=3, + rank=1, + ), + client=client, + source=ModuleSource(_module_with(("w", torch.zeros(4)))), + ) + + assert engine.is_sender is False + assert engine.model_update_group is None + assert client.order == [] + + # send_weights on a non-sender only iterates the source (packed mode needs + # no CUDA stream on non-senders), never the client. + engine.send_weights() + assert client.order == [] + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 1, + reason="Need at least 1 GPU (NCCL broadcast / CUDA stream).", +) +def test_nccl_trainer_send_weights_drives_client_in_order(): + """send_weights issues start -> update -> finish and ships per-round + metadata; the packed wire params ride the init handshake, not the + per-round update_info.""" + client = RecordingClient() + engine = NCCLTrainerWeightTransferEngine( + client=client, + source=ModuleSource(_module_with(("w", torch.zeros(4, device="cuda")))), + packed=False, + ) + # Bypass the real NCCL rendezvous; broadcast is a no-op. + engine.model_update_group = MagicMock() + + engine.send_weights() + + assert client.order == ["start", "update", "finish"] + assert client.last_update_info is not None + assert client.last_update_info["names"] == ["w"] + assert client.last_update_info["shapes"] == [[4]] + assert "packed" not in client.last_update_info + + +class _ScriptedSource(WeightSource): + """Declares `meta` but yields whatever `pairs` says — used to drive the + metadata/iteration agreement checks.""" + + def __init__(self, meta, pairs): + self._meta = meta + self._pairs = pairs + + def metadata(self): + return list(self._meta) + + def __iter__(self): + yield from self._pairs + + +def _mock_group_engine(source, monkeypatch, **kwargs): + """Unpacked trainer engine with a mocked group and stream (no GPU needed).""" + engine = NCCLTrainerWeightTransferEngine( + client=RecordingClient(), source=source, packed=False, **kwargs + ) + engine.model_update_group = MagicMock() + monkeypatch.setattr(torch.cuda, "current_stream", MagicMock()) + return engine + + +def test_nccl_trainer_init_requires_source(): + """NCCL is a full-resync backend: it cannot run without a WeightSource.""" + with pytest.raises(ValueError, match="requires a WeightSource"): + NCCLTrainerWeightTransferEngine.trainer_init( + NCCLTrainerInitInfo( + master_address="127.0.0.1", master_port=29500, world_size=2, rank=0 + ), + client=RecordingClient(), + ) + + +def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch): + """The worker sizes its buffers (and cuts packed chunks) from metadata(), so + iteration disagreeing with it must raise rather than corrupt the stream.""" + meta = [ + ParamMeta("w", torch.float32, (4,)), + ParamMeta("b", torch.float32, (2,)), + ] + reordered = [("b", torch.zeros(2)), ("w", torch.zeros(4))] + engine = _mock_group_engine(_ScriptedSource(meta, reordered), monkeypatch) + + with pytest.raises(ValueError, match="disagrees with iteration at index 0"): + engine.send_weights() + + +def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch): + """A source that declares one wire dtype and materializes another would make + the two sides disagree on every byte offset.""" + meta = [ParamMeta("w", torch.float32, (4,))] + engine = _mock_group_engine( + _ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]), + monkeypatch, + ) + + with pytest.raises(ValueError, match="disagrees with iteration"): + engine.send_weights() + + +def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch): + """Yielding fewer parameters than declared leaves the worker waiting.""" + meta = [ + ParamMeta("w", torch.float32, (4,)), + ParamMeta("b", torch.float32, (2,)), + ] + engine = _mock_group_engine( + _ScriptedSource(meta, [("w", torch.zeros(4))]), monkeypatch + ) + + with pytest.raises(ValueError, match="yielded 1 parameters"): + engine.send_weights() + + +def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch): + """NCCL sends numel elements from data_ptr(), so a non-contiguous view must + be linearized first or the worker receives unrelated memory.""" + base = torch.arange(6, dtype=torch.float32).reshape(2, 3) + view = base.t() # non-contiguous + meta = [ParamMeta("w", torch.float32, tuple(view.shape))] + engine = _mock_group_engine(_ScriptedSource(meta, [("w", view)]), monkeypatch) + + engine.send_weights() + + sent = engine.model_update_group.broadcast.call_args.args[0] + assert sent.is_contiguous() + assert torch.equal(sent, view) + + +def test_nccl_trainer_send_weights_raises_instead_of_hanging(monkeypatch): + """A failed broadcast must surface even while the inference-side + update_weights is still blocked in its matching NCCL call. + + Joining the RPC thread there would deadlock: the worker only returns once + the broadcast it is waiting for arrives, which never happens. + """ + rpc_entered = threading.Event() + release_rpc = threading.Event() + + class _BlockingClient(RecordingClient): + def update_weights(self, update_info): + rpc_entered.set() + release_rpc.wait(timeout=30) # stands in for a wedged NCCL recv + super().update_weights(update_info) + + class _FailingSource(WeightSource): + def metadata(self): + return [ParamMeta("w", torch.float32, (4,))] + + def __iter__(self): + # Fail only once the RPC is provably in flight. + rpc_entered.wait(timeout=60) + raise RuntimeError("broadcast blew up") + + engine = NCCLTrainerWeightTransferEngine( + client=_BlockingClient(), source=_FailingSource(), packed=False + ) + engine.model_update_group = MagicMock() + monkeypatch.setattr(torch.cuda, "current_stream", MagicMock()) + + started = time.perf_counter() + try: + with pytest.raises(RuntimeError, match="broadcast blew up"): + engine.send_weights() + elapsed = time.perf_counter() - started + assert rpc_entered.is_set(), "the RPC was never in flight" + assert not release_rpc.is_set(), "send_weights waited for the wedged RPC" + # Regression guard: joining the RPC thread would park here until the + # client's own timeout expires instead of raising immediately. + assert elapsed < 5.0, f"send_weights blocked {elapsed:.1f}s on the RPC" + finally: + release_rpc.set() + + +def _sparse_patch(device: str = "cpu") -> SparseWeightPatch: + return SparseWeightPatch( + name="w", + indices=torch.tensor([1, 3], dtype=torch.int32, device=device), + values=torch.tensor([1.0, 2.0], dtype=torch.float32, device=device), + full_shape=(4, 4), + ) + + +def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch): + """The sender's trainer_init drives the init handshake with the + worker-shaped init info; sparse ships no packed wire params, so the worker + keeps its unpacked defaults. Sparse takes no `source`.""" + import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod + + monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock()) + + client = RecordingClient() + engine = WeightTransferTrainerFactory.trainer_init( + init_info=SparseNCCLTrainerInitInfo( + master_address="127.0.0.1", + master_port=29500, + world_size=2, + rank=0, + ), + client=client, + ) + + assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine) + assert client.order == ["init"] + assert client.last_init_info == { + "master_address": "127.0.0.1", + "master_port": 29500, + "rank_offset": 1, + "world_size": 2, + "packed": False, + "packed_buffer_size_bytes": DEFAULT_PACKED_BUFFER_SIZE_BYTES, + "packed_num_buffers": DEFAULT_PACKED_NUM_BUFFERS, + } + + +def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch): + """One-shot send drives start, update, and finish in order.""" + client = RecordingClient() + engine = SparseNCCLTrainerWeightTransferEngine(client=client) + engine.model_update_group = MagicMock() + engine.model_update_group.device = torch.device("cpu") + # The group is a mock, so the stream is just a handle it is handed (and a + # handle _post_send_sync can synchronize). + monkeypatch.setattr(torch.cuda, "current_stream", MagicMock()) + + engine.send_weights([_sparse_patch()]) + + assert client.order == ["start", "update", "finish"] + assert client.last_update_info is not None + assert client.last_update_info["names"] == ["w"] + assert client.last_update_info["shapes"] == [[4, 4]] + assert client.last_update_info["num_updates_list"] == [2] + # One broadcast for indices + one for values per patch. + assert engine.model_update_group.broadcast.call_count == 2 + engine.shutdown() + + +def test_sparse_nccl_trainer_send_weights_empty_round_is_noop(): + """A round with no patches must not touch the client (an empty sparse + update info is invalid by construction).""" + client = RecordingClient() + engine = SparseNCCLTrainerWeightTransferEngine(client=client) + engine.model_update_group = MagicMock() + + engine.send_weights([]) + engine.send_weights() # no argument is also a no-op round + engine.send_weight_chunk([]) + engine.send_weight_chunk() + + assert client.order == [] + + +def test_sparse_nccl_trainer_send_weights_requires_full_shape(): + patch = _sparse_patch() + patch.full_shape = None + engine = SparseNCCLTrainerWeightTransferEngine(client=RecordingClient()) + engine.model_update_group = MagicMock() + engine.model_update_group.device = torch.device("cpu") + + with pytest.raises(ValueError, match="full_shape"): + engine.send_weights([patch]) + + +def test_sparse_nccl_trainer_rejects_source(): + """Sparse is a delta backend; a WeightSource would silently never be sent.""" + with pytest.raises(ValueError, match="takes no WeightSource"): + SparseNCCLTrainerWeightTransferEngine( + client=RecordingClient(), + source=ModuleSource(_module_with(("w", torch.zeros(2)))), + ) + + +def test_sparse_nccl_trainer_validates_patch_before_any_rpc(): + """Malformed patches must fail on the trainer, before start_weight_update: + the worker's own checks only run once the broadcasts are already in flight, + where a size mismatch wedges both sides instead of raising.""" + client = RecordingClient() + engine = SparseNCCLTrainerWeightTransferEngine(client=client) + engine.model_update_group = MagicMock() + engine.model_update_group.device = torch.device("cpu") + + mismatched = SparseWeightPatch( + name="w", + indices=torch.tensor([1, 3], dtype=torch.int32), + values=torch.tensor([1.0], dtype=torch.float32), + full_shape=(4, 4), + ) + with pytest.raises(ValueError, match="matching lengths"): + engine.send_weights([mismatched]) + + wrong_index_dtype = SparseWeightPatch( + name="w", + indices=torch.tensor([1, 3], dtype=torch.int64), + values=torch.tensor([1.0, 2.0], dtype=torch.float32), + full_shape=(4, 4), + ) + with pytest.raises(ValueError, match="int32 indices"): + engine.send_weights([wrong_index_dtype]) + + assert client.order == [] + + +def test_sparse_nccl_trainer_non_sender_skips_client(): + client = RecordingClient() + engine = WeightTransferTrainerFactory.trainer_init( + init_info=SparseNCCLTrainerInitInfo( + master_address="127.0.0.1", + master_port=29500, + world_size=2, + rank=1, + ), + client=client, + ) + + assert engine.is_sender is False + assert engine.model_update_group is None + assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine) + engine.send_weights([_sparse_patch()]) + engine.send_weight_chunk([_sparse_patch()]) + assert client.order == [] + + +# --- Optional ModelExpress Client Lifecycle --- + + +def _import_modelexpress_shim(): + spec = importlib.util.find_spec( + "vllm.distributed.weight_transfer.modelexpress_engine" + ) + assert spec is not None and spec.origin is not None + runpy.run_path(spec.origin, run_name="_test_modelexpress_shim") + + +@pytest.mark.parametrize( + "missing_module", + [ + "modelexpress", + "modelexpress_rl", + "modelexpress_rl.inference", + "modelexpress_rl.inference.engines", + "modelexpress_rl.inference.engines.vllm", + "modelexpress_rl.inference.engines.vllm.weight_transfer_engine", + ], +) +def test_modelexpress_missing_backend_explains_source_install( + monkeypatch, missing_module +): + original_import = builtins.__import__ + error = ModuleNotFoundError(name=missing_module) + + def import_without_backend(name, *args, **kwargs): + if name == "modelexpress": + raise error + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_backend) + with pytest.raises(ImportError, match="modelexpress_client/python") as exc_info: + _import_modelexpress_shim() + assert exc_info.value.__cause__ is error + assert "uv pip install" in str(exc_info.value) + + +@pytest.mark.parametrize("missing_module", ["grpc", "modelexpress.internal_dependency"]) +def test_modelexpress_preserves_transitive_import_errors(monkeypatch, missing_module): + original_import = builtins.__import__ + error = ModuleNotFoundError(name=missing_module) + + def import_without_dependency(name, *args, **kwargs): + if name == "modelexpress": + raise error + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_dependency) + with pytest.raises(ModuleNotFoundError) as exc_info: + _import_modelexpress_shim() + assert exc_info.value is error + + +@pytest.fixture +def mx_client(monkeypatch): + client_module = pytest.importorskip("modelexpress_rl.inference.client") + client = MagicMock() + monkeypatch.setattr( + client_module.ModelExpressGeneratorClient, + "initialize", + MagicMock(return_value=client), + ) + monkeypatch.setattr(torch.accelerator, "synchronize", MagicMock()) + return client + + +def _make_modelexpress_engine(): + return WeightTransferEngineFactory.create_engine( + WeightTransferConfig(backend="modelexpress"), + SimpleNamespace( + parallel_config=SimpleNamespace(), + model_config=SimpleNamespace(model="test/model"), + ), + torch.device("cpu"), + torch.nn.Linear(2, 2), + ) + + +@pytest.fixture +def mx_worker(mx_client, monkeypatch): + from vllm.v1.worker import gpu_worker + + monkeypatch.setattr(gpu_worker, "set_current_vllm_config", lambda _: nullcontext()) + worker = object.__new__(gpu_worker.Worker) + worker.weight_transfer_engine = _make_modelexpress_engine() + worker.vllm_config = worker.weight_transfer_engine.vllm_config + worker._weight_update_active = False + worker._weight_update_is_draft = False + worker.model_runner = MagicMock() + return worker + + +def test_modelexpress_worker_rejects_updates_before_initialization( + mx_worker, mx_client +): + with pytest.raises(RuntimeError, match="not initialized"): + mx_worker.start_weight_update() + with pytest.raises(RuntimeError, match="start_weight_update must be called"): + mx_worker.update_weights({"version_id": "version-a"}) + with pytest.raises(RuntimeError, match="without a matching"): + mx_worker.finish_weight_update() + mx_client.stage_weight.assert_not_called() + + +@pytest.mark.parametrize( + "second_update, error_type", + [ + ({"version_id": "version-a"}, RuntimeError), + ({"version_id": "version-b"}, RuntimeError), + ({"version_id": ""}, ValueError), + ({"version_id": None}, ValueError), + ({}, ValueError), + ], +) +def test_modelexpress_worker_recovers_after_rejected_update( + mx_worker, mx_client, second_update, error_type +): + """A rejected payload must release A so a fresh session can actually install B.""" + mx_worker.weight_transfer_engine.init_transfer_engine( + mx_worker.weight_transfer_engine.parse_init_info({}) + ) + staged_a = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock()) + staged_b = SimpleNamespace(version_id="version-b", metrics={}, release=MagicMock()) + mx_client.stage_weight.side_effect = [staged_a, staged_b] + mx_client.apply_weight.return_value = {} + mx_worker.start_weight_update() + mx_worker.update_weights({"version_id": "version-a"}) + + with pytest.raises(error_type): + mx_worker.update_weights(second_update) + staged_a.release.assert_called_once_with() + mx_client.apply_weight.assert_called_once_with(staged_a) + + mx_worker.start_weight_update() + mx_worker.update_weights({"version_id": "version-b"}) + mx_worker.finish_weight_update() + + assert [ + call.kwargs["version"].version_id + for call in mx_client.stage_weight.call_args_list + ] == ["version-a", "version-b"] + mx_client.apply_weight.assert_called_with(staged_b) + staged_b.release.assert_called_once_with() + + +def test_modelexpress_worker_can_retry_failed_finish(mx_worker, mx_client): + mx_worker.weight_transfer_engine.init_transfer_engine( + mx_worker.weight_transfer_engine.parse_init_info({}) + ) + staged = mx_client.stage_weight.return_value + staged.release.side_effect = [RuntimeError("release failed"), None] + mx_worker.start_weight_update() + mx_worker.update_weights({"version_id": "version-a"}) + + with pytest.raises(RuntimeError, match="release failed"): + mx_worker.finish_weight_update() + mx_worker.finish_weight_update() + + assert staged.release.call_count == 2 + assert not mx_worker._weight_update_active + mx_worker.start_weight_update() + + +def test_native_engine_applies_and_releases_exact_versions(mx_client): + from modelexpress_rl.inference.engines.vllm import ( + weight_transfer_engine as mx_engine, + ) + + from vllm.distributed.weight_transfer import modelexpress_engine + + engine = _make_modelexpress_engine() + assert type(engine) is mx_engine.ModelExpressWeightTransferEngine + assert ( + modelexpress_engine.ModelExpressWeightTransferInitInfo + is mx_engine.ModelExpressWeightTransferInitInfo + ) + assert ( + modelexpress_engine.ModelExpressWeightTransferUpdateInfo + is mx_engine.ModelExpressWeightTransferUpdateInfo + ) + assert not engine.supports_draft_weight_update + engine.init_transfer_engine(engine.parse_init_info({})) + + for version_id in ("version-a", "version-b"): + staged = SimpleNamespace(version_id=version_id, metrics={}, release=MagicMock()) + mx_client.stage_weight.return_value = staged + mx_client.apply_weight.return_value = {} + engine.start_weight_update() + engine.update_weights({"version_id": version_id}) + + assert ( + mx_client.stage_weight.call_args.kwargs["version"].version_id == version_id + ) + mx_client.apply_weight.assert_called_with(staged) + torch.accelerator.synchronize.assert_called() + staged.release.assert_not_called() + engine.finish_weight_update() + staged.release.assert_called_once_with() + + engine.shutdown() + engine.shutdown() + mx_client.close.assert_called_once_with() + + +def test_vime_init_config_reaches_mx_client(mx_client): + from modelexpress_rl.inference.client import ModelExpressGeneratorClient + + engine = _make_modelexpress_engine() + ModelExpressGeneratorClient.initialize.assert_not_called() + engine.init_transfer_engine( + engine.parse_init_info( + { + "model_name": "policy", + "server_url": "mx:8001", + "initial_serving_version_id": "serving-a", + "object_storage_type": "S3", + "initial_base_version_id": "base-a", + "seed_checkpoint_path": "/models/launch", + "refit_checkpoint_dir": "/mxdelta/receiver", + "refit_checkpoint_max_size_gb": 200, + "object_storage_endpoint_url": "http://minio:9000", + "object_storage_region_name": "us-west-2", + "registration_ttl_seconds": 90, + "lease_ttl_seconds": 60, + "max_transfer_attempts": 4, + "max_replay_chain_length": 17, + "rpc_timeout_seconds": 12.5, + } + ) + ) + config = ModelExpressGeneratorClient.initialize.call_args.args[0] + assert config.engine_context.model is engine.model + assert config.engine_context.vllm_config is engine.vllm_config + assert config.model_name == "policy" + assert config.server_url == "mx:8001" + assert config.initial_serving_version_id == "serving-a" + assert config.registration_ttl_seconds == 90 + assert config.lease_ttl_seconds == 60 + assert config.max_transfer_attempts == 4 + assert config.max_replay_chain_length == 17 + assert config.rpc_timeout_seconds == 12.5 + storage = config.object_storage + assert storage.storage_type.value == "S3" + assert storage.initial_base_version_id == "base-a" + assert storage.seed_checkpoint_path == "/models/launch" + assert storage.refit_checkpoint_dir == "/mxdelta/receiver" + assert storage.refit_checkpoint_max_size_gb == 200 + assert storage.endpoint_url == "http://minio:9000" + assert storage.region_name == "us-west-2" + + +@pytest.mark.parametrize( + "key", + [ + "object_storage_type", + "initial_base_version_id", + "seed_checkpoint_path", + "refit_checkpoint_dir", + "object_storage_endpoint_url", + "object_storage_region_name", + ], +) +def test_incomplete_storage_config_rejected_before_init(mx_client, key): + from modelexpress_rl.inference.client import ModelExpressGeneratorClient + + engine = _make_modelexpress_engine() + with pytest.raises(ValueError, match="object storage requires"): + engine.init_transfer_engine(engine.parse_init_info({key: "value"})) + ModelExpressGeneratorClient.initialize.assert_not_called() + + +@pytest.mark.parametrize("version_id", ["", " ", None, 1]) +def test_invalid_version_rejected_before_staging(mx_client, version_id): + engine = _make_modelexpress_engine() + engine.init_transfer_engine(engine.parse_init_info({})) + engine.start_weight_update() + with pytest.raises(ValueError, match="version_id is required"): + engine.update_weights({"version_id": version_id}) + mx_client.stage_weight.assert_not_called() + + +@pytest.mark.parametrize("failure", ["stage", "apply", "release"]) +def test_update_failure_preserves_error_and_releases_handle(mx_client, failure): + engine = _make_modelexpress_engine() + engine.init_transfer_engine(engine.parse_init_info({})) + staged = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock()) + mx_client.stage_weight.return_value = staged + if failure == "stage": + mx_client.stage_weight.side_effect = RuntimeError("stage failed") + else: + mx_client.apply_weight.side_effect = RuntimeError("apply failed") + if failure == "release": + staged.release.side_effect = RuntimeError("release failed") + engine.start_weight_update() + with pytest.raises(RuntimeError, match="stage failed|apply failed"): + engine.update_weights({"version_id": "version-a"}) + assert staged.release.call_count == (0 if failure == "stage" else 1) + torch.accelerator.synchronize.assert_not_called() + engine.shutdown() + mx_client.close.assert_called_once_with() + + +def test_plugin_does_not_replace_native_backend(mx_client): + from modelexpress.engines.vllm.registration import ( + register_plugin_weight_transfer_engine, + ) + from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import ( + ModelExpressWeightTransferEngine, + ) + + native_loader = WeightTransferEngineFactory._registry["modelexpress"] + register_plugin_weight_transfer_engine() + assert WeightTransferEngineFactory._registry["modelexpress"] is native_loader + assert type(_make_modelexpress_engine()) is ModelExpressWeightTransferEngine diff --git a/tests/entrypoints/launchers/test_run_batch.py b/tests/entrypoints/launchers/test_run_batch.py index 1376eb0645..31f9239c8d 100644 --- a/tests/entrypoints/launchers/test_run_batch.py +++ b/tests/entrypoints/launchers/test_run_batch.py @@ -16,12 +16,9 @@ from vllm.assets.audio import AudioAsset from vllm.connections import HTTPConnection from vllm.entrypoints.launchers.run_batch import ( BatchRequestOutput, - BatchTranscriptionRequest, download_bytes_from_url, - make_transcription_wrapper, upload_data, ) -from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import VLLMValidationError from vllm.utils.mem_constants import MiB_bytes @@ -644,122 +641,10 @@ def test_reasoning_parser(): assert len(reasoning) > 0 -def test_transcription(): - with ( - tempfile.NamedTemporaryFile("w") as input_file, - tempfile.NamedTemporaryFile("r") as output_file, - ): - input_file.write(INPUT_TRANSCRIPTION_BATCH) - input_file.flush() - proc = subprocess.Popen( - [ - "vllm", - "run-batch", - "-i", - input_file.name, - "-o", - output_file.name, - "--model", - SPEECH_LARGE_MODEL_NAME, - ], - ) - proc.communicate() - proc.wait() - assert proc.returncode == 0, f"{proc=}" - - contents = output_file.read() - print(f"\n\ncontents: {contents}\n\n") - for line in contents.strip().split("\n"): - BatchRequestOutput.model_validate_json(line) - - line_dict = json.loads(line) - assert isinstance(line_dict, dict) - assert line_dict["error"] is None - - response_body = line_dict["response"]["body"] - assert response_body is not None - assert "text" in response_body - assert "usage" in response_body -def test_transcription_http_url(): - with ( - tempfile.NamedTemporaryFile("w") as input_file, - tempfile.NamedTemporaryFile("r") as output_file, - ): - input_file.write(INPUT_TRANSCRIPTION_HTTP_BATCH) - input_file.flush() - proc = subprocess.Popen( - [ - "vllm", - "run-batch", - "-i", - input_file.name, - "-o", - output_file.name, - "--model", - SPEECH_LARGE_MODEL_NAME, - ], - ) - proc.communicate() - proc.wait() - assert proc.returncode == 0, f"{proc=}" - - contents = output_file.read() - for line in contents.strip().split("\n"): - BatchRequestOutput.model_validate_json(line) - - line_dict = json.loads(line) - assert isinstance(line_dict, dict) - assert line_dict["error"] is None - - response_body = line_dict["response"]["body"] - assert response_body is not None - assert "text" in response_body - assert "usage" in response_body - - transcription_text = response_body["text"] - assert "Mary had a little lamb" in transcription_text -def test_translation(): - with ( - tempfile.NamedTemporaryFile("w") as input_file, - tempfile.NamedTemporaryFile("r") as output_file, - ): - input_file.write(INPUT_TRANSLATION_BATCH) - input_file.flush() - proc = subprocess.Popen( - [ - "vllm", - "run-batch", - "-i", - input_file.name, - "-o", - output_file.name, - "--model", - SPEECH_SMALL_MODEL_NAME, - ], - ) - proc.communicate() - proc.wait() - assert proc.returncode == 0, f"{proc=}" - - contents = output_file.read() - for line in contents.strip().split("\n"): - BatchRequestOutput.model_validate_json(line) - - line_dict = json.loads(line) - assert isinstance(line_dict, dict) - assert line_dict["error"] is None - - response_body = line_dict["response"]["body"] - assert response_body is not None - assert "text" in response_body - - translation_text = response_body["text"] - translation_text_lower = str(translation_text).strip().lower() - assert "mary" in translation_text_lower or "lamb" in translation_text_lower def test_tool_calling(): @@ -974,28 +859,6 @@ async def test_download_bytes_backslash_bypass(): ) -@pytest.mark.asyncio -async def test_transcription_wrapper_rejects_oversized_data_url_before_decode( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) - handler = AsyncMock() - wrapped_handler = make_transcription_wrapper(is_translation=False)(handler) - request = BatchTranscriptionRequest.model_validate( - { - "model": SPEECH_LARGE_MODEL_NAME, - "file_url": f"data:audio/wav;base64,{_OVERSIZED_BASE64_AUDIO}", - "response_format": "json", - } - ) - - with patch("vllm.entrypoints.launchers.run_batch.base64.b64decode") as decode: - response = await wrapped_handler(request) - - assert isinstance(response, ErrorResponse) - assert "Maximum file size exceeded" in response.error.message - decode.assert_not_called() - handler.assert_not_awaited() @pytest.mark.asyncio @@ -1019,35 +882,6 @@ async def test_download_bytes_allows_http_body_at_audio_limit( await _close_async_connection(connection) -@pytest.mark.asyncio -async def test_transcription_wrapper_rejects_oversized_http_before_handler( - monkeypatch: pytest.MonkeyPatch, - batch_http_server: _BatchHTTPServer, -): - monkeypatch.setattr(envs, "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", 1) - connection = HTTPConnection() - monkeypatch.setattr( - "vllm.entrypoints.launchers.run_batch.global_http_connection", - connection, - ) - handler = AsyncMock() - wrapped_handler = make_transcription_wrapper(is_translation=False)(handler) - request = BatchTranscriptionRequest.model_validate( - { - "model": SPEECH_LARGE_MODEL_NAME, - "file_url": _batch_http_url(batch_http_server, "/oversized-chunked"), - "response_format": "json", - } - ) - - try: - response = await wrapped_handler(request) - finally: - await _close_async_connection(connection) - - assert isinstance(response, ErrorResponse) - assert "Maximum file size exceeded" in response.error.message - handler.assert_not_awaited() # --------------------------------------------------------------------------- diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 48b9dd5099..f367b0f661 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -11,7 +11,6 @@ import pytest import pytest_asyncio from transformers import AutoTokenizer -import vllm.envs as envs from tests.utils import RemoteOpenAIServer from vllm.config import ModelConfig from vllm.v1.engine.detokenizer import check_stop_strings @@ -140,7 +139,7 @@ async def test_generate_rejects_min_tokens_above_filled_max_tokens(client): @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="sampling mask output is not supported by the Rust frontend", ) @pytest.mark.parametrize( @@ -218,7 +217,7 @@ async def test_generate_defaults_max_tokens_when_omitted(client): @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="parallel sampling (n > 1) is not supported by the Rust frontend", ) async def test_generate_returns_all_choices_when_n_greater_than_one(client): @@ -485,7 +484,7 @@ async def test_stop_string_workflow(client, tokenizer, messages): @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="--tokens-only is not supported by the Rust frontend", ) @pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True) @@ -617,7 +616,7 @@ async def _render_then_generate(client, body) -> tuple[dict, list[int]]: # either reasoning field is dropped between render and generate. @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="the Rust frontend does not forward reasoning fields to the engine", ) @pytest.mark.parametrize( @@ -638,7 +637,7 @@ async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, mes @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="the Rust frontend does not forward reasoning fields to the engine", ) @pytest.mark.parametrize( @@ -679,7 +678,7 @@ def _text_mode_payload(token_ids, **sampling_params) -> dict: @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="output_mode is not supported by the Rust frontend", ) async def test_text_mode_matches_chat_completions(client, tokenizer, messages): @@ -707,7 +706,7 @@ async def test_text_mode_matches_chat_completions(client, tokenizer, messages): @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="output_mode is not supported by the Rust frontend", ) async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages): @@ -734,7 +733,7 @@ async def test_text_mode_stream_matches_non_stream(client, tokenizer, messages): @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="output_mode is not supported by the Rust frontend", ) @pytest.mark.parametrize( @@ -759,7 +758,7 @@ async def test_text_mode_invalid_requests_return_400(client, overrides, expected @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="--tokens-only is not supported by the Rust frontend", ) @pytest.mark.parametrize("server", [["--tokens-only"]], indirect=True) @@ -778,7 +777,7 @@ async def test_text_mode_rejected_when_tokens_only(client): @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="the Rust frontend does not serve /inference/v1/abort_requests", ) @pytest.mark.parametrize("output_mode", ["tokens", "text"]) @@ -825,7 +824,7 @@ async def test_stream_delivers_abort_finish_chunk( @pytest.mark.asyncio @pytest.mark.skipif( - envs.VLLM_USE_RUST_FRONTEND, + False, reason="the Rust frontend does not serve /inference/v1/abort_requests", ) async def test_abort_requests_is_served_without_tokens_only(client): diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py new file mode 100644 index 0000000000..4eec87945b --- /dev/null +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Isolated GSM8K evaluation script for vLLM serve endpoint.""" + +import argparse +import ast +import asyncio +import json +import os +import tempfile +import time +from collections.abc import Generator + +import aiohttp +import numpy as np +import regex as re +import requests +from tqdm.asyncio import tqdm + +from vllm.assets.base import VLLM_S3_BUCKET_URL + +INVALID = -9999999 + + +def download_and_cache_file(url: str, filename: str | None = None) -> str: + """Download and cache a file from a URL.""" + if filename is None: + filename = os.path.join(tempfile.gettempdir(), url.split("/")[-1]) + + if os.path.exists(filename): + return filename + + print(f"Downloading from {url} to {filename}") + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(filename, "wb") as f: + for chunk in response.iter_content(chunk_size=1024): + f.write(chunk) + + return filename + + +def load_gsm8k_data() -> tuple[list[dict], list[dict]]: + """Load GSM8K train and test data.""" + train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl" + test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl" + + train_file = download_and_cache_file(train_url) + test_file = download_and_cache_file(test_url) + + train_data = list(read_jsonl(train_file)) + test_data = list(read_jsonl(test_file)) + + return train_data, test_data + + +def read_jsonl(filename: str) -> Generator[dict, None, None]: + """Read a JSONL file.""" + with open(filename) as fin: + for line in fin: + if not line.startswith("#"): + yield json.loads(line) + + +def get_answer_value(answer_str: str) -> int: + """Extract the numerical answer from the response.""" + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def _optional_params(**params: object) -> dict[str, object]: + return {k: v for k, v in params.items() if v is not None} + + +async def call_vllm_api( + session: aiohttp.ClientSession, + prompt: str, + temperature: float | None, + max_tokens: int, + stop: list[str] | None = None, + url: str | None = None, + seed: int | None = None, + top_p: float | None = None, + top_k: int | None = None, +) -> tuple[str, int]: + """Call vLLM's OpenAI-compatible completions endpoint. + + Returns: + Tuple of (response_text, completion_tokens) + + """ + data = { + "prompt": prompt, + "max_tokens": max_tokens, + "stop": stop, + } + data.update( + _optional_params(temperature=temperature, seed=seed, top_p=top_p, top_k=top_k) + ) + + try: + async with session.post(f"{url}/v1/completions", json=data) as response: + response.raise_for_status() + result = await response.json() + text = result["choices"][0]["text"] + completion_tokens = result.get("usage", {}).get("completion_tokens", 0) + return text, completion_tokens + except Exception as e: + print(f"Error calling vLLM API ({type(e).__name__}): {e}") + return "", 0 + + +async def call_vllm_chat_api( + session: aiohttp.ClientSession, + model: str | None, + prompt: str, + temperature: float | None, + max_tokens: int, + stop: list[str] | None = None, + url: str | None = None, + seed: int | None = None, + top_p: float | None = None, + top_k: int | None = None, + reasoning_effort: str | None = None, + chat_template_kwargs: dict[str, object] | None = None, +) -> tuple[str, int]: + """Call vLLM's OpenAI-compatible chat completions endpoint. + + ``model``, ``temperature``, ``stop`` and the other optional fields are + omitted from the request when ``None``, so the server defaults apply (vLLM + uses its served model when ``model`` is omitted). + + Returns: + Tuple of (final answer content, completion_tokens). Reasoning returned + separately by a reasoning parser is not part of the answer. + + """ + data = { + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + } + data.update( + _optional_params( + stop=stop, + model=model, + temperature=temperature, + seed=seed, + top_p=top_p, + top_k=top_k, + reasoning_effort=reasoning_effort, + chat_template_kwargs=chat_template_kwargs, + ) + ) + + try: + async with session.post(f"{url}/v1/chat/completions", json=data) as response: + response.raise_for_status() + result = await response.json() + text = result["choices"][0]["message"]["content"] or "" + completion_tokens = result.get("usage", {}).get("completion_tokens", 0) + return text, completion_tokens + except Exception as e: + print(f"Error calling vLLM chat API ({type(e).__name__}): {e}") + return "", 0 + + +def _build_gsm8k_prompts( + num_questions: int = 1319, + num_shots: int = 5, + gen_prefix: str = "", +) -> tuple[list[str], list[int]]: + """Build few-shot GSM8K completion prompts and ground-truth labels.""" + if num_questions == 0: + return [], [] + train_data, test_data = load_gsm8k_data() + num_questions = min(num_questions, len(test_data)) + + few_shot_examples = "" + for i in range(num_shots): + few_shot_examples += ( + f"Question: {train_data[i]['question']}\n" + f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n" + ) + + prompts = [] + labels = [] + for i in range(num_questions): + prompts.append( + few_shot_examples + + f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}" + ) + labels.append(get_answer_value(test_data[i]["answer"])) + + assert all(label != INVALID for label in labels), "Some labels are invalid" + return prompts, labels + + +def _score_gsm8k( + states: list[str], + output_tokens: list[int], + labels: list[int], + num_shots: int, + max_tokens: int, + latency: float, +) -> dict[str, float | int]: + """Score GSM8K responses and return a results dict.""" + num_questions = len(labels) + preds = [get_answer_value(state) for state in states] + accuracy = np.mean(np.array(preds) == np.array(labels)) + invalid_rate = np.mean(np.array(preds) == INVALID) + total_output_tokens = sum(output_tokens) + tokens_per_second = total_output_tokens / latency if latency > 0 else 0.0 + + return { + "accuracy": accuracy, + "invalid_rate": invalid_rate, + "latency": latency, + "questions_per_second": num_questions / latency if latency > 0 else 0.0, + "total_output_tokens": total_output_tokens, + "tokens_per_second": tokens_per_second, + "num_questions": num_questions, + "num_shots": num_shots, + "max_tokens": max_tokens, + "timestamp": time.time(), + } + + +def evaluate_gsm8k( + num_questions: int = 1319, + num_shots: int = 5, + max_tokens: int = 256, + model: str | None = None, + use_chat_completions: bool = False, + host: str = "http://127.0.0.1", + port: int = 8000, + temperature: float | None = 0.0, + seed: int | None = 42, + request_timeout_seconds: float = 600, + gen_prefix: str = "", + max_concurrency: int | None = None, + top_p: float | None = None, + top_k: int | None = None, + reasoning_effort: str | None = None, + chat_template_kwargs: dict[str, object] | None = None, +) -> dict[str, float | int]: + """Evaluate GSM8K accuracy using vLLM serve endpoint. + + ``temperature``, ``top_p`` and ``top_k`` are sent only when not ``None``; + otherwise the server default applies. ``model`` is used only in chat mode, + where it may be ``None`` (the server uses its served model). + ``reasoning_effort`` and ``chat_template_kwargs`` require + ``use_chat_completions=True``. Stop strings are sent only in completions + mode. + + Returns dict with accuracy, invalid_rate, latency, etc. + """ + if not use_chat_completions and ( + reasoning_effort is not None or chat_template_kwargs is not None + ): + raise ValueError( + "reasoning_effort and chat_template_kwargs require " + "use_chat_completions=True" + ) + base_url = f"{host}:{port}" + prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) + num_questions = len(prompts) + + async def run_async_evaluation(): + states: list[str] = [""] * num_questions + output_tokens: list[int] = [0] * num_questions + + async def get_answer(session: aiohttp.ClientSession, i: int) -> tuple[str, int]: + if use_chat_completions: + # No stop strings: the chat template delimits the turn, and they + # would also cut off reasoning. + answer, tokens = await call_vllm_chat_api( + session=session, + model=model, + prompt=prompts[i], + temperature=temperature, + max_tokens=max_tokens, + url=base_url, + seed=seed, + top_p=top_p, + top_k=top_k, + reasoning_effort=reasoning_effort, + chat_template_kwargs=chat_template_kwargs, + ) + else: + answer, tokens = await call_vllm_api( + session=session, + prompt=prompts[i], + temperature=temperature, + max_tokens=max_tokens, + stop=["Question", "Assistant:", "<|separator|>"], + url=base_url, + seed=seed, + top_p=top_p, + top_k=top_k, + ) + states[i] = answer + output_tokens[i] = tokens + return answer, tokens + + timeout = aiohttp.ClientTimeout(total=request_timeout_seconds) + connector = ( + aiohttp.TCPConnector(limit=max_concurrency) + if max_concurrency is not None + else None + ) + async with aiohttp.ClientSession( + timeout=timeout, connector=connector + ) as session: + tasks = [get_answer(session, i) for i in range(num_questions)] + await tqdm.gather(*tasks, desc="Evaluating") + + return states, output_tokens + + print(f"Running GSM8K evaluation: {num_questions} questions, {num_shots}-shot") + + tic = time.perf_counter() + states, output_tokens = asyncio.run(run_async_evaluation()) + latency = time.perf_counter() - tic + + return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency) + + +def evaluate_gsm8k_offline( + llm, + num_questions: int = 1319, + num_shots: int = 5, + max_tokens: int = 256, + temperature: float = 0.0, + gen_prefix: str = "", + use_chat_completions: bool = False, + chat_template_kwargs: dict[str, object] | None = None, +) -> dict[str, float | int]: + """Evaluate GSM8K accuracy using an offline vllm.LLM object. + + Same prompts and scoring as evaluate_gsm8k(), but runs generation + directly via llm.generate() instead of calling a server over HTTP. + + When ``use_chat_completions=True``, prompts go through the chat template via + ``llm.chat()`` instead of raw completion (for instruction-tuned models). + ``chat_template_kwargs`` are forwarded to ``llm.chat()`` when provided. + """ + from vllm import SamplingParams + + prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) + sampling_params = SamplingParams( + temperature=temperature, + max_tokens=max_tokens, + stop=["Question", "Assistant:", "<|separator|>"], + ) + mode = "chat" if use_chat_completions else "completion" + print( + f"Running offline GSM8K evaluation: {len(prompts)} questions, " + f"{num_shots}-shot, {mode}" + ) + + tic = time.perf_counter() + if use_chat_completions: + conversations = [[{"role": "user", "content": p}] for p in prompts] + outputs = llm.chat( + conversations, + sampling_params, + chat_template_kwargs=chat_template_kwargs, + ) + else: + outputs = llm.generate(prompts, sampling_params) + latency = time.perf_counter() - tic + + states = [o.outputs[0].text for o in outputs] + output_tokens = [len(o.outputs[0].token_ids) for o in outputs] + + return _score_gsm8k(states, output_tokens, labels, num_shots, max_tokens, latency) + + +def main() -> None: + parser = argparse.ArgumentParser(description="GSM8K evaluation for vLLM serve") + parser.add_argument( + "--num-shots", type=int, default=5, help="Number of few-shot examples" + ) + parser.add_argument( + "--num-questions", + type=int, + default=1319, + help="Number of questions to evaluate", + ) + parser.add_argument( + "--max-tokens", type=int, default=256, help="Max tokens for generation" + ) + parser.add_argument("--host", type=str, default="http://127.0.0.1", help="Host URL") + parser.add_argument("--port", type=int, default=8000, help="Port number") + parser.add_argument( + "--temperature", + type=float, + help="Temperature for generation (default: 0 for completions; " + "server default with --use-chat-completions)", + ) + parser.add_argument( + "--top-p", type=float, help="Top-p for generation (server default if unset)" + ) + parser.add_argument( + "--top-k", type=int, help="Top-k for generation (server default if unset)" + ) + parser.add_argument( + "--use-chat-completions", + action="store_true", + help="Send each few-shot prompt as a user message to /v1/chat/completions " + "instead of /v1/completions (for chat-only models)", + ) + parser.add_argument( + "--model", + type=str, + help="Model name to send with chat requests (default: omitted, so the " + "server uses its served model)", + ) + parser.add_argument( + "--reasoning-effort", + type=str, + help="reasoning_effort for chat completions, e.g. none, low, high", + ) + parser.add_argument( + "--chat-template-kwargs", + type=json.loads, + help="JSON chat_template_kwargs, e.g. '{\"enable_thinking\": false}'", + ) + parser.add_argument( + "--seed", type=int, default=42, help="Random seed for reproducibility" + ) + parser.add_argument( + "--max-concurrency", + type=int, + help="Maximum number of concurrent requests", + ) + parser.add_argument( + "--request-timeout-seconds", + type=float, + default=600, + help="Timeout for each request, including time waiting for a connection", + ) + parser.add_argument("--save-results", type=str, help="Save results to JSON file") + + args = parser.parse_args() + temperature = args.temperature + if temperature is None and not args.use_chat_completions: + temperature = 0.0 + + result = evaluate_gsm8k( + num_questions=args.num_questions, + num_shots=args.num_shots, + max_tokens=args.max_tokens, + host=args.host, + port=args.port, + temperature=temperature, + seed=args.seed, + max_concurrency=args.max_concurrency, + request_timeout_seconds=args.request_timeout_seconds, + model=args.model, + use_chat_completions=args.use_chat_completions, + top_p=args.top_p, + top_k=args.top_k, + reasoning_effort=args.reasoning_effort, + chat_template_kwargs=args.chat_template_kwargs, + ) + + # Print results to terminal + print("\nResults:") + print(f"Accuracy: {result['accuracy']:.3f}") + print(f"Invalid responses: {result['invalid_rate']:.3f}") + print(f"Total latency: {result['latency']:.3f} s") + print(f"Questions per second: {result['questions_per_second']:.3f}") + print(f"Total output tokens: {result['total_output_tokens']}") + print(f"Output tokens per second: {result['tokens_per_second']:.3f}") + + # Optional file saving + if args.save_results: + # None means the field was not sent and the server default applied. + result["request_params"] = { + "endpoint": "chat" if args.use_chat_completions else "completions", + "model": args.model, + "temperature": temperature, + "top_p": args.top_p, + "top_k": args.top_k, + "seed": args.seed, + "stop": None + if args.use_chat_completions + else ["Question", "Assistant:", "<|separator|>"], + "reasoning_effort": args.reasoning_effort, + "chat_template_kwargs": args.chat_template_kwargs, + } + with open(args.save_results, "w") as f: + json.dump(result, f, indent=2) + print(f"Results saved to {args.save_results}") + + +if __name__ == "__main__": + main() diff --git a/tests/kernels/helion/test_benchmark_script.py b/tests/kernels/helion/test_benchmark_script.py deleted file mode 100644 index c593412843..0000000000 --- a/tests/kernels/helion/test_benchmark_script.py +++ /dev/null @@ -1,90 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest -import torch - -from vllm.utils.import_utils import has_helion - -if not has_helion(): - pytest.skip("Helion is not installed", allow_module_level=True) - -from tools import benchmark_helion_kernels - -check_correctness = benchmark_helion_kernels.check_correctness - - -class _FakeKernel: - helion_settings = None - - def __init__(self, offset: float): - self.calls = 0 - self.offset = offset - - def __call__(self, output: torch.Tensor, input: torch.Tensor) -> torch.Tensor: - self.calls += 1 - output.copy_(input + self.offset) - return input * 2 - - -def test_check_correctness_runs_once_without_mutating_benchmark_inputs(): - kernel = _FakeKernel(offset=1) - baseline_calls = 0 - - def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor: - nonlocal baseline_calls - baseline_calls += 1 - output.copy_(input + 1) - return input * 2 - - output = torch.zeros(4) - inputs = (output, torch.arange(4)) - - check_correctness(kernel, baseline, inputs, "matching") - - assert kernel.calls == 1 - assert baseline_calls == 1 - torch.testing.assert_close(output, torch.zeros(4)) - - -def test_check_correctness_reports_mutated_input_mismatch(): - kernel = _FakeKernel(offset=2) - - def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor: - output.copy_(input + 1) - return input * 2 - - inputs = (torch.zeros(4), torch.arange(4)) - - with pytest.raises(AssertionError, match="Numerics check failed for case bad"): - check_correctness(kernel, baseline, inputs, "bad") - - -def test_check_correctness_reports_return_value_mismatch(): - kernel = _FakeKernel(offset=1) - - def baseline(output: torch.Tensor, input: torch.Tensor) -> torch.Tensor: - output.copy_(input + 1) - return input * 3 - - inputs = (torch.zeros(4), torch.arange(4)) - - with pytest.raises(AssertionError, match="Numerics check failed for case bad"): - check_correctness(kernel, baseline, inputs, "bad") - - -def test_log_versions(monkeypatch): - messages = [] - - def record(message: str, *args): - messages.append(message % args) - - monkeypatch.setattr(benchmark_helion_kernels.logger, "info", record) - - benchmark_helion_kernels.log_versions() - - assert [message.split(":", 1)[0] for message in messages] == [ - "torch", - "helion", - "triton", - ] diff --git a/tests/kernels/mamba/test_mamba_mixer2.py b/tests/kernels/mamba/test_mamba_mixer2.py deleted file mode 100644 index db2274c3c9..0000000000 --- a/tests/kernels/mamba/test_mamba_mixer2.py +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import unittest - -import pytest -import torch - -from tests.utils import ensure_current_vllm_config, multi_gpu_test -from vllm.distributed.parallel_state import ( - init_distributed_environment, - initialize_model_parallel, -) -from vllm.model_executor.layers.mamba.mamba_mixer2 import Mixer2RMSNormGated -from vllm.utils.system_utils import update_environment_variables -from vllm.utils.torch_utils import set_random_seed - - -@multi_gpu_test(num_gpus=2) -@pytest.mark.parametrize("batch_size", [8]) -@pytest.mark.parametrize("seq_len", [128]) -@pytest.mark.parametrize( - "hidden_size_n_groups", - [ - (64, 1), - (64, 2), - (64, 4), # hidden_size be divisible by num_gpus - ], -) -@pytest.mark.parametrize("dtype", [torch.float16]) -def test_mixer2_gated_norm_multi_gpu( - batch_size: int, - seq_len: int, - hidden_size_n_groups: tuple[int, int], - dtype: torch.dtype, - device: str = "cuda", -): - hidden_size, n_groups = hidden_size_n_groups - num_processes = 2 - - def run_torch_spawn(fn, nprocs): - # need to use torch.mp.spawn otherwise will have problems with - # torch.distributed and cuda - torch.multiprocessing.spawn( - fn, - args=( - num_processes, - batch_size, - seq_len, - hidden_size, - n_groups, - dtype, - device, - ), - nprocs=nprocs, - ) - - run_torch_spawn(mixer2_gated_norm_tensor_parallel, 2) - - -def mixer2_gated_norm_tensor_parallel( - local_rank: int, - world_size: int, - batch_size: int, - seq_len: int, - hidden_size: int, - n_groups: int, - dtype: torch.dtype, - device: str, -): - set_random_seed(0) - - device = torch.device(f"cuda:{local_rank}") - torch.accelerator.set_device_index(device) - torch.set_default_device(device) - torch.set_default_dtype(dtype) - - update_environment_variables( - { - "RANK": str(local_rank), - "LOCAL_RANK": str(local_rank), - "WORLD_SIZE": str(world_size), - "MASTER_ADDR": "localhost", - "MASTER_PORT": "12345", - } - ) - - # initialize distributed - init_distributed_environment() - with ensure_current_vllm_config(): - initialize_model_parallel(tensor_model_parallel_size=world_size) - - # create random weights an inputs - weight = torch.rand((hidden_size,), dtype=dtype, device=device) - hidden_states = torch.randn(batch_size, seq_len, hidden_size) - gate_states = torch.randn(batch_size, seq_len, hidden_size) - - # create gated-norm with TP - with ensure_current_vllm_config(): - mixer = Mixer2RMSNormGated( - full_hidden_size=hidden_size, - full_n_groups=n_groups, - ) - mixer.weight.weight_loader(mixer.weight, weight) # load - - # create gated-norm without TP to compute reference - # - utilize mock patching to disable TP when - with ( - ensure_current_vllm_config(), - unittest.mock.patch( - "vllm.model_executor.layers.mamba.mamba_mixer2." - "get_tensor_model_parallel_world_size", - return_value=1, - ), - unittest.mock.patch( - "vllm.model_executor.layers.mamba.mamba_mixer2." - "get_tensor_model_parallel_rank", - return_value=0, - ), - ): - mixer_single_gpu = Mixer2RMSNormGated( - full_hidden_size=hidden_size, - full_n_groups=n_groups, - ) - # assign weight to single-gpu mixer - mixer_single_gpu.weight.data = weight - - # generate and compare - N = hidden_size // world_size - output = mixer( - hidden_states[..., local_rank * N : (local_rank + 1) * N], - gate_states[..., local_rank * N : (local_rank + 1) * N], - ) - ref_output = mixer_single_gpu(hidden_states, gate_states) - torch.testing.assert_close( - output, - ref_output[..., local_rank * N : (local_rank + 1) * N], - atol=5e-3, - rtol=1e-3, - ) diff --git a/tests/kernels/mamba/test_recoverssm_ple.py b/tests/kernels/mamba/test_recoverssm_ple.py index 6e2e1f0526..d0e3d7fffd 100644 --- a/tests/kernels/mamba/test_recoverssm_ple.py +++ b/tests/kernels/mamba/test_recoverssm_ple.py @@ -13,7 +13,7 @@ import pytest import torch from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first -from vllm.models.qwen4_exp.nvidia.ops.ple import ple_conv +from vllm.models.qwen4_exp.common.ops.ple import ple_conv from vllm.v1.attention.backends.ple_recoverssm import _PleConvCommit C, KSIZE, DIL, SQ, NB = 512, 4, 3, 4, 8 diff --git a/tests/kernels/moe/test_deepgemm.py b/tests/kernels/moe/test_deepgemm.py new file mode 100644 index 0000000000..e341f12a71 --- /dev/null +++ b/tests/kernels/moe/test_deepgemm.py @@ -0,0 +1,634 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP). +Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts. +""" + +import importlib +import math + +import pytest +import torch + +# vLLM fused-expert reference (Triton fallback + DeepGEMM option) +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe.activation import ( + MoEActivation, +) +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEQuantConfig, + FusedMoEQuantDesc, + fp8_w8a8_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.deep_gemm_utils import ( + deepgemm_moe_permute, + ep_gather, +) +from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( + _fp8_workspace_shape, +) +from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import ( + TritonOrDeepGemmExperts, +) +from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + per_token_group_quant_fp8, +) +from vllm.utils.deep_gemm import ( + calc_diff, + is_deep_gemm_supported, + per_block_cast_to_fp8, +) + +BLOCK_SIZE = [128, 128] + + +def test_ep_gather_uses_64_bit_row_offsets(): + hidden_size = 4096 + source_row = 1 << 18 + input_tensor = torch.empty( + (source_row + 1, hidden_size), + device="cuda", + dtype=torch.bfloat16, + ) + input_tensor[source_row].fill_(1) + output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16) + + ep_gather( + input_tensor=input_tensor, + recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64), + recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32), + input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32), + expert_map=None, + output_tensor=output, + ) + + torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0) + + +def test_ep_gather_uses_64_bit_output_row_offsets(): + hidden_size = 6144 + output_row = (1 << 31) // hidden_size + 1 + num_rows = output_row + 1 + output = torch.empty( + (num_rows, hidden_size), + device="cuda", + dtype=torch.bfloat16, + ) + + ep_gather( + input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16), + recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64), + recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32), + input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32), + expert_map=None, + output_tensor=output, + ) + + torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0) + + +@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("num_columns", [2048, 6144, 6145]) +def test_fp8_workspace_shape(workspace_dtype, num_columns): + num_rows = 17 + shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype) + + allocated_bytes = math.prod(shape) * workspace_dtype.itemsize + required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize + + assert allocated_bytes >= required_bytes + assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize + + +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init): + hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16) + activations, scales = per_token_group_quant_fp8( + hidden_states, + group_size=128, + use_ue8m0=True, + ) + topk_ids = torch.tensor([[0], [1]], device="cuda", dtype=torch.int64) + + _, permuted_scales, expert_ids, _, _ = deepgemm_moe_permute( + aq=activations, + aq_scale=scales, + topk_ids=topk_ids, + local_num_experts=2, + expert_map=None, + expert_tokens_meta=None, + ) + + padding = expert_ids < 0 + assert padding.any() + torch.testing.assert_close( + permuted_scales[padding], + torch.zeros_like(permuted_scales[padding]), + rtol=0, + atol=0, + ) + + +def make_block_quant_fp8_weights( + e: int, + n: int, + k: int, + block_size: list[int], +): + """Generate (w1, w2) expert weights and their per-block scale tensors + in FP8 block-quantized format. + + w1 shape: (E, 2N, K) + w2 shape: (E, K, N) + """ + dtype = torch.bfloat16 + fp8_max, fp8_min = ( + torch.finfo(torch.float8_e4m3fn).max, + torch.finfo(torch.float8_e4m3fn).min, + ) + + # bf16 reference weights + w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) / 10 + w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) / 10 + w1_bf16.clamp_(fp8_min, fp8_max) + w2_bf16.clamp_(fp8_min, fp8_max) + + block_n, block_k = block_size + n_tiles_w1 = math.ceil((2 * n) / block_n) + k_tiles_w1 = math.ceil(k / block_k) + n_tiles_w2 = math.ceil(k / block_n) + k_tiles_w2 = math.ceil(n / block_k) + + w1 = torch.empty_like(w1_bf16, dtype=torch.float8_e4m3fn) + w2 = torch.empty_like(w2_bf16, dtype=torch.float8_e4m3fn) + w1_s = torch.empty(e, n_tiles_w1, k_tiles_w1, device="cuda", dtype=torch.float32) + w2_s = torch.empty(e, n_tiles_w2, k_tiles_w2, device="cuda", dtype=torch.float32) + + for i in range(e): + w1[i], w1_s[i] = per_block_cast_to_fp8( + w1_bf16[i], block_size=block_size, use_ue8m0=True + ) + w2[i], w2_s[i] = per_block_cast_to_fp8( + w2_bf16[i], block_size=block_size, use_ue8m0=True + ) + + return w1, w2, w1_s, w2_s + + +@pytest.mark.parametrize("num_tokens", [16, 129, 512]) +@pytest.mark.parametrize("with_metadata", [False, True]) +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch): + """Both FP8 GEMMs consume only live rows as routing changes on replay.""" + from unittest.mock import Mock + + import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl + from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT + + if not ( + impl.current_platform.is_device_capability_family(90) + or ( + impl.current_platform.is_device_capability_family(100) + and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0 + ) + ): + pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x") + + torch.manual_seed(0) + e, n, k, topk = 8, 1024, 512, 2 + x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10 + aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True) + w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE) + quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE) + experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant) + ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64) + weights = torch.full((num_tokens, topk), 0.5, device="cuda") + counts = torch.zeros(e, device="cuda", dtype=torch.int32) + metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None + shapes = experts.workspace_shapes( + num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU + ) + + def route(mode): + ids[:, 0] = torch.arange(num_tokens, device="cuda") % e + ids[:, 1] = (ids[:, 0] + 1) % e + if mode == 1: + ids.fill_(-1) + ids[:1, 0] = e - 1 + elif mode == 2: + ids.fill_(-1) + counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int()) + + route(0) + variants = [] + for dense in (True, False): + ws1, ws2, output = [ + torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes + ] + + def invoke(output=output, ws1=ws1, ws2=ws2): + experts.apply( + output, + aq, + w1, + w2, + weights, + ids, + MoEActivation.SILU, + e, + None, + scales, + None, + ws1, + ws2, + metadata, + False, + ) + + with monkeypatch.context() as mp: + if dense: + platform = Mock(wraps=impl.current_platform) + platform.is_device_capability_family.return_value = False + mp.setattr(impl, "current_platform", platform) + gemm = impl.m_grouped_fp8_gemm_nt_contiguous + calls: list[int] = [] + + def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs): + assert kwargs.get("use_psum_layout", False) == (not dense) + calls.append(args[3].numel()) + return gemm(*args, **kwargs) + + mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout) + invoke() + assert len(calls) == 2 + if not dense: + assert calls == [e, e] + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + invoke() + variants.append((graph, output, ws1, ws2)) + + for mode in (0, 1, 2, 0): + route(mode) + for graph, _, ws1, ws2 in variants: + ws1.fill_(float("nan")) + ws2.fill_(float("nan")) + graph.replay() + torch.accelerator.synchronize() + torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0) + assert torch.isfinite(variants[1][1]).all() + + +@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)]) +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k): + """Both projection shapes skip unused tiles as counts change on replay.""" + from vllm.platforms import current_platform + from vllm.utils.deep_gemm import ( + m_grouped_fp8_gemm_nt_contiguous, + mk_alignment_scope, + ) + + if not ( + current_platform.is_device_capability_family(90) + or current_platform.is_device_capability_family(100) + ): + pytest.skip("Requires SM90 or SM10.x") + + capacity, experts, alignment = 512, 4, 128 + x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10 + aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True) + w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE) + ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32) + output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16) + + def invoke(): + with mk_alignment_scope(alignment): + m_grouped_fp8_gemm_nt_contiguous( + (aq, scales), (w, ws), output, ends, use_psum_layout=True + ) + + invoke() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + invoke() + for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)): + ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32)) + output.fill_(91) + graph.replay() + torch.accelerator.synchronize() + assert torch.all(output[allocated_end:] == 91) + if allocated_end: + assert torch.all(output[[0, 128]] != 91) + + +def run_single_case(m, n, k, topk, num_experts, block_size): + """Run one (M,N,K) configuration on a single GPU and assert DeepGEMM == + Triton baseline within tolerance. + """ + tokens_bf16 = ( + torch.randn(m, k, device="cuda", dtype=torch.bfloat16) + .clamp_min_(-1) + .clamp_max_(1) + ) + _, a1_scale = per_token_group_quant_fp8(tokens_bf16, block_size[1]) + + # expert weight tensors + w1, w2, w1_s, w2_s = make_block_quant_fp8_weights(num_experts, n, k, block_size) + + router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32) + topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1) + topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1) + + quant_config = fp8_w8a8_moe_quant_config( + w1_scale=w1_s, + w2_scale=w2_s, + a1_scale=a1_scale, + block_shape=block_size, + ) + moe_config = make_dummy_moe_config() + + deep_gemm_experts = mk.FusedMoEKernel( + prepare_finalize=maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ), + fused_experts=TritonOrDeepGemmExperts( + moe_config=moe_config, + quant_config=quant_config, + ), + ) + + # triton reference + out_triton = fused_experts( + hidden_states=tokens_bf16, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + quant_config=quant_config, + ) + + # DeepGemm + out_deepgemm = deep_gemm_experts.apply( + hidden_states=tokens_bf16, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + global_num_experts=num_experts, + activation=MoEActivation.SILU, + apply_router_weight_on_input=False, + expert_map=None, + ) + diff = calc_diff(out_deepgemm, out_triton) + assert diff < 0.001, f"Diff exceeded 1%: {diff}" + + +# Note: N <= 512 will disable the deepgemm path due to performance issues. +MNKs = [ + (1024, 768, 128), + (2048, 768, 512), + (512, 1024, 1024), + (4096, 4096, 1024), +] + +TOPKS = [2, 6] +NUM_EXPERTS = [32] + + +@pytest.mark.parametrize(("m", "n", "k"), MNKs) +@pytest.mark.parametrize("topk", TOPKS) +@pytest.mark.parametrize("num_experts", NUM_EXPERTS) +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_init): + with monkeypatch.context() as mp: + mp.setenv("VLLM_USE_DEEP_GEMM", "1") + + _DeepGemmExperts = importlib.import_module( + "vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe" + ).DeepGemmExperts + + call_counter = {"cnt": 0} + + orig_fn = _DeepGemmExperts.apply + + def _spy_apply(*args, **kwargs): + call_counter["cnt"] += 1 + return orig_fn(*args, **kwargs) + + monkeypatch.setattr(_DeepGemmExperts, "apply", _spy_apply) + if topk > num_experts: + pytest.skip(f"topk={topk} > num_experts={num_experts}") + + run_single_case( + m=m, + n=n, + k=k, + topk=topk, + num_experts=num_experts, + block_size=BLOCK_SIZE, + ) + + # ensure that the DeepGEMM path was indeed taken. + assert call_counter["cnt"] == 1, ( + f"DeepGEMM path was not executed during the test. " + f"Call counter: {call_counter['cnt']}" + ) + + +# --------------------------------------------------------------------------- +# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous) +# --------------------------------------------------------------------------- + + +def make_mxfp4_weights( + e: int, + n: int, + k: int, +): + """Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales, + plus BF16 reference weights for validation. + + w1 shape: (E, 2N, K//2) uint8 — packed FP4 + w2 shape: (E, K, N//2) uint8 — packed FP4 + w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales + w2_s shape: (E, K, N//32) float32 — per-row block-32 scales + w1_bf16: (E, 2N, K) — original BF16 for reference + w2_bf16: (E, K, N) — original BF16 for reference + """ + from deep_gemm.utils.math import per_token_cast_to_fp4 + + dtype = torch.bfloat16 + gran_k = 32 # MXFP4 block size + + # bf16 reference weights — scale by 1/sqrt(dim) for numerical stability + w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5) + w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5) + + # Quantize per-expert to FP4 + w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8) + w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8) + w1_s = torch.empty( + e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32 + ) + w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32) + + for i in range(e): + w1[i], w1_s[i] = per_token_cast_to_fp4( + w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k + ) + w2[i], w2_s[i] = per_token_cast_to_fp4( + w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k + ) + + return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 + + +def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids): + """BF16 token-loop MoE reference for correctness testing.""" + import torch.nn.functional as F + + num_tokens, hidden_size = x.shape + intermediate = w1.shape[1] // 2 + top_k = topk_ids.shape[1] + + output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device) + for t in range(num_tokens): + for kk in range(top_k): + e = topk_ids[t, kk].item() + w = topk_weights[t, kk].item() + fc1 = x[t : t + 1].float() @ w1[e].float().T + linear = fc1[:, :intermediate] + gate = fc1[:, intermediate:] + act = F.silu(gate) * linear + fc2 = act @ w2[e].float().T + output[t] += w * fc2[0] + return output.to(torch.bfloat16) + + +def run_single_fp4_case(m, n, k, topk, num_experts): + """Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert + DeepGEMM FP4 == BF16 reference within tolerance. + """ + tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5) + + # FP4 expert weight tensors + BF16 originals for reference + w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k) + + router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32) + topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1) + topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1) + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, + ) + from vllm.platforms import current_platform + + _fp8_dtype = current_platform.fp8_dtype() + _block_shape = GroupShape(128, 128) + quant_config = FusedMoEQuantConfig( + _a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None), + _a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None), + _w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None), + _w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None), + ) + moe_config = make_dummy_moe_config() + + from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( + DeepGemmFP4Experts, + ) + + deep_gemm_fp4_experts = mk.FusedMoEKernel( + prepare_finalize=maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ), + fused_experts=DeepGemmFP4Experts( + moe_config=moe_config, + quant_config=quant_config, + ), + ) + + # DeepGEMM FP4 path + out_deepgemm_fp4 = deep_gemm_fp4_experts.apply( + hidden_states=tokens_bf16, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + global_num_experts=num_experts, + activation=MoEActivation.SILU, + apply_router_weight_on_input=False, + expert_map=None, + ) + + # BF16 reference using the same original weights + out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids) + + # FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations + diff = calc_diff(out_deepgemm_fp4, out_ref) + assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}" + + +# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096. +# FP4 quantization with block_k=32 needs large K for good accuracy. +FP4_MNKs = [ + (128, 4096, 4096), # DeepSeek V4 shape + (256, 2048, 2048), # Half-size variant + (128, 384, 3584), # Kimi-K3 TP8 latent-MoE shape +] + +FP4_TOPKS = [2] +FP4_NUM_EXPERTS = [8] + + +@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs) +@pytest.mark.parametrize("topk", FP4_TOPKS) +@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS) +@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels") +def test_deepgemm_fp4_vs_triton( + m, n, k, topk, num_experts, monkeypatch, workspace_init +): + pytest.importorskip("deep_gemm.utils.math") + with monkeypatch.context() as mp: + mp.setenv("VLLM_USE_DEEP_GEMM", "1") + + _DeepGemmFP4Experts = importlib.import_module( + "vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe" + ).DeepGemmFP4Experts + + call_counter = {"cnt": 0} + + orig_fn = _DeepGemmFP4Experts.apply + + def _spy_apply(*args, **kwargs): + call_counter["cnt"] += 1 + return orig_fn(*args, **kwargs) + + monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply) + if topk > num_experts: + pytest.skip(f"topk={topk} > num_experts={num_experts}") + + run_single_fp4_case( + m=m, + n=n, + k=k, + topk=topk, + num_experts=num_experts, + ) + + # ensure that the DeepGEMM FP4 path was indeed taken. + assert call_counter["cnt"] == 1, ( + f"DeepGEMM FP4 path was not executed during the test. " + f"Call counter: {call_counter['cnt']}" + ) diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py new file mode 100644 index 0000000000..1536c4cec9 --- /dev/null +++ b/tests/kernels/moe/utils.py @@ -0,0 +1,658 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Callable + +import torch + +import vllm._custom_ops as ops +from tests.kernels.quant_utils import per_block_cast_to_int8 +from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import ( + BatchedTritonExperts, + NaiveBatchedExperts, +) +from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonExperts, +) +from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_experts, +) +from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput +from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute +from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import ( + BatchedPrepareAndFinalize, +) +from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk +from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input +from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8 +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + ref_nvfp4_quant, +) +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import per_block_cast_to_fp8 +from vllm.utils.math_utils import round_up + +DEVICE = current_platform.device_type + + +def shuffle_weight(w: torch.Tensor) -> torch.Tensor: + """Fold weights to adjacent locations for Triton MoE / SwiGLU kernel layout.""" + shape = w.shape + n = shape[-1] + first = w[..., : n // 2] + second = w[..., n // 2 :] + stacked = torch.stack((first, second), dim=-1) + return stacked.reshape(shape) + + +def make_dummy_moe_config( + num_experts: int = 1, + num_local_experts: int | None = None, + experts_per_token: int = 1, + hidden_dim: int = 1, + intermediate_size: int = 1, + in_dtype: torch.dtype = torch.bfloat16, + max_num_tokens: int = 512, + activation: MoEActivation = MoEActivation.SILU, +) -> FusedMoEConfig: + """This is a dummy config for the mk constructor interface + as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN + do not actually use this config. + + CUTLASSFp8 needs to set some params for workshapes. + """ + return FusedMoEConfig( + num_experts=num_experts, + experts_per_token=experts_per_token, + hidden_dim=hidden_dim, + intermediate_size=intermediate_size, + num_local_experts=num_local_experts + if num_local_experts is not None + else num_experts, + num_logical_experts=num_experts, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + activation=activation, + in_dtype=in_dtype, + device=DEVICE, + routing_method=RoutingMethodType.TopK, + max_num_tokens=max_num_tokens, + ) + + +def triton_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + w1_scale: torch.Tensor | None = None, + w2_scale: torch.Tensor | None = None, + a1_scale: torch.Tensor | None = None, + a2_scale: torch.Tensor | None = None, + quant_dtype: torch.dtype | None = None, + per_act_token_quant=False, + block_shape: list[int] | None = None, +) -> torch.Tensor: + quant_config = FusedMoEQuantConfig.make( + quant_dtype, + per_act_token_quant=per_act_token_quant, + block_shape=block_shape, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + return fused_experts(a, w1, w2, topk_weight, topk_ids, quant_config=quant_config) + + +def batched_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + w1_scale: torch.Tensor | None = None, + w2_scale: torch.Tensor | None = None, + a1_scale: torch.Tensor | None = None, + a2_scale: torch.Tensor | None = None, + quant_dtype: torch.dtype | None = None, + per_act_token_quant: bool = False, + block_shape: list[int] | None = None, + moe_config: FusedMoEConfig | None = None, +) -> torch.Tensor: + max_num_tokens = round_up(a.shape[0], 64) + + quant_config = FusedMoEQuantConfig.make( + quant_dtype, + per_act_token_quant=per_act_token_quant, + block_shape=block_shape, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + if moe_config is None: + moe_config = make_dummy_moe_config() + + fused_experts = FusedMoEKernel( + BatchedPrepareAndFinalize( + max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0 + ), + BatchedTritonExperts( + max_num_tokens=max_num_tokens, + num_dispatchers=1, + quant_config=quant_config, + moe_config=moe_config, + ), + ) + + return fused_experts.apply( + a, + w1, + w2, + topk_weight, + topk_ids, + global_num_experts=w1.shape[0], + activation=moe_config.activation, + apply_router_weight_on_input=False, + expert_map=None, + ) + + +def naive_batched_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + w1_scale: torch.Tensor | None = None, + w2_scale: torch.Tensor | None = None, + a1_scale: torch.Tensor | None = None, + a2_scale: torch.Tensor | None = None, + quant_dtype: torch.dtype | None = None, + per_act_token_quant: bool = False, + block_shape: list[int] | None = None, +) -> torch.Tensor: + max_num_tokens = round_up(a.shape[0], 64) + + quant_config = FusedMoEQuantConfig.make( + quant_dtype, + per_act_token_quant=per_act_token_quant, + block_shape=block_shape, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + moe_config = make_dummy_moe_config() + + fused_experts = FusedMoEKernel( + BatchedPrepareAndFinalize( + max_num_tokens, num_dispatchers=1, num_local_experts=w1.shape[0], rank=0 + ), + NaiveBatchedExperts( + max_num_tokens=max_num_tokens, + num_dispatchers=1, + quant_config=quant_config, + moe_config=moe_config, + ), + ) + + return fused_experts.apply( + a, + w1, + w2, + topk_weight, + topk_ids, + global_num_experts=w1.shape[0], + activation=moe_config.activation, + apply_router_weight_on_input=False, + expert_map=None, + ) + + +def chunk_scales( + scales: torch.Tensor | None, start: int, end: int +) -> torch.Tensor | None: + if scales is not None: + if scales.numel() == 1: + return scales + else: + return scales[start:end] + return None + + +def make_quantized_test_activations( + E: int, + m: int, + k: int, + in_dtype: torch.dtype, + quant_dtype: torch.dtype | None = None, + block_shape: list[int] | None = None, + per_act_token_quant: bool = False, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + a = torch.randn((E, m, k), device=DEVICE, dtype=in_dtype) / 10 + a_q = a + a_scale = None + + if quant_dtype is not None: + assert quant_dtype == torch.float8_e4m3fn or quant_dtype == torch.int8, ( + "only fp8/int8 supported" + ) + a_q = torch.zeros_like(a, dtype=quant_dtype) + a_scale_l = [None] * E + for e in range(E): + a_q[e], a_scale_l[e] = moe_kernel_quantize_input( + a[e], None, quant_dtype, per_act_token_quant, block_shape + ) + a_scale = torch.stack(a_scale_l) + + if not per_act_token_quant and block_shape is None: + a_scale = a_scale.view(E, 1, 1) + + return a, a_q, a_scale + + +def moe_quantize_weights_2d( + w: torch.Tensor, + w_s: torch.Tensor | None, + quant_dtype: torch.dtype | str | None, + per_token_quant: bool, + block_shape: list[int] | None, +) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + is_fp8_dtype = is_fp8(quant_dtype) + assert is_fp8_dtype or quant_dtype == torch.int8 or quant_dtype == "nvfp4", ( + "only fp8/int8/nvfp4 supported" + ) + + w_gs = None + + if block_shape is not None: + assert not per_token_quant + if quant_dtype == torch.int8: + w, w_s = per_block_cast_to_int8(w, block_shape) + elif is_fp8_dtype: + w, w_s = per_block_cast_to_fp8(w, block_shape) + elif quant_dtype == "nvfp4": + raise RuntimeError("blocked quantization not supported for nvfp4") + else: + raise RuntimeError(f"Unsupported quant type {quant_dtype}") + else: + if quant_dtype == torch.int8: + w, w_s = ops.scaled_int8_quant( + w, w_s, use_per_token_if_dynamic=per_token_quant + ) + elif is_fp8_dtype: + w, w_s = ops.scaled_fp8_quant( + w, w_s, use_per_token_if_dynamic=per_token_quant + ) + elif quant_dtype == "nvfp4": + assert not per_token_quant + w_amax = torch.abs(w).max().to(torch.float32) + w_gs = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w_amax + if current_platform.is_rocm(): + w, w_s = _scaled_fp4_quant_emulated(w, w_gs) + else: + w, w_s = ops.scaled_fp4_quant(w, w_gs) + else: + raise RuntimeError(f"Unsupported quant type {quant_dtype}") + + return w, w_s, w_gs + + +def _pack_e2m1_fp4(fp4_values: torch.Tensor) -> torch.Tensor: + assert fp4_values.shape[-1] % 2 == 0 + + abs_values = fp4_values.abs() + codes = torch.empty_like(abs_values, dtype=torch.uint8) + for code, value in enumerate((0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)): + codes[abs_values == value] = code + codes = codes | ((fp4_values < 0).to(torch.uint8) << 3) + return codes[..., 0::2] | (codes[..., 1::2] << 4) + + +def _scaled_fp4_quant_emulated( + w: torch.Tensor, w_gs: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + fp4_values, w_s = ref_nvfp4_quant(w, w_gs, block_size=16) + return _pack_e2m1_fp4(fp4_values), w_s.to(torch.float8_e4m3fn) + + +def moe_quantize_weights( + w: torch.Tensor, + w_s: torch.Tensor | None, + quant_dtype: torch.dtype | str | None, + per_token_quant: bool, + block_shape: list[int] | None, +) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + assert w.dim() == 3 + e, rows, cols = w.shape + w_l = [None] * e + w_s_l = [None] * e + w_gs_l = [None] * e + for idx in range(e): + w_l[idx], w_s_l[idx], w_gs_l[idx] = moe_quantize_weights_2d( + w[idx], None, quant_dtype, per_token_quant, block_shape + ) + + w = torch.stack(w_l) + w_s = torch.stack(w_s_l) + w_gs = torch.stack(w_gs_l) if e > 0 and w_gs_l[0] is not None else None + + if w_s.ndim == 2: + assert w_s.shape[-1] == 1 + w_s = w_s.view(-1, 1, 1) + + if block_shape is not None: + block_n, block_k = block_shape + n_tiles = (rows + block_n - 1) // block_n + k_tiles = (cols + block_k - 1) // block_k + assert w_s.shape == (e, n_tiles, k_tiles) + + return w, w_s, w_gs + + +def make_test_weight( + e: int, + rows: int, + cols: int, + in_dtype: torch.dtype = torch.bfloat16, + quant_dtype: torch.dtype | str | None = None, + block_shape: list[int] | None = None, + per_out_ch_quant: bool = False, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + w_16 = torch.randn((e, rows, cols), device=DEVICE, dtype=in_dtype) / 15 + + if quant_dtype is not None: + w, w_s, w_gs = moe_quantize_weights( + w_16, None, quant_dtype, per_out_ch_quant, block_shape + ) + else: + w = w_16 + w_s = None + w_gs = None + + return w_16, w, w_s, w_gs + + +def make_test_weights( + e: int, + n: int, + k: int, + in_dtype: torch.dtype = torch.bfloat16, + quant_dtype: torch.dtype | str | None = None, + block_shape: list[int] | None = None, + per_out_ch_quant: bool = False, + make_gate: bool = True, +) -> tuple[ + tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None], + tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None], +]: + return ( + make_test_weight( + e, + (2 if make_gate else 1) * n, + k, + in_dtype, + quant_dtype, + block_shape, + per_out_ch_quant, + ), + make_test_weight(e, k, n, in_dtype, quant_dtype, block_shape, per_out_ch_quant), + ) + + +def per_token_cast_to_fp8( + x: torch.Tensor, block_size: int = 128 +) -> tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (block_size - (n % block_size)) % block_size + x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, block_size) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def make_test_quant_config( + e: int, + n: int, + k: int, + in_dtype: torch.dtype, + quant_dtype: torch.dtype | str | None = None, + per_act_token_quant: bool = False, + block_shape: list[int] | None = None, + make_gate: bool = True, + is_scale_swizzled: bool = True, +) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]: + (_, w1, w1_s, w1_gs), (_, w2, w2_s, w2_gs) = make_test_weights( + e, + n, + k, + in_dtype, + quant_dtype, + per_out_ch_quant=per_act_token_quant, + block_shape=block_shape, + make_gate=make_gate, + ) + + # Hacky/trivial scales for nvfp4. + a1_gscale: torch.Tensor | None = None + a2_gscale: torch.Tensor | None = None + if quant_dtype == "nvfp4": + a1_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32) + a2_gscale = torch.ones((e,), device=DEVICE, dtype=torch.float32) + a1_scale = a1_gscale + a2_scale = a2_gscale + else: + a1_scale = None + a2_scale = None + + return ( + w1, + w2, + FusedMoEQuantConfig.make( + quant_dtype, + per_act_token_quant=per_act_token_quant, + block_shape=block_shape, + w1_scale=w1_s, + w2_scale=w2_s, + a1_gscale=a1_gscale, + a2_gscale=a2_gscale, + a1_scale=a1_scale, + a2_scale=a2_scale, + # TODO: make sure this is handled properly + g1_alphas=(1 / w1_gs) if w1_gs is not None else None, + g2_alphas=(1 / w2_gs) if w2_gs is not None else None, + is_scale_swizzled=is_scale_swizzled, + ), + ) + + +def fused_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + score: torch.Tensor, + topk: int, + renormalize: bool = False, + quant_config: FusedMoEQuantConfig | None = None, + global_num_experts: int = -1, + expert_map: torch.Tensor | None = None, +) -> torch.Tensor: + topk_weights, topk_ids, _ = fused_topk( + hidden_states, score.float(), topk, renormalize + ) + return fused_experts( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + global_num_experts=global_num_experts, + expert_map=expert_map, + quant_config=quant_config, + ) + + +class BaselineMM(torch.nn.Module): + def __init__( + self, + b: torch.Tensor, + out_dtype: torch.dtype, + ): + super().__init__() + self.b = torch.nn.Parameter(b.to(dtype=torch.float32)) + self.out_dtype = out_dtype + + def forward(self, a: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]: + return torch.mm(a.to(dtype=torch.float32), self.b).to(self.out_dtype), None + + +class BaselineSiluAndMul(torch.nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return torch.nn.functional.silu(x[..., :d]) * x[..., d:] + + +class TestMLP(torch.nn.Module): + def __init__( + self, + w1: torch.Tensor, + w2: torch.Tensor, + out_dtype: torch.dtype, + ): + super().__init__() + self.gate_up_proj = BaselineMM(w1, out_dtype) + self.down_proj = BaselineMM(w2, out_dtype) + self.act_fn = BaselineSiluAndMul() + + def forward(self, x): + x, _ = self.gate_up_proj(x) + x = self.act_fn(x) + x, _ = self.down_proj(x) + return x + + +def modular_triton_fused_moe( + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, +) -> FusedMoEKernel: + return FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ), + TritonExperts(moe_config, quant_config), + ) + + +def check_accuracy(a, b, atol, rtol, percent): + """Allow a mismatch percentage of 1 - percent.""" + if torch.any(torch.isnan(a)): + raise Exception("NaN in reference output") + if torch.any(torch.isnan(b)): + raise Exception("NaN in actual output") + if torch.any(torch.isinf(a)): + raise Exception("Inf in reference output") + if torch.any(torch.isinf(b)): + raise Exception("Inf in actual output") + assert a.shape == b.shape, f"Shape mismatch: {a.shape} vs {b.shape}" + + left = torch.abs(a - b) + right = atol + rtol * torch.abs(b) + count = torch.sum(left > right) + mismatch_percent = count / a.numel() + if mismatch_percent > 1 - percent: + raise Exception( + f"Mismatch percentage is {mismatch_percent:.4f} for rtol {rtol} " + f"(threshold: {1 - percent:.4f})" + ) + + +def check_deferred_moe_finalize( + moe_config: FusedMoEConfig, + run: Callable[[], torch.Tensor | UnfinalizedMoEOutput], + router_weights: torch.Tensor | None = None, + chunked: bool = False, +) -> None: + """Check a kernel that defers its finalize against the finalize it skips. + + ``run`` calls the kernel on fixed inputs, first as built and then with + ``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must + report truthfully. A deferred output reduced by ``moe_unpermute``, the + TRT-LLM finalize kernel, must give the kernel's own finalized output bit for + bit, and modular experts hand the router's weights back as-is. A call the + experts split across kernel launches finalizes instead. + """ + # The deferred output views the router's buffer, so compare with a copy. + expected_weights = None if router_weights is None else router_weights.clone() + finalized = run() + assert isinstance(finalized, torch.Tensor) + moe_config.defer_moe_finalize() + output = run() + assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked + if chunked: + torch.testing.assert_close(output, finalized, atol=0, rtol=0) + return + + assert isinstance(output, UnfinalizedMoEOutput) + if expected_weights is not None: + torch.testing.assert_close( + output.expert_weights, expected_weights, atol=0, rtol=0 + ) + reference = torch.empty_like(finalized) + moe_unpermute( + reference, + output.gemm2_permuted, + output.expert_weights.float(), + output.expanded_idx_to_permuted_idx, + # The kernel reads its valid-row count through this pointer either way. + expert_first_token_offset=output.gemm2_permuted.new_full( + (1,), output.gemm2_permuted.shape[0], dtype=torch.int64 + ), + ) + torch.testing.assert_close(reference, finalized, atol=0, rtol=0) + + +def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8): + """Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs. + + triton_kernels 3.8 returns layout instances; earlier versions return a + (layout, opts) tuple. + """ + from triton_kernels.tensor_details import layout + + from vllm.utils.import_utils import get_triton_kernels_version + + if get_triton_kernels_version() == "3.8": + w = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis) + s = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=mx_axis, num_warps=num_warps + ) + return w, {}, s, {} + + w, w_opts = layout.make_default_matmul_mxfp4_w_layout(mx_axis=mx_axis) + s, s_opts = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=mx_axis, num_warps=num_warps + ) + return w, w_opts, s, s_opts diff --git a/tests/kernels/quant_utils.py b/tests/kernels/quant_utils.py new file mode 100644 index 0000000000..188fa74b22 --- /dev/null +++ b/tests/kernels/quant_utils.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +import torch + +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + get_fp8_min_max, + group_broadcast, +) +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import _ceil_to_ue8m0, is_deep_gemm_e8m0_used +from vllm.utils.math_utils import round_up + +FP8_DTYPE = current_platform.fp8_dtype() +DEVICE = current_platform.device_type + + +def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor: + return torch.as_tensor(x, dtype=torch.float32, device=DEVICE) + + +def ref_dynamic_per_token_quant( + x: torch.Tensor, quant_dtype: torch.dtype, scale_ub: torch.Tensor | None = None +) -> tuple[torch.Tensor, torch.Tensor]: + assert quant_dtype in [torch.int8, FP8_DTYPE] + if scale_ub is not None: + assert quant_dtype == FP8_DTYPE + + if quant_dtype == torch.int8: + qtype_traits = torch.iinfo(quant_dtype) + qtype_traits_min = qtype_traits.min + qtype_traits_max = qtype_traits.max + else: + qtype_traits_min, qtype_traits_max = get_fp8_min_max() + qtype_max = as_float32_tensor(qtype_traits_max) + s_1 = as_float32_tensor(1.0) + s_512 = as_float32_tensor(512.0) + + # For fp8, in order to match the cuda kernel output, we have to do exactly + # the same operations as in the corresponding fp8 kernel to prevent + # rounding errors. + + # Compute scales + x_token_max, _ = x.abs().max(dim=-1) + x_token_max = as_float32_tensor(x_token_max) + if scale_ub is not None: + x_token_max = x_token_max.clamp(max=scale_ub) + scales = (x_token_max / qtype_max)[:, None] + + # Quant + if quant_dtype == torch.int8: + iscales = as_float32_tensor(s_1 / scales) + torch_out = as_float32_tensor(x) * iscales + torch_out = torch_out.round() + torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype) + else: + assert quant_dtype == FP8_DTYPE + min_scaling_factor = s_1 / (qtype_max * s_512) + scales = scales.clamp(min=min_scaling_factor) + torch_out = as_float32_tensor(x) / scales + torch_out = torch_out.clamp(qtype_traits_min, qtype_traits_max).to(quant_dtype) + + return torch_out, scales + + +# The int8 version is very similar. Incorporate the int8 version, like in +# ref_dynamic_per_token_quant, when we have a dynamic_per_tensor int8 quant +# kernel +def ref_dynamic_per_tensor_fp8_quant( + x: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + fp8_traits_min, fp8_traits_max = get_fp8_min_max() + fp8_max = as_float32_tensor(fp8_traits_max) + one = as_float32_tensor(1.0) + + # For fp8, in order to match the cuda kernel output, we have to do exactly + # the same operations as in the corresponding fp8 kernel to prevent + # rounding errors. + + x_max = as_float32_tensor(x.abs().max()) + ref_scale = x_max / fp8_max + ref_iscale = one / ref_scale + ref_out = ( + (as_float32_tensor(x) * ref_iscale) + .clamp(fp8_traits_min, fp8_traits_max) + .to(FP8_DTYPE) + ) + return ref_out, ref_scale.view(1) + + +def native_w8a8_block_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: list[int], + output_dtype: torch.dtype, + compute_type: torch.dtype = torch.float32, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise + quantization using native torch. + It is agnostic to the input data type and can be used for both int8 and + fp8 data types. + + It takes two input tensors `A` and `B` (int8) with scales `As` and + `Bs` (float32). + The output is returned in the specified `output_dtype`. + """ + A = A.to(compute_type) + B = B.to(compute_type) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0], f"{n_tiles} == {Bs.shape[0]}" + assert k_tiles == Bs.shape[1], f"{k_tiles} == {Bs.shape[1]}" + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=compute_type, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +def native_per_token_group_quant_fp8( + x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn +): + """Function to perform per-token-group quantization on an input tensor + `x` using native torch.""" + assert x.shape[-1] % group_size == 0, ( + "the last dimension of `x` must be divisible by `group_size`" + ) + assert x.is_contiguous(), "`x` is not contiguous" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / fp8_max + if is_deep_gemm_e8m0_used(): + x_s = _ceil_to_ue8m0(x_s) + x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +def native_per_token_group_quant_int8( + x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True +): + """Function to perform per-token-group quantization on an input tensor + `x` using native torch. + + It converts the tensor values into int8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Per-token quantization rounds to nearest; block quantization truncates. + """ + assert x.shape[-1] % group_size == 0, ( + "the last dimension of `x` must be divisible by `group_size`" + ) + assert x.is_contiguous(), "`x` is not contiguous" + + iinfo = torch.iinfo(dtype) + int8_min = iinfo.min + int8_max = iinfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + # Use float32 for scale calculation for stability + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / int8_max + x_q = x_.to(torch.float32) / x_s + if round_to_nearest: + x_q = x_q.round() + x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +DEFAULT_BLOCK_SHAPE = [128, 128] + + +def per_block_cast_to_int8( + x: torch.Tensor, + block_shape: list[int] = DEFAULT_BLOCK_SHAPE, +) -> tuple[torch.Tensor, torch.Tensor]: + block_m, block_n = block_shape + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (round_up(m, block_m), round_up(n, block_n)), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, block_m, x_padded.size(1) // block_n, block_n) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (256.0 / x_amax)).to(torch.int8) + x_scaled_sub = x_scaled.view_as(x_padded)[:m, :n].contiguous() + scales = (x_amax / 256.0).view(x_view.size(0), x_view.size(2)) + return x_scaled_sub, scales + + +def dequant( + t: torch.Tensor, + scale: torch.Tensor | None, + block_shape: list[int] | None, + per_act_token_quant: bool, + out_dtype: torch.dtype | None = torch.float32, +) -> torch.Tensor: + if scale is not None: + f32 = torch.float32 + if per_act_token_quant or block_shape is None: + return (t.to(f32) * scale).to(out_dtype) + else: + return (t.to(f32) * group_broadcast(scale, t.shape)).to(out_dtype) + else: + return t.to(out_dtype) + + +def native_batched_masked_quant_matmul( + A: torch.Tensor, + B: torch.Tensor, + C: torch.Tensor, + num_expert_tokens: torch.Tensor, + A_scale: torch.Tensor | None = None, + B_scale: torch.Tensor | None = None, + block_shape: list[int] | None = None, + per_act_token_quant: bool = False, +) -> torch.Tensor: + num_expert_tokens_cpu = num_expert_tokens.clone() + num_expert_tokens_cpu = num_expert_tokens_cpu.to(device="cpu") + num_experts = num_expert_tokens.size(0) + + for e in range(num_experts): + num_tokens = num_expert_tokens_cpu[e] + if A.dtype.itemsize == 1 and block_shape is not None: + assert A_scale is not None and B_scale is not None + tmp = native_w8a8_block_matmul( + A[e], B[e], A_scale[e], B_scale[e], block_shape, C.dtype + ) + C[e, :num_tokens, :] = tmp[:num_tokens, :] + elif A.dtype.itemsize == 1 and block_shape is None: + assert A_scale is not None and B_scale is not None + A_dq = dequant(A[e], A_scale[e], block_shape, per_act_token_quant) + B_dq = dequant(B[e], B_scale[e], block_shape, per_act_token_quant) + C[e, :num_tokens, :] = (A_dq[:num_tokens] @ B_dq.transpose(0, 1)).to( + C.dtype + ) + else: + assert A_scale is None + assert B_scale is None + C[e, :num_tokens, :] = A[e, :num_tokens, :] @ B[e].transpose(0, 1) + + return C diff --git a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py b/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py deleted file mode 100644 index a1440ae5d2..0000000000 --- a/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from pathlib import Path - -import torch -from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher -from safetensors.torch import load_file, save_file - -from vllm.engine.arg_utils import EngineArgs -from vllm.transformers_utils.repo_utils import hf_api - -from .conftest import RunaiDummyExecutor - -load_format = "runai_streamer" -test_model = "openai-community/gpt2" - - -def test_runai_model_loader_download_files_s3_mocked_with_patch( - vllm_runner, - tmp_path: Path, - monkeypatch, -): - patcher = StreamerPatcher(str(tmp_path)) - - test_mock_s3_model = "s3://my-mock-bucket/gpt2/" - - # Download model from HF - mock_model_dir = f"{tmp_path}/gpt2" - hf_api().snapshot_download(repo_id=test_model, local_dir=mock_model_dir) - - monkeypatch.setattr( - "vllm.transformers_utils.runai_utils.runai_list_safetensors", - patcher.shim_list_safetensors, - ) - monkeypatch.setattr( - "vllm.transformers_utils.runai_utils.runai_pull_files", - patcher.shim_pull_files, - ) - monkeypatch.setattr( - "vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer", - patcher.create_mock_streamer, - ) - - engine_args = EngineArgs( - model=test_mock_s3_model, - load_format=load_format, - tensor_parallel_size=1, - ) - - vllm_config = engine_args.create_engine_config() - - executor = RunaiDummyExecutor(vllm_config) - executor.driver_worker.load_model() - - -def test_runai_reload_weights_path_from_mocked_s3(tmp_path: Path, monkeypatch): - # An engine started from object storage keeps the bucket URI in - # model_config.model_weights. A reload with weights_path must stream the - # new checkpoint, not re-stream the one the engine started from. - patcher = StreamerPatcher(str(tmp_path)) - monkeypatch.setattr( - "vllm.transformers_utils.runai_utils.runai_list_safetensors", - patcher.shim_list_safetensors, - ) - monkeypatch.setattr( - "vllm.transformers_utils.runai_utils.runai_pull_files", - patcher.shim_pull_files, - ) - monkeypatch.setattr( - "vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer", - patcher.create_mock_streamer, - ) - - base_dir = tmp_path / "gpt2" - hf_api().snapshot_download( - repo_id=test_model, - local_dir=str(base_dir), - allow_patterns=["*.json", "*.txt", "model.safetensors"], - ) - # Second checkpoint under another prefix: the final layer norm scaled, so a - # reload that silently re-streams the original bucket is distinguishable. - tensors = load_file(base_dir / "model.safetensors") - original = tensors["ln_f.weight"].clone() - modified = original * 2 - tensors["ln_f.weight"] = modified - mod_dir = tmp_path / "gpt2-mod" - mod_dir.mkdir() - save_file(tensors, mod_dir / "model.safetensors", metadata={"format": "pt"}) - - vllm_config = EngineArgs( - model="s3://my-mock-bucket/gpt2/", - load_format=load_format, - tensor_parallel_size=1, - ).create_engine_config() - assert vllm_config.model_config.model_weights == "s3://my-mock-bucket/gpt2/" - - executor = RunaiDummyExecutor(vllm_config) - executor.driver_worker.load_model() - model = executor.driver_worker.model_runner.get_model() - (param,) = [p for n, p in model.named_parameters() if n.endswith("ln_f.weight")] - - def current() -> torch.Tensor: - return param.detach().float().cpu() - - torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2) - - # No weights_path: re-stream the checkpoint the engine started from, i.e. - # the bucket kept in model_weights. Zero the live tensor first so a reload - # that did nothing is distinguishable from one that streamed the bucket. - param.data.zero_() - executor.driver_worker.reload_weights() - torch.testing.assert_close(current(), original, rtol=1e-2, atol=1e-2) - - # weights_path: the new prefix must win over the stale model_weights. - executor.driver_worker.reload_weights(weights_path="s3://my-mock-bucket/gpt2-mod/") - torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2) - - # After the override, a bare reload means the new checkpoint, not the - # one the engine was started with. - param.data.zero_() - executor.driver_worker.reload_weights() - torch.testing.assert_close(current(), modified, rtol=1e-2, atol=1e-2) diff --git a/tests/models/language/generation_ppl_test/test_gemma.py b/tests/models/language/generation_ppl_test/test_gemma.py deleted file mode 100644 index b846bb7020..0000000000 --- a/tests/models/language/generation_ppl_test/test_gemma.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest - -from tests.models.utils import GenerateModelInfo - -from .ppl_utils import wikitext_ppl_test - -MODELS = [ - GenerateModelInfo("google/gemma-2b", hf_ppl=21.48524284362793), - GenerateModelInfo("google/gemma-2-2b", hf_ppl=102.59290313720703), - GenerateModelInfo("google/gemma-3-4b-it", hf_ppl=27.79648208618164), -] - - -@pytest.mark.parametrize("model_info", MODELS) -def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo): - wikitext_ppl_test(hf_runner, vllm_runner, model_info) diff --git a/tests/models/language/generation_ppl_test/test_gpt.py b/tests/models/language/generation_ppl_test/test_gpt.py deleted file mode 100644 index af0e30cc4c..0000000000 --- a/tests/models/language/generation_ppl_test/test_gpt.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest - -from tests.models.utils import GenerateModelInfo - -from .ppl_utils import wikitext_ppl_test - -MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.457056045532227)] - - -@pytest.mark.parametrize("model_info", MODELS) -def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo): - bf16_ppl = wikitext_ppl_test(hf_runner, vllm_runner, model_info) - fp32_ppl = wikitext_ppl_test( - hf_runner, - vllm_runner, - model_info, - vllm_extra_kwargs={"hf_overrides": {"head_dtype": "float32"}}, - ) - - differ = ((fp32_ppl - bf16_ppl) / bf16_ppl) * 100 - print("fp32 head difference (%):", differ) - assert differ < 0 diff --git a/tests/models/language/pooling/embed_utils.py b/tests/models/language/pooling/embed_utils.py new file mode 100644 index 0000000000..f119d42721 --- /dev/null +++ b/tests/models/language/pooling/embed_utils.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Sequence + +import openai +import pytest + +from tests.conftest import HfRunner +from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy +from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse + + +def float_embeddings(response: EmbeddingResponse) -> list[list[float]]: + embeddings = [] + for d in response.data: + assert isinstance(d.embedding, list) + embeddings.append(d.embedding) + return embeddings + + +def run_embedding_correctness_test( + hf_model: "HfRunner", + inputs: list[str], + vllm_outputs: Sequence[list[float]], + dimensions: int | None = None, +): + hf_outputs = hf_model.encode(inputs) + if dimensions: + hf_outputs = matryoshka_fy(hf_outputs, dimensions) + + check_embeddings_close( + embeddings_0_lst=hf_outputs, + embeddings_1_lst=vllm_outputs, + name_0="hf", + name_1="vllm", + tol=1e-2, + ) + + +def correctness_test_embed_models( + hf_runner, + vllm_runner, + model_info: EmbedModelInfo, + example_prompts, + vllm_extra_kwargs=None, + hf_model_callback=None, +): + pytest.skip("Debug only, ci prefers to use mteb test.") + + # The example_prompts has ending "\n", for example: + # "Write a short story about a robot that dreams for the first time.\n" + # sentence_transformers will strip the input texts, see: + # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159 + # This makes the input_ids different between hf_model and vllm_model. + # So we need to strip the input texts to avoid test failing. + example_prompts = [str(s).strip() for s in example_prompts] + + vllm_extra_kwargs = vllm_extra_kwargs or {} + vllm_extra_kwargs["dtype"] = model_info.dtype + + if model_info.hf_overrides is not None: + vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides + + with vllm_runner( + model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs + ) as vllm_model: + vllm_outputs = vllm_model.embed(example_prompts) + + with hf_runner( + model_info.name, + dtype=model_info.hf_dtype, + is_sentence_transformer=True, + ) as hf_model: + if hf_model_callback is not None: + hf_model_callback(hf_model) + + run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs) + + +async def run_client_embeddings( + client: openai.AsyncOpenAI, + model_name: str, + queries: list[str], + instruction: str = "", +) -> list[list[float]]: + outputs = await client.embeddings.create( + model=model_name, + input=[instruction + q for q in queries], + ) + return [data.embedding for data in outputs.data] diff --git a/tests/models/language/pooling_mteb_test/__init__.py b/tests/models/language/pooling_mteb_test/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/models/language/pooling_mteb_test/conftest.py b/tests/models/language/pooling_mteb_test/conftest.py deleted file mode 100644 index ce34d82563..0000000000 --- a/tests/models/language/pooling_mteb_test/conftest.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Pytest configuration for vLLM pooling MTEB tests.""" - -import gc - -import pytest -import torch - -from tests.utils import wait_for_memory_to_settle -from vllm.logger import init_logger -from vllm.platforms import current_platform - -logger = init_logger(__name__) - - -@pytest.fixture(autouse=True) -def release_gpu_memory_between_tests(): - """Reclaim cached GPU memory between tests in this shard. - - These tests start many vLLM/HF engines sequentially in one pytest - process; without explicit reclamation between tests, VRAM fragmentation - accumulates and later tests fail at engine startup ("Engine core - initialization failed"). - """ - yield - gc.collect() - if torch.accelerator.is_available(): - torch.accelerator.empty_cache() - if current_platform.is_rocm(): - return - try: - wait_for_memory_to_settle() - except ValueError as e: - # Longer-lived (class/module-scoped) engine fixtures may legitimately - # still hold VRAM; nothing further to reclaim in that case. - logger.info("Failed to clean GPU memory: %s", e) diff --git a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py deleted file mode 100644 index 0423e94d21..0000000000 --- a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py +++ /dev/null @@ -1,279 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import mteb -import numpy as np -import torch -from mteb.models import ModelMeta -from mteb.types import Array -from torch.utils.data import DataLoader - -import tests.ci_envs as ci_envs -from tests.models.utils import ( - EmbedModelInfo, - check_embeddings_close, - get_vllm_extra_kwargs, -) - -# Most embedding models on the STS12 task (See #17175): -# - Model implementation and minor changes in tensor dtype -# results in differences less than 1e-4 -# - Different model results in differences more than 1e-3 -# 5e-4 is a good tolerance threshold -MTEB_EMBED_TASKS = ["STS12"] -MTEB_EMBED_TOL = 5e-4 - - -_empty_model_meta = ModelMeta( - loader=None, - name="vllm/model", - revision="1", - release_date=None, - languages=None, - framework=[], - similarity_fn_name=None, - n_parameters=None, - memory_usage_mb=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, - use_instructions=None, - training_datasets=None, - modalities=["text"], # 'image' can be added to evaluate multimodal models -) - - -class MtebEmbedMixin(mteb.EncoderProtocol): - mteb_model_meta = _empty_model_meta - - def similarity( - self, - embeddings1: np.ndarray, - embeddings2: np.ndarray, - ) -> np.ndarray: - # Cosine similarity - norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True) - norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True) - sim = np.dot(embeddings1, embeddings2.T) / (norm1 * norm2.T) - return sim - - def similarity_pairwise( - self, - embeddings1: Array, - embeddings2: Array, - ) -> Array: - # Cosine similarity - norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True) - norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True) - sim = np.sum(embeddings1 * embeddings2, axis=1) / ( - norm1.flatten() * norm2.flatten() - ) - return sim - - -class HfMtebEncoder(MtebEmbedMixin): - def __init__(self, model): - self.model = model - - def encode( - self, - inputs: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - sentences = [text for batch in inputs for text in batch["text"]] - return self.model.encode(sentences) - - -class VllmMtebEncoder(MtebEmbedMixin): - def __init__(self, vllm_model, prompt_prefix: str | None = None): - self.llm = vllm_model - self.rng = np.random.default_rng(seed=42) - self.prompt_prefix = prompt_prefix - - def encode( - self, - inputs: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - # Hoping to discover potential scheduling - # issues by randomizing the order. - sentences = [ - self.prompt_prefix + text if self.prompt_prefix else text - for batch in inputs - for text in batch["text"] - ] - r = self.rng.permutation(len(sentences)) - sentences = [sentences[i] for i in r] - outputs = self.llm.embed(sentences, use_tqdm=False) - embeds = np.array(outputs) - embeds = embeds[np.argsort(r)] - return embeds - - -class OpenAIClientMtebEncoder(MtebEmbedMixin): - def __init__(self, model_name: str, client): - self.model_name = model_name - self.client = client - self.rng = np.random.default_rng(seed=42) - - def encode( - self, - inputs: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - # Hoping to discover potential scheduling - # issues by randomizing the order. - sentences = [text for batch in inputs for text in batch["text"]] - r = self.rng.permutation(len(sentences)) - sentences = [sentences[i] for i in r] - - embeddings = self.client.embeddings.create( - model=self.model_name, input=sentences - ) - outputs = [d.embedding for d in embeddings.data] - embeds = np.array(outputs) - embeds = embeds[np.argsort(r)] - return embeds - - -def run_mteb_embed_task(encoder: mteb.EncoderProtocol, tasks): - tasks = mteb.get_tasks(tasks=tasks) - results = mteb.evaluate( - encoder, - tasks, - cache=None, - show_progress_bar=False, - ) - - main_score = results[0].scores["test"][0]["main_score"] - return main_score - - -def mteb_test_embed_models( - hf_runner, - vllm_runner, - model_info: EmbedModelInfo, - vllm_extra_kwargs=None, - hf_model_callback=None, - atol: float | None = None, - prompt_prefix: str | None = None, - vllm_model_callback=None, -): - if atol is None: - atol = ( - model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_EMBED_TOL - ) - vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs) - - # Test embed_dims, isnan and whether to use normalize - example_prompts = ["The chef prepared a delicious meal." * 1000] - - with vllm_runner( - model_info.name, - revision=model_info.revision, - runner="pooling", - max_model_len=model_info.max_model_len, - **vllm_extra_kwargs, - ) as vllm_model: - model_config = vllm_model.llm.llm_engine.model_config - - if vllm_model_callback is not None: - vllm_model_callback(vllm_model) - - # Confirm whether vllm is using the correct architecture - if model_info.architecture: - assert model_info.architecture in model_config.architectures - - # Confirm whether the important configs in model_config are correct. - pooler_config = model_config.pooler_config - if model_info.seq_pooling_type is not None: - assert pooler_config.seq_pooling_type == model_info.seq_pooling_type - if model_info.tok_pooling_type is not None: - assert pooler_config.tok_pooling_type == model_info.tok_pooling_type - if model_info.attn_type is not None: - assert model_config.attn_type == model_info.attn_type - if model_info.is_prefix_caching_supported is not None: - assert ( - model_config.is_prefix_caching_supported - == model_info.is_prefix_caching_supported - ) - if model_info.is_chunked_prefill_supported is not None: - assert ( - model_config.is_chunked_prefill_supported - == model_info.is_chunked_prefill_supported - ) - - vllm_main_score = run_mteb_embed_task( - VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS - ) - vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype - head_dtype = model_config.head_dtype - - # Test embedding_size, isnan and whether to use normalize. - # Apply the same prompt_prefix used for scoring so this check compares - # identical effective inputs: HF's SentenceTransformer.encode() applies - # the model's default prompt (e.g. "Document: "), so vLLM must receive it - # too. This mirrors VllmMtebEncoder and changes no production behavior. - consistency_prompts = ( - [prompt_prefix + p for p in example_prompts] - if prompt_prefix - else example_prompts - ) - vllm_outputs = vllm_model.embed( - consistency_prompts, - tokenization_kwargs=dict(truncate_prompt_tokens=-1), - ) - outputs_tensor = torch.tensor(vllm_outputs) - assert not torch.any(torch.isnan(outputs_tensor)) - embedding_size = model_config.embedding_size - assert torch.tensor(vllm_outputs).shape[-1] == embedding_size - - # Accelerate mteb test by setting - # SentenceTransformers mteb score to a constant - if model_info.mteb_score is None: - with hf_runner( - model_info.name, - revision=model_info.revision, - is_sentence_transformer=True, - dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype, - ) as hf_model: - # e.g. setting default parameters for the encode method of hf_runner - if hf_model_callback is not None: - hf_model_callback(hf_model) - - st_main_score = run_mteb_embed_task( - HfMtebEncoder(hf_model), MTEB_EMBED_TASKS - ) - st_dtype = next(hf_model.model.parameters()).dtype - - # Check embeddings close to hf outputs - hf_outputs = hf_model.encode(example_prompts) - check_embeddings_close( - embeddings_0_lst=hf_outputs, - embeddings_1_lst=vllm_outputs, - name_0="hf", - name_1="vllm", - tol=1e-2, - ) - else: - st_main_score = model_info.mteb_score - st_dtype = "Constant" - - print("Model:", model_info.name) - print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score) - print("SentenceTransformers:", st_dtype, st_main_score) - print("Difference:", st_main_score - vllm_main_score) - - # We are not concerned that the vllm mteb results are better - # than SentenceTransformers, so we only perform one-sided testing. - diff = st_main_score - vllm_main_score - assert diff < atol, ( - f"diff={diff:.6g} tol={atol} model={model_info.name} " - f"(st={st_main_score}, vllm={vllm_main_score})" - ) diff --git a/tests/models/language/pooling_mteb_test/mteb_score_utils.py b/tests/models/language/pooling_mteb_test/mteb_score_utils.py deleted file mode 100644 index 2d83a2ebaa..0000000000 --- a/tests/models/language/pooling_mteb_test/mteb_score_utils.py +++ /dev/null @@ -1,319 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import tempfile -from pathlib import Path -from typing import Any - -import mteb -import numpy as np -import requests -import torch -from mteb.models import ModelMeta -from torch.utils.data import DataLoader - -from tests.conftest import HfRunner -from tests.models.utils import ( - RerankModelInfo, - get_vllm_extra_kwargs, -) - -# See #19344 -MTEB_RERANK_TASKS = ["NFCorpus"] -MTEB_RERANK_LANGS = ["eng"] -MTEB_RERANK_TOL = 2e-3 - -template_home = ( - Path(__file__).parent.parent.parent.parent.parent - / "examples/pooling/score/template" -) - -_empty_model_meta = ModelMeta( - loader=None, - name="vllm/model", - revision="1", - release_date=None, - languages=None, - framework=[], - similarity_fn_name=None, - n_parameters=None, - memory_usage_mb=None, - max_tokens=None, - embed_dim=None, - license=None, - open_weights=None, - public_training_code=None, - public_training_data=None, - use_instructions=None, - training_datasets=None, - modalities=["text"], # 'image' can be added to evaluate multimodal models -) - - -class MtebCrossEncoderMixin(mteb.CrossEncoderProtocol): - mteb_model_meta = _empty_model_meta - - -class VllmMtebCrossEncoder(MtebCrossEncoderMixin): - def __init__(self, vllm_model): - self.llm = vllm_model - self.rng = np.random.default_rng(seed=42) - self.chat_template: str | None = getattr(vllm_model, "chat_template", None) - - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - corpus = [text for batch in inputs2 for text in batch["text"]] - - # Hoping to discover potential scheduling - # issues by randomizing the order. - r = self.rng.permutation(len(queries)) - queries = [queries[i] for i in r] - corpus = [corpus[i] for i in r] - - outputs = self.llm.score( - queries, - corpus, - use_tqdm=False, - chat_template=self.chat_template, - tokenization_kwargs={"truncate_prompt_tokens": -1}, - ) - scores = np.array(outputs) - scores = scores[np.argsort(r)] - return scores - - -class ScoreClientMtebEncoder(MtebCrossEncoderMixin): - mteb_model_meta = _empty_model_meta - - def __init__(self, model_name: str, url): - self.model_name = model_name - self.url = url - - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - full_corpus = [text for batch in inputs2 for text in batch["text"]] - - outputs = [] - for query, corpus in zip(queries, full_corpus): - outputs.append(self.get_score(query, corpus)) - - scores = np.array(outputs) - return scores - - def get_score(self, query, corpus): - response = requests.post( - self.url, - json={ - "model": self.model_name, - "queries": query, - "documents": corpus, - "truncate_prompt_tokens": -1, - }, - ).json() - return response["data"][0]["score"] - - -class RerankClientMtebEncoder(ScoreClientMtebEncoder): - def get_score(self, query, corpus): - response = requests.post( - self.url, - json={ - "model": self.model_name, - "query": query, - "documents": [corpus], - "truncate_prompt_tokens": -1, - }, - ).json() - return response["results"][0]["relevance_score"] - - -class HFMtebCrossEncoder(MtebCrossEncoderMixin, HfRunner): - chat_template: str | None = None - - def __init__(self, model_name: str, dtype: str = "auto", **kwargs: Any) -> None: - HfRunner.__init__( - self, model_name=model_name, is_cross_encoder=True, dtype=dtype, **kwargs - ) - - @torch.no_grad - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - corpus = [text for batch in inputs2 for text in batch["text"]] - - if self.chat_template is not None: - tokenizer = self.model.tokenizer - prompts = [] - for query, document in zip(queries, corpus): - conversation = [ - {"role": "query", "content": query}, - {"role": "document", "content": document}, - ] - - prompt = tokenizer.apply_chat_template( - conversation=conversation, - tools=None, - chat_template=self.chat_template, - tokenize=False, - ) - prompts.append(prompt) - outputs_list = HfRunner.classify(self, prompts) - scores = np.array(outputs_list).squeeze(-1) - return scores - else: - prompts = list(zip(queries, corpus)) - outputs_tensor = HfRunner.predict(self, prompts, show_progress_bar=False) - return outputs_tensor.cpu().numpy() - - -def run_mteb_rerank(cross_encoder: mteb.CrossEncoderProtocol, tasks, languages): - with tempfile.TemporaryDirectory() as prediction_folder: - bm25s = mteb.get_model("bm25s") - eval_splits = ["test"] - - mteb_tasks: list[mteb.abstasks.AbsTaskRetrieval] = mteb.get_tasks( - tasks=tasks, languages=languages, eval_splits=eval_splits - ) - for task in mteb_tasks: - if not task.data_loaded: - task.load_data() - - mteb.evaluate( - bm25s, - mteb_tasks, - prediction_folder=prediction_folder, - show_progress_bar=False, - # don't save results for test runs - cache=None, - overwrite_strategy="always", - ) - - second_stage_tasks = [] - for task in mteb_tasks: - second_stage_tasks.append( - task.convert_to_reranking( - prediction_folder, - top_k=10, - ) - ) - - results = mteb.evaluate( - cross_encoder, - second_stage_tasks, - show_progress_bar=False, - cache=None, - ) - main_score = results[0].scores["test"][0]["main_score"] - return main_score - - -def mteb_test_rerank_models( - vllm_runner, - model_info: RerankModelInfo, - hf_runner=HFMtebCrossEncoder, - vllm_extra_kwargs=None, - vllm_mteb_encoder=VllmMtebCrossEncoder, - atol: float | None = None, -): - if atol is None: - atol = ( - model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_RERANK_TOL - ) - vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs) - - # Maybe load chat_template. - chat_template: str | None = None - if model_info.chat_template_name is not None: - chat_template = (template_home / model_info.chat_template_name).read_text() - - with vllm_runner( - model_info.name, - revision=model_info.revision, - runner="pooling", - max_model_len=None, - max_num_seqs=8, - **vllm_extra_kwargs, - ) as vllm_model: - model_config = vllm_model.llm.llm_engine.model_config - vllm_model.chat_template = chat_template - - # Confirm whether vllm is using the correct architecture - if model_info.architecture: - assert model_info.architecture in model_config.architectures - - # Score API is only enabled for num_labels == 1 - assert model_config.hf_config.num_labels == 1 - - # Confirm whether the important configs in model_config are correct. - pooler_config = model_config.pooler_config - if model_info.seq_pooling_type is not None: - assert pooler_config.seq_pooling_type == model_info.seq_pooling_type - if model_info.tok_pooling_type is not None: - assert pooler_config.tok_pooling_type == model_info.tok_pooling_type - if model_info.attn_type is not None: - assert model_config.attn_type == model_info.attn_type - if model_info.is_prefix_caching_supported is not None: - assert ( - model_config.is_prefix_caching_supported - == model_info.is_prefix_caching_supported - ) - if model_info.is_chunked_prefill_supported is not None: - assert ( - model_config.is_chunked_prefill_supported - == model_info.is_chunked_prefill_supported - ) - - vllm_main_score = run_mteb_rerank( - vllm_mteb_encoder(vllm_model), - tasks=MTEB_RERANK_TASKS, - languages=MTEB_RERANK_LANGS, - ) - vllm_dtype = model_config.dtype - head_dtype = model_config.head_dtype - - # Accelerate mteb test by setting - # SentenceTransformers mteb score to a constant - if model_info.mteb_score is None: - with hf_runner( - model_info.name, revision=model_info.revision, dtype=model_info.hf_dtype - ) as hf_model: - hf_model.chat_template = chat_template - st_main_score = run_mteb_rerank( - hf_model, - tasks=MTEB_RERANK_TASKS, - languages=MTEB_RERANK_LANGS, - ) - st_dtype = next(hf_model.model.model.parameters()).dtype - else: - st_main_score = model_info.mteb_score - st_dtype = "Constant" - - print("Model:", model_info.name) - print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score) - print("SentenceTransformers:", st_dtype, st_main_score) - print("Difference:", st_main_score - vllm_main_score) - - # We are not concerned that the vllm mteb results are better - # than SentenceTransformers, so we only perform one-sided testing. - diff = st_main_score - vllm_main_score - assert diff < atol, ( - f"diff={diff:.6g} tol={atol} model={model_info.name} " - f"(st={st_main_score}, vllm={vllm_main_score})" - ) diff --git a/tests/models/language/pooling_mteb_test/test_baai.py b/tests/models/language/pooling_mteb_test/test_baai.py deleted file mode 100644 index a3a4ffaef1..0000000000 --- a/tests/models/language/pooling_mteb_test/test_baai.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest - -from tests.models.language.pooling.embed_utils import correctness_test_embed_models -from tests.models.utils import ( - EmbedModelInfo, - RerankModelInfo, -) - -from .mteb_embed_utils import mteb_test_embed_models -from .mteb_score_utils import mteb_test_rerank_models - -MODELS = [ - ########## BertModel - EmbedModelInfo( - "BAAI/bge-base-en", - architecture="BertModel", - mteb_score=0.779336792, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo("BAAI/bge-base-zh", architecture="BertModel", enable_test=False), - EmbedModelInfo("BAAI/bge-small-en", architecture="BertModel", enable_test=False), - EmbedModelInfo("BAAI/bge-small-zh", architecture="BertModel", enable_test=False), - EmbedModelInfo("BAAI/bge-large-en", architecture="BertModel", enable_test=False), - EmbedModelInfo("BAAI/bge-large-zh", architecture="BertModel", enable_test=False), - EmbedModelInfo( - "BAAI/bge-large-zh-noinstruct", architecture="BertModel", enable_test=False - ), - EmbedModelInfo( - "BAAI/bge-base-en-v1.5", architecture="BertModel", enable_test=False - ), - EmbedModelInfo( - "BAAI/bge-base-zh-v1.5", architecture="BertModel", enable_test=False - ), - EmbedModelInfo( - "BAAI/bge-small-en-v1.5", architecture="BertModel", enable_test=False - ), - EmbedModelInfo( - "BAAI/bge-small-zh-v1.5", architecture="BertModel", enable_test=False - ), - EmbedModelInfo( - "BAAI/bge-large-en-v1.5", architecture="BertModel", enable_test=False - ), - EmbedModelInfo( - "BAAI/bge-large-zh-v1.5", architecture="BertModel", enable_test=False - ), - ########## XLMRobertaModel - EmbedModelInfo( - "BAAI/bge-m3", - architecture="XLMRobertaModel", - mteb_score=0.787343078, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - ########## Qwen2Model - EmbedModelInfo( - "BAAI/bge-code-v1", - architecture="Qwen2Model", - mteb_score=0.75724465, - seq_pooling_type="LAST", - attn_type="decoder", - is_prefix_caching_supported=True, - is_chunked_prefill_supported=True, - # Skip: model's custom tokenizer on HF hub is incompatible with - # transformers v5 (sets attrs before super().__init__, triggering - # AttributeError on 'verbose' in __getattr__). - enable_test=False, - ), -] - -RERANK_MODELS = [ - ########## XLMRobertaForSequenceClassification - RerankModelInfo( - "BAAI/bge-reranker-base", - architecture="XLMRobertaForSequenceClassification", - mteb_score=0.32398, - mteb_tol=1e-2, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - RerankModelInfo( - "BAAI/bge-reranker-large", - architecture="XLMRobertaForSequenceClassification", - enable_test=False, - ), - RerankModelInfo( - "BAAI/bge-reranker-v2-m3", - architecture="XLMRobertaForSequenceClassification", - enable_test=False, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - mteb_test_embed_models(hf_runner, vllm_runner, model_info) - - -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_correctness( - hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts -) -> None: - correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py b/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py deleted file mode 100644 index b750b622b9..0000000000 --- a/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -import mteb -import numpy as np -import pytest -import torch -from torch.utils.data import DataLoader - -from tests.conftest import HfRunner -from tests.models.utils import RerankModelInfo - -from .mteb_score_utils import ( - MtebCrossEncoderMixin, - mteb_test_rerank_models, -) - -RERANK_MODELS = [ - RerankModelInfo( - "BAAI/bge-reranker-v2-gemma", - architecture="GemmaForSequenceClassification", - hf_overrides={ - "architectures": ["GemmaForSequenceClassification"], - "classifier_from_token": ["Yes"], - "method": "no_post_processing", - }, - mteb_score=0.33757, - mteb_tol=1e-2, - seq_pooling_type="LAST", - attn_type="decoder", - is_prefix_caching_supported=True, - is_chunked_prefill_supported=True, - chat_template_name="bge-reranker-v2-gemma.jinja", - ), -] - -PROMPT = "Given a query A and a passage B, determine whether the passage contains an answer to the query by providing a prediction of either 'Yes' or 'No'." # noqa: E501 - - -class GemmaRerankerHfRunner(MtebCrossEncoderMixin, HfRunner): - def __init__( - self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any - ) -> None: - from transformers import AutoModelForCausalLM, AutoTokenizer - - HfRunner.__init__( - self, - model_name=model_name, - auto_cls=AutoModelForCausalLM, - dtype=dtype, - **kwargs, - ) - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - self.yes_loc = self.tokenizer.convert_tokens_to_ids("Yes") - - @torch.no_grad - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - corpus = [text for batch in inputs2 for text in batch["text"]] - - def get_inputs(pairs, tokenizer, prompt=None): - if prompt is None: - prompt = PROMPT - - sep = "\n" - prompt_inputs = tokenizer( - prompt, return_tensors=None, add_special_tokens=False - )["input_ids"] - sep_inputs = tokenizer(sep, return_tensors=None, add_special_tokens=False)[ - "input_ids" - ] - inputs = [] - for query, passage in pairs: - query_inputs = tokenizer( - f"A: {query}", - return_tensors=None, - add_special_tokens=False, - truncation=True, - ) - passage_inputs = tokenizer( - f"B: {passage}", - return_tensors=None, - add_special_tokens=False, - truncation=True, - ) - item = tokenizer.prepare_for_model( - [tokenizer.bos_token_id] + query_inputs["input_ids"], - sep_inputs + passage_inputs["input_ids"], - truncation="only_second", - padding=False, - return_attention_mask=False, - return_token_type_ids=False, - add_special_tokens=False, - ) - item["input_ids"] = item["input_ids"] + sep_inputs + prompt_inputs - item["attention_mask"] = [1] * len(item["input_ids"]) - inputs.append(item) - return tokenizer.pad( - inputs, - padding=True, - return_tensors="pt", - ) - - scores = [] - for query, document in zip(queries, corpus): - pairs = [(query, document)] - inputs = get_inputs(pairs, self.tokenizer) - inputs = inputs.to(self.model.device) - _n_tokens = inputs["input_ids"].shape[1] - logits = self.model(**inputs, return_dict=True).logits - _scores = ( - logits[:, -1, self.yes_loc] - .view( - -1, - ) - .float() - .sigmoid() - ) - scores.append(_scores[0].item()) - return torch.Tensor(scores) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models( - vllm_runner, - model_info, - hf_runner=GemmaRerankerHfRunner, - ) diff --git a/tests/models/language/pooling_mteb_test/test_cross_encoder.py b/tests/models/language/pooling_mteb_test/test_cross_encoder.py deleted file mode 100644 index 8f5d86fa8d..0000000000 --- a/tests/models/language/pooling_mteb_test/test_cross_encoder.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest - -from tests.models.utils import ( - RerankModelInfo, -) - -from .mteb_score_utils import mteb_test_rerank_models - -RERANK_MODELS = [ - RerankModelInfo( - "cross-encoder/ms-marco-TinyBERT-L-2-v2", - architecture="BertForSequenceClassification", - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - mteb_score=0.32898, - mteb_tol=1e-2, - ), - RerankModelInfo( - "tomaarsen/Qwen3-Reranker-0.6B-seq-cls", - architecture="Qwen3ForSequenceClassification", - seq_pooling_type="LAST", - attn_type="decoder", - is_prefix_caching_supported=True, - is_chunked_prefill_supported=True, - chat_template_name="qwen3_reranker.jinja", - mteb_score=0.33459, - mteb_tol=1e-2, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_intfloat.py b/tests/models/language/pooling_mteb_test/test_intfloat.py deleted file mode 100644 index 805a3f91ed..0000000000 --- a/tests/models/language/pooling_mteb_test/test_intfloat.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest - -from tests.models.language.pooling.embed_utils import correctness_test_embed_models -from tests.models.utils import EmbedModelInfo - -from .mteb_embed_utils import mteb_test_embed_models - -MODELS = [ - ########## BertModel - EmbedModelInfo( - "intfloat/e5-small", - architecture="BertModel", - mteb_score=0.742285423, - mteb_tol=2e-3, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo("intfloat/e5-base", architecture="BertModel", enable_test=False), - EmbedModelInfo("intfloat/e5-large", architecture="BertModel", enable_test=False), - EmbedModelInfo( - "intfloat/multilingual-e5-small", architecture="BertModel", enable_test=False - ), - ########## XLMRobertaModel - EmbedModelInfo( - "intfloat/multilingual-e5-base", - architecture="XLMRobertaModel", - mteb_score=0.779325955, - mteb_tol=2e-3, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo( - "intfloat/multilingual-e5-large", - architecture="XLMRobertaModel", - enable_test=False, - ), - EmbedModelInfo( - "intfloat/multilingual-e5-large-instruct", - architecture="XLMRobertaModel", - enable_test=False, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - mteb_test_embed_models(hf_runner, vllm_runner, model_info) - - -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_correctness( - hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts -) -> None: - correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) diff --git a/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py b/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py deleted file mode 100644 index 1ad1f68b9a..0000000000 --- a/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -import mteb -import numpy as np -import pytest -import torch -from torch.utils.data import DataLoader - -from tests.conftest import HfRunner -from tests.models.utils import RerankModelInfo - -from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models - -mxbai_rerank_hf_overrides = { - "architectures": ["Qwen2ForSequenceClassification"], - "classifier_from_token": ["0", "1"], - "method": "from_2_way_softmax", -} - -RERANK_MODELS = [ - RerankModelInfo( - "mixedbread-ai/mxbai-rerank-base-v2", - architecture="Qwen2ForSequenceClassification", - hf_overrides=mxbai_rerank_hf_overrides, - seq_pooling_type="LAST", - attn_type="decoder", - is_prefix_caching_supported=True, - is_chunked_prefill_supported=True, - chat_template_name="mxbai_rerank_v2.jinja", - mteb_score=0.33651, - mteb_tol=1e-2, - enable_test=True, - ), - RerankModelInfo( - "mixedbread-ai/mxbai-rerank-large-v2", - architecture="Qwen2ForSequenceClassification", - hf_overrides=mxbai_rerank_hf_overrides, - chat_template_name="mxbai_rerank_v2.jinja", - enable_test=False, - ), -] - - -class MxbaiRerankerHfRunner(MtebCrossEncoderMixin, HfRunner): - def __init__( - self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any - ) -> None: - from transformers import AutoModelForCausalLM, AutoTokenizer - - HfRunner.__init__( - self, - model_name=model_name, - auto_cls=AutoModelForCausalLM, - dtype=dtype, - **kwargs, - ) - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - self.yes_loc = self.tokenizer.convert_tokens_to_ids("1") - self.no_loc = self.tokenizer.convert_tokens_to_ids("0") - - @torch.no_grad - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - corpus = [text for batch in inputs2 for text in batch["text"]] - - tokenizer = self.tokenizer - prompts = [] - for query, document in zip(queries, corpus): - conversation = [ - {"role": "query", "content": query}, - {"role": "document", "content": document}, - ] - - prompt = tokenizer.apply_chat_template( - conversation=conversation, - tools=None, - chat_template=self.chat_template, - tokenize=False, - ) - prompts.append(prompt) - - def compute_logits(inputs): - logits = self.model(**inputs).logits[:, -1, :] - yes_logits = logits[:, self.yes_loc] - no_logits = logits[:, self.no_loc] - logits = yes_logits - no_logits - scores = logits.float().sigmoid() - return scores - - scores = [] - for prompt in prompts: - inputs = tokenizer([prompt], return_tensors="pt") - inputs = self.wrap_device(inputs) - score = compute_logits(inputs) - scores.append(score[0].item()) - return torch.Tensor(scores) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models(vllm_runner, model_info, hf_runner=MxbaiRerankerHfRunner) diff --git a/tests/models/language/pooling_mteb_test/test_nemotron.py b/tests/models/language/pooling_mteb_test/test_nemotron.py deleted file mode 100644 index 711224a1a2..0000000000 --- a/tests/models/language/pooling_mteb_test/test_nemotron.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.models.language.pooling_mteb_test.mteb_embed_utils import ( - mteb_test_embed_models, -) -from tests.models.language.pooling_mteb_test.mteb_score_utils import ( - mteb_test_rerank_models, -) -from tests.models.utils import ( - EmbedModelInfo, - RerankModelInfo, -) - -EMBEDDING_MODELS = [ - EmbedModelInfo( - "nvidia/llama-nemotron-embed-1b-v2", - architecture="LlamaBidirectionalModel", - mteb_score=0.689164662128673, - mteb_tol=2e-3, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - ) -] - -RERANK_MODELS = [ - RerankModelInfo( - "nvidia/llama-nemotron-rerank-1b-v2", - architecture="LlamaBidirectionalForSequenceClassification", - chat_template_name="nemotron-rerank.jinja", - mteb_score=0.33994, - mteb_tol=1e-2, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", EMBEDDING_MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - mteb_test_embed_models(hf_runner, vllm_runner, model_info) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models(vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_nomic.py b/tests/models/language/pooling_mteb_test/test_nomic.py deleted file mode 100644 index 38e34b78b0..0000000000 --- a/tests/models/language/pooling_mteb_test/test_nomic.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.models.language.pooling.embed_utils import correctness_test_embed_models -from tests.models.utils import EmbedModelInfo - -from .mteb_embed_utils import mteb_test_embed_models - -MODELS = [ - EmbedModelInfo( - "nomic-ai/nomic-embed-text-v1", - architecture="NomicBertModel", - # Fixme: - # Update nomic-embed code to support the latest - # HF version and remove revision set. - revision="720244025c1a7e15661a174c63cce63c8218e52b", - mteb_score=0.737568559, - mteb_tol=2e-3, - enable_test=True, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - ), - EmbedModelInfo( - "nomic-ai/nomic-embed-text-v1.5", - architecture="NomicBertModel", - enable_test=False, - ), - EmbedModelInfo( - "nomic-ai/CodeRankEmbed", architecture="NomicBertModel", enable_test=False - ), - EmbedModelInfo( - "nomic-ai/nomic-embed-text-v2-moe", - architecture="NomicBertModel", - mteb_score=0.715488912, - mteb_tol=2e-3, - enable_test=True, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - mteb_test_embed_models(hf_runner, vllm_runner, model_info) - - -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_correctness( - hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts -) -> None: - correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) diff --git a/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py b/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py deleted file mode 100644 index 10c90e733e..0000000000 --- a/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py +++ /dev/null @@ -1,133 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# ruff: noqa: E501 -from typing import Any - -import mteb -import numpy as np -import pytest -import torch -from torch.utils.data import DataLoader - -from tests.conftest import HfRunner -from tests.models.utils import RerankModelInfo -from tests.utils import multi_gpu_test - -from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models - -qwen3_reranker_hf_overrides = { - "architectures": ["Qwen3ForSequenceClassification"], - "classifier_from_token": ["no", "yes"], - "is_original_qwen3_reranker": True, -} - -RERANK_MODELS = [ - RerankModelInfo( - "Qwen/Qwen3-Reranker-0.6B", - architecture="Qwen3ForSequenceClassification", - hf_overrides=qwen3_reranker_hf_overrides, - chat_template_name="qwen3_reranker.jinja", - seq_pooling_type="LAST", - attn_type="decoder", - is_prefix_caching_supported=True, - is_chunked_prefill_supported=True, - mteb_score=0.33459, - mteb_tol=1e-2, - enable_test=True, - ), - RerankModelInfo( - "Qwen/Qwen3-Reranker-4B", - architecture="Qwen3ForSequenceClassification", - chat_template_name="qwen3_reranker.jinja", - hf_overrides=qwen3_reranker_hf_overrides, - enable_test=False, - ), -] - - -class Qwen3RerankerHfRunner(MtebCrossEncoderMixin, HfRunner): - def __init__( - self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any - ) -> None: - from transformers import AutoModelForCausalLM, AutoTokenizer - - HfRunner.__init__( - self, - model_name=model_name, - auto_cls=AutoModelForCausalLM, - dtype=dtype, - **kwargs, - ) - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - self.token_false_id = self.tokenizer.convert_tokens_to_ids("no") - self.token_true_id = self.tokenizer.convert_tokens_to_ids("yes") - self.max_length = 40960 - - @torch.no_grad - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - corpus = [text for batch in inputs2 for text in batch["text"]] - - tokenizer = self.tokenizer - prompts = [] - for query, document in zip(queries, corpus): - conversation = [ - {"role": "query", "content": query}, - {"role": "document", "content": document}, - ] - - prompt = tokenizer.apply_chat_template( - conversation=conversation, - tools=None, - chat_template=self.chat_template, - tokenize=False, - ) - prompts.append(prompt) - - def compute_logits(inputs): - batch_scores = self.model(**inputs).logits[:, -1, :] - true_vector = batch_scores[:, self.token_true_id] - false_vector = batch_scores[:, self.token_false_id] - batch_scores = torch.stack([false_vector, true_vector], dim=1) - batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1) - scores = batch_scores[:, 1].exp() - return scores - - scores = [] - for prompt in prompts: - inputs = tokenizer([prompt], return_tensors="pt") - inputs = self.wrap_device(inputs) - score = compute_logits(inputs) - scores.append(score[0].item()) - return torch.Tensor(scores) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models(vllm_runner, model_info, hf_runner=Qwen3RerankerHfRunner) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -@multi_gpu_test(num_gpus=2) -def test_rerank_models_mteb_tp(vllm_runner, model_info: RerankModelInfo) -> None: - assert model_info.architecture == "Qwen3ForSequenceClassification" - - vllm_extra_kwargs: dict[str, Any] = { - "tensor_parallel_size": 2, - } - - mteb_test_rerank_models( - vllm_runner, - model_info, - vllm_extra_kwargs=vllm_extra_kwargs, - hf_runner=Qwen3RerankerHfRunner, - ) diff --git a/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py b/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py deleted file mode 100644 index b68b2778b6..0000000000 --- a/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.models.language.pooling.embed_utils import correctness_test_embed_models -from tests.models.utils import EmbedModelInfo - -from .mteb_embed_utils import mteb_test_embed_models - -MODELS = [ - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-xs", - is_matryoshka=False, - architecture="BertModel", - mteb_score=0.714927797, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-s", - is_matryoshka=False, - architecture="BertModel", - enable_test=False, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-m", - is_matryoshka=False, - architecture="BertModel", - enable_test=False, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-m-long", - is_matryoshka=False, - architecture="NomicBertModel", - mteb_score=0.681146831, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-l", - is_matryoshka=False, - architecture="BertModel", - enable_test=False, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-m-v1.5", - is_matryoshka=True, - architecture="BertModel", - mteb_score=0.649088363, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-l-v2.0", - is_matryoshka=True, - architecture="XLMRobertaModel", - mteb_score=0.712258299, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo( - "Snowflake/snowflake-arctic-embed-m-v2.0", - is_matryoshka=True, - architecture="GteModel", - mteb_score=0.706622444, - mteb_tol=2e-3, - seq_pooling_type="CLS", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - mteb_test_embed_models(hf_runner, vllm_runner, model_info) - - -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_correctness( - hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts -) -> None: - correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) diff --git a/tests/models/language/pooling_mteb_test/test_st_projector.py b/tests/models/language/pooling_mteb_test/test_st_projector.py deleted file mode 100644 index 7397db8535..0000000000 --- a/tests/models/language/pooling_mteb_test/test_st_projector.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pytest - -from tests.models.utils import ( - EmbedModelInfo, -) - -from .mteb_embed_utils import mteb_test_embed_models - -# ST models with projector (Dense) layers -ST_PROJECTOR_MODELS = [ - EmbedModelInfo( - "TencentBAC/Conan-embedding-v1", - architecture="BertModel", - mteb_score=0.688611955, - mteb_tol=2e-3, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), - EmbedModelInfo( - "google/embeddinggemma-300m", - architecture="Gemma3TextModel", - mteb_score=0.7473819294684156, - mteb_tol=2e-3, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - enable_test=True, - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - mteb_test_embed_models(hf_runner, vllm_runner, model_info) diff --git a/tests/models/language/pooling_mteb_test/test_voyage.py b/tests/models/language/pooling_mteb_test/test_voyage.py deleted file mode 100644 index 628ea31ca0..0000000000 --- a/tests/models/language/pooling_mteb_test/test_voyage.py +++ /dev/null @@ -1,58 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.models.language.pooling.embed_utils import correctness_test_embed_models -from tests.models.utils import EmbedModelInfo - -from .mteb_embed_utils import mteb_test_embed_models - -MODELS = [ - EmbedModelInfo( - "voyageai/voyage-4-nano", - architecture="VoyageQwen3BidirectionalEmbedModel", - enable_test=True, - seq_pooling_type="MEAN", - attn_type="encoder_only", - is_prefix_caching_supported=False, - is_chunked_prefill_supported=False, - hf_overrides={ - "architectures": ["VoyageQwen3BidirectionalEmbedModel"], - "num_labels": 2048, - }, - mteb_score=0.7054, - mteb_tol=2e-3, - # === MTEB Results === - # STS12: 0.6613 - # STS13: 0.6906 - # STS14: 0.6556 - # STS15: 0.7843 - # STS16: 0.7340 - # STSBenchmark: 0.7063 - # Average score: 0.7054 - ), -] - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: - # Encoder-only attention models need enforce_eager=True to avoid - # CUDA graph capture issues with piecewise compilation - mteb_test_embed_models( - hf_runner, vllm_runner, model_info, vllm_extra_kwargs={"enforce_eager": True} - ) - - -@pytest.mark.parametrize("model_info", MODELS) -def test_embed_models_correctness( - hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts -) -> None: - correctness_test_embed_models( - hf_runner, - vllm_runner, - model_info, - example_prompts, - vllm_extra_kwargs={"enforce_eager": True}, - ) diff --git a/tests/models/language/pooling_mteb_test/test_zerank.py b/tests/models/language/pooling_mteb_test/test_zerank.py deleted file mode 100644 index c90c80bc4d..0000000000 --- a/tests/models/language/pooling_mteb_test/test_zerank.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any - -import mteb -import numpy as np -import pytest -import torch -from torch.utils.data import DataLoader - -from tests.conftest import HfRunner -from tests.models.utils import RerankModelInfo - -from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models - -RERANK_MODELS = [ - RerankModelInfo( - "polaria-tech/zerank-2-reranker-vllm", - architecture="Qwen3ForSequenceClassification", - chat_template_name="zerank2.jinja", - seq_pooling_type="LAST", - attn_type="decoder", - is_prefix_caching_supported=True, - is_chunked_prefill_supported=True, - mteb_score=0.34305, - mteb_tol=1e-2, - enable_test=True, - ), -] - - -class ZerankHfRunner(MtebCrossEncoderMixin, HfRunner): - # The converted checkpoint has no lm_head, so the reference is the original - # causal LM: the logit of the "Yes" token at the last position. The converted - # checkpoint returns sigmoid(logit / 5), which ranks documents identically. - original_model = "zeroentropy/zerank-2-reranker" - - def __init__( - self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any - ) -> None: - from transformers import AutoModelForCausalLM, AutoTokenizer - - HfRunner.__init__( - self, - model_name=self.original_model, - auto_cls=AutoModelForCausalLM, - dtype=dtype, - **kwargs, - ) - - self.tokenizer = AutoTokenizer.from_pretrained( - self.original_model, padding_side="left" - ) - self.token_true_id = self.tokenizer.convert_tokens_to_ids("Yes") - - @torch.no_grad - def predict( - self, - inputs1: DataLoader[mteb.types.BatchedInput], - inputs2: DataLoader[mteb.types.BatchedInput], - *args, - **kwargs, - ) -> np.ndarray: - queries = [text for batch in inputs1 for text in batch["text"]] - corpus = [text for batch in inputs2 for text in batch["text"]] - - tokenizer = self.tokenizer - prompts = [] - for query, document in zip(queries, corpus): - conversation = [ - {"role": "query", "content": query}, - {"role": "document", "content": document}, - ] - - prompt = tokenizer.apply_chat_template( - conversation=conversation, - tools=None, - chat_template=self.chat_template, - tokenize=False, - ) - prompts.append(prompt) - - scores = [] - for prompt in prompts: - inputs = tokenizer([prompt], return_tensors="pt") - inputs = self.wrap_device(inputs) - logits = self.model(**inputs).logits[:, -1, :] - scores.append(logits[0, self.token_true_id].item()) - return torch.Tensor(scores) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("model_info", RERANK_MODELS) -def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: - mteb_test_rerank_models(vllm_runner, model_info, hf_runner=ZerankHfRunner) diff --git a/tests/models/multimodal/generation/test_gemma_ragged_audio.py b/tests/models/multimodal/generation/test_gemma_ragged_audio.py deleted file mode 100644 index e51eb002ef..0000000000 --- a/tests/models/multimodal/generation/test_gemma_ragged_audio.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from tests.models.registry import HF_EXAMPLE_MODELS -from vllm.assets.audio import AudioAsset - -MODELS = { - "Gemma3nForConditionalGeneration": "google/gemma-3n-E2B-it", - "Gemma4ForConditionalGeneration": "google/gemma-4-E2B-it", -} - -# Seconds. Spread wide enough that no two clips can produce the same number of -# mel frames: equal-length clips stack normally and never reach the ragged path. -CLIP_DURATIONS = (1.0, 3.5, 7.0, 12.0) - - -@pytest.fixture(scope="module") -def ragged_clips(): - """One asset sliced to several durations, so only length varies.""" - audio, sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate - - clips = [] - for duration in CLIP_DURATIONS: - num_samples = int(duration * sample_rate) - if num_samples > len(audio): - pytest.skip(f"asset is shorter than {duration}s") - clips.append((audio[:num_samples], sample_rate)) - - assert len({len(clip) for clip, _ in clips}) == len(clips) - return clips - - -@pytest.mark.core_model -@pytest.mark.parametrize("arch", sorted(MODELS)) -@pytest.mark.parametrize("max_tokens", [8]) -def test_variable_length_audio_batching( - vllm_runner, - ragged_clips, - arch: str, - max_tokens: int, -) -> None: - """Test batching of requests with different audio durations. - - Audio features are unpadded per item so a multimodal cache entry does not - depend on the batch it was first processed in, and - `MultiModalFieldConfig.batched` stacks items only when their shapes agree. - A batch of differently sized clips therefore reaches the model as a list, - which used to take EngineCore down with - - AttributeError: 'list' object has no attribute 'squeeze' - - failing every in-flight request with EngineDeadError. - """ - from transformers import AutoProcessor - - model_name = MODELS[arch] - model_info = HF_EXAMPLE_MODELS.get_hf_info(arch) - model_info.check_available_online(on_fail="skip") - model_info.check_transformers_version(on_fail="skip") - - # The two architectures spell the audio placeholder differently; ask the - # processor rather than hardcoding either. - processor = AutoProcessor.from_pretrained(model_name) - prompt = processor.apply_chat_template( - [ - { - "role": "user", - "content": [ - {"type": "audio", "audio": None}, - {"type": "text", "text": "Transcribe this audio."}, - ], - } - ], - tokenize=False, - add_generation_prompt=True, - ) - - with vllm_runner( - model_name, - dtype="bfloat16", - enforce_eager=True, - limit_mm_per_prompt={"audio": 1}, - ) as vllm_model: - # One call so the clips are scheduled into the same forward pass; - # issued separately they never reach the ragged path. - outputs = vllm_model.generate_greedy( - [prompt] * len(ragged_clips), - max_tokens, - audios=[[clip] for clip in ragged_clips], - ) - - assert len(outputs) == len(ragged_clips) - for output in outputs: - assert len(output[1]) > 0, "Expected non-empty output" diff --git a/tests/models/multimodal/generation/test_interleaved.py b/tests/models/multimodal/generation/test_interleaved.py deleted file mode 100644 index 07ce235853..0000000000 --- a/tests/models/multimodal/generation/test_interleaved.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from vllm.assets.image import ImageAsset -from vllm.assets.video import VideoAsset -from vllm.multimodal.image import convert_image_mode - -models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"] - - -def base_prompt(modalities_str: str) -> str: - return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n" # noqa: E501 - - -INTERLEAVED_PROMPT = base_prompt("