Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+2 -1
View File
@@ -42,7 +42,8 @@ def _assert_close_ulps(
compare against the reference's local ulp spacing. ``pre_act`` (the
reference pre-activation) adds derivative-propagated slack: near
``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps."""
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps.
"""
g, r = got.float(), ref.float()
tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol)
if pre_act is not None:
@@ -123,7 +123,8 @@ def _check_writes_only_its_own_shard(
device: torch.device, tp_size: int, rank: int
) -> None:
"""Two ranks that swapped both weight rows and write offsets still sum to
the right total, so inspect each slice before the final collective."""
the right total, so inspect each slice before the final collective.
"""
transform = _build_transform(device)
runner = _tail_runner(transform, tp_size)
group = get_tp_group().device_group
@@ -130,7 +130,8 @@ def test_disabled_reproduces_the_plain_residual_sum(
def test_taps_the_consumer_layer_when_one_follows(recorder, monkeypatch):
"""The value the next layer reads is the mixture against *its* weights,
so the tap has to reach forward rather than use the current layer's."""
so the tap has to reach forward rather than use the current layer's.
"""
_set_last_rank(monkeypatch, True)
_call(_stub_model(enabled=True), 1, torch.zeros(2), None, torch.zeros(2))
@@ -181,7 +182,8 @@ def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
):
"""The kernel writes an applied delta back into the prefix in place, which
would double-add it into the live residual stream, so the pending output
has to arrive already summed into the prefix with `delta` left None."""
has to arrive already summed into the prefix with `delta` left None.
"""
_set_last_rank(monkeypatch, True)
prefix_sum = torch.tensor([1.0, 2.0])
pending = torch.tensor([0.5, 0.25])
+2 -1
View File
@@ -727,7 +727,8 @@ def test_cudagraph_capture_metadata_avoids_device_to_host_copy(
"""`build_for_cudagraph_capture` is not capture-only: a DP rank with nothing
scheduled re-stages its FULL-graph metadata through it on every dummy step,
so it must not synchronize the device. The host-side draft counts have to
come from `query_start_loc_cpu` and match the device-derived values."""
come from `query_start_loc_cpu` and match the device-derived values.
"""
num_speculative_tokens = 2
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -60,7 +60,8 @@ def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
"""KV-shared layers have q_proj instead of a packed qkv_proj; their
redundant K/V tensors in original checkpoints have no parameter and are
skipped rather than failing the load."""
skipped rather than failing the load.
"""
model = torch.nn.Module()
model.config = SimpleNamespace(num_experts=0)
model.start_layer, model.end_layer = 0, 2
@@ -92,7 +93,8 @@ def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
@pytest.mark.cpu_test
def test_gemma3n_kv_shared_layer_mapper() -> None:
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
q_proj and drop the redundant K/V tensors original checkpoints ship."""
q_proj and drop the redundant K/V tensors original checkpoints ship.
"""
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
weights = [
@@ -164,8 +164,7 @@ def test_chunked_prefill_with_parallel_sampling(
max_tokens: int,
conv_state_layout: str,
) -> None:
"""
Tests chunked prefill in conjunction with n > 1.
"""Tests chunked prefill in conjunction with n > 1.
In this case, prefill is populated with decoding tokens and
we test that it doesn't fail.
@@ -199,8 +198,7 @@ def test_mamba_cache_cg_padding(
max_tokens: int,
conv_state_layout: str,
) -> None:
"""
This test is for verifying that mamba cache is padded to CG captured
"""This test is for verifying that mamba cache is padded to CG captured
batch size. If it's not, a torch RuntimeError will be raised because
tensor dimensions aren't compatible.
"""
@@ -236,8 +234,7 @@ def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
example_prompts,
model: str,
) -> None:
"""
This test is for verifying that the hybrid inner state management doesn't
"""This test is for verifying that the hybrid inner state management doesn't
collapse in case where the number of incoming requests and
finished_requests_ids is larger than the maximum mamba block capacity.
@@ -263,8 +260,7 @@ def test_state_cleanup(
example_prompts,
model: str,
) -> None:
"""
This test is for verifying that the Hybrid state is cleaned up between
"""This test is for verifying that the Hybrid state is cleaned up between
steps.
If it's not cleaned, an error would be expected.
@@ -274,7 +274,8 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
def test_mistral_function_call_nested_json():
"""Ensure that the function-name regex captures the entire outermost
JSON block, including nested braces."""
JSON block, including nested braces.
"""
# Create a minimal stub tokenizer that provides the few attributes the
# parser accesses (`version` and `get_vocab`).
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for max_tokens_per_doc and max_tokens_per_query.
"""
"""Tests for max_tokens_per_doc and max_tokens_per_query."""
import json
import os
@@ -29,7 +29,8 @@ from vllm.v1.worker.gpu.states import RequestState
@torch.inference_mode
def test_splade_pooler_matches_reference_formula(B, T, H, V):
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
log1p(relu(logits)) -> max over sequence length (after masking)."""
log1p(relu(logits)) -> max over sequence length (after masking).
"""
torch.manual_seed(0)
# Prepare [B] sequences of shape [T, H]
@@ -23,11 +23,9 @@ NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
@pytest.mark.parametrize("dtype", ["float16"])
@pytest.mark.parametrize("max_tokens", [128])
def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
"""
This is a simple test to check if interleaved and non-interleaved prompts
"""This is a simple test to check if interleaved and non-interleaved prompts
give the same result.
"""
image_cherry = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
image_stop = convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB")
images = [image_cherry, image_stop]
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Create a reduced-layer version of the Maverick model for testing purposes.
"""Create a reduced-layer version of the Maverick model for testing purposes.
This script creates a new model with fewer layers by:
1. Loading the original Maverick model configuration
@@ -39,7 +38,6 @@ def run_maverick_serving(model: str):
"""Test Llama-4-Maverick model with vLLM LLM class using CLI equivalent
options with reduced layers.
"""
try:
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
@@ -71,8 +69,7 @@ def run_maverick_serving(model: str):
def get_rope_layers_config(model_path: str) -> list[int]:
"""
Get the interleaved RoPE configuration from HuggingFace config
"""Get the interleaved RoPE configuration from HuggingFace config.
Args:
model_path: Path to the local directory containing the reduced
@@ -81,6 +78,7 @@ def get_rope_layers_config(model_path: str) -> list[int]:
Returns:
List of 0 or 1 indicating whether each layer uses RoPE and local attn
0 indicates that RoPE is not used while 1 indicates that RoPE is used.
"""
config_path = Path(model_path) / "config.json"
model_config = json.loads(config_path.read_text())
@@ -98,8 +96,7 @@ def create_reduced_maverick_model(
vision_layers: int = 2,
force_recreate: bool = False,
) -> str:
"""
Create a reduced-layer version of the Maverick model.
"""Create a reduced-layer version of the Maverick model.
Args:
original_model_name: Name of the original Maverick model
@@ -111,8 +108,8 @@ def create_reduced_maverick_model(
Returns:
Path to the created reduced model directory
"""
"""
print(
f"Creating reduced Maverick model with {text_layers} text layers and "
f"{vision_layers} vision layers..."
@@ -178,7 +175,6 @@ def create_reduced_config(
original_config: Any, text_layers: int, num_experts: int, vision_layers: int
) -> dict[str, Any]:
"""Create a reduced configuration based on the original."""
# Convert config to dictionary
config_dict = original_config.to_dict()
@@ -220,7 +216,6 @@ def create_reduced_config(
def copy_tokenizer_files(original_model_name: str, output_path: Path) -> None:
"""Copy tokenizer files from the original model."""
try:
tokenizer = AutoTokenizer.from_pretrained(
original_model_name, trust_remote_code=True
@@ -233,7 +228,6 @@ def copy_tokenizer_files(original_model_name: str, output_path: Path) -> None:
def create_preprocessor_config(original_config: Any, output_path: Path) -> None:
"""Create preprocessor_config.json for multimodal model."""
# Try to load the original preprocessor config
try:
processor = AutoProcessor.from_pretrained(
@@ -253,7 +247,6 @@ def create_reduced_safetensors(
original_config: Any, reduced_config: dict[str, Any], output_path: Path
) -> None:
"""Create safetensors files with weights for the reduced model."""
print("Generating synthetic weights for reduced model...")
text_config = reduced_config["text_config"]
@@ -276,7 +269,6 @@ def create_reduced_safetensors(
def create_text_model_weights(text_config: dict[str, Any]) -> dict[str, torch.Tensor]:
"""Create synthetic weights for the text model with MoE structure."""
weights = {}
vocab_size = text_config["vocab_size"]
@@ -405,7 +397,6 @@ def create_vision_model_weights(
vision_config: dict[str, Any],
) -> dict[str, torch.Tensor]:
"""Create synthetic weights for the vision model."""
weights = {}
hidden_size = vision_config["hidden_size"]
@@ -473,8 +464,7 @@ def create_vision_model_weights(
def create_shared_weights(
text_config: dict[str, Any], vision_config: dict[str, Any]
) -> dict[str, torch.Tensor]:
"""Create weights for shared components (vision-language connector)"""
"""Create weights for shared components (vision-language connector)."""
weights = {}
text_hidden_size = text_config["hidden_size"]
@@ -492,7 +482,6 @@ def save_weights_to_safetensors(
weights: dict[str, torch.Tensor], output_path: Path
) -> None:
"""Save weights to safetensors files and create index."""
# Determine how to shard the weights
max_shard_size = 5 * 1024 * 1024 * 1024 # 5GB per shard
@@ -655,7 +644,6 @@ def test_dummy_maverick(
def main():
"""Main function to create and test the reduced model."""
import argparse
parser = argparse.ArgumentParser(
@@ -33,7 +33,8 @@ NUM_TOKENS = 3
class _StubGate(torch.nn.Module):
"""Router stand-in, so the test does not depend on platform-specific
``ReplicatedLinear`` weight post-processing."""
``ReplicatedLinear`` weight post-processing.
"""
def __init__(self, hidden_size: int, num_experts: int):
super().__init__()
@@ -31,8 +31,7 @@ def mask_bbox_tokens(
output: tuple[list[int], str, SampleLogprobs],
tokenizer: TokenizerLike,
) -> tuple[list[int], str, SampleLogprobs]:
"""
Always pass check_logprobs_close check for bounding box tokens
"""Always pass check_logprobs_close check for bounding box tokens
because it is reasonable for them to differ slightly.
"""
ignore_pattern = r"<[xy]_[\d.]+>"
@@ -97,7 +97,7 @@ def batch_make_image_embeddings(
processor,
llm: VllmRunner,
) -> list[Qwen2VLPromptImageEmbeddingInput]:
"""batched image embeddings for Qwen2-VL
"""Batched image embeddings for Qwen2-VL.
This will infer all images' embeddings in a single batch,
and split the result according to input batches.
@@ -108,7 +108,6 @@ def batch_make_image_embeddings(
returns: `list[Qwen2VLPromptImageEmbeddingInput]`
"""
image_batches_: list[Any] = image_batches[:]
# convert single-image batches to multiple-image batches
@@ -183,7 +182,7 @@ def batch_make_image_embeddings(
def batch_make_video_embeddings(
video_batches: PromptVideoInput, processor, llm: VllmRunner
) -> list[Qwen2VLPromptVideoEmbeddingInput]:
"""batched video embeddings for Qwen2-VL
"""Batched video embeddings for Qwen2-VL.
A NDArray represents a single video's all frames.
@@ -194,7 +193,6 @@ def batch_make_video_embeddings(
- Single-video batches: `list[NDArray]`
- Multiple-video batches: `list[list[NDArray]]`
"""
video_batches_: list[Any] = video_batches[:]
for idx in range(len(video_batches_)):
@@ -81,7 +81,8 @@ def test_transformers_audio_generation(
under pytest can cause (possibly flaky) hangs, so they are grouped under
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
deadlock and allows worker processes to terminate cleanly, and release
GPU memory between test runs until the issue is fixed."""
GPU memory between test runs until the issue is fixed.
"""
# TODO: Remove monkeypatch once
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
disable_envs_cache()
@@ -204,7 +204,6 @@ def test_variable_length_audio_batching(
@pytest.mark.asyncio
async def test_online_serving(client, audio_assets: AudioTestAssets):
"""Exercises online serving with/without chunked prefill enabled."""
messages = [
{
"role": "user",
@@ -78,7 +78,6 @@ def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
Steps run sequentially so each releases the GPU before the next starts.
"""
question = f"What's happening in these {len(audio_assets)} audio clips?"
max_tokens = 10
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
@@ -294,7 +294,8 @@ def sample_frames_with_video_metadata(
def apply_image_size_scaling(image, size: float | tuple[int, int], size_type: SizeType):
"""Applies a size scaler to one image; this can be an image size factor,
which scales the image while maintaining the aspect ratio"""
which scales the image while maintaining the aspect ratio
"""
# Special case for embeddings; if it's a tensor, it's only valid if we
# are considering size factors at constant scale, i.e., we just clone
# the tensor
@@ -156,6 +156,7 @@ def get_wrapped_test_sizes(
Args:
test_info: Test configuration to be expanded.
test_type: The type of test being filtered for.
"""
# If it is an embedding test, we always use the EMBEDDING_SIZE_FACTORS
if test_type == VLMTestType.EMBEDDING:
@@ -200,7 +200,7 @@ def process_runner_outputs(
def process_outputs(output_processor, model, outputs_per_image):
"""Applies a model specific post-processor function to a runner's output"""
"""Applies a model specific post-processor function to a runner's output."""
return [
[output_processor(res, model) for res in outputs]
for outputs in outputs_per_image
@@ -22,6 +22,7 @@ def multi_image_multi_aspect_ratio_inputs(formatter: Callable[[str], str]):
Args:
formatter: model-specific prompt formatter.
"""
stop_sign = IMAGE_ASSETS[0].pil_image
cherry_blossom = IMAGE_ASSETS[1].pil_image
@@ -65,6 +66,7 @@ def multi_video_multi_aspect_ratio_inputs(
Args:
formatter: model-specific prompt formatter.
"""
video = sample_frames_from_video(VIDEO_ASSETS[0].np_ndarrays, num_frames)
# Apply the selected formatter to the base prompts
@@ -252,6 +252,7 @@ def qwen_prompt_path_encoder(
tmp_path: Tempdir for test under consideration.
prompt: Prompt with image placeholders.
assets: list of image assets whose len equals the num placeholders.
"""
# Ensure that the number of placeholders matches the number of assets;
# If this is not true, the test is probably written incorrectly.
@@ -507,15 +508,12 @@ def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
2) Ensure IsaacModel.forward returns hidden_states
for compatibility with hidden_states_to_seq_logprobs()
"""
from perceptron.tensorstream import TextType
from perceptron.tensorstream.ops import compute_mrope_pos_tensor, modality_mask
from transformers.modeling_outputs import BaseModelOutputWithPast
def compute_position_ids_input_ids(input_ids: torch.Tensor) -> torch.Tensor:
"""
Create 3D positional indices for token input.
"""
"""Create 3D positional indices for token input."""
batch_size, seq_length = input_ids.shape
position_ids = torch.arange(seq_length, device=input_ids.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
@@ -570,8 +568,7 @@ def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
cache_position=None,
**kwargs,
):
"""
Forward pass with MRoPE position embeddings.
"""Forward pass with MRoPE position embeddings.
Computes position embeddings once and passes them through all layers.
"""
output_hidden_states = (
@@ -1184,7 +1181,6 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
HfRunner.generate calls batch_decode on the full sequence (prompt +
generated).
"""
import io
import pybase64 as base64
@@ -87,7 +87,7 @@ def _run_test(
*,
dtype: str,
) -> None:
"""SET PYTHONPATH"""
"""SET PYTHONPATH."""
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
@@ -108,8 +108,7 @@ def _normalize_image(image_val: str) -> str:
def create_score_multimodal_param(
content_parts: list[dict],
) -> list[ScoreMultiModalParam]:
"""
Create a ScoreMultiModalParam from a list of content dictionaries.
"""Create a ScoreMultiModalParam from a list of content dictionaries.
Each dict supports the following formats:
- Text: {'text': 'content'}
@@ -257,7 +256,7 @@ def test_model_text_image(
model: str,
dtype: str,
) -> None:
"""Visual Documents Reranking"""
"""Visual Documents Reranking."""
_run_test(
hf_runner,
vllm_runner,
@@ -280,7 +279,7 @@ def test_model_text_text(
model: str,
dtype: str,
) -> None:
"""Textual Documents Reranking"""
"""Textual Documents Reranking."""
_run_test(
hf_runner,
vllm_runner,
@@ -303,7 +302,7 @@ def test_model_image_text(
model: str,
dtype: str,
) -> None:
"""Image Querying for Textual Documents"""
"""Image Querying for Textual Documents."""
_run_test(
hf_runner,
vllm_runner,
@@ -326,7 +325,7 @@ def test_model_image_image(
model: str,
dtype: str,
) -> None:
"""Image Querying for Image Documents"""
"""Image Querying for Image Documents."""
_run_test(
hf_runner,
vllm_runner,
@@ -349,7 +348,7 @@ def test_model_text_mixed_documents(
model: str,
dtype: str,
) -> None:
"""Text Query for Mixed Text and Image Documents"""
"""Text Query for Mixed Text and Image Documents."""
_run_test(
hf_runner,
vllm_runner,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for the LlamaNemotronVL model family:
"""Tests for the LlamaNemotronVL model family:
- nvidia/llama-nemotron-embed-vl-1b-v2 (LlamaNemotronVLForCausalLM / embed)
- nvidia/llama-nemotron-rerank-vl-1b-v2
(LlamaNemotronVLForSequenceClassification / rerank)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
"""Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
caching.
Tests the use_audio_in_video feature where audio is extracted from video and
@@ -50,8 +49,7 @@ def create_mm_data(num_videos: int) -> dict[str, list]:
@pytest.mark.parametrize("model_id", MODELS)
@pytest.mark.parametrize("num_videos", [1, 2])
def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
"""
Regression test for https://github.com/vllm-project/vllm/pull/36800
"""Regression test for https://github.com/vllm-project/vllm/pull/36800.
MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
on a cache hit, so mm_kwargs["video"] items become None on the second call.
@@ -117,8 +115,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
@pytest.mark.parametrize("model_id", MODELS)
def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
"""
A video with no audio track, combined with `use_audio_in_video=True`,
"""A video with no audio track, combined with `use_audio_in_video=True`,
must raise a clear `ValueError` naming the mismatch, rather than an
opaque `StopIteration`-derived error from the underlying HF processor.
"""
@@ -33,9 +33,7 @@ from ...registry import (
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
Add metadata to video mm_data
"""
"""Add metadata to video mm_data."""
def create_metadata(frames: np.ndarray):
num_frames = len(frames)
@@ -61,8 +59,7 @@ def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
Patch the multimodal data for GLM-ASR model.
"""Patch the multimodal data for GLM-ASR model.
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
"""
if "audio" in mm_data:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
"""Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
images that are small enough to not require cropping produce an empty
@@ -109,7 +108,8 @@ class TestDeepseekOCREmptyImagesCrop:
def test_mismatched_image_size_raises(self, processor):
"""Deliberately wrong image_size binding should still be caught
by TensorSchema validation."""
by TensorSchema validation.
"""
small_image = Image.new("RGB", (100, 100), color="green")
result = processor(
@@ -143,7 +143,8 @@ class TestDeepseekOCRInputValidation:
def test_mismatched_image_count_raises_value_error(self, processor):
"""Two ``<image>`` tokens with one image must raise ValueError, not
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
drop the middle text segment)."""
drop the middle text segment).
"""
image = Image.new("RGB", (100, 100), color="red")
with pytest.raises(ValueError, match="does not match number of images"):
processor(prompt="<image> and <image>", images=[image])
@@ -152,8 +152,7 @@ def test_video_loader_consistency(
fps: int,
backend: str,
):
"""
Ensure dynamic video loader (pre-sampled by loader) and normal video
"""Ensure dynamic video loader (pre-sampled by loader) and normal video
loader (post-sampled by processor) produce same video processing outputs.
"""
ctx = build_model_context(
@@ -116,8 +116,7 @@ def _test_image_prompt_replacements(
num_imgs: int,
image_sizes: list[ImageSize],
) -> None:
"""
Ensure LlavaNextMultiModalProcessor
"""Ensure LlavaNextMultiModalProcessor
handles prompt replacement properly for input images.
"""
failed_size_excs = list[tuple[ImageSize, Exception]]()
@@ -16,7 +16,8 @@ from ...utils import build_model_context
class _StubModel:
"""Carries only the attributes the token-count method reads from
`self`, so the real methods can be exercised without constructing
the full `nn.Module` (vision tower, language model, etc.)."""
the full `nn.Module` (vision tower, language model, etc.).
"""
patch_grid_length: int
pooled_grid_length: int
@@ -31,7 +32,8 @@ get_mm_lora_token_counts = (
def test_num_mm_tokens_match_real_config(model_id):
"""The stored grid lengths must match what `__init__` derives from
the real HF config, and the LoRA token counts must preserve the
frame-level scaling exactly."""
frame-level scaling exactly.
"""
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
config = ctx.model_config.hf_config
@@ -122,8 +122,7 @@ def _test_image_prompt_replacements(
num_imgs: int,
image_sizes: list[ImageSize],
) -> None:
"""
Ensure LlavaOnevisionMultiModalProcessor
"""Ensure LlavaOnevisionMultiModalProcessor
handles prompt replacement properly for input images.
"""
failed_size_excs = list[tuple[ImageSize, Exception]]()
@@ -70,7 +70,8 @@ def test_dummy_video_num_frames_override_honors_min_of_two(
):
"""A ``num_frames`` override below 2 must be ignored (the model needs at
least 2 frames), matching the "cannot be less than 2, will be ignored"
warning."""
warning.
"""
builder = object.__new__(Molmo2DummyInputsBuilder)
builder.info = SimpleNamespace(
get_hf_processor=lambda: SimpleNamespace(
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
"""Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
correctly assigned to audio/image/video token positions.
Regression test for: https://github.com/vllm-project/vllm/issues/34506
@@ -40,8 +39,7 @@ def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tens
def make_token_seq(
audio_n: int, image_n: int, video_n: int, text_prefix: int = 3, text_sep: int = 2
):
"""
Build a flat token sequence:
"""Build a flat token sequence:
[text_prefix] [AUDIO * audio_n] [text_sep] [IMAGE * image_n]
[text_sep] [VIDEO * video_n] [text_sep]
Returns (input_ids tensor, is_multimodal mask, positions dict).
@@ -67,9 +65,8 @@ def make_token_seq(
def make_interleaved_seq(
video_chunks: list[int], audio_chunks: list[int], text_prefix: int = 2
):
"""
Build an interleaved sequence like use_audio_in_video:
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
"""Build an interleaved sequence like use_audio_in_video:
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
"""
tokens = [TEXT_TOKEN_ID] * text_prefix
for v, a in zip(video_chunks, audio_chunks):
@@ -140,8 +137,7 @@ class TestCheckInterleavedAudioVideo:
)
def test_batched_non_interleaved_no_false_positive(self):
"""
Regression test for https://github.com/vllm-project/vllm/issues/35394.
"""Regression test for https://github.com/vllm-project/vllm/issues/35394.
5 identical non-interleaved mixed-modality requests batched together:
each has [audio][image][video] in separate blocks with text between them.
@@ -172,8 +168,7 @@ class TestCheckInterleavedAudioVideo:
def make_mock_model(hidden: int = 8):
"""
Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
"""Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
that has enough structure to run embed_input_ids.
"""
from vllm.model_executor.models.qwen2_5_omni_thinker import (
@@ -236,8 +231,7 @@ def make_mock_model(hidden: int = 8):
def build_mm_embeds(
audio_n, image_n, video_n, hidden, audio_val=10.0, image_val=20.0, video_val=30.0
):
"""
Build multimodal_embeddings list in position order (audio, image, video).
"""Build multimodal_embeddings list in position order (audio, image, video).
Each embedding is filled with a distinct constant so we can verify placement.
"""
embs = []
@@ -252,8 +246,7 @@ def build_mm_embeds(
class TestEmbedInputIds:
def _run(self, audio_n, image_n, video_n, hidden=8):
"""
Run embed_input_ids for a non-interleaved mixed-modality sequence.
"""Run embed_input_ids for a non-interleaved mixed-modality sequence.
Returns (result_embeds, input_ids, is_multimodal).
"""
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
@@ -288,8 +281,7 @@ class TestEmbedInputIds:
)
def test_mixed_modalities_audio_goes_to_audio_pos(self):
"""
Regression test for GitHub issue #34506:
"""Regression test for GitHub issue #34506:
With audio + image + video (non-interleaved), audio positions must
receive audio embeddings (not image or video embeddings).
"""
@@ -329,8 +321,7 @@ class TestEmbedInputIds:
)
def test_interleaved_use_audio_in_video(self):
"""
Interleaved (use_audio_in_video): video chunks interleaved with audio.
"""Interleaved (use_audio_in_video): video chunks interleaved with audio.
Video embeddings must go to video positions, audio to audio positions.
"""
hidden = 8
@@ -232,7 +232,6 @@ def test_mm_device_do_normalize(
num_imgs: int,
):
"""Ensure that enable mm_device_do_normalize yields the correct result."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": num_imgs},
@@ -25,8 +25,7 @@ def test_processor_with_audio_sample_rate(
audio_sample_rate: int,
audio_duration_sec: float,
) -> None:
"""
Test that vLLM's processor generates expected outputs with audio_sample_rate.
"""Test that vLLM's processor generates expected outputs with audio_sample_rate.
This validates that the processor correctly handles audio_sample_rate
passed via hf_processor_mm_kwargs and generates audio tokens.
@@ -70,8 +69,7 @@ def test_processor_with_audio_sample_rate(
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
def test_longer_audio_generates_more_tokens(model_id: str) -> None:
"""
Test that longer audio generates more tokens than shorter audio.
"""Test that longer audio generates more tokens than shorter audio.
This validates that audio_sample_rate is being used correctly by checking
that audio duration affects token count as expected.
@@ -31,7 +31,8 @@ def _build_video_mm_data(
``total_num_frames`` is set equal to the ndarray frame count so
that HF's ``sample_frames`` indices stay within bounds of the
actual tensor that is passed."""
actual tensor that is passed.
"""
video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
metadata = {
"fps": original_fps,
@@ -237,7 +238,8 @@ def _build_video_embeds_mm_data(
grid_thw: tuple[int, int, int] = (2, 4, 4),
) -> dict[str, Any]:
"""Create an embeds-only video item as an EPD consumer receives it:
pre-computed embeddings plus the metadata published by the encoder."""
pre-computed embeddings plus the metadata published by the encoder.
"""
import torch
t, h, w = grid_thw
@@ -256,7 +258,8 @@ def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
"""Embeds-only video input must size the placeholder range from the
grid and the real timestamps published by the encoder (EC consumer
path); synthesized or missing timestamps would change the token count
and break embedding merging downstream."""
and break embedding merging downstream.
"""
# `build_model_context` forces enable_mm_embeds off, so build the
# config directly; keep the same online-availability skip behavior.
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
@@ -301,7 +304,8 @@ def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
"""Timestamps are required metadata for video embeds: they size the
placeholder range, so omitting them must fail loudly at parse time
instead of silently producing a wrong prompt."""
instead of silently producing a wrong prompt.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
@@ -116,7 +116,8 @@ def test_audio_multimodal_processor(model_id, processor_cls):
@pytest.mark.parametrize("separator", [" and ", ""])
def test_audio_multiple_inputs(separator, processor_cls):
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
and multi-modal item by the Transformers modelling backend.
"""
model_id = "ibm-granite/granite-speech-3.3-2b"
mm_processor = create_processor(model_id, processor_cls)
@@ -83,7 +83,8 @@ def _process_two_images(processor_cls, separator: str):
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_multiple_inputs(processor_cls):
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
and multi-modal item by the Transformers modelling backend.
"""
result = _process_two_images(processor_cls, separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
@@ -102,7 +103,8 @@ def test_image_adjacent_inputs(processor_cls):
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_batch_padding_removed_from_image_items(processor_cls):
"""Emu3 pads every image up to the largest in the batch, which would leave an
item's data dependent on what it was processed with and so uncacheable."""
item's data dependent on what it was processed with and so uncacheable.
"""
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
image_token = mm_processor.info.get_hf_processor().image_token
@@ -144,7 +146,8 @@ def _process_one_gemma3_image(processor_cls):
@offsets_only
def test_non_embedding_tokens_excluded_from_placeholders():
"""Gemma3 wraps each image in text that carries no embeddings, which must be
inside the placeholder range but masked out of it."""
inside the placeholder range but masked out of it.
"""
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
@@ -156,7 +159,8 @@ def test_legacy_placeholders_hold_only_image_tokens():
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
marker in the unexpanded prompt, not the token the expansion repeats."""
marker in the unexpanded prompt, not the token the expansion repeats.
"""
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
@@ -172,7 +176,8 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
embeddings. Those belong inside the placeholder and masked out, because the token
count the processor reports is over the whole span. Idefics3 also refuses a prompt
holding `<image>` when no images are passed, which is how the offsets path has to
tokenize it before splicing in the expansion."""
tokenize it before splicing in the expansion.
"""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
)
@@ -192,7 +197,8 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
@offsets_only
def test_missing_replacement_offsets_names_the_processor():
"""A processor that reports no replacement offsets cannot be served, which must
be said plainly rather than surfacing later as a field config mismatch."""
be said plainly rather than surfacing later as a field config mismatch.
"""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
hf_processor_cls = type(mm_processor.info.get_hf_processor())
@@ -276,7 +282,8 @@ def test_repeated_image_hits_the_processor_cache():
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
"""The offsets path splices the expansion into a prompt tokenized without any
multi-modal data, so its token ids have to come out the same as the ones the HF
processor produces itself, which is what the legacy path returns."""
processor produces itself, which is what the legacy path returns.
"""
prompt_ids = []
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
mm_processor = create_processor(model_id, processor_cls)
@@ -298,7 +305,8 @@ def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
def test_nested_image_fields_split_per_image(processor_cls):
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
belonging to each image one dimension further in. Slicing the batch dimension
instead handed the first image every row and the second an empty tensor."""
instead handed the first image every row and the second an empty tensor.
"""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
)
@@ -29,7 +29,8 @@ PROCESSOR_CLASSES = [
def create_processor(model_id: str, processor_cls):
"""Build a processor directly, because the registry only ever builds the one the
installed transformers version selects, leaving the other path untested."""
installed transformers version selects, leaving the other path untested.
"""
model_config = ModelConfig(model=model_id, model_impl="transformers")
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
@@ -40,7 +41,8 @@ def create_processor(model_id: str, processor_cls):
def create_cached_processor(model_id: str, processor_cls):
"""Build a processor backed by a real multi-modal processor cache, and hand the
cache back so a test can check it was actually used."""
cache back so a test can check it was actually used.
"""
model_config = ModelConfig(model=model_id, model_impl="transformers")
model_config.multimodal_config.mm_processor_cache_gb = 4
ctx = InputProcessingContext(
+2 -1
View File
@@ -18,7 +18,8 @@ from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGenerat
def test_get_num_audio_tokens_streaming_estimate(audio_duration_s, expected_tokens):
"""The duration-based estimate must convert ``window_stride`` (seconds) to a
sample hop and divide by the encoder subsampling factor. Values are pinned to
concrete numbers so a rounding regression is actually caught."""
concrete numbers so a rounding regression is actually caught.
"""
model_config = SimpleNamespace(
hf_config=SimpleNamespace(
preprocessor={"window_stride": 0.01, "sample_rate": 16000},
+5 -8
View File
@@ -78,7 +78,8 @@ def test_cosmos3_modelopt_quantizer_weights_mapper():
"""ModelOpt/Diffusers FP8 checkpoints ship native fake-quant buffers
(``*_quantizer._amax`` / ``._scale``) alongside the vLLM-consumable
``weight_scale`` / ``input_scale`` sidecars. vLLM must drop the former
(it has no parameter for them) while keeping the latter."""
(it has no parameter for them) while keeping the latter.
"""
from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration
mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper
@@ -215,7 +216,7 @@ def test_bailing_vl_mapper_handles_module_and_parameter_names():
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
"""Create weights from safetensors checkpoint metadata."""
metadata = try_get_safetensors_metadata(repo)
weight_names = list(metadata.weight_map.keys())
with torch.device("meta"):
@@ -223,9 +224,7 @@ def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
def create_dummy_base_model(repo: str, model_arch: str) -> PreTrainedModel:
"""
Create weights from a dummy meta deserialized hf base model with name conversion
"""
"""Create weights from a dummy meta-deserialized HF base model, with renaming."""
config = AutoConfig.from_pretrained(repo)
with torch.device("meta"):
model = AutoModel.from_config(config)
@@ -233,9 +232,7 @@ def create_dummy_base_model(repo: str, model_arch: str) -> PreTrainedModel:
def create_dummy_model(repo: str, model_arch: str) -> PreTrainedModel:
"""
Create weights from a dummy meta deserialized hf model with name conversion
"""
"""Create weights from a dummy meta deserialized hf model with name conversion."""
model_cls: PreTrainedModel = getattr(transformers, model_arch)
config = AutoConfig.from_pretrained(repo)
with torch.device("meta"):
@@ -81,7 +81,8 @@ class _FakeTensor:
"""Sentinel stand-in for torch.Tensor in load_weights tests. Supports the
.detach().clone() chain used by load_weights for buffered mm weights;
both methods return self so identity (and the existing equality
assertions) are preserved through cloning."""
assertions) are preserved through cloning.
"""
def detach(self):
return self
@@ -165,7 +166,8 @@ def _make_mm_items_with_video_bytes(
def test_extract_audio_from_videos_passes_max_duration():
"""_extract_audio_from_videos must forward VLLM_MAX_AUDIO_DECODE_DURATION_S
to load_audio_pyav so decompression-bomb audio is rejected."""
to load_audio_pyav so decompression-bomb audio is rejected.
"""
dummy_audio = (np.zeros(16000, dtype=np.float32), 16000.0)
mm_items = _make_mm_items_with_video_bytes(b"\x00" * 64)
@@ -185,7 +187,8 @@ def test_extract_audio_from_videos_passes_max_duration():
def test_extract_audio_from_videos_rejects_oversized_audio():
"""When load_audio_pyav raises due to duration limit the video is
marked as having no audio instead of crashing the server."""
marked as having no audio instead of crashing the server.
"""
mm_items = _make_mm_items_with_video_bytes(b"\x00" * 64)
processor = object.__new__(NanoNemotronVLMultiModalProcessor)
+2 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end accuracy test for GPT-OSS model quantization.
"""End-to-end accuracy test for GPT-OSS model quantization.
Config:
Task: gsm8k_platinum
@@ -170,8 +169,7 @@ def test_aiter_mxfp4_moe_ignores_padded_rows(
monkeypatch: pytest.MonkeyPatch,
dist_init,
) -> None:
"""
Garbage in cudagraph padding rows must not reach the unpadded outputs.
"""Garbage in cudagraph padding rows must not reach the unpadded outputs.
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
+2 -1
View File
@@ -144,7 +144,8 @@ def test_qwen4_exp_mtp_override_sets_draft_config(
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
rest of the architecture is PP-capable, so the refusal must be conditional
-- and must land before the engine spends time loading weights."""
-- and must land before the engine spends time loading weights.
"""
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
+2 -5
View File
@@ -127,8 +127,7 @@ class _HfExamplesInfo:
check_min_version: bool = True,
check_max_version: bool = True,
) -> str | None:
"""
If the installed transformers version does not meet the requirements,
"""If the installed transformers version does not meet the requirements,
perform the given action.
"""
if (
@@ -185,9 +184,7 @@ class _HfExamplesInfo:
*,
on_fail: Literal["error", "skip"],
) -> None:
"""
If the model is not available online, perform the given action.
"""
"""If the model is not available online, perform the given action."""
if not self.is_available_online:
msg = "Model is not available online"
+11 -6
View File
@@ -103,7 +103,8 @@ def test_fi_backend_strings_are_registered_mega_moe_backends():
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
instead of silently falling through to the generic FusedMoE path."""
instead of silently falling through to the generic FusedMoE path.
"""
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
@@ -118,8 +119,9 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
)
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
"""vLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
models validate their own constraints at construction time."""
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
models validate their own constraints at construction time.
"""
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
@@ -132,7 +134,8 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
"""Every mega backend must qualify for sequence-parallel MoE at
TP>1/EP: the predicate once matched only the native backend string,
which silently ran the fi backends full-batch with an all-reduce on
every rank — 0.42-0.65x native e2e at TP8."""
every rank — 0.42-0.65x native e2e at TP8.
"""
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
vllm_config = SimpleNamespace(
@@ -176,7 +179,8 @@ def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
device it already bound via BootstrapConfig.device."""
device it already bound via BootstrapConfig.device.
"""
import vllm.utils.flashinfer_moe_ep as mod
pg = object()
@@ -241,7 +245,8 @@ def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
"""One 32-element scale group per row: low nibble is the even element,
high nibble the odd one, ue8m0 scale applies to the whole group."""
high nibble the odd one, ue8m0 scale applies to the whole group.
"""
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
+4 -2
View File
@@ -783,7 +783,8 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
"""The loader invokes the model-level PWAL hook for every load format,
so it must finalize megamoe + mhc broadcast weights to cover dummy
load, which skips load_weights()."""
load, which skips load_weights().
"""
calls = []
stub = SimpleNamespace(
model=SimpleNamespace(
@@ -800,7 +801,8 @@ def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
"""MTP/DSpark drafters load as their own top-level models, so each needs
its own PWAL hook now that the megamoe forward no longer finalizes
weights lazily on first use."""
weights lazily on first use.
"""
calls = []
mtp = SimpleNamespace(finalize_mega_moe_weights=lambda: calls.append("mtp"))
DeepSeekV4MTP.process_weights_after_loading(mtp)
+2 -1
View File
@@ -215,7 +215,8 @@ def test_v41_dspark_loads_linear_scales(
):
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
loader (see tests/quantization/test_modelopt.py), not in load_weights.
"""
from vllm.models.deepseek_v41.nvidia import dspark
mxfp8 = scale_dtype != torch.float32
+2 -3
View File
@@ -64,7 +64,6 @@ def can_initialize(
The spawn process causes the _initialize_kv_caches_v1 function below to
become ineffective.
"""
model_info = EXAMPLE_MODELS.get_hf_info(model_arch)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
@@ -199,13 +198,13 @@ def can_initialize(
@pytest.mark.parametrize("model_arch", MINIMAL_MODEL_ARCH_LIST)
def test_can_initialize_small_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
"""Test initializing small subset of supported models"""
"""Test initializing small subset of supported models."""
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
@pytest.mark.parametrize("model_arch", OTHER_MODEL_ARCH_LIST)
def test_can_initialize_large_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
"""Test initializing large subset of supported models
"""Test initializing large subset of supported models.
This test covers the complement of the tests covered in the "small subset"
test.
+6 -3
View File
@@ -73,7 +73,8 @@ def test_mtp_override_extracts_n_predict_from_multimodal_wrapper(
model_type: str, expected_arch: str
) -> None:
"""Verify that multimodal wrapper checkpoints with mtp_num_hidden_layers
in text_config resolve n_predict and architecture correctly."""
in text_config resolve n_predict and architecture correctly.
"""
cfg = SpeculativeConfig.hf_config_override(
_multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
)
@@ -93,7 +94,8 @@ def test_mtp_override_top_level_precedence_over_nested_text_config(
model_type: str, expected_arch: str
) -> None:
"""Verify that an explicit top-level mtp_num_hidden_layers takes precedence
over a nested text_config value."""
over a nested text_config value.
"""
cfg = _multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
cfg.mtp_num_hidden_layers = 3
overridden = SpeculativeConfig.hf_config_override(cfg)
@@ -113,7 +115,8 @@ def test_mtp_override_downloads_real_hf_hub_configs(
model_id: str, expected_arch: str
) -> None:
"""Verify that unmodified real-world checkpoints downloaded via
AutoConfig.from_pretrained resolve n_predict=1 from text_config."""
AutoConfig.from_pretrained resolve n_predict=1 from text_config.
"""
hf_config = AutoConfig.from_pretrained(model_id)
cfg = SpeculativeConfig.hf_config_override(hf_config)
assert cfg.model_type == "qwen3_5_mtp"
+2 -1
View File
@@ -197,7 +197,8 @@ def test_merge_multimodal_embeddings_no_sync():
def test_get_rename_mapper_keeps_only_renames():
"""`None` means "do not load", which is meaningless to the consumers of
this mapper (LoRA name parsing, quantization config layer lists), and
applying it would silently shrink their lists."""
applying it would silently shrink their lists.
"""
mapper = WeightsMapper(
orig_to_new_regex={re.compile(r"^drop_regex\."): None},
orig_to_new_substr={"drop_substr": None, "keep_substr": "kept"},
+5 -9
View File
@@ -41,9 +41,7 @@ pytestmark = pytest.mark.cpu_test
def test_resolve_visual_encoder_outputs(
select_layers, num_layers_loaded, max_possible_layers, expected_features
):
"""
Test that offsets are correctly handled for vision feature layers.
"""
"""Test that offsets are correctly handled for vision feature layers."""
encoder_outputs = [torch.tensor([idx]) for idx in range(num_layers_loaded + 1)]
output_tensor = resolve_visual_encoder_outputs(
encoder_outputs=encoder_outputs,
@@ -94,11 +92,9 @@ def test_run_dp_sharded_vision_model(batch_size: int):
def run_dp_sharded_vision_model_vs_direct(
local_rank: int, world_size: int, batch_size: int, master_port: int
):
"""
Test that run_dp_sharded_vision_model produces the same results as
"""Test that run_dp_sharded_vision_model produces the same results as
calling the model directly.
"""
# Set random seed for reproducibility
set_random_seed(0)
@@ -282,8 +278,7 @@ def test_run_dp_sharded_mrope_vision_model(batch_size: int):
def run_dp_sharded_mrope_vision_model_vs_direct(
local_rank: int, world_size: int, batch_size: int, master_port: int
):
"""
Test that run_dp_sharded_mrope_vision_model produces the same results as
"""Test that run_dp_sharded_mrope_vision_model produces the same results as
calling the model directly.
"""
# Set random seed for reproducibility
@@ -517,7 +512,8 @@ def _reference_input_norm(
def test_fused_input_norm_matches_reference(num_patches: int):
"""FusedInputNorm must equal the plain affine, including for num_patches
above the cuDNN batch-norm grid limit (~65535) that previously raised
CUDNN_STATUS_INTERNAL_ERROR (issue #51717)."""
CUDNN_STATUS_INTERNAL_ERROR (issue #51717).
"""
channel = 3
patch_size = 14 * 14
image_mean = [0.48145466, 0.4578275, 0.40821073]
+46 -23
View File
@@ -251,7 +251,8 @@ class UnderscoreInplaceParallelLinears(FourParallelLinears):
class PlainQKV(nn.Module):
"""q/k/v returned as a bare tuple: no reshape stands between the calls and
the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it."""
the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it.
"""
def __init__(self, hidden: int = 32, heads: int = 4, head_dim: int = 8):
super().__init__()
@@ -338,7 +339,8 @@ class GuardedVAttention(FakeAttention):
"""Gemma 4-style: `v_proj` is called once but also read in a `None` guard.
The guard is a fusion invariant (the fuser only binds instances where the
projection exists), so it folds to `True` and the call still fuses."""
projection exists), so it folds to `True` and the call still fuses.
"""
def forward(
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
@@ -368,7 +370,8 @@ class TruthyGuardAttention(FakeAttention):
An `nn.Linear` is always truthy, so the guard folds to `True` and the call
still fuses (the projection is a proven invariant, exactly as for a `None`
guard)."""
guard).
"""
def forward(
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
@@ -399,7 +402,8 @@ class StatementGuardVAttention(FakeAttention):
The `if`'s test is not an `ast.Compare`, so the identity-check scan skips it;
`_in_boolean_context` folds the test to `True` (an `nn.Linear` is always
truthy) and the call inside the now-`if True:` body still fuses, with the
live `else` surviving the rewrite."""
live `else` surviving the rewrite.
"""
def forward(
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
@@ -428,7 +432,8 @@ class BranchedKVAttention(FakeAttention):
The three calls do not share a block, so the fused GEMM must be hoisted to
the innermost block that dominates all of them (the function body). The
live `if`/`else` must survive the rewrite."""
live `if`/`else` must survive the rewrite.
"""
is_kv_shared_layer = False
@@ -459,7 +464,8 @@ class RebindArgAttention(FakeAttention):
The branch is dead at trace time so the match succeeds, but the source
rewrite must refuse: hoisting the GEMM above the rebind would feed k/v a
different input. Result: no fusion."""
different input. Result: no fusion.
"""
recompute = False
@@ -490,7 +496,8 @@ class InPlaceMutatedArgAttention(FakeAttention):
Like `RebindArgAttention`, but the mutation keeps the name in a `Load`
context (`hidden_states.mul_(2)`), so a bare `Name`-store check would miss it.
The rewrite must still refuse: hoisting the GEMM above the mutation changes
the value k/v see. Result: no fusion."""
the value k/v see. Result: no fusion.
"""
recompute = False
@@ -520,7 +527,8 @@ class AliasMutatedArgAttention(FakeAttention):
`hidden_states.view(-1)` shares storage with its base, so `alias.mul_(2)`
changes what k/v read without ever naming `hidden_states` as a target. Only
tracking names that may alias the input catches it. Result: no fusion."""
tracking names that may alias the input catches it. Result: no fusion.
"""
recompute = False
@@ -551,7 +559,8 @@ class FunctionalInplaceArgAttention(FakeAttention):
`F.relu(hidden_states, inplace=True)` writes through an argument, so the
name appears only in a `Load` context inside a call that is not a method on
it -- invisible to a trailing-underscore method check. Result: no fusion."""
it -- invisible to a trailing-underscore method check. Result: no fusion.
"""
recompute = False
@@ -581,7 +590,8 @@ class SubscriptMutatedArgAttention(FakeAttention):
`hidden_states[..., 0] = 0` leaves the name in a `Load` context on the
subscript's value, so it too evades a bare `Name`-store check. The rewrite
must refuse for the same reason. Result: no fusion."""
must refuse for the same reason. Result: no fusion.
"""
recompute = False
@@ -611,7 +621,8 @@ class NonFoldableRefAttention(FakeAttention):
Deleting `v_proj` would break `isinstance(self.v_proj, nn.Linear)`: it reads
the projection's type, not just whether it exists, so the fuser cannot fold
it and must refuse rather than rewrite it."""
it and must refuse rather than rewrite it.
"""
def forward(
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
@@ -836,7 +847,8 @@ def test_glu_identifies_down_projection():
It is forced to `RowParallelLinear` in `update_attrs` so its sharded input
matches the column-parallel merged gate/up; `None` when there is no such
projection to force (fusion of gate/up still applies)."""
projection to force (fusion of gate/up still applies).
"""
with torch.device("meta"):
assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
@@ -997,7 +1009,8 @@ def test_bypass_existence_guard_folds_identity_none_check(expr, expected):
The projection is a proven invariant (it exists as an `nn.Linear`), so an
`is (not) None` guard has a constant truth value; folding it lets the call be
rewritten away without changing the guard's outcome. `==`/`!=` are excluded
(see the refusal test) because a subclass may override `__eq__`/`__ne__`."""
(see the refusal test) because a subclass may override `__eq__`/`__ne__`.
"""
funcdef, ref = _guard_funcdef(expr)
bypass_existence_guard(funcdef, ref, "v_proj")
test = next(node for node in ast.walk(funcdef) if isinstance(node, ast.IfExp)).test
@@ -1016,7 +1029,8 @@ def test_bypass_existence_guard_folds_bare_truthiness(expr):
An `nn.Linear` has no `__bool__`/`__len__`, so it is always truthy; in a pure
boolean position (a test, or a `not` operand) the reference can be replaced by
`True`, letting the projection be deleted."""
`True`, letting the projection be deleted.
"""
funcdef, ref = _guard_funcdef(expr)
bypass_existence_guard(funcdef, ref, "v_proj")
# No `self.v_proj` reference survives; a literal `True` took its place.
@@ -1031,7 +1045,8 @@ def test_bypass_existence_guard_folds_bare_if_statement():
The `len(node.ops) == 1` `Compare` scan skips it (it is not a comparison at
all); `_in_boolean_context` then matches the `if`'s test and folds it, so the
bare-truthiness statement form is handled, not silently dropped."""
bare-truthiness statement form is handled, not silently dropped.
"""
funcdef = ast.parse(
"def f(self, a, b):\n if self.v_proj:\n return a\n return b"
).body[0]
@@ -1063,7 +1078,8 @@ def test_bypass_existence_guard_refuses_non_guard_reference(expr):
Rewriting it is unsafe (its value depends on more than the projection's
existence), so the fuser refuses rather than change semantics. `==`/`!=` are
refused too: `is_linear` accepts `nn.Linear` subclasses that could override
`__eq__`/`__ne__`, so equality need not agree with `is (not) None`."""
`__eq__`/`__ne__`, so equality need not agree with `is (not) None`.
"""
funcdef, ref = _guard_funcdef(expr)
with pytest.raises(ValueError, match="outside an existence guard"):
bypass_existence_guard(funcdef, ref, "v_proj")
@@ -1103,7 +1119,8 @@ def test_merged_column_fuser_folds_existence_guard():
Guard folding lives on `StackedFuser`, not just `QKVFuser`, because every
stacked fuser deletes the projections it merges -- a surviving reference to
one would fail at runtime."""
one would fail at runtime.
"""
with torch.device("meta"):
module = GuardedParallelLinears()
fuser = MergedColumnParallelFuser.match(trace(module), module)
@@ -1129,7 +1146,8 @@ def test_merged_column_fuser_rejects_input_mutation(cls):
The mutation may be direct (`x.add_(1)`), through a view that shares the
input's storage, or through a call that writes an argument in place
(`F.relu(x, inplace=True)`, `torch.relu_(x)`). Each one silently changes the
fused result, so the rewrite must refuse rather than fuse."""
fused result, so the rewrite must refuse rather than fuse.
"""
with torch.device("meta"):
module = cls()
fuser = MergedColumnParallelFuser.match(trace(module), module)
@@ -1187,7 +1205,8 @@ def test_merged_column_fuser_supports_any_number_of_linears(
def test_merged_name_is_unique_per_linear_names():
"""The computed name must not collide across different projections, so
stacking two unrelated fusions can never clobber packed_modules_mapping."""
stacking two unrelated fusions can never clobber packed_modules_mapping.
"""
ab_c = MergedColumnParallelFuser(source_cls="M", linear_names=("a_b", "c"))
a_bc = MergedColumnParallelFuser(source_cls="M", linear_names=("a", "b_c"))
assert ab_c.merged_name != a_bc.merged_name
@@ -1203,7 +1222,8 @@ def test_heterogeneous_instances_fall_back_to_merged_column_fuser(
layer whose out_features aren't a multiple of its own head_dim (as a
misconfigured or heterogeneous checkpoint might have) fails QKVFuser's
validation and must still fuse via the generic fallback, while a
well-formed sibling of the same class keeps using QKVFuser."""
well-formed sibling of the same class keeps using QKVFuser.
"""
with torch.device("meta"):
container = nn.Module()
container.compatible = PlainQKV(head_dim=8)
@@ -1224,7 +1244,8 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
"""A single projection split into q/k/v must be re-sharded, not merged.
Only the split sizes change: `QKVParallelLinear` loads the packed
checkpoint weight as-is, and shards q by heads while replicating k/v."""
checkpoint weight as-is, and shards q by heads while replicating k/v.
"""
with torch.device("meta"):
meta = PackedQKVAttention(kv_heads=kv_heads)
fuser = get_fuser(meta, PackedQKVFuser)
@@ -1255,7 +1276,8 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
def test_per_head_split_is_not_packed_qkv():
"""The split must consume the whole projection, else its sizes are head
widths and re-sharding by them would be wrong."""
widths and re-sharding by them would be wrong.
"""
with torch.device("meta"):
assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
@@ -1375,7 +1397,8 @@ def test_act_and_mul_derived_from_module(default_vllm_config):
def _wider_model_config(head_dim: int) -> SimpleNamespace:
"""A model whose global head size is twice `head_dim`, as a wider layer
elsewhere in a heterogeneous checkpoint would make it."""
elsewhere in a heterogeneous checkpoint would make it.
"""
return SimpleNamespace(
model_config=SimpleNamespace(get_head_size=lambda: 2 * head_dim),
quant_config=None,
+10 -5
View File
@@ -45,7 +45,8 @@ def _attention(q_lora_rank: int | None):
def _match(q_lora_rank: int | None) -> MLAFuser | None:
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
`get_fuser` cache, so both q_lora variants of the same class are seen).
"""
attn = _attention(q_lora_rank)
return MLAFuser.match(trace(attn), attn)
@@ -81,7 +82,8 @@ def test_discovers_modules_with_q_lora():
def test_q_lora_stacks_qkv_a_proj():
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
down-projection, so both checkpoint weights must remap into it."""
down-projection, so both checkpoint weights must remap into it.
"""
fuser = _match(q_lora_rank=64)
assert isinstance(fuser, MLAFuser)
prefix = "model.layers.0.self_attn"
@@ -102,7 +104,8 @@ def test_update_forward_rewrites_real_attention(q_lora_rank):
The q-LoRA path hoists the merged down-projection to a top-level statement
while its calls sit inside the `q_lora_rank` branches, so the region it
spans is wide; a check that over-approximates aliasing would silently refuse
here and cost the fusion rather than fail loudly."""
here and cost the fusion rather than fail loudly.
"""
fuser = _match(q_lora_rank)
assert isinstance(fuser, MLAFuser)
fuser.update_forward(_attention(q_lora_rank))
@@ -126,7 +129,8 @@ class _Norm(nn.Module):
class RenamedMLA(nn.Module):
"""An MLA-shaped attention whose children have non-standard names, proving
discovery is by structure and not attribute name."""
discovery is by structure and not attribute name.
"""
def __init__(self):
super().__init__()
@@ -153,7 +157,8 @@ class RenamedMLA(nn.Module):
def test_discovers_modules_under_arbitrary_names():
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
names, and `match` still locates each projection by dataflow."""
names, and `match` still locates each projection by dataflow.
"""
with torch.device("meta"):
module = RenamedMLA()
fuser = MLAFuser.match(trace(module), module)
+12 -6
View File
@@ -277,7 +277,8 @@ def test_moe_fuser_matches_scaled_router():
def test_moe_fuser_matches_grouped_router():
"""Group masking between the score and the routing top-k still matches: the
scorer is anchored on the routing (last) top-k, not the group one."""
scorer is anchored on the routing (last) top-k, not the group one.
"""
with torch.device("meta"):
block = MoEBlock(GroupedRouter)
fuser = MoEBlockFuser.match(block, "experts")
@@ -287,7 +288,8 @@ def test_moe_fuser_matches_grouped_router():
def test_moe_fuser_matches_correction_router():
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
rebuilt gate carries it in fp32 for FusedMoE's biased routers.
"""
with torch.device("meta"):
block = MoEBlock(CorrectionRouter)
fuser = MoEBlockFuser.match(block, "experts")
@@ -298,7 +300,8 @@ def test_moe_fuser_matches_correction_router():
def test_moe_fuser_reads_router_dtype_from_the_gate():
"""A router that computes its logits in fp32 must keep routing in fp32 when
rebuilt, even though no config field names the dtype. The cast back to the
activation dtype after the top-k must not be mistaken for the routing dtype."""
activation dtype after the top-k must not be mistaken for the routing dtype.
"""
with torch.device("meta"):
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
@@ -317,7 +320,8 @@ def test_moe_fuser_detects_shared_experts():
def test_moe_fuser_skips_shared_detection_without_extra_children():
"""With only experts and gate, shared-expert detection (and its block trace)
is skipped, so a gate-derived add is not misread as a shared expert."""
is skipped, so a gate-derived add is not misread as a shared expert.
"""
with torch.device("meta"):
block = MoEBlockNoShared()
fuser = MoEBlockFuser.match(block, "experts")
@@ -362,7 +366,8 @@ def test_moe_fuser_declines_unsupported(block_cls):
def test_moe_fuser_ignores_nested_returns():
"""A tuple `return` inside a nested helper must not decline a block whose own
forward returns a single tensor."""
forward returns a single tensor.
"""
with torch.device("meta"):
block = MoEBlockNestedTupleReturn()
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
@@ -370,7 +375,8 @@ def test_moe_fuser_ignores_nested_returns():
def test_moe_fuser_router_requires_connected_dataflow():
"""A gate with linear + softmax + top-k present but not wired as a router
(top-k selects over the input, not the scored logits) is not detected."""
(top-k selects over the input, not the scored logits) is not detected.
"""
with torch.device("meta"):
block = MoEBlock(DisconnectedRouter)
assert MoEBlockFuser.match(block, "experts") is None
@@ -83,7 +83,8 @@ class ToleranceRMSNorm(NamedEpsRMSNorm):
class LiteralEpsRMSNorm(RMSNorm):
"""eps is a literal in the source, and an unrelated attribute happens to
hold the same value: matching on the value alone would bind to it."""
hold the same value: matching on the value alone would bind to it.
"""
def __init__(self, hidden: int = 16, eps: float = 1e-5):
super().__init__(hidden, eps)
@@ -95,7 +96,8 @@ class LiteralEpsRMSNorm(RMSNorm):
class AmbiguousEpsRMSNorm(nn.Module):
"""Two attributes hold the eps value, and the forward reads the second, so
their order cannot pick it."""
their order cannot pick it.
"""
def __init__(self, hidden: int = 16, eps: float = 1e-6):
super().__init__()
@@ -231,7 +233,8 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config):
def test_fused_rms_norm_op_default_eps(default_vllm_config):
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`.
"""
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
with torch.device("meta"):
@@ -247,7 +250,8 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
def test_eps_is_derived_per_instance(default_vllm_config):
"""Two instances of the same norm class with different eps must fuse to their
own eps: the type-cached fuser holds only structure, not this value."""
own eps: the type-cached fuser holds only structure, not this value.
"""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
@@ -259,7 +263,8 @@ def test_eps_is_derived_per_instance(default_vllm_config):
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
"""The eps attribute is identified by holding the traced value, so a norm
stays per-instance correct whatever it names it."""
stays per-instance correct whatever it names it.
"""
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
@@ -271,7 +276,8 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
"""A literal eps is recognised as coming from no attribute, even when one
holds the same value, and is taken from the traced source instead."""
holds the same value, and is taken from the traced source instead.
"""
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
@@ -284,7 +290,8 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
"""When several attributes hold the eps value, the one the forward actually
reads is identified, and they are all left as they were found."""
reads is identified, and they are all left as they were found.
"""
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
@@ -301,7 +308,8 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
def test_fused_norm_is_gather_capable(default_vllm_config):
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
plumbing. A full-width input skips the gather and equals a plain norm."""
plumbing. A full-width input skips the gather and equals a plain norm.
"""
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers import rms_norm
@@ -323,7 +331,8 @@ def test_fused_norm_is_gather_capable(default_vllm_config):
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
"""A sharded input (narrower than the full-width weight) that does not tile
the weight evenly across ranks is rejected before any collective."""
the weight evenly across ranks is rejected before any collective.
"""
from vllm.model_executor.models.transformers.fusers import rms_norm
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
@@ -28,7 +28,8 @@ def fake_hw_layernorm(monkeypatch):
A `SimpleNamespace` stands in for the module: `importlib.import_module`
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
attributes are set at construction (no `ModuleType` attribute-set that mypy
rejects, no constant `setattr` that ruff rejects)."""
rejects, no constant `setattr` that ruff rejects).
"""
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
monkeypatch.setitem(sys.modules, HW_MODULE, module)
return module
@@ -81,7 +82,8 @@ def test_act_and_mul_falls_back_for_unknown_activation(
@pytest.fixture(scope="module")
def tiny_llama_path(tmp_path_factory):
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
tokenizer) so vLLM can load it like any local checkpoint."""
tokenizer) so vLLM can load it like any local checkpoint.
"""
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
+4 -4
View File
@@ -29,8 +29,7 @@ def check_outputs_equal(
name_0: str,
name_1: str,
):
"""
Compare the two sequences generated by different models,
"""Compare the two sequences generated by different models,
which should be equal.
"""
assert len(outputs_0_lst) == len(outputs_1_lst)
@@ -131,6 +130,7 @@ def check_logprobs_close(
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
mismatch between the two sequences
always_check_logprobs: If true, check logprobs even when tokens match
"""
assert len(outputs_0_lst) == len(outputs_1_lst)
@@ -290,6 +290,7 @@ def build_model_context(
Returns:
InputProcessingContext for the model being considered.
"""
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
@@ -447,8 +448,7 @@ def dummy_hf_overrides(
exist_overrides: dict[str, Any] | None = None,
use_original_num_layers: bool = False,
) -> PretrainedConfig:
"""
Dummy HF overrides function used to create dummy model
"""Dummy HF overrides function used to create dummy model
with only minimum nums of layer.
"""
# Copy because this helper is called more than once