Merge branch 'main' into patches
This commit is contained in:
@@ -110,7 +110,7 @@ def test_modernbert_models(
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -17,6 +18,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
@@ -348,3 +350,56 @@ def test_encoder_chunk_no_free_memory_falls_back_to_one():
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
|
||||
model = object.__new__(Gemma4ForConditionalGeneration)
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=16, pooling_kernel_size=3, default_output_length=280
|
||||
)
|
||||
model.vision_tower = SimpleNamespace(config=vision_config)
|
||||
model.config = SimpleNamespace(vision_config=vision_config)
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
|
||||
)
|
||||
def test_encoder_cudagraph_capture_rows_fit_token_budget(
|
||||
token_budget: int, row_tokens: int
|
||||
):
|
||||
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
|
||||
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
inputs = model.prepare_encoder_cudagraph_capture_inputs(
|
||||
token_budget=token_budget,
|
||||
max_batch_size=58,
|
||||
max_frames_per_batch=58 * 32,
|
||||
device="meta",
|
||||
dtype=torch.bfloat16,
|
||||
axis_keys=(row_tokens,),
|
||||
)
|
||||
|
||||
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
|
||||
assert row_patches == row_tokens * 9
|
||||
assert num_rows == max(1, token_budget // row_tokens)
|
||||
|
||||
|
||||
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
mm_kwargs = {
|
||||
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
|
||||
"pixel_position_ids": [
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
torch.zeros(630, 2, dtype=torch.long),
|
||||
],
|
||||
}
|
||||
|
||||
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
|
||||
assert [spec.output_tokens for spec in specs] == [280, 70]
|
||||
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
|
||||
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4/V4.1 VL wrappers stream the language model's weights to it
|
||||
as one contiguous group."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA implementations"
|
||||
)
|
||||
|
||||
|
||||
class _FakeFinalizingLanguageModel(nn.Module):
|
||||
finalizes_weights_during_load = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tensor_a = nn.Parameter(torch.zeros(1))
|
||||
self.tensor_c = nn.Parameter(torch.zeros(1))
|
||||
self.load_calls = 0
|
||||
self.finalized_values: list[tuple[float, float]] = []
|
||||
|
||||
def load_weights(self, weights) -> set[str]:
|
||||
self.load_calls += 1
|
||||
loaded = set()
|
||||
for name, value in weights:
|
||||
getattr(self, name).data.copy_(value)
|
||||
loaded.add(name)
|
||||
self.finalized_values.append((self.tensor_a.item(), self.tensor_c.item()))
|
||||
return loaded
|
||||
|
||||
|
||||
def _v4_wrapper():
|
||||
from vllm.models.deepseek_v4.common.vl_model import (
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
|
||||
return DeepseekV4ForConditionalGeneration
|
||||
|
||||
|
||||
def _v41_wrapper():
|
||||
from vllm.models.deepseek_v41.nvidia.vl_model import DeepseekV41ForCausalLM
|
||||
|
||||
return DeepseekV41ForCausalLM
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapper", [_v4_wrapper, _v41_wrapper], ids=["v4", "v41"])
|
||||
def test_load_weights_streams_language_model_group(wrapper) -> None:
|
||||
model = object.__new__(wrapper())
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = _FakeFinalizingLanguageModel()
|
||||
model.vision = nn.Module()
|
||||
model.vision.tensor_b = nn.Parameter(torch.zeros(1))
|
||||
model.hf_to_vllm_mapper = WeightsMapper()
|
||||
|
||||
def interleaved_weights():
|
||||
yield "language_model.tensor_a", torch.tensor([1.0])
|
||||
# Loaded before the stream is drained, not after.
|
||||
assert model.language_model.tensor_a.item() == 1.0
|
||||
# Splits the language model's weights unless the wrapper regroups them.
|
||||
yield "vision.tensor_b", torch.tensor([2.0])
|
||||
yield "language_model.tensor_c", torch.tensor([3.0])
|
||||
|
||||
loaded = model.load_weights(interleaved_weights())
|
||||
|
||||
assert loaded == {
|
||||
"language_model.tensor_a",
|
||||
"vision.tensor_b",
|
||||
"language_model.tensor_c",
|
||||
}
|
||||
assert model.language_model.load_calls == 1
|
||||
assert model.language_model.finalized_values == [(1.0, 3.0)]
|
||||
assert model.vision.tensor_b.item() == 2.0
|
||||
Reference in New Issue
Block a user