Merge branch 'main' into patches
This commit is contained in:
@@ -455,6 +455,31 @@ def test_per_head_quant_scales_backend_selection(
|
||||
assert backend_name in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
CudaPlatform is None, reason="CUDA platform is required for this test"
|
||||
)
|
||||
@pytest.mark.parametrize("head_size", [64, 80, 96, 128, 192, 256])
|
||||
def test_int4_per_token_head_head_size_selection(head_size: int):
|
||||
"""INT4 per-token-head KV cache is selectable at any even head size.
|
||||
|
||||
The Hadamard rotation the INT4 write path applies is now defined for
|
||||
non-power-of-two rows as well (it runs block-diagonally, see
|
||||
``fast_hadamard_transform``), so a model such as Phi-3 (head_size=96) must
|
||||
keep selecting TRITON_ATTN instead of being rejected.
|
||||
"""
|
||||
vllm_config = VllmConfig(cache_config=CacheConfig(block_size=64))
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch("vllm.platforms.current_platform", CudaPlatform()),
|
||||
):
|
||||
backend = get_attn_backend(
|
||||
head_size=head_size,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="int4_per_token_head",
|
||||
)
|
||||
assert backend.get_name() == "TRITON_ATTN"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,use_non_causal,should_succeed",
|
||||
[
|
||||
|
||||
@@ -713,8 +713,8 @@ def test_low_latency_table_capability_routing(
|
||||
def test_qwen4_exp_hopper_plans_are_valid() -> None:
|
||||
plans = qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS
|
||||
|
||||
assert len(plans) == 9
|
||||
assert sum(map(len, plans.values())) == 31
|
||||
assert len(plans) == 7
|
||||
assert sum(map(len, plans.values())) == 24
|
||||
assert (320, 10240) in plans
|
||||
assert (10240, 320) not in plans
|
||||
for (n, k), shape_plans in plans.items():
|
||||
@@ -728,10 +728,11 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"capability,expected_plans",
|
||||
[
|
||||
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
|
||||
((10, 3), qwen4_exp_gemm.QWEN4_EXP_SM103_GEMM_PLANS),
|
||||
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
|
||||
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
|
||||
((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS),
|
||||
((12, 0), {}),
|
||||
((8, 0), {}),
|
||||
],
|
||||
)
|
||||
@@ -742,11 +743,16 @@ def test_qwen4_exp_gemm_capability_routing(
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
qwen4_exp_gemm.current_platform,
|
||||
"is_device_capability",
|
||||
lambda target: capability == target,
|
||||
"get_device_capability",
|
||||
lambda: capability,
|
||||
)
|
||||
|
||||
assert qwen4_exp_gemm._gemm_plans() == expected_plans
|
||||
assert (
|
||||
qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS_BY_CAPABILITY.get(
|
||||
qwen4_exp_gemm.current_platform.get_device_capability(), {}
|
||||
)
|
||||
== expected_plans
|
||||
)
|
||||
|
||||
|
||||
def test_installation_is_shape_specific_and_unquantized(
|
||||
|
||||
@@ -69,13 +69,8 @@ AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
|
||||
def test_transformers_audio_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
monkeypatch,
|
||||
model_id: str,
|
||||
):
|
||||
@pytest.fixture(autouse=True)
|
||||
def use_spawn_for_audio_models(monkeypatch):
|
||||
"""Single-process workaround for V1 fork safety deadlock issue
|
||||
(vllm-project/vllm/issues/17676). Running multiple audio models together
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
@@ -87,6 +82,13 @@ def test_transformers_audio_generation(
|
||||
disable_envs_cache()
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
|
||||
def test_transformers_audio_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model_id: str,
|
||||
):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
audio_lora_path = settings.get("audio_lora_path")
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.envs import disable_envs_cache
|
||||
|
||||
from ....conftest import HfRunner, VllmRunner
|
||||
from ...utils import check_logprobs_close
|
||||
from .vlm_utils import model_utils
|
||||
|
||||
VIDEO_ASSET = VideoAsset("baby_reading", num_frames=8)
|
||||
|
||||
VIDEO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
|
||||
"Qwen/Qwen3-VL-2B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf": {
|
||||
"prompt": (
|
||||
"<|im_start|>user <video>\nDescribe this video.<|im_end|>"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"CohereLabs/North-Micro-Vision-Instruct": {
|
||||
"prompt": (
|
||||
"<BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>"
|
||||
"<|VISION_START|><|VIDEO_PAD|><|VISION_END|>Describe this video."
|
||||
"<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
|
||||
),
|
||||
},
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def use_spawn_for_video_models(monkeypatch):
|
||||
"""Single-process workaround for V1 fork safety deadlock issue
|
||||
(vllm-project/vllm/issues/17676). Running multiple video models together
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
|
||||
deadlock and allows worker processes to terminate cleanly, and release
|
||||
GPU memory between test runs until the issue is fixed."""
|
||||
# TODO: Remove monkeypatch once
|
||||
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
|
||||
disable_envs_cache()
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_transformers_video_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model_id: str,
|
||||
):
|
||||
prompt = VIDEO_MODEL_SETTINGS[model_id]["prompt"]
|
||||
video = (VIDEO_ASSET.np_ndarrays, VIDEO_ASSET.metadata)
|
||||
|
||||
with vllm_runner(
|
||||
model_id,
|
||||
model_impl="transformers",
|
||||
dtype="bfloat16",
|
||||
max_model_len=8192,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[prompt], 128, num_logprobs=10, videos=[video]
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model_id, dtype="bfloat16", auto_cls=AutoModelForImageTextToText
|
||||
) as hf_model:
|
||||
hf_model = model_utils.qwen3_vl_patch_hf_runner(hf_model)
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
[prompt], 128, num_logprobs=10, videos=[video]
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import Qwen3VLVideoProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
|
||||
VIDEO_MODEL_SETTINGS = {
|
||||
"Qwen/Qwen3-VL-2B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf": {
|
||||
"prompt": (
|
||||
"<|im_start|>user <video>\nDescribe this video.<|im_end|>"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"CohereLabs/North-Micro-Vision-Instruct": {
|
||||
"prompt": (
|
||||
"<BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>"
|
||||
"<|VISION_START|><|VIDEO_PAD|><|VISION_END|>Describe this video."
|
||||
"<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
|
||||
),
|
||||
},
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _video(
|
||||
num_frames: int = 8,
|
||||
seed: int = 0,
|
||||
fps: float = 1.0,
|
||||
do_sample_frames: bool = False,
|
||||
) -> tuple[np.ndarray, dict]:
|
||||
rng = np.random.default_rng(seed)
|
||||
frames = rng.integers(256, size=(num_frames, 64, 64, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": fps,
|
||||
"total_num_frames": num_frames,
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"do_sample_frames": do_sample_frames,
|
||||
}
|
||||
return frames, metadata
|
||||
|
||||
|
||||
def _hf_video_kwargs(*videos):
|
||||
return {
|
||||
"videos": [frames for frames, _ in videos],
|
||||
"video_metadata": [
|
||||
{k: v for k, v in metadata.items() if k != "do_sample_frames"}
|
||||
for _, metadata in videos
|
||||
],
|
||||
"do_sample_frames": videos[0][1]["do_sample_frames"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_video_multimodal_processor(model_id):
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
|
||||
video = _video()
|
||||
frames, _ = video
|
||||
result = mm_processor(
|
||||
prompt=VIDEO_MODEL_SETTINGS[model_id]["prompt"],
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": video}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
hf_inputs = hf_processor(
|
||||
text=VIDEO_MODEL_SETTINGS[model_id]["prompt"],
|
||||
**_hf_video_kwargs(video),
|
||||
return_mm_token_type_ids=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist()
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["video"]
|
||||
(num_video_tokens,) = hf_processor._get_num_multimodal_tokens(
|
||||
video_sizes=[frames.shape[:3]]
|
||||
)["num_video_tokens"]
|
||||
assert placeholder.get_num_embeds() == num_video_tokens
|
||||
assert placeholder.get_num_embeds() == int(
|
||||
(hf_inputs["mm_token_type_ids"] == 2).sum()
|
||||
)
|
||||
|
||||
(item,) = result["mm_kwargs"]["video"]
|
||||
rows = item["pixel_values_videos"].data.shape[0]
|
||||
if "video_grid_thw" in item:
|
||||
assert rows == int(item["num_video_patches"].data)
|
||||
else:
|
||||
assert rows == len(frames)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_video_multiple_inputs(model_id):
|
||||
"""Multiple videos per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modeling backend."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video"},
|
||||
{"type": "video"},
|
||||
{"type": "text", "text": "Compare these videos."},
|
||||
],
|
||||
}
|
||||
]
|
||||
prompt = hf_processor.apply_chat_template(
|
||||
messages, add_generation_prompt=True, tokenize=False
|
||||
)
|
||||
videos = [_video(8, seed=0), _video(8, seed=1)]
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": videos}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["video"]) == 2
|
||||
assert len(result["mm_kwargs"]["video"]) == 2
|
||||
hf_inputs = hf_processor(
|
||||
text=prompt, **_hf_video_kwargs(*videos), return_tensors="pt"
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist()
|
||||
|
||||
|
||||
def test_video_fields_not_claimed_by_image():
|
||||
"""A prompt holding an image and a video keeps the fields of each on its own
|
||||
modality, so the image branch does not claim the video processor's outputs."""
|
||||
model_id = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt=(
|
||||
"<|vision_start|><|image_pad|><|vision_end|> and "
|
||||
"<|vision_start|><|video_pad|><|vision_end|> What do these show?"
|
||||
),
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": _video()[0][0], "video": _video()}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
(image_item,) = result["mm_kwargs"]["image"]
|
||||
(video_item,) = result["mm_kwargs"]["video"]
|
||||
assert {"pixel_values", "image_grid_thw"} <= set(image_item.keys())
|
||||
assert {"pixel_values_videos", "video_grid_thw"} <= set(video_item.keys())
|
||||
assert len(result["mm_placeholders"]["image"]) == 1
|
||||
assert len(result["mm_placeholders"]["video"]) == 1
|
||||
|
||||
|
||||
def test_repeated_video_hits_the_processor_cache():
|
||||
"""Check that mm caching is actually working."""
|
||||
model_config = ModelConfig(
|
||||
model="Qwen/Qwen3-VL-2B-Instruct", model_impl="transformers"
|
||||
)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
video = _video()
|
||||
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=VIDEO_MODEL_SETTINGS["Qwen/Qwen3-VL-2B-Instruct"]["prompt"],
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": video}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
first, second = process(), process()
|
||||
|
||||
assert cache.make_stats().hits > 0
|
||||
assert first["prompt_token_ids"] == second["prompt_token_ids"]
|
||||
assert first["mm_hashes"] == second["mm_hashes"]
|
||||
|
||||
|
||||
def test_video_unsupported_when_processor_cannot_count_tokens(monkeypatch):
|
||||
"""A processor that cannot count video tokens is served as image-only."""
|
||||
monkeypatch.delattr(Qwen3VLVideoProcessor, "get_num_of_video_patches")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="Qwen/Qwen3-VL-2B-Instruct", model_impl="transformers")
|
||||
)
|
||||
|
||||
assert "video" not in mm_processor.info.get_supported_mm_limits()
|
||||
|
||||
|
||||
def test_merged_token_fields_split_per_video():
|
||||
"""VideoLLaMA3's compression mask has one row per merged token, not per patch."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="lkhl/VideoLLaMA3-2B-Image-HF", model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user\n<|video_pad|>\n<|video_pad|>\nDescribe.<|im_end|>\n",
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": [_video(8), _video(4)]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
items = result["mm_kwargs"]["video"]
|
||||
assert [len(item["video_compression_mask"].data) for item in items] == [128, 64]
|
||||
|
||||
|
||||
def test_video_sampled_by_processor():
|
||||
"""Frames left for the processor to sample yield the tokens HF does."""
|
||||
model_id = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
prompt = VIDEO_MODEL_SETTINGS[model_id]["prompt"]
|
||||
video = _video(64, fps=30.0, do_sample_frames=True)
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": video}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
hf_inputs = hf_processor(
|
||||
text=prompt,
|
||||
**_hf_video_kwargs(video),
|
||||
return_mm_token_type_ids=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist()
|
||||
(placeholder,) = result["mm_placeholders"]["video"]
|
||||
assert placeholder.get_num_embeds() == int(
|
||||
(hf_inputs["mm_token_type_ids"] == 2).sum()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_dummy_video_has_the_most_tokens(model_id):
|
||||
"""Profiling must build a dummy video with exactly the most tokens a video
|
||||
can produce, or memory is under- or over-reserved."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
info = mm_processor.info
|
||||
seq_len = info.ctx.model_config.max_model_len
|
||||
|
||||
result = mm_processor.get_dummy_mm_inputs({"video": 1})
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["video"]
|
||||
max_tokens = info.get_max_video_tokens(seq_len, {"video": 1})
|
||||
assert placeholder.get_num_embeds() == max_tokens
|
||||
# Shrinking the dummy frames must never cost tokens
|
||||
num_frames = info.get_num_frames_with_most_features(seq_len, {"video": 1})
|
||||
(huge_frame_tokens,) = info.get_hf_processor()._get_num_multimodal_tokens(
|
||||
video_sizes=[[num_frames, 10_000, 10_000]]
|
||||
)["num_video_tokens"]
|
||||
assert max_tokens >= huge_frame_tokens
|
||||
@@ -74,13 +74,14 @@ def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
|
||||
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
|
||||
slots[-1] = -1
|
||||
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
|
||||
qkv_width = 2 * (MAIN_HQ + MAIN_HK) * MAIN_D
|
||||
return dict(
|
||||
main_qkv=torch.randn(
|
||||
num_tokens,
|
||||
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
|
||||
qkv_width + (HQ + 1) * D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
),
|
||||
)[:, :qkv_width],
|
||||
main_q_norm_weight=norm_weights[0],
|
||||
main_k_norm_weight=norm_weights[1],
|
||||
main_eps=EPS,
|
||||
@@ -316,9 +317,12 @@ def test_qsa_fused_prepare_matches_unfused(
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
# TP=4 QKVG width makes the indexer slice row-strided, as in the merged
|
||||
# projection. The exact width is unimportant; the larger stride is tested.
|
||||
main_qkvg_width = 3584
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
num_tokens, main_qkvg_width + (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)[:, main_qkvg_width:]
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
|
||||
@@ -587,12 +587,14 @@ class MarkingStub(SupportsMultiModal, nn.Module):
|
||||
_pre_trained_model_classes = Base._pre_trained_model_classes
|
||||
|
||||
|
||||
def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
|
||||
def build_marked_model(
|
||||
image_limit: int, video_limit: int = 0, skip_tokenizer_init: bool = False
|
||||
):
|
||||
"""Build the HF model inside the marking context and return it."""
|
||||
model_config = ModelConfig(
|
||||
model=MULTIMODAL_MODEL,
|
||||
model_impl="transformers",
|
||||
limit_mm_per_prompt={"image": image_limit},
|
||||
limit_mm_per_prompt={"image": image_limit, "video": video_limit},
|
||||
)
|
||||
# Set after construction: building the config itself needs the tokenizer
|
||||
model_config.skip_tokenizer_init = skip_tokenizer_init
|
||||
@@ -606,10 +608,15 @@ def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
|
||||
return stub.model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("image_limit", "skipped"), [(0, True), (4, False)])
|
||||
def test_tower_weights_skipped_when_modality_disabled(image_limit, skipped):
|
||||
"""`--limit-mm-per-prompt image=0` should drop the vision tower's weights."""
|
||||
vision_tower = build_marked_model(image_limit).vision_tower
|
||||
@pytest.mark.parametrize(
|
||||
("image_limit", "video_limit", "skipped"),
|
||||
[(0, 0, True), (4, 0, False), (0, 4, False)],
|
||||
)
|
||||
def test_tower_weights_skipped_when_modality_disabled(
|
||||
image_limit, video_limit, skipped
|
||||
):
|
||||
"""Only `--limit-mm-per-prompt image=0,video=0` drops the tower they share."""
|
||||
vision_tower = build_marked_model(image_limit, video_limit).vision_tower
|
||||
assert isinstance(vision_tower, StageMissingLayer) is skipped
|
||||
|
||||
|
||||
|
||||
@@ -1552,6 +1552,53 @@ def test_chunked_prefill_reads_live_block_ids() -> None:
|
||||
assert len(meta2.store_gpu_blocks) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
|
||||
@pytest.mark.parametrize("same_output", [False, True])
|
||||
def test_eager_store_resets_block_cursor_after_preemption(
|
||||
use_v2_model_runner: bool,
|
||||
same_output: bool,
|
||||
) -> None:
|
||||
"""A resumed request stores its confirmed tail after preemption."""
|
||||
fix = make_scheduler(num_cpu_blocks=8, num_gpu_blocks=8, lazy=False)
|
||||
sched = fix.scheduler
|
||||
req = make_request(num_blocks=3)
|
||||
|
||||
initial_blocks = _alloc_and_register(fix, req, num_blocks=2)
|
||||
sched.update_state_after_alloc(req, initial_blocks, num_external_tokens=0)
|
||||
initial_output = make_scheduler_output(
|
||||
{req.request_id: 2 * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: initial_blocks.get_block_ids()},
|
||||
)
|
||||
initial_meta = sched.build_connector_meta(initial_output)
|
||||
assert len(initial_meta.store_gpu_blocks) == 2
|
||||
simulate_store_completion(sched, initial_meta.store_event)
|
||||
|
||||
if not same_output:
|
||||
preemption_output = make_scheduler_output({})
|
||||
preemption_output.preempted_req_ids = {req.request_id}
|
||||
sched.build_connector_meta(preemption_output)
|
||||
fix.gpu_block_pool.free_blocks(initial_blocks.blocks[0])
|
||||
|
||||
resumed_blocks = _alloc_and_register(fix, req, num_blocks=3)
|
||||
sched.update_state_after_alloc(req, resumed_blocks, num_external_tokens=0)
|
||||
if use_v2_model_runner:
|
||||
resumed_output = make_scheduler_output(
|
||||
{req.request_id: 3 * BLOCK_SIZE},
|
||||
new_reqs={req.request_id: resumed_blocks.get_block_ids()},
|
||||
)
|
||||
else:
|
||||
resumed_output = make_scheduler_output(
|
||||
{req.request_id: 3 * BLOCK_SIZE},
|
||||
cached_req_new_blocks={req.request_id: resumed_blocks.get_block_ids()},
|
||||
)
|
||||
resumed_output.scheduled_cached_reqs.resumed_req_ids.add(req.request_id)
|
||||
if same_output:
|
||||
resumed_output.preempted_req_ids = {req.request_id}
|
||||
|
||||
resumed_meta = sched.build_connector_meta(resumed_output)
|
||||
assert resumed_meta.store_gpu_blocks == [resumed_blocks.blocks[0][2].block_id]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 10: Partial GPU prefix hit + CPU load + new compute blocks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -152,6 +152,46 @@ def test_scoped_max_split_ignores_async_allocator(monkeypatch):
|
||||
set_settings.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_memory_bytes", [None, 1])
|
||||
def test_memory_profile_bounds_decode_logits_rows(monkeypatch, kv_cache_memory_bytes):
|
||||
"""ROCm decode workspace uses max_num_seqs during startup profiling."""
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _max_decode_logits_rows
|
||||
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=5),
|
||||
)
|
||||
|
||||
class Profiled(Exception):
|
||||
pass
|
||||
|
||||
def profile_run(*, randomize_inputs=False):
|
||||
assert randomize_inputs is True
|
||||
# Without the config context, the helper falls back to all 32768
|
||||
# batched tokens, which would reserve 128 GiB at 1M context.
|
||||
assert _max_decode_logits_rows(32768) == 16 * (1 + 5)
|
||||
raise Profiled
|
||||
|
||||
worker = SimpleNamespace(
|
||||
vllm_config=config,
|
||||
randomize_dummy_inputs=True,
|
||||
cache_config=SimpleNamespace(kv_cache_memory_bytes=kv_cache_memory_bytes),
|
||||
model_runner=SimpleNamespace(profile_run=profile_run, model_memory_usage=0),
|
||||
init_snapshot=SimpleNamespace(free_memory=2),
|
||||
_scoped_allocator_max_split=lambda **kwargs: nullcontext(),
|
||||
)
|
||||
monkeypatch.setattr(gpu_worker, "maybe_apply_startup_plan", lambda _: None)
|
||||
monkeypatch.setattr(
|
||||
gpu_worker, "memory_profiling", lambda *args, **kwargs: nullcontext()
|
||||
)
|
||||
|
||||
assert get_current_vllm_config_or_none() is None
|
||||
with pytest.raises(Profiled):
|
||||
gpu_worker.Worker.determine_available_memory(worker)
|
||||
assert get_current_vllm_config_or_none() is None
|
||||
|
||||
|
||||
# Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and
|
||||
# saved by Worker.determine_available_memory / compile_or_warm_up_model.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user