Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _force_cpu_default_device():
|
||||
# get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
|
||||
# Ensure the default device is CPU so the rest of the test tensors match.
|
||||
original = torch.get_default_device()
|
||||
torch.set_default_device("cpu")
|
||||
yield
|
||||
torch.set_default_device(original)
|
||||
|
||||
|
||||
TEXT_TOKEN_ID = 1
|
||||
VISION_START_TOKEN_ID = 100
|
||||
VISION_END_TOKEN_ID = 101
|
||||
IMAGE_TOKEN_ID = 200
|
||||
VIDEO_TOKEN_ID = 201
|
||||
AUDIO_TOKEN_ID = 202
|
||||
AUDIO_START_TOKEN_ID = 300
|
||||
AUDIO_END_TOKEN_ID = 301
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyVisionConfig:
|
||||
spatial_merge_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyConfig:
|
||||
position_id_per_seconds: int = 25
|
||||
vision_config: DummyVisionConfig = dataclasses.field(
|
||||
default_factory=DummyVisionConfig
|
||||
)
|
||||
|
||||
|
||||
def make_model() -> Qwen3OmniMoeThinkerForConditionalGeneration:
|
||||
model = Qwen3OmniMoeThinkerForConditionalGeneration.__new__(
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration
|
||||
)
|
||||
model.config = DummyConfig()
|
||||
return model
|
||||
|
||||
|
||||
def _elem(data):
|
||||
return MultiModalFieldElem(
|
||||
data=torch.tensor(data),
|
||||
field=None, # HACK.
|
||||
)
|
||||
|
||||
|
||||
def make_image_feature(offset: int, grid_thw: tuple[int, int, int]):
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem({"image_grid_thw": _elem(grid_thw)}),
|
||||
modality="image",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset, length=int(torch.tensor(grid_thw).prod())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_video_feature(
|
||||
offset: int, grid_thw: tuple[int, int, int], use_audio_in_video: bool = False
|
||||
):
|
||||
data = {"video_grid_thw": _elem(grid_thw)}
|
||||
if use_audio_in_video:
|
||||
data["use_audio_in_video"] = _elem(True)
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(data),
|
||||
modality="video",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=offset, length=int(torch.tensor(grid_thw).prod())
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def make_audio_feature(offset: int, audio_feature_length: int):
|
||||
return MultiModalFeatureSpec(
|
||||
data=MultiModalKwargsItem(
|
||||
{"audio_feature_lengths": _elem(audio_feature_length)}
|
||||
),
|
||||
modality="audio",
|
||||
identifier="DUMMY",
|
||||
mm_position=PlaceholderRange(offset=offset, length=0),
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_positions_image_at_prompt_end():
|
||||
"""A prompt ending with <|vision_end|> must still produce seq_len positions."""
|
||||
model = make_model()
|
||||
# [<|vision_start|>, <|image_pad|> x 4, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[VISION_START_TOKEN_ID] + [IMAGE_TOKEN_ID] * 4 + [VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [make_image_feature(offset=1, grid_thw=(1, 2, 2))]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_audio_at_prompt_end():
|
||||
model = make_model()
|
||||
# audio_feature_length=16 -> 2 audio tokens.
|
||||
# [<|audio_start|>, <|audio_pad|> x 2, <|audio_end|>]
|
||||
input_tokens = [AUDIO_START_TOKEN_ID] + [AUDIO_TOKEN_ID] * 2 + [AUDIO_END_TOKEN_ID]
|
||||
mm_features = [make_audio_feature(offset=1, audio_feature_length=16)]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_video_at_prompt_end():
|
||||
model = make_model()
|
||||
# [T, <|vision_start|>, <|video_pad|> x 8, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID]
|
||||
+ [VISION_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 8
|
||||
+ [VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [make_video_feature(offset=2, grid_thw=(2, 2, 2))]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_audio_in_video_at_prompt_end():
|
||||
model = make_model()
|
||||
# video grid (1, 2, 2) -> 4 video tokens; audio_feature_length=16 -> 2
|
||||
# audio tokens, interleaved after the video tokens.
|
||||
# [T, T, <|vision_start|>, <|audio_start|>, V x 4, A x 2,
|
||||
# <|audio_end|>, <|vision_end|>]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [VISION_START_TOKEN_ID, AUDIO_START_TOKEN_ID]
|
||||
+ [VIDEO_TOKEN_ID] * 4
|
||||
+ [AUDIO_TOKEN_ID] * 2
|
||||
+ [AUDIO_END_TOKEN_ID, VISION_END_TOKEN_ID]
|
||||
)
|
||||
mm_features = [
|
||||
make_video_feature(offset=3, grid_thw=(1, 2, 2), use_audio_in_video=True),
|
||||
make_audio_feature(offset=3, audio_feature_length=16),
|
||||
]
|
||||
|
||||
positions, _ = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
assert positions.shape == (3, len(input_tokens))
|
||||
|
||||
|
||||
def test_mrope_positions_image_then_text():
|
||||
"""Media followed by text: positions must match the expected layout exactly.
|
||||
|
||||
Before the fix this case passed the length guard but every position from
|
||||
the first <|image_pad|> onward was silently shifted by one slot.
|
||||
"""
|
||||
model = make_model()
|
||||
# [T, T, <|vision_start|>, <|image_pad|> x 4, <|vision_end|>, T, T, T]
|
||||
input_tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [VISION_START_TOKEN_ID]
|
||||
+ [IMAGE_TOKEN_ID] * 4
|
||||
+ [VISION_END_TOKEN_ID]
|
||||
+ [TEXT_TOKEN_ID] * 3
|
||||
)
|
||||
mm_features = [make_image_feature(offset=3, grid_thw=(1, 2, 2))]
|
||||
|
||||
positions, delta = model.get_mrope_input_positions(input_tokens, mm_features)
|
||||
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3, 3, 3, 3, 5, 6, 7, 8],
|
||||
[0, 1, 2, 3, 3, 4, 4, 5, 6, 7, 8],
|
||||
[0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8],
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(positions, expected)
|
||||
assert delta == 9 - len(input_tokens)
|
||||
@@ -11,8 +11,16 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEKernelMonolithicImpl,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
|
||||
MoEPrepareAndFinalizeNoDPEPMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
RoutedExpertsSink,
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
@@ -174,45 +182,139 @@ def test_public_binding_supports_direct_capture_source():
|
||||
assert calls == [(3, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
self.layer_id = 3
|
||||
self.router = _make_router()
|
||||
# Use a concrete monolithic expert and override its capability
|
||||
# instead of instantiating the abstract base class directly.
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsFp8,
|
||||
)
|
||||
_SINK_CONFIG = SimpleNamespace(
|
||||
use_ep=False,
|
||||
dp_size=1,
|
||||
ep_size=1,
|
||||
max_num_tokens=4,
|
||||
experts_per_token=2,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8)
|
||||
self.routed_experts = types.SimpleNamespace(
|
||||
quant_method=types.SimpleNamespace(
|
||||
is_monolithic=True,
|
||||
moe_kernel=types.SimpleNamespace(
|
||||
impl=types.SimpleNamespace(fused_experts=fused_experts)
|
||||
),
|
||||
)
|
||||
)
|
||||
self._quant_method = self.routed_experts.quant_method
|
||||
self._quant_method.moe_kernel.impl.fused_experts = fused_experts
|
||||
fused_experts.supports_routing_replay_capture = lambda: False
|
||||
|
||||
class DummyCapturer:
|
||||
def capture(self, layer_id, topk_ids):
|
||||
pass
|
||||
class _ToyMonolithicExperts(CPUExpertsInt4):
|
||||
"""A monolithic kernel instance, as a weight reload rebuilds it: routes
|
||||
token t to experts (t + offset, t + offset + 1)."""
|
||||
|
||||
dummy_module = DummyFusedMoE()
|
||||
moe_config = _SINK_CONFIG
|
||||
quant_config = None
|
||||
|
||||
def __init__(self, offset: int = 0, supports_capture: bool = True):
|
||||
self.offset = offset
|
||||
self.supports_capture = supports_capture
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return self.supports_capture
|
||||
|
||||
def apply(self, hidden_states, *args, routing_replay_out=None, **kwargs):
|
||||
if routing_replay_out is not None:
|
||||
t = torch.arange(hidden_states.shape[0])
|
||||
routing_replay_out[: len(t)] = torch.stack([t, t + 1], 1) + self.offset
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _MonolithicMoE:
|
||||
"""The parts of a MoE layer that capture binding reads."""
|
||||
|
||||
layer_id = 5
|
||||
|
||||
def __init__(self, experts: _ToyMonolithicExperts):
|
||||
self.router = _make_router()
|
||||
kernel = SimpleNamespace(impl=SimpleNamespace(fused_experts=experts))
|
||||
self._quant_method = SimpleNamespace(is_monolithic=True, moe_kernel=kernel)
|
||||
self.routed_experts = SimpleNamespace(
|
||||
quant_method=self._quant_method, routing_sink=None
|
||||
)
|
||||
|
||||
|
||||
def _bind_monolithic(
|
||||
monkeypatch, experts: _ToyMonolithicExperts, capture
|
||||
) -> _MonolithicMoE:
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", _MonolithicMoE)
|
||||
layer = _MonolithicMoE(experts)
|
||||
bind_routed_experts_capturer(
|
||||
SimpleNamespace(modules=lambda: [layer]), SimpleNamespace(capture=capture)
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def test_public_binding_gives_monolithic_layers_a_sink(monkeypatch):
|
||||
"""A monolithic kernel routes internally, so binding gives the layer, which
|
||||
outlives kernel rebuilds, a sink that captures under its layer id."""
|
||||
calls = []
|
||||
layer = _bind_monolithic(
|
||||
monkeypatch, _ToyMonolithicExperts(), lambda *args: calls.append(args)
|
||||
)
|
||||
ids = torch.tensor([[1, 2]])
|
||||
layer.routed_experts.routing_sink.capture_fn(ids)
|
||||
assert calls == [(_MonolithicMoE.layer_id, ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
with pytest.raises(ValueError, match="monolithic MoE kernel"):
|
||||
bind_routed_experts_capturer(
|
||||
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
|
||||
_bind_monolithic(
|
||||
monkeypatch, _ToyMonolithicExperts(supports_capture=False), Mock()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_ep", "dp_size", "ep_size", "rows"),
|
||||
[(False, 1, 1, 4), (False, 2, 4, 8), (True, 2, 4, 16)],
|
||||
ids=["single-rank", "dp", "ep"],
|
||||
)
|
||||
def test_routed_experts_sink_holds_a_dispatched_batch(use_ep, dp_size, ep_size, rows):
|
||||
"""One int16 buffer, allocated once, large enough for the batch the kernel
|
||||
sees after a DP or EP gather."""
|
||||
config = SimpleNamespace(
|
||||
**{
|
||||
**vars(_SINK_CONFIG),
|
||||
"use_ep": use_ep,
|
||||
"dp_size": dp_size,
|
||||
"ep_size": ep_size,
|
||||
}
|
||||
)
|
||||
sink = RoutedExpertsSink(config, Mock())
|
||||
|
||||
assert sink.buffer.shape == (rows, 2) and sink.buffer.dtype == torch.int16
|
||||
|
||||
|
||||
def test_monolithic_capture_survives_kernel_rebuilds():
|
||||
"""The bug in #59449: a weight reload rebuilds the kernel. Through the
|
||||
production kernel path, the layer's sink captures what the current kernel
|
||||
routed, before and after a rebuild, and a rebuilt kernel that cannot
|
||||
capture is refused instead of leaving stale rows."""
|
||||
captured = []
|
||||
sink = RoutedExpertsSink(_SINK_CONFIG, lambda ids: captured.append(ids.tolist()))
|
||||
|
||||
def forward(experts: _ToyMonolithicExperts) -> None:
|
||||
kernel = FusedMoEKernelMonolithicImpl(
|
||||
MoEPrepareAndFinalizeNoDPEPMonolithic(), experts
|
||||
)
|
||||
kernel.apply(
|
||||
torch.zeros(3, 8),
|
||||
w1=None,
|
||||
w2=None,
|
||||
router_logits=torch.zeros(3, 16),
|
||||
activation=None,
|
||||
global_num_experts=16,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=sink,
|
||||
)
|
||||
|
||||
for offset in (0, 7): # the kernel before and after a reload
|
||||
forward(_ToyMonolithicExperts(offset))
|
||||
assert captured[-1] == [[t + offset, t + offset + 1] for t in range(3)]
|
||||
with pytest.raises(ValueError, match="not supported"):
|
||||
forward(_ToyMonolithicExperts(supports_capture=False))
|
||||
|
||||
|
||||
def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
"""dp_metadata is None: capture writes the full topk_ids rows."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
|
||||
Reference in New Issue
Block a user