Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``--mamba-ssm-cache-dtype`` must reach GLM-5.3-Flash's KDA recurrent state
|
||||
(it used to be dropped, leaving the state float32 unconditionally)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.glm5next.common.kda import Glm5NextLinearAttention
|
||||
from vllm.models.glm5next.common.model import (
|
||||
Glm5NextForCausalLM,
|
||||
Glm5NextForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ssm_dtype", "expected"),
|
||||
[("auto", torch.float32), ("float32", torch.float32), ("bfloat16", torch.bfloat16)],
|
||||
)
|
||||
def test_kda_recurrent_state_dtype_follows_ssm_cache_dtype(ssm_dtype, expected):
|
||||
model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
cache_config = SimpleNamespace(
|
||||
mamba_cache_dtype="auto",
|
||||
mamba_ssm_cache_dtype=ssm_dtype,
|
||||
use_recoverssm=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=model_config, cache_config=cache_config)
|
||||
want = (torch.bfloat16, expected)
|
||||
|
||||
assert Glm5NextForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == want
|
||||
assert (
|
||||
Glm5NextForConditionalGeneration.get_mamba_state_dtype_from_config(vllm_config)
|
||||
== want
|
||||
)
|
||||
# The layer's own dtype must agree with the model-level one, since the
|
||||
# KV-cache spec and the kernels read them separately.
|
||||
layer = Glm5NextLinearAttention.__new__(Glm5NextLinearAttention)
|
||||
layer.model_config, layer.cache_config = model_config, cache_config
|
||||
assert layer.get_state_dtype() == want
|
||||
@@ -289,3 +289,59 @@ def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
"""The fused-decode weight copies must not depend on the weight loaders.
|
||||
|
||||
Loaders that bypass ``weight_loader`` (e.g. ``--load-format dummy``) left
|
||||
them uninitialized, so decode could read NaN. Loading runs with grad mode
|
||||
on, so the copies must also stay out of autograd.
|
||||
"""
|
||||
_requires_kernel()
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.model_executor.model_loader.utils import process_weights_after_loading
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
config = KimiLinearConfig(
|
||||
hidden_size=256,
|
||||
linear_attn_config={
|
||||
"kda_layers": [1],
|
||||
"full_attn_layers": [],
|
||||
"head_dim": HEAD_DIM,
|
||||
"num_heads": 12,
|
||||
"short_conv_kernel_size": CONV_WIDTH,
|
||||
"use_full_rank_gate": True,
|
||||
"gate_lower_bound": GATE_LOWER_BOUND,
|
||||
},
|
||||
)
|
||||
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", dtype=DTYPE)
|
||||
with set_current_vllm_config(vllm_config), torch.device("cuda"):
|
||||
layer = KimiK3DeltaAttention(
|
||||
config, vllm_config, prefix="model.layers.0.self_attn"
|
||||
)
|
||||
assert layer.decode_conv1d_weight is not None
|
||||
with torch.no_grad():
|
||||
for param in layer.parameters():
|
||||
param.normal_()
|
||||
layer.decode_conv1d_weight.fill_(float("nan"))
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
|
||||
process_weights_after_loading(
|
||||
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
|
||||
)
|
||||
|
||||
dim = layer.local_projection_size
|
||||
conv_weights = layer.conv1d.weight.squeeze(1)
|
||||
expected_conv = torch.stack(
|
||||
[conv_weights[i * dim : (i + 1) * dim].transpose(0, 1) for i in range(3)]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
|
||||
@@ -692,8 +692,10 @@ def test_kda_spec_decode_correctness(
|
||||
pytest.param(True, False, None, True, id="aligned"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("accepted", [(2, 8), (3, 4), (5, 8)])
|
||||
@torch.inference_mode()
|
||||
def test_kda_recoverssm_verify_and_group_commit(
|
||||
accepted: tuple[int, int],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
lower_bound: float | None,
|
||||
use_request_indices: bool,
|
||||
@@ -738,7 +740,6 @@ def test_kda_recoverssm_verify_and_group_commit(
|
||||
state_indices = torch.tensor(
|
||||
[5, 6] if align_mode else [1, 2], dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
accepted = [2, 8]
|
||||
if use_request_indices:
|
||||
global_num_accepted = torch.tensor(
|
||||
[0, accepted[0], 0, accepted[1]],
|
||||
@@ -891,7 +892,9 @@ def test_kda_recoverssm_verify_and_group_commit(
|
||||
if align_mode:
|
||||
assert block_table is not None
|
||||
row = request_indices[seq_idx] if use_request_indices else seq_idx
|
||||
final_block = block_table[row, (4 + commit_len) // 8]
|
||||
final_block = block_table[
|
||||
row, {2: 0, 3: 0, 4: 0, 5: 1, 8: 1}[commit_len]
|
||||
]
|
||||
committed_states[final_block] = committed_state.transpose(-1, -2)
|
||||
if align_mode and 4 + commit_len >= 8:
|
||||
_, boundary_state = naive_recurrent_kda(
|
||||
@@ -944,7 +947,7 @@ def test_kda_recoverssm_verify_and_group_commit(
|
||||
if align_mode:
|
||||
assert block_table is not None
|
||||
row = request_indices[seq_idx] if use_request_indices else seq_idx
|
||||
block = block_table[row, (4 + commit_len) // 8]
|
||||
block = block_table[row, {2: 0, 3: 0, 4: 0, 5: 1, 8: 1}[commit_len]]
|
||||
source_block = state_indices[seq_idx] if align_mode else block
|
||||
if conv_state_dim_first:
|
||||
actual_conv = layer.kv_cache[0][block, :, :history_len]
|
||||
|
||||
@@ -198,7 +198,8 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
||||
|
||||
builder_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(linear_key_head_dim=32)
|
||||
dtype=torch.bfloat16,
|
||||
hf_text_config=SimpleNamespace(linear_key_head_dim=32),
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_cache_mode="align",
|
||||
@@ -219,7 +220,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=4),
|
||||
additional_config={},
|
||||
num_speculative_tokens=2,
|
||||
use_v2_model_runner=False,
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
builder = KimiK3KDAMetadataBuilder(
|
||||
kv_cache_spec=kv_cache_spec,
|
||||
|
||||
@@ -4,7 +4,12 @@
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from math import lcm
|
||||
@@ -60,6 +65,16 @@ SPEC_CONFIG = {
|
||||
}
|
||||
|
||||
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
|
||||
MOONCAKE = {
|
||||
"kv_connector": "MooncakeStoreConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_load_failure_policy": "recompute",
|
||||
"kv_connector_extra_config": {
|
||||
"load_async": True,
|
||||
"lookup_async": True,
|
||||
"enable_cross_layers_blocks": False,
|
||||
},
|
||||
}
|
||||
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
|
||||
OFFLOAD = {
|
||||
"kv_connector": "SimpleCPUOffloadConnector",
|
||||
@@ -243,10 +258,16 @@ def _tokens_text_logprobs(response: dict):
|
||||
|
||||
|
||||
def _reset_gpu_prefix_cache(url: str) -> None:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
if response.json()["success"]:
|
||||
return
|
||||
assert time.monotonic() < deadline, "GPU cache reset timed out"
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -354,6 +375,35 @@ def _format(turns: list[Turn]) -> str:
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _check_resend_reuses_prompt_checkpoint(
|
||||
deployment: Deployment, prompt_len: int
|
||||
) -> None:
|
||||
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
|
||||
mode = Mode(128, True)
|
||||
salt = str(uuid4())
|
||||
rng = random.Random(0)
|
||||
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
|
||||
with _serve(deployment, mode) as servers:
|
||||
url = servers[0].url_root
|
||||
cold = _complete(url, prompt, salt, max_tokens=1)
|
||||
if deployment.offload:
|
||||
_reset_gpu_prefix_cache(url)
|
||||
resend = _complete(url, prompt, salt, max_tokens=1)
|
||||
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
|
||||
|
||||
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
|
||||
expected = (prompt_len // 128 - 1) * 128
|
||||
actual = _cached(resend)
|
||||
assert _cached(cold) == _cached(recompute) == 0
|
||||
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[_tokens_text_logprobs(recompute)],
|
||||
outputs_1_lst=[_tokens_text_logprobs(resend)],
|
||||
name_0="recompute",
|
||||
name_1="resend",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_len",
|
||||
[
|
||||
@@ -368,28 +418,67 @@ def _format(turns: list[Turn]) -> str:
|
||||
],
|
||||
)
|
||||
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
|
||||
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
|
||||
mode = Mode(128, True)
|
||||
salt = str(uuid4())
|
||||
rng = random.Random(0)
|
||||
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
|
||||
deployment = DEPLOYMENTS["plain"]
|
||||
with _serve(deployment, mode) as servers:
|
||||
url = servers[0].url_root
|
||||
cold = _complete(url, prompt, salt, max_tokens=1)
|
||||
resend = _complete(url, prompt, salt, max_tokens=1)
|
||||
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
|
||||
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
|
||||
|
||||
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
|
||||
expected = (prompt_len // 128 - 1) * 128
|
||||
actual = _cached(resend)
|
||||
assert _cached(cold) == _cached(recompute) == 0
|
||||
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[_tokens_text_logprobs(recompute)],
|
||||
outputs_1_lst=[_tokens_text_logprobs(resend)],
|
||||
name_0="recompute",
|
||||
name_1="resend",
|
||||
|
||||
@pytest.fixture
|
||||
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
|
||||
"""Run a Mooncake master and point the servers' store config at it."""
|
||||
if shutil.which("mooncake_master") is None:
|
||||
pytest.skip("mooncake_master is not installed")
|
||||
port = get_open_port()
|
||||
log_path = tmp_path / "mooncake_master.log"
|
||||
with open(log_path, "w") as log:
|
||||
master = subprocess.Popen(
|
||||
[
|
||||
"mooncake_master",
|
||||
f"--port={port}",
|
||||
f"--metrics_port={get_open_port()}",
|
||||
"--enable_metric_reporting=false",
|
||||
"--default_kv_lease_ttl=60000",
|
||||
],
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
assert master.poll() is None, log_path.read_text()
|
||||
with (
|
||||
contextlib.suppress(OSError),
|
||||
socket.create_connection(("127.0.0.1", port)),
|
||||
):
|
||||
break
|
||||
assert time.monotonic() < deadline, "Mooncake master did not start"
|
||||
time.sleep(0.1)
|
||||
config_path = tmp_path / "mooncake.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"metadata_server": "P2PHANDSHAKE",
|
||||
"master_server_address": f"127.0.0.1:{port}",
|
||||
"protocol": "rdma",
|
||||
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "1GB",
|
||||
}
|
||||
)
|
||||
)
|
||||
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
|
||||
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
|
||||
yield
|
||||
finally:
|
||||
master.kill()
|
||||
master.wait()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("mooncake_store")
|
||||
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
|
||||
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
|
||||
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
|
||||
remain reusable from the store after a GPU cache reset."""
|
||||
_check_resend_reuses_prompt_checkpoint(
|
||||
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.model import (
|
||||
KimiK3ForConditionalGeneration,
|
||||
KimiLinearForCausalLM,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeKimiLinearModel(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tensor_a = nn.Parameter(torch.zeros(1))
|
||||
self.tensor_c = nn.Parameter(torch.zeros(1))
|
||||
self.finalized_values: list[tuple[float, float]] = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
params = dict(self.named_parameters())
|
||||
loaded = set()
|
||||
for name, value in weights:
|
||||
params[name].data.copy_(value)
|
||||
loaded.add(name)
|
||||
return loaded
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
self.finalized_values.append((self.tensor_a.item(), self.tensor_c.item()))
|
||||
# MegaMoE finalization replaces its original weight parameters.
|
||||
self.tensor_a = None
|
||||
self.tensor_c = None
|
||||
|
||||
|
||||
def test_interleaved_composite_weights_finalize_kimi_once_after_loading() -> None:
|
||||
language_model = object.__new__(KimiLinearForCausalLM)
|
||||
nn.Module.__init__(language_model)
|
||||
language_model.config = SimpleNamespace(tie_word_embeddings=False)
|
||||
language_model.model = _FakeKimiLinearModel()
|
||||
|
||||
model = object.__new__(KimiK3ForConditionalGeneration)
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = language_model
|
||||
model.vision_tower = nn.Module()
|
||||
model.vision_tower.tensor_b = nn.Parameter(torch.zeros(1))
|
||||
|
||||
loaded = model.load_weights(
|
||||
iter(
|
||||
[
|
||||
("language_model.model.tensor_a", torch.tensor([1.0])),
|
||||
("vision_tower.tensor_b", torch.tensor([2.0])),
|
||||
("language_model.model.tensor_c", torch.tensor([3.0])),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
assert loaded == {
|
||||
"language_model.model.tensor_a",
|
||||
"vision_tower.tensor_b",
|
||||
"language_model.model.tensor_c",
|
||||
}
|
||||
assert language_model.model.finalized_values == []
|
||||
|
||||
model.process_weights_after_loading()
|
||||
|
||||
assert language_model.model.finalized_values == [(1.0, 3.0)]
|
||||
assert model.vision_tower.tensor_b.item() == 2.0
|
||||
@@ -342,10 +342,13 @@ def test_full_cuda_graph(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Leave headroom for the HF model that ran earlier in this process on a
|
||||
# 32.5 GiB MIG.
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
gpu_memory_utilization=0.9,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
|
||||
@@ -55,6 +55,8 @@ def test_max_truncation_size(vllm_model):
|
||||
|
||||
prompt_tokens = vllm_output[0].prompt_token_ids
|
||||
|
||||
# Pooling requests emit an embedding rather than generated tokens, so -1
|
||||
# truncates to the full context length.
|
||||
assert len(prompt_tokens) == max_model_len
|
||||
|
||||
|
||||
|
||||
@@ -13,8 +13,8 @@ import torch
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
|
||||
from vllm.utils.mem_utils import format_mib
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The shared Clef pooler against a port of the checkpoint's joint schema head,
|
||||
which loops over questions where the pooler batches them."""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.clef_joint_schema import (
|
||||
JointSchemaHead,
|
||||
JointSchemaPooler,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
HIDDEN, VOCAB = 64, 500
|
||||
TYPES = [0, 1, 1, 2]
|
||||
COUNTS = [2, 3, 1, 5]
|
||||
|
||||
|
||||
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
|
||||
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
|
||||
for one record, with ``spans`` the option spans of each question."""
|
||||
hidden = head.hidden_norm(hidden)
|
||||
memory = head.memory_projection(hidden).unsqueeze(0)
|
||||
global_vector = hidden[-1]
|
||||
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
|
||||
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
|
||||
lexicals = [
|
||||
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
|
||||
]
|
||||
options = torch.cat(
|
||||
[
|
||||
head.option_context_projection(c)
|
||||
+ head.option_lexical_projection(lex)
|
||||
+ head.option_question_projection(questions[i]).unsqueeze(0)
|
||||
for i, (c, lex) in enumerate(zip(contexts, lexicals))
|
||||
]
|
||||
).unsqueeze(0)
|
||||
for layer in head.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
routed = torch.split(options[0], [len(q) for q in spans])
|
||||
|
||||
base = head.question_projection(questions)
|
||||
summaries = []
|
||||
for field, opts in zip(base, routed):
|
||||
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
|
||||
summaries.append((weights[:, None] * opts).sum(0))
|
||||
fields = (
|
||||
base
|
||||
+ head.option_summary_norm(torch.stack(summaries))
|
||||
+ head.global_projection(global_vector)
|
||||
+ head.type_embedding(torch.tensor(TYPES, device=hidden.device))
|
||||
).unsqueeze(0)
|
||||
for layer in head.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = head.field_norm(fields[0])
|
||||
|
||||
logits = []
|
||||
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
|
||||
anchor = F.normalize(questions[i] + global_vector, dim=-1)
|
||||
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
|
||||
opts = head.option_norm(opts)
|
||||
field = field.expand_as(opts)
|
||||
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
|
||||
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
|
||||
joint = joint + head.residual_scorer(features).squeeze(-1)
|
||||
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
|
||||
return logits
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pooler():
|
||||
torch.manual_seed(0)
|
||||
head = JointSchemaHead(
|
||||
hidden_size=HIDDEN,
|
||||
width=32,
|
||||
routing_layers=2,
|
||||
layers=2,
|
||||
heads=4,
|
||||
feedforward=48,
|
||||
).eval()
|
||||
with torch.no_grad():
|
||||
for p in head.parameters():
|
||||
p.add_(torch.randn_like(p) * 0.2)
|
||||
head.to(DEVICE_TYPE)
|
||||
embedding = torch.randn(VOCAB, HIDDEN, device=DEVICE_TYPE)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(
|
||||
enable_chunked_prefill=False, async_scheduling=False
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=config,
|
||||
):
|
||||
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
|
||||
return pooler, embedding
|
||||
|
||||
|
||||
def schema(length: int):
|
||||
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
|
||||
pos = 20
|
||||
question_spans: list[tuple[int, int]] = []
|
||||
spans: list[list[tuple[int, int]]] = []
|
||||
for count in COUNTS:
|
||||
question_spans.append((pos, pos + 4))
|
||||
pos += 6
|
||||
spans.append([])
|
||||
for k in range(count):
|
||||
n = 1 + (k * 3) % 4
|
||||
spans[-1].append((pos, pos + n))
|
||||
pos += n + 1
|
||||
assert pos < length
|
||||
extra = {
|
||||
"question_types": TYPES,
|
||||
"question_spans": [[s - length, e - length] for s, e in question_spans],
|
||||
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
|
||||
"option_counts": COUNTS,
|
||||
}
|
||||
return {"joint_schema": extra}, question_spans, spans
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_pooler_matches_reference(pooler):
|
||||
pooler, embedding = pooler
|
||||
lengths = [120, 90, 100]
|
||||
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
|
||||
hidden = torch.randn(sum(lengths), HIDDEN, device=DEVICE_TYPE)
|
||||
first, first_questions, first_spans = schema(lengths[0])
|
||||
last, last_questions, last_spans = schema(lengths[2])
|
||||
params = [
|
||||
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
|
||||
# A prompt without a schema, like the profiling run's.
|
||||
PoolingParams(task="token_classify", use_activation=True),
|
||||
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
|
||||
]
|
||||
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
|
||||
for i, ids in enumerate(token_ids):
|
||||
padded[i, : len(ids)] = ids
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor(lengths),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=padded,
|
||||
pooling_params=params,
|
||||
pooling_states=[PoolingStates() for _ in lengths],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
torch.tensor(lengths).numpy(),
|
||||
seq_lens_cpu=torch.tensor(lengths),
|
||||
device=hidden.device,
|
||||
)
|
||||
probs, empty, logits = pooler(hidden, metadata)
|
||||
|
||||
first_hidden, _, last_hidden = torch.split(hidden, lengths)
|
||||
expected = reference_logits(
|
||||
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
|
||||
)
|
||||
expected_probs = torch.cat([q.softmax(-1) for q in expected])
|
||||
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
|
||||
assert empty.numel() == 0
|
||||
expected = reference_logits(
|
||||
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
|
||||
)
|
||||
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
|
||||
@@ -14,12 +14,12 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import VllmRunner
|
||||
|
||||
|
||||
@@ -14,11 +14,11 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
|
||||
from ....conftest import VllmRunner
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""d3-edge against the checkpoint's own runtime (`trust_remote_code`).
|
||||
|
||||
The runtime reads a question's options from the 255 code logits of its readout
|
||||
at the last prompt token. vLLM's classify head must give the same code logits
|
||||
for the same prompt tokens, and for an image prompt that vLLM processes itself.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL = "vllm-sr/d3-edge"
|
||||
STATE = (
|
||||
"The order arrived damaged yesterday. The customer has a receipt and asks "
|
||||
"for a replacement today."
|
||||
)
|
||||
QUESTIONS = {
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team should handle this request?",
|
||||
"criteria": {
|
||||
"returns": "Refunds, replacements and damaged deliveries",
|
||||
"billing": "Payments, invoices and charges",
|
||||
"technical": "Product setup and faults",
|
||||
},
|
||||
},
|
||||
"receipt": {"type": "noul", "instructions": "Does the customer have a receipt?"},
|
||||
"urgency": {
|
||||
"type": "score",
|
||||
"instructions": "How urgent is this request?",
|
||||
"criteria": ["Routine", "Soon", "Today"],
|
||||
},
|
||||
"department": {
|
||||
"type": "choice",
|
||||
"criteria": {f"department {i}": None for i in range(40)},
|
||||
},
|
||||
}
|
||||
IMAGE_QUESTIONS = {"photo": {"type": "noul", "instructions": "Is this a photograph?"}}
|
||||
|
||||
|
||||
def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
|
||||
return torch.softmax(logits[:count].float(), dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [MODEL])
|
||||
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
|
||||
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
|
||||
runtime = reference.runtime
|
||||
text = runtime.prepare(STATE, QUESTIONS)
|
||||
keys = text.runnable
|
||||
counts = [len(text.questions[key].keys) for key in keys]
|
||||
sequences = [text.sequences[key] for key in keys]
|
||||
ref_logits = list(runtime.logits(sequences, counts).cpu())
|
||||
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
visual = runtime.prepare("Describe the attached image.", IMAGE_QUESTIONS, [image])
|
||||
(image_key,) = visual.runnable
|
||||
image_prompt = visual.texts[image_key]
|
||||
ref_logits += list(
|
||||
runtime.image_logits(
|
||||
[image_prompt], visual.images, [2], visual.lengths[image_key]
|
||||
).cpu()
|
||||
)
|
||||
counts.append(2)
|
||||
del reference, runtime
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
with vllm_runner(model, runner="pooling", max_model_len=4096) as vllm_model:
|
||||
outputs = vllm_model.llm.classify(
|
||||
[{"prompt_token_ids": ids} for ids in sequences]
|
||||
+ [{"prompt": image_prompt, "multi_modal_data": {"image": image}}],
|
||||
use_tqdm=False,
|
||||
)
|
||||
# The head returns raw code logits by default.
|
||||
vllm_logits = [torch.tensor(output.outputs.probs) for output in outputs]
|
||||
assert all(len(logits) == 255 for logits in vllm_logits)
|
||||
|
||||
for ref, ours, count in zip(ref_logits, vllm_logits, counts):
|
||||
ref_probs, our_probs = option_probs(ref, count), option_probs(ours, count)
|
||||
torch.testing.assert_close(our_probs, ref_probs, atol=0.02, rtol=0)
|
||||
@@ -5,12 +5,12 @@ import pytest
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageEmbedsParam,
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
|
||||
from ....conftest import HfRunner, VllmRunner
|
||||
|
||||
|
||||
@@ -18,13 +18,13 @@ import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoModel, AutoModelForSequenceClassification, AutoProcessor
|
||||
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
SIGLIP_MEAN,
|
||||
SIGLIP_STD,
|
||||
|
||||
@@ -23,13 +23,13 @@ from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.entrypoints.chat_utils import _get_embeds_data
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
|
||||
@@ -3,13 +3,27 @@
|
||||
"""DeepSeek-V4 VL processing helpers."""
|
||||
|
||||
from PIL import Image
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import (
|
||||
DeepseekV4VLProcessingInfo,
|
||||
DeepseekV4VLProcessor,
|
||||
build_image_block_pad_free,
|
||||
)
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
|
||||
# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
|
||||
VISION_CONFIG = dict(
|
||||
vision_n_layers=24,
|
||||
vision_dim=1024,
|
||||
vision_n_heads=16,
|
||||
vision_inter_dim=2816,
|
||||
vision_patch_size=14,
|
||||
vision_rope_theta=10000.0,
|
||||
vision_downsample_ratio=3,
|
||||
vision_max_n_token=384,
|
||||
vision_min_pixels=147456,
|
||||
vision_max_wh_ratio=8,
|
||||
)
|
||||
|
||||
|
||||
class _ConfigCtx:
|
||||
@@ -34,7 +48,7 @@ def test_get_image_size_with_most_features_beats_wide_real_image():
|
||||
|
||||
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
|
||||
"""
|
||||
config = DeepseekV4Config(vision_n_layers=24)
|
||||
config = DeepseekV4Config(**VISION_CONFIG)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
processor = DeepseekV4VLProcessor(config)
|
||||
|
||||
@@ -47,7 +61,7 @@ def test_get_image_size_with_most_features_beats_wide_real_image():
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_respects_max_wh_ratio():
|
||||
config = DeepseekV4Config(vision_n_layers=24)
|
||||
config = DeepseekV4Config(**VISION_CONFIG)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for LightOnOCR precomputed image embeddings."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("tensor_input", [False, True])
|
||||
@pytest.mark.parametrize("prompt_as_tokens", [False, True])
|
||||
def test_lightonocr_image_embeddings_preserve_lengths_and_values(
|
||||
tensor_input: bool, prompt_as_tokens: bool
|
||||
):
|
||||
"""Reserve each image's feature count and pass projected embeddings through."""
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
ctx.model_config.get_multimodal_config().enable_mm_embeds = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
hidden_size = ctx.model_config.get_hidden_size()
|
||||
feature_sizes = [3, 3] if tensor_input else [3, 7]
|
||||
embeddings = [
|
||||
torch.full((size, hidden_size), float(i))
|
||||
for i, size in enumerate(feature_sizes)
|
||||
]
|
||||
image_input = torch.stack(embeddings) if tensor_input else embeddings
|
||||
prompt = " ".join([hf_processor.image_token] * len(embeddings))
|
||||
if prompt_as_tokens:
|
||||
prompt = ctx.get_tokenizer().encode(prompt)
|
||||
|
||||
items = processor.info.parse_mm_data({"image": image_input})
|
||||
processed = processor(prompt, mm_items=items)
|
||||
|
||||
assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum(
|
||||
feature_sizes
|
||||
)
|
||||
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
|
||||
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
|
||||
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Nemotron-Parse's device-side normalisation path."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_MODEL_ID = "nvidia/NVIDIA-Nemotron-Parse-2.0"
|
||||
|
||||
|
||||
def _noise_image(width: int, height: int) -> Image.Image:
|
||||
rng = np.random.default_rng(0)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(("width", "height"), [(1240, 1754), (310, 470)])
|
||||
def test_mm_device_do_normalize(width: int, height: int):
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
images = [_noise_image(width, height)]
|
||||
prompt = [0]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
|
||||
"pixel_values"
|
||||
]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
|
||||
"pixel_values"
|
||||
]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
assert raw_values.shape == normalized_values.shape
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [False, True])
|
||||
@pytest.mark.parametrize("layout", ["ndarray_hwc", "tensor_hwc", "tensor_chw"])
|
||||
def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = mm_device_do_normalize
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = _noise_image(310, 470)
|
||||
array = np.asarray(image)
|
||||
item = {
|
||||
"ndarray_hwc": array,
|
||||
"tensor_hwc": torch.from_numpy(array),
|
||||
"tensor_chw": torch.from_numpy(array).permute(2, 0, 1),
|
||||
}[layout]
|
||||
|
||||
def pixel_values(img: object) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [img]})
|
||||
return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_explicit_kwargs_without_device_normalize():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_kwargs={"do_rescale": False, "do_normalize": False},
|
||||
)
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
|
||||
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
|
||||
# The remote processor ignores these kwargs and still normalises on the host.
|
||||
assert values.is_floating_point()
|
||||
@@ -0,0 +1,181 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.models.qwen3_asr import Qwen3ASRProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
|
||||
from .test_common import _assert_inputs_equal
|
||||
|
||||
_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
|
||||
|
||||
@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"])
|
||||
def asr_processor(request):
|
||||
config = ModelConfig(
|
||||
request.param,
|
||||
dtype="float32",
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"audio": 2},
|
||||
mm_processor_cache_gb=0,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(config)
|
||||
native = Qwen3ASRProcessor.from_pretrained(request.param)
|
||||
return processor, native
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001]
|
||||
)
|
||||
@pytest.mark.parametrize("num_audios", [1, 2])
|
||||
def test_audio_processing_matches_hf(asr_processor, samples, num_audios):
|
||||
"""Each clip must match HF independently of other clips in the request."""
|
||||
processor, native = asr_processor
|
||||
rng = np.random.RandomState(42)
|
||||
audios = [
|
||||
rng.randn(samples + index * 160).astype(np.float32)
|
||||
for index in range(num_audios)
|
||||
]
|
||||
expected = [
|
||||
native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios
|
||||
]
|
||||
actual = processor(
|
||||
_AUDIO_PROMPT * num_audios,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
expected_ids = [
|
||||
token for item in expected for token in item["input_ids"][0].tolist()
|
||||
]
|
||||
assert actual["prompt_token_ids"] == expected_ids
|
||||
masks = [item["input_features_mask"][0] for item in expected]
|
||||
expected_features = torch.cat(
|
||||
[
|
||||
item["input_features"][0, :, mask.bool()]
|
||||
for item, mask in zip(expected, masks)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
actual_features = actual["mm_kwargs"].get_data()
|
||||
torch.testing.assert_close(
|
||||
actual_features["input_audio_features"], expected_features, rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_features["audio_feature_lengths"],
|
||||
torch.stack([mask.sum() for mask in masks]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]])
|
||||
def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices):
|
||||
"""Partial cache hits must not change features, masks, or prompt expansion."""
|
||||
processor, _ = asr_processor
|
||||
config = processor.info.ctx.model_config
|
||||
config.get_multimodal_config().mm_processor_cache_gb = 1
|
||||
cache = MultiModalProcessorOnlyCache(config)
|
||||
rng = np.random.RandomState(42)
|
||||
audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)]
|
||||
for index in cached_indices:
|
||||
processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios[index]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
for batch in (audios, audios[::-1]):
|
||||
kwargs = dict(
|
||||
mm_items=processor.info.parse_mm_data({"audio": batch}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
_assert_inputs_equal(
|
||||
processor(_AUDIO_PROMPT * 2, **kwargs),
|
||||
processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache),
|
||||
)
|
||||
|
||||
|
||||
def test_text_only_processing(asr_processor):
|
||||
"""Empty media input must not invoke the audio feature extractor."""
|
||||
processor, native = asr_processor
|
||||
prompt = "Transcribe the audio."
|
||||
actual = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt)
|
||||
assert actual["mm_kwargs"].get_data() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}])
|
||||
def test_hf_processor_without_audio(asr_processor, hf_data):
|
||||
"""The media hook must not require audio or call HF for absent waveforms."""
|
||||
processor, _ = asr_processor
|
||||
assert not processor._call_hf_processor(hf_data, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}])
|
||||
def test_postprocessing_without_audio_features(asr_processor, hf_data):
|
||||
"""Non-audio output must pass through even when HF input is nonempty."""
|
||||
processor, _ = asr_processor
|
||||
processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])})
|
||||
actual = processor._postprocess_hf_mm_data(hf_data, {}, processed)
|
||||
assert actual is processed
|
||||
assert set(actual) == {"input_ids"}
|
||||
torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]]))
|
||||
|
||||
|
||||
def test_chat_template_processing(asr_processor):
|
||||
"""The generic processor loader must still render audio chat prompts."""
|
||||
processor, native = asr_processor
|
||||
chat_processor = cached_processor_from_config(processor.info.ctx.model_config)
|
||||
messages = [
|
||||
{"role": "system", "content": ""},
|
||||
{"role": "user", "content": [{"type": "audio", "audio": "unused"}]},
|
||||
]
|
||||
prompt = chat_processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
expected_prompt = native.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
assert prompt == expected_prompt
|
||||
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
|
||||
expected = native(text=expected_prompt, audio=audio, return_tensors="pt")
|
||||
actual = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
|
||||
def test_precomputed_audio_features(asr_processor):
|
||||
"""Precomputed features must bypass waveform extraction unchanged."""
|
||||
processor, _ = asr_processor
|
||||
processor.info.ctx.get_mm_config().enable_mm_embeds = True
|
||||
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
|
||||
expected = processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
features = expected["mm_kwargs"].get_data()
|
||||
actual = processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": features}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
|
||||
assert actual["mm_placeholders"] == expected["mm_placeholders"]
|
||||
assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features)
|
||||
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
|
||||
f"Got short={short_tokens}, long={long_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
|
||||
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
|
||||
"""Audio and video tokens interleave by the fps the video was processed
|
||||
at, as in HF, not by the default of one frame a second."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
rng = np.random.RandomState(42)
|
||||
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
|
||||
audio = rng.rand(32000).astype(np.float32)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
"use_audio_in_video": True,
|
||||
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [video], "audio": [(audio, 16000)]}
|
||||
),
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
)
|
||||
expected = processor.info.get_hf_processor()(
|
||||
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
|
||||
)
|
||||
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing.context import BaseProcessingInfo
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
@@ -336,65 +337,136 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
("hf_mm_kwargs", "scalar_kwargs"),
|
||||
[
|
||||
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
|
||||
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
|
||||
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_kwargs", [False, True])
|
||||
@pytest.mark.parametrize("use_uuids", [False, True])
|
||||
@pytest.mark.parametrize("scoped", [False, True])
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
scalar_kwargs: list[dict[str, Any]],
|
||||
model_kwargs: bool,
|
||||
use_uuids: bool,
|
||||
scoped: bool,
|
||||
) -> None:
|
||||
"""List-valued video kwargs must be scalarized per item without mutation.
|
||||
|
||||
Each video in the multi-video request must match processing the same input
|
||||
independently with its corresponding scalar ``fps``/``num_frames`` value.
|
||||
"""
|
||||
"""Per-video options must survive duplicate items and partial cache hits."""
|
||||
if scoped:
|
||||
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = (
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
)
|
||||
mm_data = {
|
||||
"video": [
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
]
|
||||
}
|
||||
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
|
||||
videos = [
|
||||
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
|
||||
for _ in range(3)
|
||||
]
|
||||
for i, (frames, _) in enumerate(videos):
|
||||
frames.fill(i)
|
||||
|
||||
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
single_mm_data = _build_video_mm_data(num_frames=32)
|
||||
single = processor(
|
||||
single_prompt,
|
||||
mm_items=processor.info.parse_mm_data(single_mm_data),
|
||||
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
|
||||
hf_processor_mm_kwargs=hf_kwargs,
|
||||
)
|
||||
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
|
||||
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
|
||||
assert not torch.equal(expected_grids[0], expected_grids[1])
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
|
||||
def run(indices, *, cache=None, cached_only=False):
|
||||
return processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [None if cached_only else videos[i] for i in indices]}
|
||||
),
|
||||
mm_uuid_items=(
|
||||
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
|
||||
),
|
||||
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
|
||||
expected = run(indices)
|
||||
actual = run(indices, cache=cache)
|
||||
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
|
||||
assert actual["mm_placeholders"] == expected["mm_placeholders"]
|
||||
assert actual["mm_kwargs"] == expected["mm_kwargs"]
|
||||
assert actual["mm_hashes"] == expected["mm_hashes"]
|
||||
assert len(set(actual["mm_hashes"]["video"])) == 2
|
||||
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
|
||||
if use_uuids:
|
||||
cached_only = run([2, 0], cache=cache, cached_only=True)
|
||||
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
|
||||
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
|
||||
assert hf_mm_kwargs == hf_mm_kwargs_before
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
|
||||
@pytest.mark.parametrize("length", [0, 1])
|
||||
@pytest.mark.parametrize("use_cache", [False, True])
|
||||
@pytest.mark.parametrize("scoped", [False, True])
|
||||
def test_processor_rejects_short_per_video_options(
|
||||
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
|
||||
) -> None:
|
||||
"""Reject malformed option lists with and without a warm processor cache."""
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
|
||||
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
|
||||
|
||||
def process(values: list[float | int]) -> None:
|
||||
kwargs = {key: values}
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"video": videos}),
|
||||
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
if use_cache:
|
||||
process([value, value])
|
||||
|
||||
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
|
||||
process([value] * length)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
|
||||
def test_processor_profiles_fewer_videos_than_configured_options(
|
||||
kwargs: dict[str, list[float] | list[int]],
|
||||
) -> None:
|
||||
"""Profiling may use fewer videos than the configured per-video list."""
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
mm_processor_kwargs=kwargs,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
profiled = processor.get_dummy_mm_inputs({"video": 1})
|
||||
|
||||
assert len(profiled["mm_placeholders"]["video"]) == 1
|
||||
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
|
||||
|
||||
|
||||
def _build_video_embeds_mm_data(
|
||||
|
||||
@@ -97,6 +97,7 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
|
||||
object.__setattr__(model, "language_model", language_model)
|
||||
object.__setattr__(model, "mlp1", _AdapterModule())
|
||||
object.__setattr__(model, "vision_model", _MissingMultiModalModule())
|
||||
object.__setattr__(model, "vision_final_layernorm", _MissingMultiModalModule())
|
||||
object.__setattr__(model, "sound_encoder", None)
|
||||
|
||||
language_weight = object()
|
||||
@@ -106,6 +107,8 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
|
||||
("mlp1.0.weight", object()),
|
||||
("vision_model.radio_model.encoder.weight", object()),
|
||||
("sound_encoder.encoder.weight", object()),
|
||||
("vision_projector.vision_final_layernorm.weight", object()),
|
||||
("vision_projector.vision_final_layernorm.bias", object()),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -120,6 +123,7 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
|
||||
object.__setattr__(model, "language_model", language_model)
|
||||
object.__setattr__(model, "mlp1", _AdapterModule())
|
||||
object.__setattr__(model, "vision_model", vision_model)
|
||||
object.__setattr__(model, "vision_final_layernorm", None)
|
||||
object.__setattr__(model, "sound_encoder", None)
|
||||
|
||||
language_weight = object()
|
||||
@@ -145,12 +149,65 @@ def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
|
||||
object.__setattr__(model, "language_model", language_model)
|
||||
object.__setattr__(model, "mlp1", _AdapterModule())
|
||||
object.__setattr__(model, "vision_model", vision_model)
|
||||
object.__setattr__(model, "vision_final_layernorm", None)
|
||||
object.__setattr__(model, "sound_encoder", None)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
model.load_weights([("sound_encoder.encoder.weight", object())])
|
||||
|
||||
|
||||
def _vision_norm_model(use_layernorm=True):
|
||||
model = object.__new__(NemotronH_Nano_VL_V2)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.model_config = _ImageOnlyModelConfig()
|
||||
model.language_model = _LanguageModel()
|
||||
model.vision_model = _VisionModel()
|
||||
model.sound_encoder = None
|
||||
model.mlp1 = torch.nn.Identity()
|
||||
model.vision_final_layernorm = (
|
||||
torch.nn.LayerNorm(4, eps=1e-6, dtype=torch.bfloat16) if use_layernorm else None
|
||||
)
|
||||
model.patch_size = 2
|
||||
model.downsample_ratio = 0.5
|
||||
model.ps_version = "v2"
|
||||
model.video_temporal_patch_size = 2
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["image", "dynamic", "video"])
|
||||
@pytest.mark.parametrize("use_layernorm", [True, False])
|
||||
def test_vision_layernorm_is_loaded_and_applied_before_pixel_shuffle(
|
||||
mode, use_layernorm
|
||||
):
|
||||
model = _vision_norm_model(use_layernorm)
|
||||
weight = torch.tensor([2.0, 3.0, 4.0, 5.0], dtype=torch.bfloat16)
|
||||
bias = torch.tensor([0.5, -0.5, 1.0, -1.0], dtype=torch.bfloat16)
|
||||
model.load_weights(
|
||||
[
|
||||
("vision_projector.vision_final_layernorm.weight", weight),
|
||||
("vision_projector.vision_final_layernorm.bias", bias),
|
||||
]
|
||||
if use_layernorm
|
||||
else []
|
||||
)
|
||||
features = torch.arange(16, dtype=torch.bfloat16).reshape(1, 4, 4)
|
||||
expected = (
|
||||
torch.nn.functional.layer_norm(features, (4,), weight, bias, eps=1e-6)
|
||||
if use_layernorm
|
||||
else features
|
||||
)
|
||||
expected = model.pixel_shuffle(expected.reshape(1, 2, 2, 4)).reshape(1, 1, 16)
|
||||
model.vision_model = lambda *args, **kwargs: (None, features)
|
||||
pixels = torch.zeros(2 if mode == "video" else 1, 3, 4, 4)
|
||||
if mode == "dynamic":
|
||||
output = model.extract_feature_dynamic(pixels, imgs_sizes=[(4, 4)])
|
||||
else:
|
||||
output = model.extract_feature(
|
||||
pixels, num_frames=2 if mode == "video" else None
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _make_mm_items_with_video_bytes(
|
||||
video_bytes: bytes,
|
||||
) -> MultiModalDataItems:
|
||||
|
||||
@@ -12,6 +12,7 @@ import pytest
|
||||
import torch
|
||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import CacheConfig, ParallelConfig
|
||||
from vllm.config.speculative import SpeculativeConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
Qwen3_5ForConditionalGenerationConfig,
|
||||
@@ -53,23 +54,54 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||
return Qwen4ExpTextConfig(**values)
|
||||
|
||||
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
def test_moe_sp_rejects_pipeline_parallel() -> None:
|
||||
"""Automatic MoE SP must validate HC topology even without --enable-hc-sp."""
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(num_experts=4, ple_layer_ids=[]),
|
||||
),
|
||||
cache_config=CacheConfig(),
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=2,
|
||||
data_parallel_size=2,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda", return_value=True),
|
||||
pytest.raises(ValueError, match="MoE SP requires PP=1"),
|
||||
):
|
||||
Qwen4ExpForConditionalGenerationConfig.verify_and_update_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_sp", [False, True])
|
||||
def test_qwen4_exp_mtp_returns_sample_and_multi_streams(use_sp: bool) -> None:
|
||||
"""MTP returns complete, contiguous outputs after gathering padded SP shards."""
|
||||
from vllm.models.qwen4_exp.nvidia.mtp import (
|
||||
Qwen4ExpMultiTokenPredictor,
|
||||
)
|
||||
|
||||
model = object.__new__(Qwen4ExpMultiTokenPredictor)
|
||||
model.use_hc_sequence_parallel = use_sp
|
||||
model.use_sequence_parallel = False
|
||||
torch.nn.Module.__init__(model)
|
||||
model.hc_count = 2
|
||||
model.hidden_size = 4
|
||||
model.num_mtp_layers = 1
|
||||
model.layers = [
|
||||
lambda **kwargs: (
|
||||
kwargs["hidden_states"],
|
||||
kwargs["hidden_states"],
|
||||
torch.zeros(kwargs["hidden_states"].shape[0], 2),
|
||||
),
|
||||
]
|
||||
model.pre_fc_norm_embedding = torch.nn.Identity()
|
||||
model.pre_fc_norm_hidden = torch.nn.Identity()
|
||||
model.fc_embedding = torch.nn.Identity()
|
||||
model.fc_hidden = torch.nn.Identity()
|
||||
|
||||
def decoder(**kwargs):
|
||||
"""Check that the embedding residual shares the hidden state's token slice."""
|
||||
hidden = kwargs["hidden_states"]
|
||||
assert kwargs["prev_block_output"].shape == (hidden.shape[0], model.hidden_size)
|
||||
return hidden, hidden, torch.zeros(hidden.shape[0], model.hc_count)
|
||||
|
||||
model.layers = [decoder]
|
||||
model.hyper_connection_mixer = SimpleNamespace(
|
||||
combine_and_mix=lambda hidden_states, block_output, injection: (
|
||||
hidden_states,
|
||||
@@ -77,24 +109,32 @@ def test_qwen4_exp_mtp_returns_sample_and_multi_streams() -> None:
|
||||
None,
|
||||
),
|
||||
)
|
||||
multi_hidden = torch.arange(16, dtype=torch.float32).reshape(2, 8)
|
||||
pp_group = SimpleNamespace(is_first_rank=False, is_last_rank=True)
|
||||
multi_hidden = torch.arange(24, dtype=torch.float32).reshape(3, 8)
|
||||
expected_sample = multi_hidden.unflatten(-1, (2, 4)).mean(-2)
|
||||
pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
|
||||
|
||||
with patch(
|
||||
"vllm.models.qwen4_exp.nvidia.mtp.get_pp_group",
|
||||
return_value=pp_group,
|
||||
def gather(local):
|
||||
"""Supply rank 1's last token and padding for a simulated two-rank gather."""
|
||||
assert local.shape[0] == 2
|
||||
remote = torch.cat([expected_sample[2:], multi_hidden[2:]], dim=-1)
|
||||
return torch.cat([local, remote, torch.full_like(remote, float("nan"))])
|
||||
|
||||
with (
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.get_pp_group", return_value=pp_group),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_shard", side_effect=lambda x: x[:2]),
|
||||
patch("vllm.models.qwen4_exp.nvidia.mtp.sp_all_gather", side_effect=gather),
|
||||
):
|
||||
sample_hidden, returned_multi_hidden = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
intermediate_tensors={"hidden_states": multi_hidden},
|
||||
positions=torch.arange(3),
|
||||
hidden_states=multi_hidden,
|
||||
inputs_embeds=torch.zeros(3, 4),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
sample_hidden,
|
||||
multi_hidden.unflatten(-1, (2, 4)).mean(dim=-2),
|
||||
)
|
||||
assert returned_multi_hidden is multi_hidden
|
||||
torch.testing.assert_close(sample_hidden, expected_sample)
|
||||
torch.testing.assert_close(returned_multi_hidden, multi_hidden)
|
||||
assert sample_hidden.is_contiguous()
|
||||
assert returned_multi_hidden.is_contiguous()
|
||||
|
||||
|
||||
@spawn_new_process_for_each_test
|
||||
@@ -155,9 +195,7 @@ def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> Non
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
multimodal_config=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, enable_dbo=False, ubatch_size=1
|
||||
),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
speculative_config=None,
|
||||
)
|
||||
with patch.object(
|
||||
|
||||
@@ -48,6 +48,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFileGatherEmbedding,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
@@ -181,9 +182,10 @@ def _mock_etp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int = 1,
|
||||
all_reduce=lambda tensor: tensor,
|
||||
rank: int = 0,
|
||||
) -> None:
|
||||
group = SimpleNamespace(
|
||||
rank_in_group=0,
|
||||
rank_in_group=rank,
|
||||
world_size=world_size,
|
||||
all_reduce=all_reduce,
|
||||
)
|
||||
@@ -332,6 +334,162 @@ def test_ngram_embedding_loads_fp8_shards_and_global_scale() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _make_nvfp4_ngram_embedding(monkeypatch, *, rank=0, device="cpu", pinned=False):
|
||||
_mock_etp_group(monkeypatch, world_size=2, rank=rank)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_rank", lambda: rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
cls = Qwen4ExpPLEPinnedHostEmbedding if pinned else Qwen4ExpPLEDeviceEmbedding
|
||||
with torch.device(device):
|
||||
embedding = cls(
|
||||
8,
|
||||
160,
|
||||
params_dtype=torch.bfloat16,
|
||||
padding_size=2,
|
||||
prefix="test.ple_embedding.ngram_embedding",
|
||||
embedding_method=Qwen4ExpPLENvFp4EmbeddingMethod(),
|
||||
num_ngram_heads=2,
|
||||
max_total_tokens=4,
|
||||
)
|
||||
module = _make_ngram_embedding_for_load_test()
|
||||
module.ngram_embedding = embedding
|
||||
module.split_ngram_parts = 3
|
||||
codes = torch.arange(8 * 80).reshape(8, 80).to(torch.uint8)
|
||||
scales = ((torch.arange(80).reshape(8, 10) + 1) / 4).to(torch.float8_e4m3fn)
|
||||
tensors = []
|
||||
# Checkpoint shards cross ETP boundaries; scales can arrive before weights.
|
||||
for shard in (2, 0, 1):
|
||||
for suffix, tensor in (("weight_scale", scales), ("weight", codes)):
|
||||
tensors.append(
|
||||
(f"ngram_embedding.shard_{shard}.{suffix}", tensor[shard * 3 :][:3])
|
||||
)
|
||||
tensors.append(("ngram_embedding.weight_scale_2", torch.tensor(0.37)))
|
||||
return module, tensors, codes, scales
|
||||
|
||||
|
||||
def _reference_nvfp4_ple(codes, scales):
|
||||
nibbles = torch.stack((codes & 15, codes >> 4), dim=-1).reshape(8, 160)
|
||||
values = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
|
||||
decoded = values[(nibbles & 7).long()] * torch.where(nibbles < 8, 1.0, -1.0)
|
||||
return (decoded * scales.float().repeat_interleave(16, -1) * 0.37).bfloat16()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
def test_nvfp4_ple_loads_streamed_shards_across_etp_boundaries(monkeypatch, rank):
|
||||
module, tensors, codes, scales = _make_nvfp4_ngram_embedding(monkeypatch, rank=rank)
|
||||
loaded = set()
|
||||
for start in range(0, len(tensors), 3):
|
||||
loaded.update(module.load_weights(iter(tensors[start : start + 3])))
|
||||
layer = module.ngram_embedding
|
||||
layer.embedding_method.process_weights_after_loading(layer)
|
||||
|
||||
assert loaded == {
|
||||
"ngram_embedding.weight",
|
||||
"ngram_embedding.weight_scale",
|
||||
"ngram_embedding.weight_scale_2",
|
||||
}
|
||||
assert layer.weight.dtype == torch.uint8
|
||||
assert layer.weight_scale.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight.numel() + layer.weight_scale.numel() == 4 * (80 + 10)
|
||||
torch.testing.assert_close(layer.weight, codes[rank * 4 :][:4])
|
||||
torch.testing.assert_close(
|
||||
layer.weight_scale.float(), scales[rank * 4 :][:4].float()
|
||||
)
|
||||
ids = torch.tensor([[3, 4], [7, 0], [4, 3]])
|
||||
output = layer(ids)
|
||||
expected = _reference_nvfp4_ple(codes, scales)[ids]
|
||||
expected[(ids < rank * 4) | (ids >= (rank + 1) * 4)] = 0
|
||||
torch.testing.assert_close(output, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_nvfp4_ple_rejects_missing_global_scale(monkeypatch):
|
||||
module, tensors, _, _ = _make_nvfp4_ngram_embedding(monkeypatch)
|
||||
module.load_weights(tensors[:-1])
|
||||
layer = module.ngram_embedding
|
||||
with pytest.raises(ValueError, match="missing its global scale"):
|
||||
layer.embedding_method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_nvfp4_ple_resolves_from_mixed_precision_quantized_layers():
|
||||
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
"quant_algo": "MIXED_PRECISION",
|
||||
"kv_cache_quant_algo": None,
|
||||
"quantized_layers": {
|
||||
prefix: {"quant_algo": "NVFP4", "group_size": 16},
|
||||
"model.language_model.layers.0.mlp.experts": {
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(config, prefix),
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required")
|
||||
@pytest.mark.parametrize("pinned", [False, True])
|
||||
def test_nvfp4_ple_cuda_lookup_and_graph_replay(monkeypatch, pinned):
|
||||
# Rank 1 exercises both the vocab offset and out-of-shard masking.
|
||||
rank = 1
|
||||
module, tensors, codes, scales = _make_nvfp4_ngram_embedding(
|
||||
monkeypatch, rank=rank, device="cuda", pinned=pinned
|
||||
)
|
||||
module.load_weights(iter(tensors))
|
||||
layer = module.ngram_embedding
|
||||
layer.embedding_method.process_weights_after_loading(layer)
|
||||
ids = torch.tensor([[3, 4], [7, 0]], device="cuda")
|
||||
lookup_fn = layer._lookup if pinned else torch.compile(layer, fullgraph=True)
|
||||
|
||||
def lookup():
|
||||
return lookup_fn(ids)
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
lookup()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = lookup()
|
||||
for new_ids in ([[3, 4], [7, 0]], [[4, 3], [0, 7]]):
|
||||
ids.copy_(torch.tensor(new_ids, device="cuda"))
|
||||
graph.replay()
|
||||
cpu_ids = ids.cpu()
|
||||
expected = _reference_nvfp4_ple(codes, scales)[cpu_ids]
|
||||
expected[(cpu_ids < rank * 4) | (cpu_ids >= (rank + 1) * 4)] = 0
|
||||
torch.testing.assert_close(output.cpu(), expected, atol=0, rtol=0)
|
||||
if pinned:
|
||||
layer.start_prefetch(ids)
|
||||
torch.testing.assert_close(
|
||||
layer.finalize_prefetch(total_tokens=ids.shape[0]).cpu(),
|
||||
expected.flatten(-2),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
assert layer.weight.device.type == ("cpu" if pinned else "cuda")
|
||||
assert layer.weight_scale.device == layer.weight.device
|
||||
assert layer.weight_scale_2.device.type == "cuda"
|
||||
assert lookup_fn(ids[:0]).shape == (0, 2, 160)
|
||||
padded_ids = torch.tensor([[8, -1]], device="cuda")
|
||||
torch.testing.assert_close(
|
||||
lookup_fn(padded_ids),
|
||||
torch.zeros(1, 2, 160, device="cuda", dtype=torch.bfloat16),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("dp_rank", "local_tokens"), [(0, 2), (1, 3)])
|
||||
def test_etp_lookup_gathers_and_returns_dp_local_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -618,46 +776,65 @@ def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_finalizes_prefetched_output(
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("reduce_results", [False, True])
|
||||
def test_pinned_embedding_finalize_prefetch_optionally_reduces_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dtype: torch.dtype,
|
||||
reduce_results: bool,
|
||||
) -> None:
|
||||
"""Deferring reduction must preserve full token rows and wait for the lookup."""
|
||||
total_tokens = 3
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = torch.empty(4, 2, 3, dtype=torch.float8_e4m3fn)
|
||||
values = torch.arange(1, 13).reshape(4, 3)
|
||||
embedding._prefetch_buffer = torch.stack(
|
||||
(values, torch.zeros_like(values)), dim=1
|
||||
).to(dtype)
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
expected = torch.arange(12).reshape(2, 6).to(torch.float8_e4m3fn)
|
||||
|
||||
def finalize_prefetched(
|
||||
prefetch_output: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert prefetch_output is embedding._prefetch_buffer
|
||||
output.copy_(expected)
|
||||
|
||||
embedding.tp_size = 2
|
||||
embedding.reduce_results = reduce_results
|
||||
embedding.etp_data_parallel_size = 1
|
||||
embedding._prefetch_stream = object()
|
||||
waited_streams: list[object] = []
|
||||
reduced_dtypes = []
|
||||
monkeypatch.setattr(
|
||||
embedding,
|
||||
"_finalize_prefetch",
|
||||
finalize_prefetched,
|
||||
torch.cuda,
|
||||
"current_stream",
|
||||
lambda: SimpleNamespace(wait_stream=waited_streams.append),
|
||||
)
|
||||
|
||||
output = embedding(hidden_states)
|
||||
def all_reduce(tensor: torch.Tensor) -> torch.Tensor:
|
||||
assert waited_streams == [embedding._prefetch_stream]
|
||||
reduced_dtypes.append(tensor.dtype)
|
||||
# The remote rank owns the other head's vocabulary rows.
|
||||
return tensor + tensor.flip(1)
|
||||
|
||||
assert output.dtype == embedding._prefetch_buffer.dtype
|
||||
assert torch.equal(output, expected)
|
||||
embedding.parallel_group = SimpleNamespace(all_reduce=all_reduce)
|
||||
output = embedding.finalize_prefetch(total_tokens=total_tokens)
|
||||
|
||||
expected = (
|
||||
values.unsqueeze(1).expand(-1, 2, -1).to(dtype)
|
||||
if reduce_results
|
||||
else embedding._prefetch_buffer
|
||||
)
|
||||
expected = expected[:total_tokens].flatten(-2)
|
||||
comm_dtype = torch.int8 if dtype == torch.float8_e4m3fn else dtype
|
||||
assert reduced_dtypes == ([comm_dtype] if reduce_results else [])
|
||||
assert waited_streams == [embedding._prefetch_stream]
|
||||
assert output.dtype == dtype
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
|
||||
"""Forward before any start_prefetch must fail loudly, not read garbage."""
|
||||
def test_pinned_embedding_finalize_prefetch_requires_prior_start_prefetch() -> None:
|
||||
"""Finalizing before start_prefetch must fail instead of reading garbage."""
|
||||
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
|
||||
nn.Module.__init__(embedding)
|
||||
embedding._prefetch_buffer = None
|
||||
embedding._prefetch_stream = None
|
||||
embedding._output_dim = 6
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="prior start_prefetch"):
|
||||
embedding(hidden_states)
|
||||
embedding.finalize_prefetch(total_tokens=2)
|
||||
|
||||
|
||||
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
|
||||
@@ -788,15 +965,10 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
def test_pinned_embedding_start_prefetch_allocates_lazily(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
|
||||
|
||||
Exercises the full pinned-prefetch contract end to end: first call
|
||||
allocates the side stream and buffer with the constructor geometry,
|
||||
and forward finalizes the lookup into the returned output.
|
||||
"""
|
||||
"""Allocate prefetch storage on first use and finalize the lookup results."""
|
||||
_mock_etp_group(monkeypatch)
|
||||
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
@@ -832,8 +1004,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
)
|
||||
|
||||
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
|
||||
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
|
||||
embedding.start_prefetch(hidden_states, ngram_ids)
|
||||
embedding.start_prefetch(ngram_ids)
|
||||
|
||||
buffer = embedding._prefetch_buffer
|
||||
assert buffer is not None
|
||||
@@ -842,7 +1013,7 @@ def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
|
||||
assert buffer.device.type == "cuda"
|
||||
assert embedding._prefetch_stream is not None
|
||||
|
||||
output = embedding(hidden_states)
|
||||
output = embedding.finalize_prefetch(total_tokens=ngram_ids.shape[0])
|
||||
|
||||
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
@@ -859,7 +1030,7 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
"quantized_layers": {
|
||||
prefix: {"quant_algo": "FP8"},
|
||||
"model.language_model.layers.2.moe.gate_proj": {
|
||||
"quant_algo": "NVFP4"
|
||||
"quant_algo": "MXFP8"
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -870,13 +1041,12 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
# A quant_algo the PLE table cannot load must not fall back to unquantized.
|
||||
with pytest.raises(ValueError, match="Unsupported quant_algo 'MXFP8'"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
"model.language_model.layers.2.moe.gate_proj",
|
||||
),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _patch_ple_construction(
|
||||
@@ -919,6 +1089,7 @@ def _make_file_gather_embedding(
|
||||
)
|
||||
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||
nn.Module.__init__(module)
|
||||
module.use_sequence_parallel = False
|
||||
module.split_ngram_parts = 4
|
||||
module.eos_token_id = 0
|
||||
module.ngram_size = 2
|
||||
@@ -1215,7 +1386,7 @@ def test_file_gather_forward_reads_staging_without_host_work(
|
||||
step = (torch.arange(6), torch.tensor([0, 6]), torch.tensor([[1]]))
|
||||
hidden_states = torch.zeros(6, 8, dtype=torch.bfloat16)
|
||||
|
||||
assert module.start_prefetch(hidden_states, *step) is None
|
||||
assert module.start_prefetch(*step) is None
|
||||
output = module(hidden_states, *step)
|
||||
|
||||
assert torch.all(staging.view(torch.uint8) == 0xFF)
|
||||
@@ -1500,7 +1671,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
|
||||
prefetched: list[torch.Tensor] = []
|
||||
module.ngram_embedding = SimpleNamespace(
|
||||
supports_prefetch=True,
|
||||
start_prefetch=lambda _, ids: prefetched.append(weak_ref_tensor(ids)),
|
||||
start_prefetch=lambda ids: prefetched.append(weak_ref_tensor(ids)),
|
||||
)
|
||||
expected = _reference_ngram_ids(input_ids, query_start_loc, ngram_context, **params)
|
||||
|
||||
@@ -1508,7 +1679,7 @@ def test_ngram_prefetch_ids_outlive_start_prefetch() -> None:
|
||||
# like a later CUDA graph segment would.
|
||||
pool = torch.cuda.MemPool()
|
||||
with torch.cuda.use_mem_pool(pool):
|
||||
module.start_prefetch(None, input_ids, query_start_loc, ngram_context)
|
||||
module.start_prefetch(input_ids, query_start_loc, ngram_context)
|
||||
torch.full_like(expected, -1)
|
||||
|
||||
assert torch.equal(prefetched[0], expected)
|
||||
@@ -2210,12 +2381,18 @@ def _make_conv_metadata(
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"layer_cls", [Qwen4ExpPLELayer, Qwen4ExpPLELayerAMD], ids=["nvidia", "amd"]
|
||||
("layer_cls", "fuse_residual"),
|
||||
[
|
||||
pytest.param(Qwen4ExpPLELayer, True, id="nvidia-fused"),
|
||||
pytest.param(Qwen4ExpPLELayer, False, id="nvidia-unfused"),
|
||||
pytest.param(Qwen4ExpPLELayerAMD, True, id="amd-fused"),
|
||||
],
|
||||
)
|
||||
def test_fused_conv_correctness(
|
||||
case: _ConvBatchCase,
|
||||
state_layout: str,
|
||||
layer_cls: type[nn.Module],
|
||||
fuse_residual: bool,
|
||||
) -> None:
|
||||
device = torch.device("cuda")
|
||||
metadata, num_real_tokens = _make_conv_metadata(case, device)
|
||||
@@ -2257,12 +2434,15 @@ def test_fused_conv_correctness(
|
||||
)
|
||||
null_state = conv_state[NULL_BLOCK_ID].clone()
|
||||
residual_kernel = residual.clone()
|
||||
residual_reference = residual.clone()
|
||||
# The unfused kernel must overwrite the buffer without reading its contents.
|
||||
residual_reference = (
|
||||
residual.clone() if fuse_residual else torch.zeros_like(residual)
|
||||
)
|
||||
|
||||
module._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual_kernel,
|
||||
outer_residual=outer_residual,
|
||||
outer_residual=outer_residual if fuse_residual else None,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=weights,
|
||||
@@ -2276,7 +2456,8 @@ def test_fused_conv_correctness(
|
||||
conv_state_len=module.conv_state_len,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
residual_reference = outer_residual + residual_reference
|
||||
if fuse_residual:
|
||||
residual_reference = outer_residual + residual_reference
|
||||
|
||||
assert torch.equal(
|
||||
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
|
||||
@@ -2286,7 +2467,7 @@ def test_fused_conv_correctness(
|
||||
if case.graph_padding:
|
||||
assert torch.equal(
|
||||
residual_kernel[num_real_tokens:],
|
||||
(outer_residual + residual)[num_real_tokens:],
|
||||
residual_reference[num_real_tokens:],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Multi-GPU equivalence tests for Qwen4Exp HC and MoE sequence parallelism."""
|
||||
|
||||
from dataclasses import MISSING, fields
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Qwen4Exp SP requires CUDA", allow_module_level=True)
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import (
|
||||
EngramConfig,
|
||||
ParallelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia import model as qwen4_model
|
||||
from vllm.models.qwen4_exp.nvidia.model import Qwen4ExpDecoderLayer
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.v1.attention.backends.short_conv_attn import PleShortConvAttentionMetadata
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
from .test_ple import _ConvBatchCase, _make_conv_metadata
|
||||
|
||||
|
||||
class _TPAttentionProjection(nn.Module):
|
||||
"""Keep real TP communication without setting up an attention KV cache."""
|
||||
|
||||
def __init__(self, config, *, reduce_results: bool, **kwargs) -> None:
|
||||
super().__init__()
|
||||
self.proj = RowParallelLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
input_is_parallel=False,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor):
|
||||
return self.proj(hidden_states)[0]
|
||||
|
||||
|
||||
def _make_config(rank: int, *, hc_sp: bool = False, moe_sp: bool = False) -> VllmConfig:
|
||||
"""Configure HC SP explicitly or MoE SP through TP=2, DP=2 and EP."""
|
||||
config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=2 if moe_sp else 1,
|
||||
data_parallel_rank=rank // 2,
|
||||
enable_expert_parallel=moe_sp,
|
||||
enable_hc_sp=hc_sp,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
),
|
||||
)
|
||||
config.model_config = SimpleNamespace(
|
||||
architecture="Qwen4ExpForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
hf_text_config=Qwen4ExpTextConfig(
|
||||
vocab_size=128,
|
||||
eos_token_id=0,
|
||||
hidden_size=128,
|
||||
hc_count=2,
|
||||
hc_lowrank=16,
|
||||
num_hidden_layers=1,
|
||||
layer_types=["linear_attention"],
|
||||
num_experts=4,
|
||||
num_experts_per_tok=2,
|
||||
moe_intermediate_size=256,
|
||||
shared_expert_intermediate_size=256,
|
||||
ple_layer_ids=[] if moe_sp else [1],
|
||||
ple_embed_dim=128,
|
||||
heads_per_ngram=2,
|
||||
ngram_vocab_size_base=32,
|
||||
make_ngram_vocab_size_divisible_by=64,
|
||||
),
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
config.engram_config = EngramConfig(cpu_offload=False)
|
||||
return config
|
||||
|
||||
|
||||
def _make_decoder(config: VllmConfig, rank: int) -> Qwen4ExpDecoderLayer:
|
||||
"""Use the production constructor and identical initial weights for both modes."""
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
patch.object(qwen4_model, "QwenGatedDeltaNetAttention", _TPAttentionProjection),
|
||||
):
|
||||
layer = Qwen4ExpDecoderLayer(
|
||||
config, config.model_config.hf_text_config.layer_types[0], "model.layers.0"
|
||||
)
|
||||
torch.manual_seed(12)
|
||||
for param in layer.parameters():
|
||||
param.normal_(std=0.08)
|
||||
# Distinct rank contributions expose missing or duplicate reductions.
|
||||
layer.linear_attn.proj.weight.add_(rank * 0.01)
|
||||
experts = layer.mlp.experts.routed_experts
|
||||
if config.parallel_config.enable_expert_parallel:
|
||||
experts.w13_weight.add_(rank * 0.01)
|
||||
experts.quant_method.process_weights_after_loading(experts)
|
||||
return layer
|
||||
|
||||
|
||||
def _check_decoder_ple(rank: int) -> None:
|
||||
"""Compare HC SP with full tokens across a PLE prefill and cached decode."""
|
||||
configs = [_make_config(rank, hc_sp=enabled) for enabled in (False, True)]
|
||||
layers = [_make_decoder(config, rank) for config in configs]
|
||||
states = []
|
||||
for layer in layers:
|
||||
ple = layer.ple
|
||||
assert ple is not None
|
||||
state = torch.zeros(2, ple.hc_hidden_size, ple.conv_state_len)
|
||||
if not is_conv_state_dim_first():
|
||||
state = state.transpose(-1, -2).contiguous()
|
||||
ple.kv_cache = (state,)
|
||||
states.append(state)
|
||||
|
||||
# Three prefill tokens straddle the TP=2 boundary and require SP padding.
|
||||
for case in (
|
||||
_ConvBatchCase(prefill_query_lens=(3,)),
|
||||
_ConvBatchCase(num_decodes=1),
|
||||
):
|
||||
layout, num_tokens = _make_conv_metadata(case, torch.device("cuda", rank))
|
||||
metadata_args = {
|
||||
f.name: None
|
||||
for f in fields(PleShortConvAttentionMetadata)
|
||||
if f.default is MISSING and f.default_factory is MISSING
|
||||
}
|
||||
metadata = PleShortConvAttentionMetadata(**(metadata_args | vars(layout)))
|
||||
ids = torch.randint(0, 128, (num_tokens,))
|
||||
positions = torch.arange(num_tokens).expand(3, -1)
|
||||
qsl = torch.tensor([0, num_tokens], dtype=torch.int32)
|
||||
context = torch.zeros(1, layers[0].config.ngram_size - 1, dtype=torch.int64)
|
||||
hidden = torch.randn(num_tokens, 256)
|
||||
block = torch.randn(num_tokens, 128)
|
||||
injection = torch.randn(num_tokens, 2)
|
||||
outputs = []
|
||||
for config, layer in zip(configs, layers):
|
||||
assert layer.ple is not None
|
||||
inputs = (hidden, block, injection)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
inputs = tuple(sp_shard(x) for x in inputs)
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context({layer.ple.prefix: metadata}, config),
|
||||
):
|
||||
layer.ple.start_prefetch(ids, qsl, context)
|
||||
output = layer(
|
||||
*inputs,
|
||||
positions,
|
||||
input_ids=ids,
|
||||
query_start_loc=qsl,
|
||||
ngram_context=context,
|
||||
)
|
||||
if config.parallel_config.enable_hc_sp:
|
||||
output = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
# A later MoE call may reuse workspace-backed output storage.
|
||||
outputs.append(tuple(x.clone() for x in output))
|
||||
torch.testing.assert_close(outputs[1], outputs[0], atol=0.002, rtol=0.02)
|
||||
torch.testing.assert_close(states[1], states[0], atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _check_moe_sp(rank: int, config: VllmConfig) -> None:
|
||||
"""Compare native SP with full-token GR and the MoE wrapper's public interface."""
|
||||
layer = _make_decoder(config, rank)
|
||||
dp_rank = config.parallel_config.data_parallel_rank
|
||||
counts = torch.tensor([3, 5], device="cpu")
|
||||
num_tokens = int(counts[dp_rank])
|
||||
torch.manual_seed(30 + dp_rank)
|
||||
hidden = torch.randn(num_tokens, 256)
|
||||
positions = torch.arange(num_tokens)
|
||||
# Include runner padding in addition to the padding introduced by SP.
|
||||
padding = sp_padding_mask(positions == num_tokens - 1, hidden)
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
set_forward_context(
|
||||
None,
|
||||
config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=counts,
|
||||
is_padding=padding,
|
||||
),
|
||||
):
|
||||
# The reference keeps full GR rows. MoE handles its own chunk and gather.
|
||||
full_hidden, attn_input, injection = layer.attn_hyper_connection.mix(hidden)
|
||||
attn_output = tensor_model_parallel_all_reduce(layer.linear_attn(attn_input))
|
||||
full_hidden, moe_input, injection = layer.mlp_hyper_connection.combine_and_mix(
|
||||
full_hidden, attn_output, injection
|
||||
)
|
||||
expected = tuple(
|
||||
x.clone() for x in (full_hidden, layer.mlp(moe_input), injection)
|
||||
)
|
||||
output = layer(
|
||||
sp_shard(hidden),
|
||||
None,
|
||||
None,
|
||||
positions,
|
||||
input_ids=None,
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
actual = tuple(sp_all_gather(x)[:num_tokens] for x in output)
|
||||
torch.testing.assert_close(actual, expected, atol=0.002, rtol=0.02)
|
||||
|
||||
|
||||
def _run_sp(rank: int, port: int, moe_sp: bool) -> None:
|
||||
"""Initialize rank-local runtime state and run one equivalence check."""
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
config = _make_config(rank, moe_sp=moe_sp)
|
||||
init_distributed_environment(
|
||||
world_size=4 if moe_sp else 2,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{port}",
|
||||
)
|
||||
try:
|
||||
# Spawned ranks bypass the GPU worker's workspace initialization.
|
||||
init_workspace_manager(torch.device(f"cuda:{rank}"))
|
||||
with set_current_vllm_config(config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=2)
|
||||
with torch.device(f"cuda:{rank}"), torch.inference_mode():
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
if moe_sp:
|
||||
_check_moe_sp(rank, config)
|
||||
else:
|
||||
_check_decoder_ple(rank)
|
||||
finally:
|
||||
reset_workspace_manager()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_hc_sp_decoder_and_ple() -> None:
|
||||
"""HC SP preserves decoder output and PLE cache across a token-shard boundary."""
|
||||
mp.spawn(_run_sp, args=(get_open_port(), False), nprocs=2)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_moe_sp_equivalence(monkeypatch) -> None:
|
||||
"""Native MoE SP preserves outputs with unequal DP batches and padding."""
|
||||
monkeypatch.setenv("VLLM_MOE_SKIP_PADDING", "1")
|
||||
mp.spawn(_run_sp, args=(get_open_port(), True), nprocs=4)
|
||||
+32
-11
@@ -197,6 +197,10 @@ class _HfExamplesInfo:
|
||||
_TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
# [Decoder-only]
|
||||
"AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),
|
||||
"AliceAIForCausalLM": _HfExamplesInfo(
|
||||
"yandex/AliceAI-Foundation-80B-A3B-Base",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
|
||||
"ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
|
||||
"AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),
|
||||
@@ -266,6 +270,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
|
||||
"ExaoneMoEForCausalLM": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
hf_overrides={"architectures": ["ExaoneMoEForCausalLM"]},
|
||||
),
|
||||
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
|
||||
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
|
||||
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
|
||||
@@ -356,6 +364,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"KimiLinearForCausalLM": _HfExamplesInfo(
|
||||
"moonshotai/Kimi-Linear-48B-A3B-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Kolibri1ForCausalLM": _HfExamplesInfo(
|
||||
"Aleph-Alpha/Kolibri-1", min_transformers_version="5.20.0"
|
||||
),
|
||||
"LagunaForCausalLM": _HfExamplesInfo("poolside/Laguna-XS.2"),
|
||||
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
|
||||
"Lfm2MoeForCausalLM": _HfExamplesInfo(
|
||||
@@ -718,6 +729,11 @@ _SEQUENCE_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
"BertForSequenceClassification": _HfExamplesInfo(
|
||||
"cross-encoder/ms-marco-MiniLM-L-6-v2"
|
||||
),
|
||||
"D3ForSequenceClassification": _HfExamplesInfo(
|
||||
"vllm-sr/d3-edge",
|
||||
extras={"d3": "vllm-sr/d3", "d3-mini": "vllm-sr/d3-mini"},
|
||||
max_model_len=4096,
|
||||
),
|
||||
"GPT2ForSequenceClassification": _HfExamplesInfo(
|
||||
"nie3e/sentiment-polish-gpt2-small"
|
||||
),
|
||||
@@ -798,6 +814,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Salesforce/blip2-opt-2.7b",
|
||||
extras={"6b": "Salesforce/blip2-opt-6.7b"},
|
||||
),
|
||||
"ClefFlashForJointSchema": _HfExamplesInfo(
|
||||
"Cloudflare/clef-flash",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
|
||||
@@ -1114,6 +1134,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_6",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_7",
|
||||
min_transformers_version="5.18.0",
|
||||
is_available_online=False,
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
trust_remote_code=True,
|
||||
@@ -1342,6 +1367,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"ClefOmniForJointSchema": _HfExamplesInfo(
|
||||
"Cloudflare/clef-omni",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
@@ -1417,7 +1446,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"CohereLabs/cohere-transcribe-03-2026", trust_remote_code=True
|
||||
),
|
||||
"NemotronParseForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/NVIDIA-Nemotron-Parse-v1.2", trust_remote_code=True
|
||||
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
|
||||
extras={"2.0": "nvidia/NVIDIA-Nemotron-Parse-2.0"},
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"WhisperForConditionalGeneration": _HfExamplesInfo(
|
||||
"openai/whisper-large-v3-turbo",
|
||||
@@ -1429,16 +1460,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
|
||||
|
||||
_SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
# [Medusa]
|
||||
"MedusaModel": _HfExamplesInfo(
|
||||
"JackFram/llama-68m", speculative_model="abhigoyal/vllm-medusa-llama-68m-random"
|
||||
),
|
||||
# Temporarily disabled.
|
||||
# TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
# "MLPSpeculatorPreTrainedModel": _HfExamplesInfo(
|
||||
# "JackFram/llama-160m",
|
||||
# speculative_model="ibm-ai-platform/llama-160m-accelerator"
|
||||
# ),
|
||||
# [DFlash]
|
||||
"DFlashDraftModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-4B",
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||
from vllm.model_executor.models import alice_ai as alice_ai_module
|
||||
from vllm.model_executor.models.alice_ai import (
|
||||
AliceAIDecoderBlock,
|
||||
AliceAIForCausalLM,
|
||||
AliceAIModel,
|
||||
_AttentionResidualFinal,
|
||||
_load_split_kda_conv_weights,
|
||||
)
|
||||
from vllm.model_executor.offloader import PrefetchOffloader, UVAOffloader
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend,error",
|
||||
[
|
||||
("uva", "requires UVA"),
|
||||
("prefetch", "does not support prefetch"),
|
||||
],
|
||||
)
|
||||
def test_unsupported_offloading_is_rejected(monkeypatch, backend, error):
|
||||
from vllm.model_executor.offloader import uva
|
||||
|
||||
monkeypatch.setattr(uva, "is_uva_available", lambda: False)
|
||||
monkeypatch.setenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0")
|
||||
if backend == "uva":
|
||||
offloader = UVAOffloader(cpu_offload_max_bytes=1024)
|
||||
else:
|
||||
offloader = PrefetchOffloader.__new__(PrefetchOffloader)
|
||||
monkeypatch.setattr(alice_ai_module, "get_offloader", lambda: offloader)
|
||||
config = SimpleNamespace(lora_config=None)
|
||||
|
||||
with pytest.raises(NotImplementedError, match=error):
|
||||
AliceAIForCausalLM(vllm_config=config, prefix="target")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source,target,shard_id",
|
||||
[
|
||||
("linear_attn.q_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 0),
|
||||
("linear_attn.k_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 1),
|
||||
("linear_attn.v_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 2),
|
||||
("linear_attn.b_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 3),
|
||||
("linear_attn.f_a_proj.weight", "linear_attn.in_proj_qkvgfab.weight", 4),
|
||||
("self_attn.q_proj.weight", "self_attn.qkv_proj.weight", "q"),
|
||||
("linear_attn.a_log_bias", "linear_attn.A_log", None),
|
||||
],
|
||||
)
|
||||
def test_weight_mapping_preserves_shard_ids(source, target, shard_id):
|
||||
tensor = torch.empty(1)
|
||||
[(mapped, mapped_tensor)] = list(
|
||||
AliceAIModel.hf_to_vllm_mapper.apply([(f"model.layers.0.{source}", tensor)])
|
||||
)
|
||||
assert mapped == f"model.layers.0.{target}"
|
||||
assert getattr(mapped_tensor, "shard_id", None) == shard_id
|
||||
|
||||
|
||||
def test_split_kda_conv_weights_use_fused_loader_shard_ids() -> None:
|
||||
calls: list[tuple[torch.Tensor, int]] = []
|
||||
param = torch.nn.Parameter(torch.empty(3, 1, 1))
|
||||
|
||||
def weight_loader(
|
||||
target: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
shard_id: int,
|
||||
) -> None:
|
||||
assert target is param
|
||||
calls.append((weight, shard_id))
|
||||
|
||||
param.weight_loader = weight_loader
|
||||
target_name = "model.layers.0.linear_attn.conv1d.weight"
|
||||
weights = [
|
||||
("model.layers.0.linear_attn.q_conv1d.weight", torch.full((1, 1), 1.0)),
|
||||
("model.layers.0.linear_attn.k_conv1d.weight", torch.full((1, 1), 2.0)),
|
||||
("model.layers.0.linear_attn.v_conv1d.weight", torch.full((1, 1), 3.0)),
|
||||
("model.embed_tokens.weight", torch.full((1, 1), 4.0)),
|
||||
]
|
||||
loaded: set[str] = set()
|
||||
|
||||
remaining = list(
|
||||
_load_split_kda_conv_weights(weights, {target_name: param}, loaded)
|
||||
)
|
||||
|
||||
assert loaded == {target_name}
|
||||
assert [shard_id for _, shard_id in calls] == [0, 1, 2]
|
||||
assert [weight.item() for weight, _ in calls] == [1.0, 2.0, 3.0]
|
||||
assert remaining == [weights[-1]]
|
||||
|
||||
|
||||
def test_kda_cache_contract_is_fp32_recurrent() -> None:
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
hf_text_config=SimpleNamespace(
|
||||
linear_num_key_heads=32,
|
||||
linear_key_head_dim=128,
|
||||
linear_conv_kernel_dim=4,
|
||||
),
|
||||
),
|
||||
cache_config=SimpleNamespace(mamba_cache_dtype="auto"),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
speculative_config=None,
|
||||
)
|
||||
|
||||
assert AliceAIForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == (
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
)
|
||||
conv_shape = (12288, 3) if is_conv_state_dim_first() else (3, 12288)
|
||||
assert AliceAIForCausalLM.get_mamba_state_shape_from_config(vllm_config) == (
|
||||
conv_shape,
|
||||
(32, 128, 128),
|
||||
)
|
||||
|
||||
|
||||
class _FixedOutput(torch.nn.Module):
|
||||
def __init__(self, value: float) -> None:
|
||||
super().__init__()
|
||||
self.value = value
|
||||
self.inputs: list[torch.Tensor] = []
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, positions: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
del positions
|
||||
self.inputs.append(hidden_states.detach().clone())
|
||||
return torch.full_like(hidden_states, self.value)
|
||||
|
||||
|
||||
def _make_state_layer(
|
||||
layer_idx: int,
|
||||
block_size: int,
|
||||
attention_value: float,
|
||||
mlp_value: float,
|
||||
) -> AliceAIDecoderBlock:
|
||||
layer = AliceAIDecoderBlock.__new__(AliceAIDecoderBlock)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.layer_idx = layer_idx
|
||||
layer.layer_type = "linear_attention"
|
||||
layer.layer_scale = False
|
||||
layer.attn_res_block_size = block_size
|
||||
layer.attn_res_rms_norm_eps = 1e-6
|
||||
layer.input_layernorm = torch.nn.Identity()
|
||||
layer.post_attention_layernorm = torch.nn.Identity()
|
||||
layer.linear_attn = _FixedOutput(attention_value)
|
||||
layer.mlp = _FixedOutput(mlp_value)
|
||||
if layer_idx > 0:
|
||||
layer.attn_res_proj = torch.nn.Linear(1, 1, bias=False)
|
||||
layer.attn_res_norm_weight = torch.nn.Parameter(torch.ones(1))
|
||||
layer.mlp_res_proj = torch.nn.Linear(1, 1, bias=False)
|
||||
layer.mlp_res_norm_weight = torch.nn.Parameter(torch.ones(1))
|
||||
return layer
|
||||
|
||||
|
||||
def _make_final_attn_res() -> _AttentionResidualFinal:
|
||||
final = _AttentionResidualFinal.__new__(_AttentionResidualFinal)
|
||||
torch.nn.Module.__init__(final)
|
||||
final.rms_norm_eps = 1e-6
|
||||
final.res_proj = torch.nn.Linear(1, 1, bias=False)
|
||||
final.res_norm_weight = torch.nn.Parameter(torch.ones(1))
|
||||
return final
|
||||
|
||||
|
||||
def test_attn_res_block_grouping_and_final_norm(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
calls: list[list[torch.Tensor]] = []
|
||||
|
||||
def reference_attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
):
|
||||
del delta, norm_weight, qk_weight, output_norm_weight
|
||||
del block_write_idx, eps, output_norm_eps
|
||||
sources = [blocks[:, index] for index in range(num_blocks)] + [prefix]
|
||||
calls.append([source.detach().clone() for source in sources])
|
||||
return torch.stack(sources).sum(dim=0)
|
||||
|
||||
monkeypatch.setattr(alice_ai_module, "attn_res", reference_attn_res)
|
||||
monkeypatch.setattr(
|
||||
alice_ai_module,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
raw_outputs = [(1, 2), (3, 4), (5, 6), (7, 8), (9, 10)]
|
||||
model = AliceAIModel.__new__(AliceAIModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.attn_res_block_size = 4
|
||||
model.start_layer = 0
|
||||
model.end_layer = len(raw_outputs)
|
||||
model.layers = torch.nn.ModuleList(
|
||||
[
|
||||
_make_state_layer(index, 4, attention, mlp)
|
||||
for index, (attention, mlp) in enumerate(raw_outputs)
|
||||
]
|
||||
)
|
||||
model.config = SimpleNamespace(num_hidden_layers=len(raw_outputs))
|
||||
model.attnres_final = _make_final_attn_res()
|
||||
model.norm = torch.nn.Linear(1, 1, bias=False)
|
||||
with torch.no_grad():
|
||||
model.norm.weight.fill_(2)
|
||||
|
||||
output = AliceAIModel.forward(
|
||||
model,
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
inputs_embeds=torch.full((2, 1), 10.0),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.full((2, 1), 130.0))
|
||||
torch.testing.assert_close(
|
||||
model.layers[0].linear_attn.inputs[0], torch.full((2, 1), 10.0)
|
||||
)
|
||||
assert [len(sources) for sources in calls] == [2] * 8 + [3, 3]
|
||||
assert [source[0, 0].item() for source in calls[-1]] == [10, 36, 19]
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for AliceAI configuration."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.config import AliceAIForCausalLMConfig
|
||||
|
||||
|
||||
def _make_config(**overrides) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
**{
|
||||
"kda_allow_negative_eigenvalues": False,
|
||||
"router_score_function": "sigmoid",
|
||||
"router_bias_correction": True,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_model_config_warns_when_mtp_is_ignored(caplog) -> None:
|
||||
config = _make_config(mtp_num_hidden_layers=1)
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(
|
||||
SimpleNamespace(hf_config=config)
|
||||
)
|
||||
assert "ignores the MTP configuration and weights" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,error_type,error",
|
||||
[
|
||||
({}, None, None),
|
||||
(
|
||||
{"kda_allow_negative_eigenvalues": True},
|
||||
NotImplementedError,
|
||||
"kda_allow_negative_eigenvalues=false",
|
||||
),
|
||||
(
|
||||
{"router_score_function": "softmax"},
|
||||
ValueError,
|
||||
"router_score_function='sigmoid'",
|
||||
),
|
||||
(
|
||||
{"router_bias_correction": False},
|
||||
ValueError,
|
||||
"router_bias_correction=true",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_model_config_rejects_unsupported_alice_contract(
|
||||
overrides, error_type, error
|
||||
) -> None:
|
||||
model_config = SimpleNamespace(hf_config=_make_config(**overrides))
|
||||
|
||||
if error is None:
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
|
||||
else:
|
||||
with pytest.raises(error_type, match=error):
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pipeline_parallel_size,speculative_method,error",
|
||||
[
|
||||
(1, None, None),
|
||||
(2, None, "pipeline_parallel_size=1"),
|
||||
(1, "eagle3", "eagle3"),
|
||||
(1, "dflash", "dflash"),
|
||||
(1, "dspark", "dspark"),
|
||||
(1, "extract_hidden_states", "extract_hidden_states"),
|
||||
],
|
||||
)
|
||||
def test_runtime_config_rejects_unsupported_alice_modes(
|
||||
pipeline_parallel_size, speculative_method, error
|
||||
) -> None:
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(pipeline_parallel_size=pipeline_parallel_size),
|
||||
speculative_config=(
|
||||
None
|
||||
if speculative_method is None
|
||||
else SimpleNamespace(method=speculative_method)
|
||||
),
|
||||
)
|
||||
|
||||
if error is None:
|
||||
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
|
||||
else:
|
||||
with pytest.raises(NotImplementedError, match=error):
|
||||
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
|
||||
@@ -0,0 +1,329 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""What the AMD replay forward does with a prefill longer than the window.
|
||||
|
||||
The cut in this fixture is layer 1, so replay starts at layer 1. That is the
|
||||
same boundary as production layer 20, on a 4-layer stand-in. The final
|
||||
``mhc_post`` and the aux capture at that cut both run on the window rows.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import ReplayBatch
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
|
||||
WINDOW = 128
|
||||
FULL = 300
|
||||
# Trailing window of a 300-token prefill.
|
||||
ROWS = list(range(FULL - WINDOW, FULL))
|
||||
HC, HIDDEN = 4, 8
|
||||
DEVICE = torch.device("cuda")
|
||||
# gfx942 writes the V4 record (584 B). Family-100 writes the V4.1 record (528 B).
|
||||
# The SWA layer uses block size 32, which is the size the KV-only insert checks.
|
||||
_BLOCK = 32
|
||||
_HEAD_DIM = 512
|
||||
|
||||
|
||||
class _RocmKvInsert:
|
||||
"""SWA cache from ``forward_kv``, checked against the KV-only insert."""
|
||||
|
||||
def __init__(self, num_tokens: int):
|
||||
from vllm.models.deepseek_v41.attention import _use_v41_mxfp8_kv_record
|
||||
|
||||
if not hasattr(torch.ops._C, "fused_deepseek_v4_kv_rope_insert"):
|
||||
pytest.skip("fused DeepseekV4 KV insert op is not built")
|
||||
|
||||
self.num_tokens = num_tokens
|
||||
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
|
||||
bytes_per = 528 if self.kv_mxfp8 else 584
|
||||
num_blocks = (num_tokens + _BLOCK - 1) // _BLOCK + 1
|
||||
cache = torch.full(
|
||||
(num_blocks, _BLOCK, bytes_per),
|
||||
0xA5,
|
||||
dtype=torch.uint8,
|
||||
device=DEVICE,
|
||||
)
|
||||
self.poison = cache.clone()
|
||||
self.kv = torch.randn(
|
||||
num_tokens, _HEAD_DIM, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
prefix = "replay.swa"
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=DEVICE)
|
||||
self.metadata = {
|
||||
prefix: SimpleNamespace(slot_mapping=slot_mapping, block_size=_BLOCK)
|
||||
}
|
||||
self.context = ForwardContext({}, self.metadata, {})
|
||||
inv_freq = 1.0 / (
|
||||
10000 ** (torch.arange(0, 64, 2, dtype=torch.float32, device=DEVICE) / 64)
|
||||
)
|
||||
freqs = torch.outer(
|
||||
torch.arange(num_tokens + 8, dtype=torch.float32, device=DEVICE), inv_freq
|
||||
)
|
||||
self.attn = SimpleNamespace(
|
||||
kv_mxfp8=self.kv_mxfp8,
|
||||
compressor=None,
|
||||
indexer=None,
|
||||
rotary_emb=SimpleNamespace(
|
||||
cos_sin_cache=torch.cat((freqs.cos(), freqs.sin()), dim=-1)
|
||||
),
|
||||
swa_cache_layer=SimpleNamespace(
|
||||
prefix=prefix, kv_cache=cache, block_size=_BLOCK
|
||||
),
|
||||
_run_parallel_input_projections=lambda hidden: (hidden, None, None),
|
||||
_split_qkv_and_norm=lambda qr_kv: (None, None, self.kv[: qr_kv.shape[0]]),
|
||||
)
|
||||
self.last_rows: int | None = None
|
||||
self.last_positions: torch.Tensor | None = None
|
||||
|
||||
def write(
|
||||
self,
|
||||
x,
|
||||
positions,
|
||||
input_ids,
|
||||
pre_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
):
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV41ROCMAiterMLAAttention,
|
||||
)
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
assert (
|
||||
DeepseekV41ROCMAiterMLAAttention.forward_kv
|
||||
is DeepseekV4Attention.forward_kv
|
||||
)
|
||||
self.last_rows = x.shape[0]
|
||||
self.last_positions = positions
|
||||
DeepseekV41ROCMAiterMLAAttention.forward_kv(self.attn, positions, x)
|
||||
|
||||
def assert_matches_forward(self):
|
||||
assert self.last_rows == self.num_tokens
|
||||
assert self.last_positions is not None
|
||||
written = self.attn.swa_cache_layer.kv_cache
|
||||
assert not torch.equal(written, self.poison)
|
||||
ref = self.poison.clone()
|
||||
meta = self.metadata[self.attn.swa_cache_layer.prefix]
|
||||
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
|
||||
self.kv,
|
||||
ref,
|
||||
meta.slot_mapping,
|
||||
self.last_positions,
|
||||
self.attn.rotary_emb.cos_sin_cache,
|
||||
_BLOCK,
|
||||
None,
|
||||
self.kv_mxfp8,
|
||||
)
|
||||
assert torch.equal(written, ref)
|
||||
assert torch.equal(written[-1], self.poison[-1])
|
||||
|
||||
|
||||
class _Layer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
idx: int,
|
||||
compressed: str | None = None,
|
||||
indexer: str | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.idx = idx
|
||||
self.seen: list[torch.Tensor] = []
|
||||
self.attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(prefix=f"layers.{idx}.swa"),
|
||||
compressed_cache_prefix=compressed,
|
||||
indexer=(
|
||||
None
|
||||
if indexer is None
|
||||
else SimpleNamespace(k_cache=SimpleNamespace(prefix=indexer))
|
||||
),
|
||||
compress_ratio=1,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
pre_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
residual,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
):
|
||||
self.seen.append(hidden_states[:, 0, 0].detach().clone())
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
if pre_mix is None:
|
||||
pre_mix = hidden_states[:, :, 0]
|
||||
if post_mix is None:
|
||||
post_mix = pre_mix
|
||||
if res_mix is None:
|
||||
res_mix = pre_mix
|
||||
return hidden_states, residual, post_mix, res_mix, pre_mix
|
||||
|
||||
|
||||
def _model(monkeypatch):
|
||||
import vllm.models.deepseek_v41.amd.model as amd
|
||||
|
||||
monkeypatch.setattr(
|
||||
amd,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=False, is_last_rank=False),
|
||||
)
|
||||
monkeypatch.setattr(amd.EngramLayout, "from_config", lambda config: None)
|
||||
monkeypatch.setattr(amd, "MHCPostOp", lambda: SimpleNamespace())
|
||||
|
||||
# Layer 0 sits before the cut, so its keys are not collected. Layer 2
|
||||
# shares layer 1's compressed cache. Layer 3 has neither.
|
||||
layers = [
|
||||
_Layer(0, "layers.0.compressed", "layers.0.indexer"),
|
||||
_Layer(1, "layers.1.compressed", "layers.1.indexer"),
|
||||
_Layer(2, "layers.1.compressed", "layers.2.indexer"),
|
||||
_Layer(3),
|
||||
]
|
||||
# Layer 1 is the KV source, so the replay callback is this layer's write.
|
||||
insert = _RocmKvInsert(FULL)
|
||||
layers[1].write_kv = insert.write
|
||||
monkeypatch.setattr(amd, "make_layers", lambda n, factory, prefix: (0, n, layers))
|
||||
|
||||
config = SimpleNamespace(
|
||||
vocab_size=32,
|
||||
hc_eps=1e-6,
|
||||
hc_mult=HC,
|
||||
hidden_size=HIDDEN,
|
||||
rms_norm_eps=1e-6,
|
||||
index_topk=4,
|
||||
kv_source_layer_ids=[1],
|
||||
num_hidden_layers=4,
|
||||
sliding_window=WINDOW,
|
||||
engram_layer_ids=(),
|
||||
candidate_source_layer_id=-1,
|
||||
candidate_topk_blocks=0,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=config, dtype=torch.bfloat16),
|
||||
quant_config=None,
|
||||
cache_config=SimpleNamespace(swa_bounded_replay=True),
|
||||
use_v2_model_runner=True,
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
use_ubatching=False,
|
||||
enable_expert_parallel=False,
|
||||
),
|
||||
speculative_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=FULL),
|
||||
)
|
||||
model = amd.DeepseekV4Model(vllm_config=vllm_config)
|
||||
# Production requests aux at the first replay layer. Here that id is the cut.
|
||||
model.aux_hidden_state_layers = (model.decoder_replay_start,)
|
||||
model.topk_indices_buffer = model.topk_indices_buffer.to(DEVICE)
|
||||
|
||||
calls: list[torch.Tensor] = []
|
||||
|
||||
def mhc_post(hidden, residual, post_mix, res_mix):
|
||||
calls.append(hidden[:, 0, 0].detach().clone())
|
||||
return hidden
|
||||
|
||||
model.mhc_post = mhc_post
|
||||
saved_aux: list[list[torch.Tensor]] = []
|
||||
run_layers = model._run_layers
|
||||
|
||||
def wrapped(*args, **kwargs):
|
||||
result = run_layers(*args, **kwargs)
|
||||
# Snapshot: forward() extends the early list with the scattered aux.
|
||||
saved_aux.append(list(result[5]))
|
||||
return result
|
||||
|
||||
model._run_layers = wrapped
|
||||
scattered: list[tuple[torch.Tensor, ...]] = []
|
||||
replay = model.decoder_replay_layers
|
||||
run = replay._run
|
||||
|
||||
def wrapped_run(*args, **kwargs):
|
||||
result = run(*args, **kwargs)
|
||||
scattered.append(result)
|
||||
return result
|
||||
|
||||
replay._run = wrapped_run
|
||||
return model, layers, calls, saved_aux, scattered, insert
|
||||
|
||||
|
||||
def _forward(model, insert: _RocmKvInsert):
|
||||
token_ids = torch.arange(FULL, device=DEVICE, dtype=torch.float32)
|
||||
hidden = token_ids[:, None, None].expand(FULL, HC, HIDDEN).contiguous()
|
||||
pre_mix = torch.ones(FULL, HC, device=DEVICE)
|
||||
rows = torch.tensor(ROWS, device=DEVICE)
|
||||
model.decoder_replay_layers.replay_batch = ReplayBatch(
|
||||
rows,
|
||||
ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={}),
|
||||
)
|
||||
positions = torch.arange(FULL, device=DEVICE)
|
||||
with override_forward_context(insert.context):
|
||||
out = model(
|
||||
input_ids=positions.to(torch.int32),
|
||||
positions=positions,
|
||||
intermediate_tensors=IntermediateTensors(
|
||||
{"hidden_states": hidden, "pre_mix": pre_mix}
|
||||
),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def test_amd_replay_forward_trims_to_the_window(monkeypatch):
|
||||
model, layers, calls, saved_aux, scattered, insert = _model(monkeypatch)
|
||||
replay_layers = model.decoder_replay_layers
|
||||
assert model.decoder_replay_start == 1
|
||||
assert replay_layers.first_swa_prefix == "layers.1.swa"
|
||||
assert replay_layers.metadata_prefixes == {
|
||||
"layers.1.swa",
|
||||
"layers.1.compressed",
|
||||
"layers.1.indexer",
|
||||
"layers.2.swa",
|
||||
"layers.2.indexer",
|
||||
"layers.3.swa",
|
||||
}
|
||||
assert replay_layers.replay_batch is None
|
||||
out = _forward(model, insert)
|
||||
insert.assert_matches_forward()
|
||||
replay = model.decoder_replay_layers.replay_batch
|
||||
assert replay is not None and replay.rows.tolist() == ROWS
|
||||
|
||||
early = [layer.seen[0] for layer in layers[: model.decoder_replay_start]]
|
||||
late = [layer.seen[0] for layer in layers[model.decoder_replay_start :]]
|
||||
full_ids = torch.arange(FULL, device=DEVICE, dtype=torch.float32)
|
||||
window_ids = full_ids[ROWS]
|
||||
assert [t.tolist() for t in early] == [full_ids.tolist()]
|
||||
assert [t.tolist() for t in late] == [
|
||||
window_ids.tolist(),
|
||||
window_ids.tolist(),
|
||||
window_ids.tolist(),
|
||||
]
|
||||
|
||||
# Scatter writes the replay rows back and leaves the trimmed rows at zero.
|
||||
hidden = out["hidden_states"][:, 0, 0]
|
||||
assert torch.equal(hidden[ROWS], window_ids)
|
||||
assert hidden[: FULL - WINDOW].eq(0).all()
|
||||
|
||||
# calls[0] is the cut-aux capture. calls[1] is the final mhc_post.
|
||||
# Both run inside the replay batch, before the scatter back to FULL.
|
||||
assert len(calls) == 2
|
||||
assert [t.shape[0] for t in calls] == [WINDOW, WINDOW]
|
||||
assert torch.equal(calls[0], window_ids)
|
||||
assert torch.equal(calls[1], window_ids)
|
||||
|
||||
# The early loop skips this id. The replay loop does not see it either.
|
||||
assert saved_aux[0] == []
|
||||
assert saved_aux[1] == []
|
||||
_hidden, _pre_mix, aux = scattered[0]
|
||||
assert aux.shape[0] == FULL
|
||||
assert torch.equal(aux[ROWS, 0], window_ids)
|
||||
assert aux[: FULL - WINDOW].eq(0).all()
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
ForwardContext,
|
||||
@@ -144,8 +145,18 @@ def test_piecewise_graphs_break_out_by_size():
|
||||
write_batch_kv.assert_called_once_with(*states)
|
||||
|
||||
|
||||
def test_replay_graph_matches_eager(monkeypatch):
|
||||
"""The replay graph of the next captured size pads and runs the rows."""
|
||||
@pytest.mark.parametrize(
|
||||
("moe_backend", "ids_dtype"),
|
||||
[
|
||||
("auto", torch.int32),
|
||||
*((b, torch.int64) for b in sorted(NATIVE_MEGA_MOE_BACKENDS)),
|
||||
*((b, torch.int32) for b in sorted(FLASHINFER_MOE_EP_BACKENDS)),
|
||||
],
|
||||
ids=str,
|
||||
)
|
||||
def test_replay_graph_matches_eager(monkeypatch, moe_backend, ids_dtype):
|
||||
"""The replay graph of the next captured size pads and runs the rows. Input ids
|
||||
arrive as the model passes them: int32, cast to int64 for native MegaMoE only."""
|
||||
monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock)
|
||||
capture = torch.cuda.stream(torch.cuda.Stream())
|
||||
monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture)
|
||||
@@ -154,9 +165,10 @@ def test_replay_graph_matches_eager(monkeypatch):
|
||||
# Mocking it true would capture into the cuMem pool, which breaks on ROCm.
|
||||
cfg.use_cumem_cudagraph_pool = False
|
||||
cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64)
|
||||
cfg.cache_config.use_kda_recoverssm = False
|
||||
cfg.cache_config.use_recoverssm = False
|
||||
cfg.model_config.hf_config = MagicMock(hc_mult=HC, hidden_size=HIDDEN)
|
||||
cfg.model_config.dtype = torch.float32
|
||||
cfg.kernel_config.moe_backend = moe_backend
|
||||
|
||||
def run_layers(hidden, positions, ids, pre, post, mix, residual):
|
||||
return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None]
|
||||
@@ -173,6 +185,7 @@ def test_replay_graph_matches_eager(monkeypatch):
|
||||
torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype)
|
||||
for b in manager.input_buffers
|
||||
]
|
||||
states[2] = torch.arange(12, device=DEVICE, dtype=ids_dtype) # input_ids
|
||||
rows = torch.tensor([11, 2, 7], device=DEVICE)
|
||||
desc = manager.dispatch(2, 3, None, 0)
|
||||
with override_forward_context(_graph_context(desc.num_tokens)):
|
||||
|
||||
@@ -290,6 +290,81 @@ def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
|
||||
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129]
|
||||
|
||||
|
||||
def test_zero_prefix_window_warms_gather_and_skips_pad(monkeypatch):
|
||||
"""ROCm leaves the prefix-cache window at 0. Replay still needs replay_start
|
||||
and a warmed gather kernel. The pad kernel stays on the prefix-cache path."""
|
||||
import vllm.models.deepseek_v41.nvidia.model_state as model_state
|
||||
|
||||
calls = {"pad_warmup": 0, "gather_warmup": 0, "pad_launch": 0}
|
||||
|
||||
def pad_warmup(*args, **kwargs):
|
||||
calls["pad_warmup"] += 1
|
||||
|
||||
def gather_warmup(*args, **kwargs):
|
||||
calls["gather_warmup"] += 1
|
||||
|
||||
class _Pad:
|
||||
warmup = staticmethod(pad_warmup)
|
||||
|
||||
def __getitem__(self, grid):
|
||||
def launch(*args, **kwargs):
|
||||
calls["pad_launch"] += 1
|
||||
|
||||
return launch
|
||||
|
||||
monkeypatch.setattr(model_state, "_pad_replayed_slots_kernel", _Pad())
|
||||
monkeypatch.setattr(
|
||||
model_state._gather_replay_batch_kernel, "warmup", gather_warmup
|
||||
)
|
||||
|
||||
cfg = MagicMock()
|
||||
cfg.model_config.enable_prompt_embeds = False
|
||||
cfg.model_config.uses_mrope = False
|
||||
cfg.model_config.is_multimodal_model = False
|
||||
cfg.scheduler_config.max_num_seqs = 16
|
||||
cfg.scheduler_config.max_num_batched_tokens = 1024
|
||||
cfg.parallel_config.data_parallel_size = 1
|
||||
cfg.compilation_config.fast_moe_cold_start = False
|
||||
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
|
||||
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
|
||||
groups = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(
|
||||
layer_names=["swa"],
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=False, prefix_replay_tokens=0
|
||||
),
|
||||
),
|
||||
SimpleNamespace(
|
||||
layer_names=["mla"],
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
prefix_cacheable=True, prefix_replay_tokens=0
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
|
||||
return {"swa": kw["model_specific_attn_metadata"]}
|
||||
|
||||
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
|
||||
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
|
||||
state._replay_start_np[0] = 16
|
||||
batch = _input_batch([8], [24], [True])
|
||||
state.prepare_attn(
|
||||
batch,
|
||||
CUDAGraphMode.NONE,
|
||||
BLOCK_TABLES,
|
||||
_slot_mappings(batch.num_tokens_after_padding),
|
||||
[],
|
||||
groups,
|
||||
)
|
||||
assert state._replay[0] == 0
|
||||
assert calls == {"pad_warmup": 0, "gather_warmup": 1, "pad_launch": 0}
|
||||
assert state._replay_start[:1].tolist() == [16]
|
||||
|
||||
|
||||
def test_dp_ranks_pad_to_one_replay_graph(dp_state):
|
||||
"""Every rank pads its replay batch to the graph fitting the largest rank's."""
|
||||
dp_state.replay_cudagraphs = MagicMock()
|
||||
|
||||
@@ -29,6 +29,7 @@ def _make_experts(
|
||||
intermediate_size=INTER,
|
||||
max_num_batched_tokens=3000,
|
||||
swiglu_limit=None,
|
||||
fuse_shared_expert=False,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
mega_moe_module,
|
||||
@@ -47,6 +48,7 @@ def _make_experts(
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size=intermediate_size,
|
||||
swiglu_limit=swiglu_limit,
|
||||
fuse_shared_expert=fuse_shared_expert,
|
||||
)
|
||||
|
||||
|
||||
@@ -108,6 +110,49 @@ def test_experts_load_into_local_slots(monkeypatch):
|
||||
assert torch.all(w2 == 10 * expert + 3)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"tokens_across_dp, expected",
|
||||
[([4, 128], "decode"), ([4, 129], "prefill")],
|
||||
)
|
||||
def test_runtime_selected_by_max_tokens_across_dp(
|
||||
monkeypatch, tokens_across_dp, expected
|
||||
):
|
||||
experts = _make_experts(monkeypatch)
|
||||
experts._runtimes = [(128, "decode"), (4096, "prefill")]
|
||||
monkeypatch.setattr(mega_moe_module, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
mega_moe_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor(tokens_across_dp)
|
||||
)
|
||||
),
|
||||
)
|
||||
# Every rank must take the same runtime, so a small local batch still gets
|
||||
# the prefill runtime when another DP rank is over the decode capacity.
|
||||
assert experts.select_runtime(4) == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
def test_shared_expert_route_appended(monkeypatch):
|
||||
experts = _make_experts(
|
||||
monkeypatch, ep_rank=3, ep_size=8, num_experts=384, fuse_shared_expert=True
|
||||
)
|
||||
ids = torch.randint(0, 384, (37, 2), device="cuda", dtype=torch.int32)
|
||||
ids[::5, 1] = -1
|
||||
weights = torch.rand(37, 2, device="cuda")
|
||||
out_weights, out_ids = experts.append_shared_expert(weights, ids)
|
||||
|
||||
shared = torch.full((37, 1), 3 * 49 + 48, device="cuda", dtype=torch.int32)
|
||||
expected_ids = torch.cat([torch.where(ids >= 0, ids + ids // 48, ids), shared], 1)
|
||||
torch.testing.assert_close(out_ids, expected_ids)
|
||||
torch.testing.assert_close(
|
||||
out_weights, torch.cat([weights, torch.ones(37, 1, device="cuda")], 1)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
@pytest.mark.parametrize("hash_routing", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 37])
|
||||
|
||||
@@ -42,10 +42,6 @@ def _disable_distributed_loader_paths(monkeypatch: pytest.MonkeyPatch):
|
||||
"fused_moe_make_expert_params_mapping",
|
||||
lambda *args, **kwargs: [],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dspark_module, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(dspark_module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
|
||||
@@ -142,6 +142,7 @@ def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
|
||||
is_act_and_mul=True,
|
||||
activation=MoEActivation.SILU,
|
||||
has_hash_routing=False,
|
||||
require_decomposed_backend=False,
|
||||
routing_method=RoutingMethodType.DeepseekV4,
|
||||
router_logits_dtype=torch.float32,
|
||||
hidden_dim=256,
|
||||
|
||||
@@ -5,6 +5,7 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
DeepGemmSm100MegaMoEBackend,
|
||||
@@ -12,8 +13,9 @@ from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_weight_scale_block,
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
get_mixture_of_experts_model,
|
||||
is_mixture_of_experts,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.dspark import DSparkDeepseekV4ForCausalLM
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
@@ -23,7 +25,6 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
prepare_mega_gate_routing_metadata,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
@@ -38,6 +39,15 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def sm100_mega_moe_backend(monkeypatch):
|
||||
"""Build the SM100 kernel layout on any GPU; kernels are gated per test."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.model.get_deep_gemm_mega_moe_backend",
|
||||
lambda *args: DeepGemmSm100MegaMoEBackend(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def v41_moe_config(dist_init):
|
||||
return SimpleNamespace(
|
||||
@@ -298,18 +308,6 @@ def test_deepseek_v4_mega_moe_expert_mapping():
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_ue8m0_uint8_to_float():
|
||||
raw = torch.tensor([0, 126, 127, 128], dtype=torch.uint8)
|
||||
|
||||
decoded = DeepseekV4MegaMoEExperts._ue8m0_uint8_to_float(raw)
|
||||
|
||||
assert torch.equal(decoded.view(torch.int32), raw.to(torch.int32) << 23)
|
||||
assert decoded[0].item() == 0.0
|
||||
assert decoded[1].item() == 0.5
|
||||
assert decoded[2].item() == 1.0
|
||||
assert decoded[3].item() == 2.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_kimi", [False, True])
|
||||
def test_deep_gemm_mega_moe_capture_precedes_eplb(monkeypatch, use_kimi):
|
||||
experts_cls = DeepseekV4MegaMoEExperts
|
||||
@@ -422,16 +420,141 @@ def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
assert torch.equal(experts.w13_weight[0, :128], w1)
|
||||
assert torch.equal(experts.w13_weight[0, 128:], w3)
|
||||
assert torch.equal(experts.w2_weight[0], w2)
|
||||
interleaved = experts.w13_weight[0].view(torch.uint8).view(16, 2, 8, 64)
|
||||
assert torch.equal(interleaved[:, 0], w1.view(16, 8, 64))
|
||||
assert torch.equal(interleaved[:, 1], w3.view(16, 8, 64))
|
||||
assert torch.equal(experts.w2_weight[0].view(torch.uint8), w2)
|
||||
assert torch.count_nonzero(experts.w13_weight[1]) == 0
|
||||
|
||||
|
||||
def _reference_mega_moe_layout(w13, s13, w2, s2):
|
||||
"""CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales."""
|
||||
|
||||
def interleave(t):
|
||||
g, n, k = t.shape
|
||||
halves = t.view(g, 2, n // 16, 8, k)
|
||||
return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k)
|
||||
|
||||
def utccp(sf):
|
||||
g, mn, k = sf.shape
|
||||
return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k)
|
||||
|
||||
return (
|
||||
interleave(w13),
|
||||
utccp(interleave(s13.view(torch.int32))),
|
||||
w2,
|
||||
utccp(s2.view(torch.int32)),
|
||||
)
|
||||
|
||||
|
||||
def _make_mega_moe_experts(num_local_experts, num_logical_experts):
|
||||
return DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=num_local_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=384,
|
||||
num_logical_experts=num_logical_experts,
|
||||
)
|
||||
|
||||
|
||||
def _load_mega_moe_checkpoint_shuffled(experts, num_logical_experts, seed):
|
||||
hidden, inter = experts.hidden_size, experts.intermediate_size
|
||||
shapes = {
|
||||
"w13_weight": (num_logical_experts, 2 * inter, hidden // 2),
|
||||
"w13_weight_scale": (num_logical_experts, 2 * inter, hidden // 32),
|
||||
"w2_weight": (num_logical_experts, hidden, inter // 2),
|
||||
"w2_weight_scale": (num_logical_experts, hidden, inter // 32),
|
||||
}
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
ckpt = {
|
||||
name: torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
for name, shape in shapes.items()
|
||||
}
|
||||
shards = []
|
||||
for expert_id in range(num_logical_experts):
|
||||
for name, tensor in ckpt.items():
|
||||
if name.startswith("w2_"):
|
||||
shards.append((name, tensor[expert_id], "w2", expert_id))
|
||||
else:
|
||||
shards.append((name, tensor[expert_id, :inter], "w1", expert_id))
|
||||
shards.append((name, tensor[expert_id, inter:], "w3", expert_id))
|
||||
for i in torch.randperm(len(shards), generator=generator).tolist():
|
||||
name, loaded_weight, shard_id, expert_id = shards[i]
|
||||
if name.endswith("scale"):
|
||||
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
|
||||
assert experts.weight_loader(
|
||||
getattr(experts, name),
|
||||
loaded_weight,
|
||||
f"experts.{name}",
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
return ckpt
|
||||
|
||||
|
||||
def _expected_mega_moe_layout(ckpt, num_local_experts, num_logical_experts):
|
||||
physical = torch.arange(num_local_experts) % num_logical_experts
|
||||
ckpt = {name: tensor[physical] for name, tensor in ckpt.items()}
|
||||
expected = _reference_mega_moe_layout(*ckpt.values())
|
||||
if is_deep_gemm_supported():
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
w13, s13, w2, s2 = (t.cuda() for t in ckpt.values())
|
||||
hidden, inter = w2.shape[1], w2.shape[2] * 2
|
||||
scales = [
|
||||
deep_gemm.transform_sf_into_required_layout(
|
||||
(sf.to(torch.int32) << 23).view(torch.float32),
|
||||
mn,
|
||||
k,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
for sf, mn, k in ((s13, 2 * inter, hidden), (s2, hidden, inter))
|
||||
]
|
||||
(l1, l1_sf), (l2, l2_sf) = deep_gemm.transform_weights_for_mega_moe(
|
||||
(w13.view(torch.int8), scales[0]), (w2.view(torch.int8), scales[1])
|
||||
)
|
||||
for actual, reference in zip((l1, l1_sf, l2, l2_sf), expected):
|
||||
assert torch.equal(actual.cpu().view(reference.dtype), reference)
|
||||
return expected
|
||||
|
||||
|
||||
def _assert_mega_moe_layout(experts, expected):
|
||||
actual = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
|
||||
for tensor, reference in zip(actual, expected):
|
||||
assert torch.equal(tensor.view(reference.dtype), reference)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_loads_shards_into_kernel_layout():
|
||||
"""Shards in any order, including EPLB replicas, land in DeepGEMM's layout."""
|
||||
experts = _make_mega_moe_experts(num_local_experts=4, num_logical_experts=3)
|
||||
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 3, seed=0)
|
||||
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 4, 3))
|
||||
# DeepGEMM's MN-major scale strides.
|
||||
assert experts.w13_weight_scale.stride() == (768 * 2, 1, 768)
|
||||
assert experts.w2_weight_scale.stride() == (256 * 3, 1, 256)
|
||||
|
||||
|
||||
def test_deepseek_v4_mega_moe_reload_overwrites_kernel_layout_in_place():
|
||||
"""A second load must update the live kernel tensors without reallocating."""
|
||||
experts = _make_mega_moe_experts(num_local_experts=2, num_logical_experts=2)
|
||||
_load_mega_moe_checkpoint_shuffled(experts, 2, seed=0)
|
||||
data_ptrs = [param.data_ptr() for param in experts.parameters()]
|
||||
ckpt = _load_mega_moe_checkpoint_shuffled(experts, 2, seed=1)
|
||||
assert [param.data_ptr() for param in experts.parameters()] == data_ptrs
|
||||
_assert_mega_moe_layout(experts, _expected_mega_moe_layout(ckpt, 2, 2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate_size", [512, 2304])
|
||||
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(
|
||||
monkeypatch, intermediate_size
|
||||
):
|
||||
def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(intermediate_size):
|
||||
"""Keep native widths so V4.1 routed and shared experts can fuse."""
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
@@ -445,216 +568,149 @@ def test_deepseek_v4_mega_moe_preserves_checkpoint_dimensions(
|
||||
hidden_size=128,
|
||||
intermediate_size=intermediate_size,
|
||||
)
|
||||
originals = []
|
||||
for param in (
|
||||
experts.w13_weight,
|
||||
experts.w13_weight_scale,
|
||||
experts.w2_weight,
|
||||
experts.w2_weight_scale,
|
||||
):
|
||||
param.data.random_(1, 128)
|
||||
originals.append(param.data.clone())
|
||||
monkeypatch.setattr(
|
||||
experts, "_ensure_backend", lambda: DeepGemmSm100MegaMoEBackend()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm",
|
||||
lambda: SimpleNamespace(
|
||||
transform_sf_into_required_layout=lambda sf, *args: sf,
|
||||
transform_weights_for_mega_moe=lambda l1, l2: (l1, l2),
|
||||
),
|
||||
)
|
||||
|
||||
experts.finalize_weights()
|
||||
assert experts.intermediate_size == intermediate_size
|
||||
assert experts.w13_weight.shape == (2, 2 * intermediate_size, 64)
|
||||
assert experts.w13_weight_scale.shape == (2, 2 * intermediate_size, 1)
|
||||
assert experts.w2_weight.shape == (2, 128, intermediate_size // 2)
|
||||
assert experts.w2_weight_scale.shape == (2, 128, intermediate_size // 128)
|
||||
transformed = (*experts._transformed_l1_weights, *experts._transformed_l2_weights)
|
||||
for actual, original in zip(transformed, originals):
|
||||
expected = (
|
||||
original
|
||||
if actual.dtype == torch.int8
|
||||
else experts._ue8m0_uint8_to_float(original)
|
||||
)
|
||||
assert torch.equal(actual.view(expected.dtype), expected)
|
||||
for actual, param in zip(transformed, experts.parameters()):
|
||||
assert actual.data_ptr() == param.data_ptr()
|
||||
|
||||
transformed_l1 = experts._transformed_l1_weights
|
||||
experts.finalize_weights()
|
||||
assert experts._transformed_l1_weights is transformed_l1
|
||||
|
||||
def test_deepseek_v4_mega_moe_loads_shared_experts_into_kernel_layout(monkeypatch):
|
||||
"""Shared expert shards land directly in the MegaMoE kernel layout."""
|
||||
monkeypatch.setattr(
|
||||
DeepGemmSm100MegaMoEBackend, "supports_shared_experts", lambda *args: True
|
||||
)
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=384,
|
||||
num_shared_experts=1,
|
||||
)
|
||||
assert experts.has_fused_shared_experts
|
||||
hidden, inter = 256, 384
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
|
||||
def rand(*shape):
|
||||
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
|
||||
w1, w3, w2 = rand(inter, hidden), rand(inter, hidden), rand(hidden, inter)
|
||||
s1, s3, s2 = (
|
||||
rand(inter, hidden // 32),
|
||||
rand(inter, hidden // 32),
|
||||
rand(hidden, inter // 32),
|
||||
)
|
||||
fp8, e8m0 = torch.float8_e4m3fn, torch.float8_e8m0fnu
|
||||
# The down shard arrives as `.w2.` (MTP, DSpark) or `.down_proj.` (mapped).
|
||||
for name, data in [
|
||||
("shared_experts.w1.weight", w1.view(fp8)),
|
||||
("shared_experts.w3.weight", w3.view(fp8)),
|
||||
("shared_experts.down_proj.weight", w2.view(fp8)),
|
||||
("shared_experts.w1.weight_scale", s1.view(e8m0)),
|
||||
("shared_experts.w3.weight_scale", s3.view(e8m0)),
|
||||
("shared_experts.w2.weight_scale", s2.view(e8m0)),
|
||||
]:
|
||||
loaded = experts.load_shared_expert_weight(f"ffn.{name}", data)
|
||||
# Weight tracking checks this name against the registered parameters.
|
||||
assert loaded in {f"ffn.experts.{n}" for n, _ in experts.named_parameters()}
|
||||
|
||||
def l1_halves():
|
||||
return experts.shared_l1_weight.view(torch.uint8).view(-1, 2, 8, hidden)
|
||||
|
||||
# Every tensor matches DeepGEMM's batched transform of the checkpoint.
|
||||
expected = _reference_mega_moe_layout(
|
||||
*(t.unsqueeze(0) for t in (torch.cat([w1, w3]), torch.cat([s1, s3]), w2, s2))
|
||||
)
|
||||
actual = (
|
||||
experts.shared_l1_weight,
|
||||
experts.shared_l1_weight_scale,
|
||||
experts.shared_l2_weight,
|
||||
experts.shared_l2_weight_scale,
|
||||
)
|
||||
for tensor, reference in zip(actual, expected):
|
||||
assert torch.equal(tensor.view(reference.dtype), reference[0])
|
||||
# A reload of one shard overwrites in place and leaves the other half alone.
|
||||
ptr = experts.shared_l1_weight.data_ptr()
|
||||
w1b = rand(inter, hidden)
|
||||
experts.load_shared_expert_weight("shared_experts.w1.weight", w1b.view(fp8))
|
||||
assert experts.shared_l1_weight.data_ptr() == ptr
|
||||
assert torch.equal(l1_halves()[:, 0], w1b.view(-1, 8, hidden))
|
||||
assert torch.equal(l1_halves()[:, 1], w3.view(-1, 8, hidden))
|
||||
with pytest.raises(ValueError):
|
||||
experts.load_shared_expert_weight("shared_experts.gate_up_proj.weight", w1)
|
||||
|
||||
|
||||
def _fake_deep_gemm(with_shared: bool):
|
||||
def old_buffer(group): ...
|
||||
|
||||
def old_kernel(y): ...
|
||||
|
||||
def new_buffer(group, num_shared_experts=0): ...
|
||||
|
||||
def new_kernel(y, shared_l1_weights=None, shared_l2_weights=None): ...
|
||||
|
||||
return SimpleNamespace(
|
||||
get_symm_buffer_for_mega_moe=new_buffer if with_shared else old_buffer,
|
||||
fp8_fp4_mega_moe=new_kernel if with_shared else old_kernel,
|
||||
get_block_m_for_mega_moe=lambda *args: 0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,intermediate_size,block_size,mxfp8,packed",
|
||||
"weight_dtype,scale_dtype,scale_shape,with_shared,fused",
|
||||
[
|
||||
pytest.param(128, 512, 128, False, False, id="aligned-block128"),
|
||||
pytest.param(128, 512, 128, False, True, id="aligned-block128-packed"),
|
||||
pytest.param(128, 512, 128, True, False, id="aligned-block128-mxfp8"),
|
||||
pytest.param(5120, 2304, 32, False, False, id="deepseek-v41-flash"),
|
||||
pytest.param(128, 2304, 128, False, False, id="native-block128"),
|
||||
pytest.param(128, 2304, 128, False, True, id="native-block128-packed"),
|
||||
pytest.param(5120, 2304, 32, True, False, id="deepseek-v41-flash-mxfp8"),
|
||||
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), True, True),
|
||||
(torch.float8_e4m3fn, torch.uint8, (256, 8), True, True),
|
||||
(torch.float8_e4m3fn, torch.float8_e8m0fnu, (2, 2), False, False),
|
||||
(torch.bfloat16, None, None, True, False),
|
||||
(torch.float8_e4m3fn, torch.float32, (2, 2), True, False),
|
||||
(torch.float8_e4m3fn, torch.uint8, (256, 16), True, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v4_mega_moe_finalizes_native_shared_expert_weights(
|
||||
monkeypatch, hidden_size, intermediate_size, block_size, mxfp8, packed
|
||||
def test_deepseek_v4_mega_moe_decides_shared_fusion_at_construction(
|
||||
monkeypatch, weight_dtype, scale_dtype, scale_shape, with_shared, fused
|
||||
):
|
||||
"""Shared fusion preserves checkpoint weights and scales at native widths."""
|
||||
if packed and not is_deep_gemm_supported():
|
||||
pytest.skip("Packing the shared scales requires DeepGEMM")
|
||||
|
||||
class FakeDeepGemm:
|
||||
transformed_dims: list[tuple[int, int]] = []
|
||||
scale_inputs: list[tuple[int, ...]] = []
|
||||
scale_values: list[torch.Tensor] = []
|
||||
|
||||
@staticmethod
|
||||
def get_symm_buffer_for_mega_moe(*args, num_shared_experts=0, **kwargs):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_block_m_for_mega_moe(*args, **kwargs):
|
||||
return 128
|
||||
|
||||
@staticmethod
|
||||
def fp8_fp4_mega_moe(
|
||||
y,
|
||||
l1_weights,
|
||||
l2_weights,
|
||||
sym_buffer,
|
||||
shared_l1_weights=None,
|
||||
shared_l2_weights=None,
|
||||
**kwargs,
|
||||
):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def transform_sf_into_required_layout(cls, sf, mn, k, *args, **kwargs):
|
||||
cls.scale_inputs.append(tuple(sf.shape))
|
||||
cls.scale_values.append(sf)
|
||||
return torch.empty((sf.shape[0], mn, k // 128), dtype=torch.int32)
|
||||
|
||||
@classmethod
|
||||
def transform_weights_for_mega_moe(cls, l1_weights, l2_weights):
|
||||
cls.transformed_dims.append((l1_weights[0].dim(), l2_weights[0].dim()))
|
||||
if l1_weights[0].dim() == 2:
|
||||
return (l1_weights[0].clone(), l1_weights[1]), l2_weights
|
||||
return l1_weights, l2_weights
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
"""Fuse only shared experts the installed kernel and the 1x32 layout fit."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.deep_gemm._import_deep_gemm", lambda: _fake_deep_gemm(with_shared)
|
||||
)
|
||||
|
||||
def linear():
|
||||
scale = (
|
||||
None if scale_dtype is None else torch.empty(scale_shape).to(scale_dtype)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
weight=torch.empty(256, 256, dtype=weight_dtype), weight_scale_inv=scale
|
||||
)
|
||||
|
||||
shared = SimpleNamespace(gate_up_proj=linear(), down_proj=linear())
|
||||
experts = DeepseekV4MegaMoEExperts(
|
||||
vllm_config,
|
||||
SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
),
|
||||
num_experts=2,
|
||||
num_local_experts=1,
|
||||
num_local_experts=2,
|
||||
experts_start_idx=0,
|
||||
top_k=1,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
hidden_size=256,
|
||||
intermediate_size=128,
|
||||
num_shared_experts=1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
experts, "_ensure_backend", lambda: DeepGemmSm100MegaMoEBackend()
|
||||
shared_experts=shared,
|
||||
)
|
||||
|
||||
def fp8_parameter(*shape):
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(*shape, dtype=torch.uint8)
|
||||
.random_(1, 119)
|
||||
.view(torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def scale_parameter(*shape, dtype=torch.int32):
|
||||
return torch.nn.Parameter(torch.ones(*shape, dtype=dtype), requires_grad=False)
|
||||
|
||||
shared_experts = SimpleNamespace(
|
||||
gate_up_proj=SimpleNamespace(
|
||||
weight=fp8_parameter(2 * intermediate_size, hidden_size),
|
||||
weight_block_size=(block_size, block_size),
|
||||
weight_scale_inv=scale_parameter(
|
||||
2 * intermediate_size // block_size,
|
||||
hidden_size // block_size,
|
||||
dtype=torch.float8_e8m0fnu,
|
||||
),
|
||||
),
|
||||
down_proj=SimpleNamespace(
|
||||
weight=fp8_parameter(hidden_size, intermediate_size),
|
||||
weight_block_size=(block_size, block_size),
|
||||
weight_scale_inv=scale_parameter(
|
||||
hidden_size // block_size,
|
||||
intermediate_size // block_size,
|
||||
dtype=torch.float8_e8m0fnu,
|
||||
),
|
||||
),
|
||||
)
|
||||
if mxfp8:
|
||||
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
|
||||
linear.weight_block_size = (1, 32)
|
||||
linear.weight_scale = torch.nn.Parameter(
|
||||
linear.weight_scale_inv.view(torch.uint8)
|
||||
.repeat_interleave(block_size, dim=0)
|
||||
.repeat_interleave(block_size // 32, dim=1),
|
||||
requires_grad=False,
|
||||
)
|
||||
del linear.weight_scale_inv
|
||||
originals = []
|
||||
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
|
||||
scale = linear.weight_scale if mxfp8 else linear.weight_scale_inv
|
||||
scale.data.view(torch.uint8).random_(124, 130)
|
||||
block_m, block_k = linear.weight_block_size
|
||||
scale_1x32 = (
|
||||
experts._ue8m0_uint8_to_float(scale.view(torch.uint8))
|
||||
.repeat_interleave(block_m, dim=0)
|
||||
.repeat_interleave(block_k // 32, dim=1)
|
||||
)
|
||||
originals.append((linear.weight.view(torch.uint8).clone(), scale_1x32))
|
||||
if packed:
|
||||
linear.weight_scale_inv = torch.nn.Parameter(
|
||||
deepgemm_post_process_weight_scale_block(
|
||||
scale.data.cuda().unsqueeze(0),
|
||||
*linear.weight.shape,
|
||||
quant_block_shape=linear.weight_block_size,
|
||||
num_groups=1,
|
||||
)
|
||||
.squeeze(0)
|
||||
.cpu(),
|
||||
requires_grad=False,
|
||||
)
|
||||
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: FakeDeepGemm)
|
||||
|
||||
original_gate_up_ptr = shared_experts.gate_up_proj.weight.data_ptr()
|
||||
experts.finalize_weights(shared_experts)
|
||||
|
||||
assert experts.has_fused_shared_experts
|
||||
assert experts.intermediate_size == intermediate_size
|
||||
assert FakeDeepGemm.transformed_dims == [(3, 3), (2, 2)]
|
||||
assert FakeDeepGemm.scale_inputs[-2:] == [
|
||||
(1, 2 * intermediate_size, hidden_size // 32),
|
||||
(1, hidden_size, intermediate_size // 32),
|
||||
]
|
||||
for transformed, actual_scale, (weight, scale) in zip(
|
||||
(
|
||||
experts._transformed_shared_l1_weights,
|
||||
experts._transformed_shared_l2_weights,
|
||||
),
|
||||
FakeDeepGemm.scale_values[-2:],
|
||||
originals,
|
||||
):
|
||||
assert torch.equal(transformed[0].view(torch.uint8), weight)
|
||||
assert torch.equal(actual_scale[0], scale)
|
||||
assert shared_experts.gate_up_proj.weight.data_ptr() != original_gate_up_ptr
|
||||
assert (
|
||||
experts._transformed_shared_l1_weights[0].data_ptr()
|
||||
== shared_experts.gate_up_proj.weight.data_ptr()
|
||||
)
|
||||
assert (
|
||||
experts._transformed_shared_l2_weights[0].data_ptr()
|
||||
== shared_experts.down_proj.weight.data_ptr()
|
||||
)
|
||||
|
||||
transformed_l1 = experts._transformed_shared_l1_weights
|
||||
experts.finalize_weights(shared_experts)
|
||||
assert experts._transformed_shared_l1_weights is transformed_l1
|
||||
assert experts.has_fused_shared_experts == fused
|
||||
assert hasattr(experts, "shared_l1_weight") == fused
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fused", [False, True])
|
||||
@@ -671,10 +727,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
|
||||
class FakeExperts(torch.nn.Module):
|
||||
has_fused_shared_experts = fused
|
||||
_backend = DeepGemmSm100MegaMoEBackend()
|
||||
|
||||
def _ensure_backend(self):
|
||||
return self._backend
|
||||
backend = DeepGemmSm100MegaMoEBackend()
|
||||
|
||||
def forward(self, hidden_states, *args, **kwargs):
|
||||
return torch.ones_like(hidden_states)
|
||||
@@ -691,7 +744,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
moe.use_native_mega_moe = True
|
||||
moe.gate = FakeGate()
|
||||
moe.experts = FakeExperts()
|
||||
moe.shared_experts = FakeSharedExperts()
|
||||
moe.shared_experts = None if fused else FakeSharedExperts()
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = 1
|
||||
moe.renormalize = True
|
||||
@@ -710,7 +763,8 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
|
||||
expected = 1 if fused else 3
|
||||
assert torch.all(output == expected)
|
||||
assert moe.shared_experts.calls == (0 if fused else 1)
|
||||
if not fused:
|
||||
assert moe.shared_experts.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
@@ -882,35 +936,110 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
|
||||
assert torch.all(fused_shared_x_sf[~populated] == -1)
|
||||
|
||||
|
||||
def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
|
||||
def test_deepseek_v4_pwal_hook_finalizes_mhc_broadcast():
|
||||
"""The loader invokes the model-level PWAL hook for every load format,
|
||||
so it must finalize megamoe + mhc broadcast weights to cover dummy
|
||||
load, which skips load_weights()."""
|
||||
so it must finalize mhc broadcast weights to cover dummy load, which
|
||||
skips load_weights()."""
|
||||
calls = []
|
||||
stub = SimpleNamespace(
|
||||
model=SimpleNamespace(
|
||||
finalize_mega_moe_weights=lambda: calls.append("mega_moe"),
|
||||
finalize_mhc_broadcast_weights=lambda: calls.append("mhc"),
|
||||
)
|
||||
)
|
||||
|
||||
DeepseekV4ForCausalLM.process_weights_after_loading(stub)
|
||||
|
||||
assert calls == ["mega_moe", "mhc"]
|
||||
assert calls == ["mhc"]
|
||||
|
||||
|
||||
def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
|
||||
"""MTP/DSpark drafters load as their own top-level models, so each needs
|
||||
its own PWAL hook now that the megamoe forward no longer finalizes
|
||||
weights lazily on first use."""
|
||||
calls = []
|
||||
mtp = SimpleNamespace(finalize_mega_moe_weights=lambda: calls.append("mtp"))
|
||||
DeepSeekV4MTP.process_weights_after_loading(mtp)
|
||||
@pytest.fixture
|
||||
def v4_dspark_config(dist_init):
|
||||
hf_config = DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"],
|
||||
hidden_size=128,
|
||||
num_hidden_layers=2,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
num_hash_layers=0,
|
||||
n_shared_experts=1,
|
||||
moe_intermediate_size=128,
|
||||
hidden_act="silu",
|
||||
swiglu_limit=10.0,
|
||||
norm_topk_prob=True,
|
||||
topk_method="noaux_tc",
|
||||
routed_scaling_factor=1.5,
|
||||
hc_mult=1,
|
||||
hc_eps=1e-5,
|
||||
rms_norm_eps=1e-5,
|
||||
dspark_target_layer_ids=[0],
|
||||
dspark_markov_rank=8,
|
||||
index_topk=4,
|
||||
head_dim=64,
|
||||
num_attention_heads=4,
|
||||
vocab_size=256,
|
||||
n_mtp_layers=2,
|
||||
enable_confidence_head=False,
|
||||
layer_types=["sliding_attention"] * 2,
|
||||
q_lora_rank=32,
|
||||
o_lora_rank=32,
|
||||
qk_rope_head_dim=16,
|
||||
o_groups=4,
|
||||
sliding_window=128,
|
||||
hc_sinkhorn_iters=1,
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16,
|
||||
hf_config=hf_config,
|
||||
max_model_len=2048,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=model_config,
|
||||
quant_config=None,
|
||||
kernel_config=SimpleNamespace(
|
||||
moe_backend="deep_gemm_mega_moe",
|
||||
enable_jit_warmup=False,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=True,
|
||||
enable_eplb=True,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=4),
|
||||
),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(block_size=64, cache_dtype="auto"),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=4),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
speculative_config=SimpleNamespace(draft_model_config=model_config),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
dspark = SimpleNamespace(_finalize_moe=lambda: calls.append("dspark"))
|
||||
DSparkDeepseekV4ForCausalLM.process_weights_after_loading(dspark)
|
||||
|
||||
assert calls == ["mtp", "dspark"]
|
||||
def test_dspark_draft_registers_mixture_of_experts(
|
||||
v4_dspark_config, monkeypatch, dist_init
|
||||
):
|
||||
"""DSV4 DSpark MoE drafts expose MixtureOfExperts for EPLB registration."""
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("DeepGEMM MegaMoE requires SM100")
|
||||
|
||||
hf_config = v4_dspark_config.model_config.hf_config
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4.nvidia.dspark.get_current_vllm_config",
|
||||
lambda: v4_dspark_config,
|
||||
)
|
||||
|
||||
with (
|
||||
set_default_torch_dtype(v4_dspark_config.model_config.dtype),
|
||||
torch.device("cuda"),
|
||||
):
|
||||
draft = DSparkDeepseekV4ForCausalLM(vllm_config=v4_dspark_config)
|
||||
|
||||
assert is_mixture_of_experts(draft)
|
||||
assert get_mixture_of_experts_model(draft) is draft
|
||||
assert len(draft.moe_layers) == draft.model.num_dspark_layers == 2
|
||||
assert draft.num_routed_experts == hf_config.n_routed_experts
|
||||
assert draft.num_redundant_experts == 4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -242,7 +242,14 @@ def test_v41_dspark_loads_linear_scales(
|
||||
linear_scale_name=scale_name,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
|
||||
layers=[
|
||||
SimpleNamespace(
|
||||
ffn=SimpleNamespace(
|
||||
use_native_mega_moe=False,
|
||||
experts=SimpleNamespace(has_fused_shared_experts=False),
|
||||
)
|
||||
)
|
||||
],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [(runtime_name, param)],
|
||||
@@ -251,8 +258,6 @@ def test_v41_dspark_loads_linear_scales(
|
||||
draft._remap_dspark_name = lambda name: (
|
||||
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
|
||||
)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_world_size", lambda: 4)
|
||||
monkeypatch.setattr(dspark, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
dspark, "fused_moe_make_expert_params_mapping", lambda *a, **kw: []
|
||||
)
|
||||
@@ -266,6 +271,55 @@ def test_v41_dspark_loads_linear_scales(
|
||||
torch.testing.assert_close(param, checkpoint_scale)
|
||||
|
||||
|
||||
def test_dsv4_dspark_fused_moe_expert_mapping_includes_redundant_experts(
|
||||
monkeypatch,
|
||||
):
|
||||
from vllm.models.deepseek_v4.nvidia import dspark
|
||||
|
||||
captured: dict[str, int] = {}
|
||||
|
||||
def capture_expert_mapping(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return []
|
||||
|
||||
draft = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
num_attention_heads=4,
|
||||
n_routed_experts=8,
|
||||
expert_dtype="fp4",
|
||||
),
|
||||
quant_config=None,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[
|
||||
SimpleNamespace(
|
||||
ffn=SimpleNamespace(
|
||||
use_native_mega_moe=False,
|
||||
use_mega_moe=False,
|
||||
n_redundant_experts=4,
|
||||
)
|
||||
)
|
||||
],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [],
|
||||
process_weights_after_loading=lambda: None,
|
||||
)
|
||||
draft._remap_dspark_name = lambda name: (
|
||||
dspark.DSparkDeepseekV4ForCausalLM._remap_dspark_name(draft, name)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dspark,
|
||||
"fused_moe_make_expert_params_mapping",
|
||||
capture_expert_mapping,
|
||||
)
|
||||
|
||||
dspark.DSparkDeepseekV4ForCausalLM.load_weights(draft, [])
|
||||
|
||||
assert captured["num_experts"] == 8
|
||||
assert captured["num_redundant_experts"] == 4
|
||||
|
||||
|
||||
def test_dsv4_context_wkv_weights_are_duplicated_by_draft_layer():
|
||||
weights = [
|
||||
("mtp.0.attn.wkv.weight", torch.arange(4)),
|
||||
@@ -343,7 +397,7 @@ def test_k3_dspark_mla_kv_cache_spec_groups_with_target_mla():
|
||||
"""The draft's MLA layers must share a KV cache group with the target's."""
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
|
||||
from vllm.v1.core.kv_cache_utils import _get_kv_cache_groups_uniform_page_size
|
||||
from vllm.v1.core.kv_cache_planning import _get_kv_cache_groups_uniform_page_size
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=None, cache_config=SimpleNamespace(block_size=64)
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4.1 mono decode layer runs only where its kernels are exact:
|
||||
decode-only steps of at most MAX_ROWS rows with causal SWA windows, eager or in
|
||||
a FULL graph. A PIECEWISE capture is replayed for mixed batches, so it must
|
||||
never record the mono launches. Layers outside the whole-layer kernels run the
|
||||
FFN launch on the same steps, from wo_b's unreduced output."""
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.models.deepseek_v41.amd import mono_decode as md
|
||||
|
||||
M = 6
|
||||
|
||||
|
||||
class _Runner:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
self.calls.append((args, kwargs))
|
||||
return ("mono outputs",)
|
||||
|
||||
def first_pre_mix(self, rows):
|
||||
return torch.zeros(rows, 4)
|
||||
|
||||
def ffn(self, *args):
|
||||
self.calls.append((args, {}))
|
||||
return ("ffn outputs",)
|
||||
|
||||
|
||||
def _layer(comp="comp", indexer=None):
|
||||
cache = lambda: torch.zeros(4, 128 * md.RECORD, dtype=torch.uint8) # noqa: E731
|
||||
attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa", kv_cache=cache()),
|
||||
compressed_cache_prefix=comp,
|
||||
topk_indices_buffer=torch.zeros(64, 512, dtype=torch.int32),
|
||||
compress_ratio=1,
|
||||
indexer=indexer,
|
||||
_compressed_kv_cache=cache,
|
||||
)
|
||||
return SimpleNamespace(attn=attn, engram=None, hc_mult=4)
|
||||
|
||||
|
||||
def _swa(**overrides):
|
||||
fields = dict(
|
||||
num_prefills=0,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=M,
|
||||
decode_swa_width=md.SWA_WIDTH,
|
||||
decode_swa_indices=torch.zeros(64, 1, md.SWA_WIDTH, dtype=torch.int32),
|
||||
decode_swa_lens=torch.zeros(64, dtype=torch.int32),
|
||||
token_to_req_indices=torch.zeros(64, dtype=torch.int32),
|
||||
slot_mapping=torch.zeros(64, dtype=torch.int64),
|
||||
block_size=32,
|
||||
)
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
|
||||
def _inputs(rows=M):
|
||||
return dict(
|
||||
x=torch.zeros(rows, 5120, dtype=torch.bfloat16),
|
||||
positions=torch.zeros(rows, dtype=torch.int64),
|
||||
residual=torch.zeros(rows, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.zeros(rows, 4, 1),
|
||||
res_mix=torch.zeros(rows, 4, 4),
|
||||
pre_mix=torch.zeros(rows, 4),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run(monkeypatch):
|
||||
"""Call the mono path for one layer under a forward context built from
|
||||
``mode`` and ``metadata``; returns (result, runner calls)."""
|
||||
|
||||
def _run(
|
||||
mode=CUDAGraphMode.FULL,
|
||||
metadata=None,
|
||||
ffn_only=False,
|
||||
entry=None,
|
||||
layer=None,
|
||||
**inputs,
|
||||
):
|
||||
if metadata is None:
|
||||
metadata = {
|
||||
"swa": _swa(),
|
||||
"comp": SimpleNamespace(block_size=128, block_table=None),
|
||||
}
|
||||
fc = SimpleNamespace(cudagraph_runtime_mode=mode, attn_metadata=metadata)
|
||||
runner = _Runner()
|
||||
monkeypatch.setattr(md, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(md, "get_forward_context", lambda: fc)
|
||||
monkeypatch.setattr(md, "_mono_runner", lambda device: runner)
|
||||
monkeypatch.setattr(md.MonoDecodeLayer, "weights", lambda self, layer: None)
|
||||
args = {**_inputs(), **inputs}
|
||||
mono = md.MonoDecodeLayer(ffn_only)
|
||||
if (entry or ("ffn" if ffn_only else "layer")) == "ffn":
|
||||
# x stands for wo_b's unreduced output; the first layer's attention
|
||||
# has no compressed cache
|
||||
part, _ = args.pop("x"), args.pop("positions")
|
||||
return mono.ffn(_layer(comp=None), part, **args), runner.calls
|
||||
return mono(layer or _layer(), **args), runner.calls
|
||||
|
||||
return _run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
|
||||
def test_decode_step_runs_mono(run, mode):
|
||||
out, calls = run(mode)
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
# the caches reach the kernels as [blocks, block, record] bytes
|
||||
assert args[8].shape == (4, 32, md.RECORD) and args[8].stride() == (
|
||||
128 * md.RECORD,
|
||||
md.RECORD,
|
||||
1,
|
||||
)
|
||||
assert kwargs["comp_cache"].shape == (4, 128, md.RECORD)
|
||||
assert kwargs["fold"] and "select" not in kwargs
|
||||
|
||||
|
||||
def test_first_layer_runs_mono_from_the_embedding(run):
|
||||
"""The first layer's residual is the embedding broadcast to the four
|
||||
streams, which K1 takes as it is (no owed post to fold)."""
|
||||
out, calls = run(residual=None, post_mix=None, res_mix=None, pre_mix=None)
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
assert args[2].shape == (M, 4, 5120) and not kwargs["fold"]
|
||||
|
||||
|
||||
def test_index_source_selects_between_the_kernels(run):
|
||||
"""An index source hands the runner its compressor and indexer to run
|
||||
between K1 and K2."""
|
||||
out, calls = run(layer=_layer(indexer=object()))
|
||||
assert out == ("mono outputs",) and callable(calls[0][1]["select"])
|
||||
|
||||
|
||||
def test_padded_decode_step_runs_mono(run):
|
||||
"""A FULL graph pads the step to its capture size: the kernels take it and
|
||||
skip the pad rows (their slot is -1)."""
|
||||
swa = _swa(num_decode_tokens=M - 2)
|
||||
comp = SimpleNamespace(block_size=128, block_table=None)
|
||||
out, calls = run(metadata={"swa": swa, "comp": comp})
|
||||
assert out == ("mono outputs",) and len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
dict(mode=CUDAGraphMode.PIECEWISE),
|
||||
dict(
|
||||
metadata={
|
||||
"swa": _swa(num_prefills=1),
|
||||
"comp": SimpleNamespace(block_size=128),
|
||||
}
|
||||
),
|
||||
dict(
|
||||
metadata={
|
||||
"swa": _swa(decode_swa_width=256),
|
||||
"comp": SimpleNamespace(block_size=128),
|
||||
}
|
||||
),
|
||||
dict(
|
||||
metadata={
|
||||
"swa": _swa(num_decode_tokens=M + 1),
|
||||
"comp": SimpleNamespace(block_size=128),
|
||||
}
|
||||
),
|
||||
dict(metadata={}), # a profile / dummy run: no attention metadata
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
# a later pipeline stage's first layer: its stream arrives without
|
||||
# the mixes
|
||||
dict(residual=None, x=torch.zeros(M, 4, 5120, dtype=torch.bfloat16)),
|
||||
],
|
||||
ids=[
|
||||
"piecewise",
|
||||
"prefill",
|
||||
"noncausal-window",
|
||||
"tokens-past-rows",
|
||||
"no-metadata",
|
||||
"rows",
|
||||
"stage-input",
|
||||
],
|
||||
)
|
||||
def test_other_steps_take_the_original_path(run, case):
|
||||
out, calls = run(**case)
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _decoder_layer(engram=None, **attn):
|
||||
attn = (
|
||||
dict(
|
||||
layer_id=21,
|
||||
compressor=None,
|
||||
indexer=None,
|
||||
compress_ratio=1,
|
||||
kv_mxfp8=False,
|
||||
kv_cache_dtype="fp8_ds_mla",
|
||||
)
|
||||
| attn
|
||||
)
|
||||
backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
quant_method = SimpleNamespace(mxfp4_backend=backend)
|
||||
ffn = SimpleNamespace(
|
||||
experts=SimpleNamespace(
|
||||
routed_experts=SimpleNamespace(quant_method=quant_method)
|
||||
),
|
||||
shared_experts=object(),
|
||||
gate=SimpleNamespace(tid2eid=None),
|
||||
n_routed_experts=384,
|
||||
n_activated_experts=6,
|
||||
routed_scaling_factor=1.5,
|
||||
swiglu_limit=10.0,
|
||||
scoring_func="sqrtsoftplus",
|
||||
renormalize=True,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
attn=SimpleNamespace(**attn),
|
||||
ffn=ffn,
|
||||
engram=engram,
|
||||
use_sequence_parallel=False,
|
||||
fuse_seam_norm=True,
|
||||
)
|
||||
|
||||
|
||||
def _indexer():
|
||||
from vllm.models.deepseek_v41.amd.rocm import DeepseekV41RocmMxfp4Indexer
|
||||
|
||||
return object.__new__(DeepseekV41RocmMxfp4Indexer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer, path",
|
||||
[
|
||||
(_decoder_layer(), "whole"),
|
||||
(_decoder_layer(compress_ratio=0), "whole"),
|
||||
(_decoder_layer(engram=object()), "whole"),
|
||||
(_decoder_layer(compressor=object(), indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(indexer=_indexer()), "whole"),
|
||||
(_decoder_layer(compressor=object()), "ffn"),
|
||||
(_decoder_layer(indexer=object()), "ffn"),
|
||||
(_decoder_layer(kv_cache_dtype="auto"), "ffn"),
|
||||
(_decoder_layer(layer_id=40), None),
|
||||
],
|
||||
ids=[
|
||||
"standard",
|
||||
"window",
|
||||
"engram",
|
||||
"kv-source",
|
||||
"index-source",
|
||||
"compressor-only",
|
||||
"other-indexer",
|
||||
"kv",
|
||||
"draft",
|
||||
],
|
||||
)
|
||||
def test_layers_take_whole_or_ffn_launch(monkeypatch, layer, path):
|
||||
"""The whole-layer kernels serve every backbone layer of the checkpoint (an
|
||||
index source's compressor and indexer run between them); a layer outside
|
||||
them keeps vLLM's attention and runs its FFN half as one launch."""
|
||||
_deployment(monkeypatch)
|
||||
mono = md.MonoDecodeLayer.create(layer, _config())
|
||||
got = None if mono is None else "ffn" if mono.ffn_only else "whole"
|
||||
assert got == path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [CUDAGraphMode.NONE, CUDAGraphMode.FULL])
|
||||
def test_decode_step_runs_ffn_launch(run, mode):
|
||||
out, calls = run(mode, ffn_only=True, metadata={"swa": _swa()})
|
||||
assert out == ("ffn outputs",) and len(calls) == 1
|
||||
assert calls[0][0][1].shape == (M, 5120) # wo_b's unreduced output
|
||||
|
||||
|
||||
def test_ffn_layer_never_runs_whole(run):
|
||||
out, calls = run(ffn_only=True, entry="layer")
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
dict(mode=CUDAGraphMode.PIECEWISE),
|
||||
dict(metadata={"swa": _swa(num_prefills=1)}),
|
||||
dict(metadata={"swa": _swa(num_decode_tokens=M + 1)}),
|
||||
dict(metadata={}),
|
||||
dict(**_inputs(md.MAX_ROWS + 6)),
|
||||
],
|
||||
ids=["piecewise", "prefill", "tokens-past-rows", "no-metadata", "rows"],
|
||||
)
|
||||
def test_other_steps_reduce_wo_b_themselves(run, case):
|
||||
"""None: the layer all-reduces wo_b's partial and runs the FFN as before."""
|
||||
case.setdefault("metadata", {"swa": _swa()})
|
||||
out, calls = run(ffn_only=True, **case)
|
||||
assert out is None and not calls
|
||||
|
||||
|
||||
def _config(monokernel=True, **parallel):
|
||||
parallel = (
|
||||
dict(enable_expert_parallel=False, enable_eplb=False, data_parallel_size=1)
|
||||
| parallel
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(num_hidden_layers=40)),
|
||||
parallel_config=SimpleNamespace(**parallel),
|
||||
kernel_config=SimpleNamespace(monokernel=monokernel),
|
||||
)
|
||||
|
||||
|
||||
def _deployment(monkeypatch, cdna=4, cus=256):
|
||||
"""A deployment the kernels take (the runner module stubbed: no FlyDSL)."""
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
|
||||
monkeypatch.setattr(rocm, "get_cdna_version", lambda: cdna)
|
||||
monkeypatch.setattr(md, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(md, "_compute_units", lambda: cus)
|
||||
name = "vllm.models.deepseek_v41.amd.mono.runner"
|
||||
runner = ModuleType(name)
|
||||
runner.BLOCKS = 256 # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, name, runner)
|
||||
|
||||
|
||||
def test_off_without_the_flag(monkeypatch):
|
||||
"""No --monokernel: no mono layer, whatever the deployment."""
|
||||
_deployment(monkeypatch)
|
||||
assert (
|
||||
md.MonoDecodeLayer.create(_decoder_layer(), _config(monokernel=False)) is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cdna", [3, 5])
|
||||
def test_opt_in_outside_the_kernels_cdna_raises(monkeypatch, cdna):
|
||||
_deployment(monkeypatch, cdna=cdna)
|
||||
with pytest.raises(ValueError, match="needs CDNA4"):
|
||||
md.MonoDecodeLayer.create(_decoder_layer(), _config())
|
||||
|
||||
|
||||
def _other_backend():
|
||||
layer = _decoder_layer()
|
||||
experts = layer.ffn.experts.routed_experts
|
||||
experts.quant_method.mxfp4_backend = Mxfp4MoeBackend.TRITON
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case, match",
|
||||
[
|
||||
(dict(parallel=dict(enable_expert_parallel=True)), "no expert"),
|
||||
(dict(parallel=dict(data_parallel_size=2)), "no expert"),
|
||||
(dict(parallel=dict(enable_eplb=True)), "no expert"),
|
||||
(dict(a4w4=True), "A8W4"),
|
||||
(dict(layer=_other_backend), "AITER_MXFP4_BF16"),
|
||||
(dict(cus=128), "256 compute units"),
|
||||
],
|
||||
ids=["expert-parallel", "data-parallel", "eplb", "a4w4", "moe-backend", "cus"],
|
||||
)
|
||||
def test_opt_in_outside_the_kernels_moe_raises(monkeypatch, case, match):
|
||||
"""The kernels read all 384 experts on each rank in AITER's A8W4 layout, with
|
||||
all 256 CTAs resident: any other deployment is an error, not garbage."""
|
||||
_deployment(monkeypatch, cus=case.get("cus", 256))
|
||||
if case.get("a4w4"):
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1")
|
||||
layer = case.get("layer", _decoder_layer)()
|
||||
with pytest.raises(ValueError, match=match):
|
||||
md.MonoDecodeLayer.create(layer, _config(**case.get("parallel", {})))
|
||||
@@ -0,0 +1,340 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4.1 mono decode path through the real model: a five-layer
|
||||
DeepSeek-V4.1-Flash at TP2 and TP4 on CDNA4, served with ``--monokernel`` on
|
||||
and off on the same random weights: every decoder layer's outputs at every
|
||||
decode step must match. Unlike ``test_dsv41_mono_numerics.py``, which
|
||||
checks the kernels against vLLM's ops one stage at a time, this runs
|
||||
``DeepseekV4DecoderLayer.forward`` as the engine does -- the model's hooks,
|
||||
the per-step path choice, the weights taken off the layer, the metadata and
|
||||
the indexer's top-k -- so a change to the decoder layer that the kernels do
|
||||
not follow fails here.
|
||||
|
||||
Every layer takes the whole-layer path (K1 + K2); the five hold each variant:
|
||||
- 0: the first layer, sliding window only (K1 takes the embedding settled)
|
||||
- 1: ratio-2 KV and index source (vLLM's compressor and indexer between K1
|
||||
and K2)
|
||||
- 2: ratio-2 consumer with an Engram (K1 takes the residual settled)
|
||||
- 3: ratio-1 KV, index and candidate source
|
||||
- 4: ratio-1 index source over 3's candidates and KV
|
||||
"""
|
||||
|
||||
import zlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader import register_model_loader
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
|
||||
LOAD_FORMAT = "dsv41_mono_random"
|
||||
TRUNCATED = {
|
||||
"num_hidden_layers": 5,
|
||||
# one entry a layer, the three MTP layers' included
|
||||
"compress_ratios": [0, 2, 2, 1, 1, 0, 0, 0],
|
||||
"kv_source_layer_ids": [1, 3],
|
||||
# 4: an indexer of its own over 3's candidates (a REINDEX layer)
|
||||
"index_source_layer_ids": [1, 3, 4],
|
||||
"candidate_source_layer_id": 3,
|
||||
# an Engram on a whole layer (K1 then takes its residual settled), its
|
||||
# tables shrunk: 24 prime-sized ranges from 1000
|
||||
"engram_layer_ids": [2],
|
||||
"engram_num_embeddings": [32768],
|
||||
"engram_vocab_size": 1000,
|
||||
"dspark_target_layer_ids": [2, 3, 4],
|
||||
}
|
||||
WHOLE, FFN = (0, 1, 2, 3, 4), ()
|
||||
STEPS = 5 # decode steps after the prefill's token
|
||||
|
||||
|
||||
def _on_cdna4() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() == 4
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def _fill(name: str, t: torch.Tensor) -> None:
|
||||
"""A random value for every weight, seeded by its name (alike on every
|
||||
rank and in every run), in the ranges the checkpoint's formats hold."""
|
||||
if t.device.type == "meta":
|
||||
return # materialized and filled by the dummy loader's own path
|
||||
g = torch.Generator(device=t.device).manual_seed(zlib.crc32(name.encode()))
|
||||
|
||||
def randn(scale=1.0, shift=0.0):
|
||||
return torch.randn(t.shape, generator=g, device=t.device) * scale + shift
|
||||
|
||||
if t.dtype == torch.uint8 and "scale" in name:
|
||||
# E8M0 2^-12 .. 2^-8, alike over each 32 rows as in the checkpoint (the
|
||||
# loader folds per-row scales into 32 x 32 blocks only then). Larger
|
||||
# random layers make the stack chaotic: atomics' order alone moves
|
||||
# vLLM's own logits run to run.
|
||||
rows = t.shape[-2] if t.dim() > 1 else 1
|
||||
shape = (*t.shape[:-2], -(-rows // 32), t.shape[-1]) if t.dim() > 1 else t.shape
|
||||
e8m0 = torch.randint(115, 120, shape, generator=g, device=t.device)
|
||||
if t.dim() > 1:
|
||||
e8m0 = e8m0.repeat_interleave(32, dim=-2)[..., :rows, :]
|
||||
t.copy_(e8m0)
|
||||
elif t.dtype in (torch.uint8, torch.float4_e2m1fn_x2):
|
||||
u8 = t.view(torch.uint8)
|
||||
u8.copy_(torch.randint(0, 256, u8.shape, generator=g, device=t.device))
|
||||
elif t.dtype == torch.float8_e4m3fn:
|
||||
t.copy_(randn().clamp(-6, 6))
|
||||
elif not torch.is_floating_point(t):
|
||||
return
|
||||
elif "norm" in name:
|
||||
t.copy_(randn(0.1, 1.0))
|
||||
elif name.endswith(("hc_attn_scale", "hc_ffn_scale")):
|
||||
t.copy_(randn(0.5).abs() + 0.5)
|
||||
elif name.endswith(("hc_attn_base", "hc_ffn_base", "attn_sink")):
|
||||
t.copy_(randn(0.1))
|
||||
elif name.endswith("embed_tokens.weight"):
|
||||
t.copy_(randn())
|
||||
else:
|
||||
t.copy_(randn(0.02))
|
||||
|
||||
|
||||
@register_model_loader(LOAD_FORMAT)
|
||||
class RandomWeightLoader(DummyModelLoader):
|
||||
"""Dummy loading with weights in the checkpoint's ranges, before post-load
|
||||
processing, so every processed copy follows (plain dummy weights leave the
|
||||
E8M0 scales zero, and both paths would agree on zeros)."""
|
||||
|
||||
def load_weights(self, model, model_config) -> None:
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
|
||||
super().load_weights(model, model_config)
|
||||
for name, t in model.state_dict().items():
|
||||
_fill(name, t)
|
||||
# the routed experts' intermediate past this rank's share is the
|
||||
# loader's zero padding (576 -> 640 at TP4): the kernels skip it
|
||||
inter = model_config.hf_text_config.moe_intermediate_size
|
||||
inter //= get_tensor_model_parallel_world_size()
|
||||
for m in model.modules():
|
||||
w13, w2 = getattr(m, "w13_weight", None), getattr(m, "w2_weight", None)
|
||||
if w13 is None or w2 is None or w13.device.type == "meta":
|
||||
continue
|
||||
padded = w13.shape[1] // 2
|
||||
if padded > inter:
|
||||
w13, w2 = w13.data.view(torch.uint8), w2.data.view(torch.uint8)
|
||||
for half in (0, padded):
|
||||
w13[:, half + inter : half + padded] = 0
|
||||
w2[..., inter // 2 :] = 0
|
||||
|
||||
|
||||
class MonoProbe:
|
||||
"""Worker extension (``worker_extension_cls``): it also brings this module,
|
||||
and so the loader, into the workers."""
|
||||
|
||||
def _decoder_layers(self) -> dict:
|
||||
from vllm.models.deepseek_v41.amd.model import DeepseekV4DecoderLayer
|
||||
|
||||
model = self.model_runner.model # type: ignore[attr-defined]
|
||||
return {
|
||||
m.attn.layer_id: m
|
||||
for m in model.modules()
|
||||
if isinstance(m, DeepseekV4DecoderLayer)
|
||||
}
|
||||
|
||||
def capture_steps(self, lens: list[int]) -> None:
|
||||
"""Keep each decoder layer's outputs (x, residual) for every decode
|
||||
token taken in a decode-only step (the steps the mono path takes),
|
||||
keyed by (layer, prompt length, position): a request's step and row
|
||||
can differ between engines (requests reach the engine core while it
|
||||
runs), its prompt length and position cannot. ``lens``: the prompt
|
||||
lengths, all distinct; a row's request is the longest prompt not past
|
||||
its position."""
|
||||
from vllm.forward_context import get_forward_context
|
||||
|
||||
self._captured: dict = {}
|
||||
lens = sorted(lens)
|
||||
|
||||
def hook(layer_id):
|
||||
def keep(module, args, kwargs, out):
|
||||
md = get_forward_context().attn_metadata
|
||||
if not isinstance(md, dict):
|
||||
return
|
||||
swa = md.get(module.attn.swa_cache_layer.prefix)
|
||||
if swa is None or swa.num_prefills != 0:
|
||||
return
|
||||
pos = kwargs.get("positions", args[1] if len(args) > 1 else None)
|
||||
for row, p in enumerate(pos.tolist()):
|
||||
n = max(n for n in lens if n <= p)
|
||||
kept = [t[row].float().cpu() for t in out[:2]]
|
||||
self._captured.setdefault((layer_id, n, p), kept)
|
||||
|
||||
return keep
|
||||
|
||||
def gap_hook(layer_id, layer):
|
||||
def keep(ffn, args, kwargs):
|
||||
# vLLM's path only (the mono kernels route inside): each row's
|
||||
# top-6 margin, sqrt(softplus(logits)) + bias as vLLM routes
|
||||
md = get_forward_context().attn_metadata
|
||||
if not isinstance(md, dict):
|
||||
return
|
||||
swa = md.get(layer.attn.swa_cache_layer.prefix)
|
||||
if swa is None or swa.num_prefills != 0:
|
||||
return
|
||||
h = args[0] if args else kwargs["hidden_states"]
|
||||
logits = h.float() @ ffn.gate.weight.float().T
|
||||
score = torch.nn.functional.softplus(logits).sqrt()
|
||||
score = score + ffn.gate.e_score_correction_bias.float()
|
||||
top = score.topk(7, dim=-1).values
|
||||
gap = (top[:, 5] - top[:, 6]).tolist()
|
||||
pos = self._positions
|
||||
for row, p in enumerate(pos.tolist()):
|
||||
n = max(n for n in lens if n <= p)
|
||||
self._gaps.setdefault((layer_id, n, p), gap[row])
|
||||
|
||||
return keep
|
||||
|
||||
def pos_hook(module, args, kwargs):
|
||||
self._positions = kwargs.get(
|
||||
"positions", args[1] if len(args) > 1 else None
|
||||
)
|
||||
|
||||
self._gaps: dict = {}
|
||||
for i, layer in self._decoder_layers().items():
|
||||
layer.register_forward_hook(hook(i), with_kwargs=True)
|
||||
layer.register_forward_pre_hook(pos_hook, with_kwargs=True)
|
||||
gap = gap_hook(i, layer)
|
||||
layer.ffn.register_forward_pre_hook(gap, with_kwargs=True)
|
||||
|
||||
def save_capture(self, path: str) -> None:
|
||||
rank = self.rank # type: ignore[attr-defined]
|
||||
torch.save((self._captured, self._gaps), f"{path}.rank{rank}")
|
||||
|
||||
def mono_layers(self) -> dict:
|
||||
from vllm.models.deepseek_v41.amd import mono_decode
|
||||
|
||||
layers = {
|
||||
i: None
|
||||
if m.mono is None
|
||||
else ["ffn" if m.mono.ffn_only else "whole", m.mono._weights is not None]
|
||||
for i, m in self._decoder_layers().items()
|
||||
}
|
||||
runner = mono_decode._runner
|
||||
return {
|
||||
"layers": layers,
|
||||
"epoch": -1 if runner is None else int(runner.epoch[0]),
|
||||
}
|
||||
|
||||
|
||||
def _decode(vllm_runner, monkeypatch, tp: int, mono: bool, prompts, path: str):
|
||||
"""Greedy decode of ``prompts`` with the mono path on or off: (each
|
||||
prompt's output, the workers' mono layers); each decoder layer's outputs
|
||||
for the decode tokens of decode-only steps, and vLLM's routing margins,
|
||||
saved under ``path``."""
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import TokensPrompt
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
tensor_parallel_size=tp,
|
||||
load_format=LOAD_FORMAT,
|
||||
hf_overrides=TRUNCATED,
|
||||
worker_extension_cls=f"{__name__}.MonoProbe",
|
||||
moe_backend="aiter",
|
||||
monokernel=mono,
|
||||
language_model_only=True,
|
||||
tokenizer_mode="deepseek_v41",
|
||||
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
|
||||
block_size=128,
|
||||
enforce_eager=True,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=len(prompts),
|
||||
# a prompt prefills in one step, so its decode tokens never share one
|
||||
max_num_batched_tokens=16384,
|
||||
enable_prefix_caching=False,
|
||||
gpu_memory_utilization=0.5,
|
||||
) as runner:
|
||||
runner.llm.collective_rpc("capture_steps", args=([len(p) for p in prompts],))
|
||||
params = SamplingParams(temperature=0.0, max_tokens=STEPS + 1)
|
||||
outs = runner.llm.generate(
|
||||
[TokensPrompt(prompt_token_ids=p) for p in prompts], params
|
||||
)
|
||||
runner.llm.collective_rpc("save_capture", args=(path,))
|
||||
probe = runner.llm.collective_rpc("mono_layers")
|
||||
return [o.outputs[0] for o in outs], probe
|
||||
|
||||
|
||||
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
return ((a - b).norm() / b.norm()).item()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
|
||||
@pytest.mark.parametrize("tp", [2, 4])
|
||||
def test_mono_decode_matches_the_model(vllm_runner, monkeypatch, tmp_path, tp):
|
||||
if torch.accelerator.device_count() < tp:
|
||||
pytest.skip(f"needs {tp} GPUs")
|
||||
pytest.importorskip("flydsl")
|
||||
g = torch.Generator().manual_seed(0)
|
||||
# contexts past both ratios' 512 compressed entries, so top-k selects
|
||||
lens = torch.randperm(1700, generator=g)[:8].add(300).tolist() # distinct
|
||||
prompts = [torch.randint(1000, 60000, (n,), generator=g).tolist() for n in lens]
|
||||
|
||||
on, probe_on = _decode(
|
||||
vllm_runner, monkeypatch, tp, True, prompts, f"{tmp_path}/on"
|
||||
)
|
||||
off, probe_off = _decode(
|
||||
vllm_runner, monkeypatch, tp, False, prompts, f"{tmp_path}/off"
|
||||
)
|
||||
|
||||
# both layer kinds took the mono path on every rank, at every decode step
|
||||
launches = len(WHOLE + FFN) # one launch (pair) a layer a step
|
||||
for rank in probe_on:
|
||||
assert rank["layers"] == {
|
||||
i: ["whole" if i in WHOLE else "ffn", True] for i in WHOLE + FFN
|
||||
}, rank
|
||||
assert rank["epoch"] >= launches * STEPS, rank
|
||||
assert rank["epoch"] % launches == 0, rank
|
||||
assert all(set(r["layers"].values()) == {None} for r in probe_off)
|
||||
# each decode token's input matches between the runs while the greedy
|
||||
# tokens before it do: (prompt length, position) pairs to compare
|
||||
same: set[tuple[int, int]] = set()
|
||||
for prompt, a, b in zip(prompts, on, off):
|
||||
for step in range(STEPS):
|
||||
if a.token_ids[: step + 1] != b.token_ids[: step + 1]:
|
||||
break
|
||||
same.add((len(prompt), len(prompt) + step))
|
||||
|
||||
# each layer's outputs for those tokens, mono against vLLM's path, token by
|
||||
# token. Layer 0 sees the same inputs on both paths, but its FFN input
|
||||
# differs by up to test_dsv41_mono_numerics' 1.2%, which random weights
|
||||
# amplify through the MoE: 3%. Each later layer takes the previous one's
|
||||
# slightly different output, which random weights amplify (2-4% by layer
|
||||
# 4): 8%. A token whose top-6 routing is a near-tie on vLLM's path may pick
|
||||
# another expert: a 0.4% input difference moves scores near 0.8 by about
|
||||
# 3e-3, so 6th / 7th scores within 5e-3 at layer 0 and, past it, where the
|
||||
# input carries the earlier layers' differences (0.4-0.7%), within 1e-2.
|
||||
# Once it has, its residual stream differs and the later layers are exempt
|
||||
# for it. The residual stream: 3% (0.5% seen). A layer the kernels no
|
||||
# longer follow is off by far more, on every token.
|
||||
for rank in range(tp):
|
||||
got, _ = torch.load(f"{tmp_path}/on.rank{rank}")
|
||||
want, gaps = torch.load(f"{tmp_path}/off.rank{rank}")
|
||||
keys: list[tuple[int, int]] = sorted(
|
||||
same & {k[1:] for k in got} & {k[1:] for k in want}
|
||||
)
|
||||
assert len(keys) >= len(prompts), keys
|
||||
flipped: set = set()
|
||||
for i in sorted(WHOLE + FFN):
|
||||
for k in keys:
|
||||
if k in flipped:
|
||||
continue
|
||||
(x, res), (x_ref, res_ref) = got[(i, *k)], want[(i, *k)]
|
||||
err = _rel(res, res_ref)
|
||||
assert err < 0.03, (rank, i, k, "residual", err)
|
||||
bound = 0.03 if i == 0 else 0.08
|
||||
err = _rel(x, x_ref)
|
||||
if err >= bound:
|
||||
tie = 5e-3 if i == 0 else 1e-2
|
||||
assert gaps[(i, *k)] < tie, (rank, i, k, "x", err)
|
||||
flipped.add(k)
|
||||
assert len(flipped) <= len(keys) // 4, (rank, sorted(flipped))
|
||||
@@ -0,0 +1,533 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Numerics of the DeepSeek-V4.1 mono decode kernels at TP2 on CDNA4 against the
|
||||
ops vLLM runs for the same decode step: random weights in the checkpoint's
|
||||
formats, the step's KV caches as vLLM's insert writes them.
|
||||
|
||||
- The whole layer (K1 + K2) against the mHC seams, the ROCm attention op chain,
|
||||
the TP all-reduce and ``DeepseekV4MoE``.
|
||||
- The FFN launch against the same FFN half, from wo_b's unreduced output.
|
||||
|
||||
Each stage is compared on its own input, so a failure names the stage; the MoE
|
||||
is checked on the kernels' own FFN input, where routing cannot differ."""
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash"
|
||||
HIDDEN, Q_RANK, HEAD_DIM, NOPE, ROPE = 5120, 1280, 512, 448, 64
|
||||
N_HEADS, O_GROUPS, O_RANK = 64, 8, 1024
|
||||
WINDOW, TOPK, SWA_BLOCK, CACHE_BLOCK = 128, 512, 32, 128
|
||||
RECORD, DATA, ALIGN = 584, 576, 576
|
||||
REPLAYS = 300
|
||||
HC = 4
|
||||
|
||||
|
||||
def _on_cdna4() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() == 4
|
||||
|
||||
|
||||
def _cos(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
a, b = a.float().reshape(-1), b.float().reshape(-1)
|
||||
return torch.nn.functional.cosine_similarity(a, b, dim=0).item()
|
||||
|
||||
|
||||
def _rel(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
"""Relative L2 error of ``a`` against ``b``: unlike the cosine, it sees a
|
||||
wrong scale."""
|
||||
a, b = a.float().reshape(-1), b.float().reshape(-1)
|
||||
return ((a - b).norm() / b.norm()).item()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- caches
|
||||
|
||||
|
||||
def _paged_cache(num_blocks: int, block: int, device) -> torch.Tensor:
|
||||
"""[blocks, block, 584] u8 over pages of round_up(block * 584, 576) bytes,
|
||||
the way vLLM binds a packed V4 cache."""
|
||||
page = -(-block * RECORD // ALIGN) * ALIGN
|
||||
raw = torch.zeros(num_blocks * page, dtype=torch.uint8, device=device)
|
||||
return torch.as_strided(raw, (num_blocks, block, RECORD), (page, RECORD, 1))
|
||||
|
||||
|
||||
def _rope(kv: torch.Tensor, positions: torch.Tensor, cos_sin: torch.Tensor):
|
||||
"""GPT-J RoPE of the last 64 dims (fp32 math, bf16 out)."""
|
||||
cs = cos_sin[positions.long()]
|
||||
cos, sin = cs[:, :32], cs[:, 32:]
|
||||
x = kv.float()
|
||||
e, o = x[:, NOPE::2].clone(), x[:, NOPE + 1 :: 2].clone()
|
||||
x[:, NOPE::2] = e * cos - o * sin
|
||||
x[:, NOPE + 1 :: 2] = e * sin + o * cos
|
||||
return x.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _write_records(cache, block: int, slots, kv):
|
||||
"""fp8_ds_mla records of roped bf16 kv rows at ``slots``: the 448 nope dims
|
||||
in e4m3 with a UE8M0 per 64, the rope dims in bf16, as vLLM's insert."""
|
||||
nope = kv.float()[:, :NOPE].view(-1, 7, 64)
|
||||
exp = torch.ceil(torch.log2(nope.abs().amax(-1).clamp_min(1e-4) / 448.0))
|
||||
q = (nope * torch.exp2(-exp)[..., None]).clamp(-448, 448).reshape(-1, NOPE)
|
||||
data = torch.cat(
|
||||
[
|
||||
q.to(torch.float8_e4m3fn).view(torch.uint8),
|
||||
kv[:, NOPE:].contiguous().view(torch.uint8),
|
||||
],
|
||||
1,
|
||||
)
|
||||
scales = torch.zeros(kv.shape[0], 8, dtype=torch.uint8, device=kv.device)
|
||||
scales[:, :7] = (exp + 127).clamp(0, 255).to(torch.uint8)
|
||||
n = cache.untyped_storage().nbytes() - cache.storage_offset()
|
||||
flat = torch.as_strided(cache, (n,), (1,))
|
||||
slots = slots.to(kv.device).long()
|
||||
b, o = slots // block, slots % block
|
||||
d0 = b * cache.stride(0) + o * DATA
|
||||
s0 = b * cache.stride(0) + block * DATA + o * 8
|
||||
ar = torch.arange(DATA, device=kv.device)
|
||||
flat[(d0[:, None] + ar).reshape(-1)] = data.reshape(-1)
|
||||
ar = torch.arange(8, device=kv.device)
|
||||
flat[(s0[:, None] + ar).reshape(-1)] = scales.reshape(-1)
|
||||
|
||||
|
||||
def _normed_kv(n: int, kv_norm, g, device):
|
||||
x = torch.randn(n, HEAD_DIM, generator=g).to(device)
|
||||
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True)) * kv_norm.float()
|
||||
return x.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _decode_step(ratio: int, R: int, T: int, kv_norm, cos_sin, seed: int):
|
||||
"""R requests x T tokens past random contexts: the step's metadata as
|
||||
vLLM's builders lay it out, its window rows and (ratio > 0) the compressed
|
||||
cache filled, a random top-512 a token."""
|
||||
g = torch.Generator().manual_seed(seed)
|
||||
dev = kv_norm.device
|
||||
ctx = [int(c) for c in torch.randint(2000, 40000, (R,), generator=g)]
|
||||
M = R * T
|
||||
spans = [
|
||||
(max(0, c - WINDOW - T) // SWA_BLOCK, (c + T - 1) // SWA_BLOCK) for c in ctx
|
||||
]
|
||||
n_swa = 1 + sum(hi - lo + 1 for lo, hi in spans)
|
||||
perm = (torch.randperm(n_swa - 1, generator=g) + 1).tolist()
|
||||
bt = torch.zeros(R, max(hi for _, hi in spans) + 1, dtype=torch.int64)
|
||||
for r, (lo, hi) in enumerate(spans):
|
||||
for lb in range(lo, hi + 1):
|
||||
bt[r, lb] = perm.pop()
|
||||
swa = _paged_cache(n_swa, SWA_BLOCK, dev)
|
||||
|
||||
def slot(r, p):
|
||||
return int(bt[r, p // SWA_BLOCK]) * SWA_BLOCK + p % SWA_BLOCK
|
||||
|
||||
pos = [ctx[r] + t for r in range(R) for t in range(T)]
|
||||
t2r = [r for r in range(R) for _ in range(T)]
|
||||
swa_idx = torch.full((M, WINDOW), -1, dtype=torch.int32)
|
||||
swa_len = torch.zeros(M, dtype=torch.int32)
|
||||
for i in range(M):
|
||||
ps = range(max(pos[i] - WINDOW + 1, 0), pos[i] + 1)
|
||||
swa_idx[i, : len(ps)] = torch.tensor([slot(t2r[i], p) for p in ps])
|
||||
swa_len[i] = len(ps)
|
||||
hist = [
|
||||
(r, p) for r in range(R) for p in range(max(0, ctx[r] - WINDOW - T), ctx[r])
|
||||
]
|
||||
hp = torch.tensor([p for _, p in hist], device=dev)
|
||||
kv = _rope(_normed_kv(len(hist), kv_norm, g, dev), hp, cos_sin)
|
||||
_write_records(swa, SWA_BLOCK, torch.tensor([slot(r, p) for r, p in hist]), kv)
|
||||
|
||||
step = dict(
|
||||
M=M,
|
||||
positions=torch.tensor(pos, dtype=torch.int64, device=dev),
|
||||
slot_mapping=torch.tensor(
|
||||
[slot(t2r[i], pos[i]) for i in range(M)], dtype=torch.int64, device=dev
|
||||
),
|
||||
token_to_req=torch.tensor(t2r, dtype=torch.int32, device=dev),
|
||||
swa_indices=swa_idx.to(dev),
|
||||
swa_lens=swa_len.to(dev),
|
||||
swa_cache=swa,
|
||||
topk=None,
|
||||
comp_cache=None,
|
||||
comp_block_table=None,
|
||||
comp_entries=0,
|
||||
)
|
||||
if ratio:
|
||||
entries = CACHE_BLOCK // ratio
|
||||
lens = [(c + T) // ratio for c in ctx]
|
||||
nblk = [-(-n // entries) for n in lens]
|
||||
perm = (torch.randperm(sum(nblk), generator=g) + 1).tolist()
|
||||
cbt = torch.zeros(R, max(nblk), dtype=torch.int32)
|
||||
for r in range(R):
|
||||
for b in range(nblk[r]):
|
||||
cbt[r, b] = perm.pop()
|
||||
comp = _paged_cache(1 + sum(nblk), entries, dev)
|
||||
for r in range(R):
|
||||
idx = torch.arange(lens[r])
|
||||
slots = cbt[r, idx // entries].long() * entries + idx % entries
|
||||
kv = _rope(
|
||||
_normed_kv(lens[r], kv_norm, g, dev), (idx * ratio).to(dev), cos_sin
|
||||
)
|
||||
_write_records(comp, entries, slots, kv)
|
||||
topk = torch.full((M, TOPK), -1, dtype=torch.int32)
|
||||
for i in range(M):
|
||||
cand = (pos[i] + 1) // ratio
|
||||
n = min(cand, TOPK)
|
||||
topk[i, :n] = torch.randperm(cand, generator=g)[:n].to(torch.int32)
|
||||
step.update(
|
||||
topk=topk.to(dev),
|
||||
comp_cache=comp,
|
||||
comp_block_table=cbt.to(dev),
|
||||
comp_entries=entries,
|
||||
)
|
||||
return step
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- weights
|
||||
|
||||
|
||||
def _mxfp8(n: int, k: int, g, device):
|
||||
"""A random MXFP8 weight [n, k] e4m3 and its E8M0 32 x 32 block scales."""
|
||||
w = torch.randn(n, k, generator=g).clamp(-6, 6).to(torch.float8_e4m3fn)
|
||||
s = torch.randint(118, 123, (n // 32, k // 32), generator=g, dtype=torch.uint8)
|
||||
return w.to(device), s.to(device)
|
||||
|
||||
|
||||
def _random_moe(moe, gd, gr, inter: int) -> None:
|
||||
"""Random parameters in the loaded layout, before post-load processing: this
|
||||
rank's shards (``gd``, a device generator) of the MXFP4 routed experts
|
||||
(random nibbles, E8M0 scales; the intermediate past ``inter`` zero, as the
|
||||
loader pads it) and the MXFP8 shared expert (one E8M0 a 32 x 32 block); the
|
||||
replicated router from ``gr``, seeded alike on every rank."""
|
||||
u8 = torch.uint8
|
||||
for lin in (moe.shared_experts.gate_up_proj, moe.shared_experts.down_proj):
|
||||
n = lin.weight.shape[0]
|
||||
w = torch.randn(lin.weight.shape, generator=gd, device=gd.device)
|
||||
lin.weight.data.copy_(w.clamp(-6, 6).to(lin.weight.dtype))
|
||||
sc = lin.weight_scale.data.view(u8)
|
||||
rows = -(-n // 32) if sc.shape[0] == n else sc.shape[0]
|
||||
blocks = torch.randint(
|
||||
118, 123, (rows, sc.shape[1]), generator=gd, device=gd.device, dtype=u8
|
||||
)
|
||||
sc.copy_(blocks.repeat_interleave(32, 0)[:n] if sc.shape[0] == n else blocks)
|
||||
e = moe.experts.routed_experts
|
||||
for name, lo, hi in (("weight", 0, 256), ("weight_scale", 118, 122)):
|
||||
for w in ("w13", "w2"):
|
||||
t = getattr(e, f"{w}_{name}").data.view(u8)
|
||||
t.copy_(
|
||||
torch.randint(lo, hi, t.shape, generator=gd, device=gd.device, dtype=u8)
|
||||
)
|
||||
padded = e.w13_weight.shape[1] // 2
|
||||
if padded > inter:
|
||||
w13, w2 = e.w13_weight.data.view(u8), e.w2_weight.data.view(u8)
|
||||
for half in (0, padded):
|
||||
w13[:, half + inter : half + padded] = 0
|
||||
w2[..., inter // 2 :] = 0
|
||||
for t in (moe.gate.weight, moe.gate.e_score_correction_bias):
|
||||
t.data.copy_(torch.randn(t.shape, generator=gr, device=gr.device) * 0.02)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the worker
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _mono_numerics(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
# the checkpoint's MXFP4 experts run on AITER's MoE; its flags are read
|
||||
# at import, so refresh them
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
init_workspace_manager(device)
|
||||
_check_layers(rank, tp_size, device)
|
||||
|
||||
|
||||
def _check_layers(rank: int, tp: int, device) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_block32_gemm import (
|
||||
rocm_mxfp8_block32_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.mhc import MHCPreDelayedOp
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import process_weights_after_loading
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.deepseek_v41.amd.model import DeepseekV4MoE
|
||||
from vllm.models.deepseek_v41.amd.mono.layer import scratch_layout
|
||||
from vllm.models.deepseek_v41.amd.mono.runner import (
|
||||
AttnWeights,
|
||||
DSV41MonoLayer,
|
||||
MonoLayerWeights,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.rope import build_deepseek_v4_rope
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
build_ragged_indices_from_dense,
|
||||
rocm_inverse_rope_mxfp8_rows,
|
||||
)
|
||||
|
||||
vc = EngineArgs(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tp,
|
||||
moe_backend="aiter",
|
||||
load_format="dummy",
|
||||
max_model_len=65536,
|
||||
language_model_only=True,
|
||||
attention_config={"indexer_kv_dtype": "mxfp4", "indexer_sparse_logits": True},
|
||||
).create_engine_config()
|
||||
cfg = vc.model_config.hf_config
|
||||
cfg = getattr(cfg, "text_config", cfg)
|
||||
# replicated tensors alike on every rank; each rank's own TP shards
|
||||
g = torch.Generator().manual_seed(1000)
|
||||
gs = torch.Generator().manual_seed(1001 + rank)
|
||||
H, G = N_HEADS // tp, O_GROUPS // tp
|
||||
bf = torch.bfloat16
|
||||
|
||||
with set_current_vllm_config(vc), set_default_torch_dtype(bf), device:
|
||||
moe = DeepseekV4MoE(vc, prefix="model.layers.21.ffn")
|
||||
mhc = MHCPreDelayedOp()
|
||||
_random_moe(
|
||||
moe,
|
||||
torch.Generator(device=device).manual_seed(2001 + rank),
|
||||
torch.Generator(device=device).manual_seed(2000),
|
||||
inter=2304 // tp,
|
||||
)
|
||||
process_weights_after_loading(moe, vc.model_config, device)
|
||||
|
||||
def f32(*shape, s=1.0, gen=g):
|
||||
return (torch.randn(*shape, generator=gen) * s).to(device)
|
||||
|
||||
mix = (2 + HC) * HC
|
||||
seams = {
|
||||
sub: dict(
|
||||
fn=f32(mix, HC * HIDDEN, s=0.01),
|
||||
scale=f32(3, s=0.5).abs() + 0.5,
|
||||
base=f32(mix, s=0.1),
|
||||
norm=(1 + f32(HIDDEN, s=0.1)).to(bf),
|
||||
)
|
||||
for sub in ("attn", "ffn")
|
||||
}
|
||||
|
||||
def seam(sub, residual, x, post, comb, pre):
|
||||
s = seams[sub]
|
||||
return mhc(
|
||||
residual, s["fn"], s["scale"], s["base"], cfg.rms_norm_eps,
|
||||
cfg.hc_eps, cfg.hc_eps, 2.0, cfg.hc_sinkhorn_iters,
|
||||
pre_mix=pre, sublayer_out=x, post_layer_mix=post, comb_res_mix=comb,
|
||||
norm_weight=s["norm"], norm_eps=cfg.rms_norm_eps,
|
||||
) # fmt: skip
|
||||
|
||||
def moe_out(x):
|
||||
with set_forward_context(None, vc):
|
||||
ids = torch.zeros(x.shape[0], dtype=torch.int64, device=device)
|
||||
return moe(x.clone(), ids)
|
||||
|
||||
wqkv, wqkv_s = _mxfp8(1792, HIDDEN, g, device)
|
||||
wq_b, wq_b_s = _mxfp8(H * HEAD_DIM, Q_RANK, gs, device)
|
||||
wo_a, wo_a_s = _mxfp8(G * O_RANK, H // G * HEAD_DIM, gs, device)
|
||||
wo_b, wo_b_s = _mxfp8(HIDDEN, G * O_RANK, gs, device)
|
||||
q_norm, kv_norm = (1 + f32(Q_RANK, s=0.1)).to(bf), (1 + f32(HEAD_DIM, s=0.1)).to(bf)
|
||||
sink = f32(H, gen=gs)
|
||||
e, sh = moe.experts.routed_experts, moe.shared_experts
|
||||
u8 = lambda t: t.view(torch.uint8) # noqa: E731
|
||||
runner = DSV41MonoLayer(tp, rank, get_tp_group().cpu_group, device)
|
||||
gw, bias = moe.gate.weight.float(), moe.gate.e_score_correction_bias.float()
|
||||
|
||||
def top6(x):
|
||||
"""The top-6 experts as vLLM routes: sqrt(softplus(logits)) + bias."""
|
||||
sc = torch.nn.functional.softplus(x.float() @ gw.T).sqrt() + bias
|
||||
return sc.topk(6, dim=1).indices.sort(1).values
|
||||
|
||||
steps = [] # each case's launches, on fixed buffers, for the replay stress
|
||||
for ratio, R, T in ((1, 1, 6), (2, 8, 6)):
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
rope = build_deepseek_v4_rope(
|
||||
cfg, head_dim=HEAD_DIM, rope_head_dim=ROPE,
|
||||
max_position_embeddings=cfg.max_position_embeddings,
|
||||
compress_ratio=ratio,
|
||||
) # fmt: skip
|
||||
cos_sin = rope.cos_sin_cache.to(device=device, dtype=torch.float32)
|
||||
s = _decode_step(ratio, R, T, kv_norm, cos_sin, seed=7 + ratio)
|
||||
M = s["M"]
|
||||
mw = MonoLayerWeights(
|
||||
attn=AttnWeights(
|
||||
layer_id=21, wqkv=wqkv, wqkv_scale=wqkv_s, q_norm=q_norm,
|
||||
kv_norm=kv_norm, wq_b=wq_b, wq_b_scale=wq_b_s, wo_a=wo_a,
|
||||
wo_a_scale=wo_a_s, wo_b=wo_b, wo_b_scale=wo_b_s, attn_sink=sink,
|
||||
cos_sin=cos_sin, ratio=ratio,
|
||||
),
|
||||
hc_attn_fn=seams["attn"]["fn"], hc_attn_scale=seams["attn"]["scale"],
|
||||
hc_attn_base=seams["attn"]["base"], attn_norm=seams["attn"]["norm"],
|
||||
hc_ffn_fn=seams["ffn"]["fn"], hc_ffn_scale=seams["ffn"]["scale"],
|
||||
hc_ffn_base=seams["ffn"]["base"], ffn_norm=seams["ffn"]["norm"],
|
||||
gate_w=moe.gate.weight, bias=moe.gate.e_score_correction_bias,
|
||||
w13=e.w13_weight, w13_s=e.w13_weight_scale, w2=e.w2_weight,
|
||||
w2_s=e.w2_weight_scale, sgu=sh.gate_up_proj.weight,
|
||||
sgu_s=u8(sh.gate_up_proj.weight_scale), sw2=sh.down_proj.weight,
|
||||
sw2_s=u8(sh.down_proj.weight_scale),
|
||||
) # fmt: skip
|
||||
|
||||
# the layer's inputs at the attention seam (the same on every rank)
|
||||
gi = torch.Generator().manual_seed(11)
|
||||
res0 = torch.randn(M, HC, HIDDEN, generator=gi)
|
||||
res0 = (
|
||||
(res0 * torch.rsqrt(res0.pow(2).mean(-1, keepdim=True))).to(bf).to(device)
|
||||
)
|
||||
x0 = (torch.randn(M, HIDDEN, generator=gi) * 0.5).to(bf).to(device)
|
||||
comb0 = torch.rand(M, HC, HC, generator=gi) + 0.1
|
||||
for _ in range(20):
|
||||
comb0 = comb0 / comb0.sum(-1, keepdim=True)
|
||||
comb0 = comb0 / comb0.sum(-2, keepdim=True)
|
||||
comb0 = comb0.to(device)
|
||||
post0 = (torch.sigmoid(torch.randn(M, HC, 1, generator=gi)) * 2).to(device)
|
||||
pre0 = (torch.sigmoid(torch.randn(M, HC, generator=gi)) + 1e-6).to(device)
|
||||
|
||||
# vLLM's decode chain
|
||||
res, post, comb, xa, attn_pre = seam("attn", res0, x0, post0, comb0, pre0)
|
||||
swa_ind, swa_ptr = build_ragged_indices_from_dense(
|
||||
s["swa_indices"].reshape(M, -1), s["swa_lens"]
|
||||
)
|
||||
top_ind = top_ptr = None
|
||||
if ratio:
|
||||
valid = torch.ones(M, dtype=torch.bool, device=device)
|
||||
top_ind, top_ptr, _ = compute_global_topk_ragged_indices_and_indptr(
|
||||
s["topk"], s["token_to_req"], s["comp_block_table"],
|
||||
s["comp_entries"], valid,
|
||||
) # fmt: skip
|
||||
x8, xs = mxfp8_e4m3_quantize(xa)
|
||||
qr_kv = rocm_mxfp8_block32_gemm(x8, xs, wqkv, wqkv_s, bf)
|
||||
qr, kv = fused_q_kv_rmsnorm(
|
||||
qr_kv[:, :Q_RANK], qr_kv[:, Q_RANK:], q_norm, kv_norm, cfg.rms_norm_eps
|
||||
)
|
||||
q8, qs = mxfp8_e4m3_quantize(qr)
|
||||
q = rocm_mxfp8_block32_gemm(q8, qs, wq_b, wq_b_s, bf).view(M, H, HEAD_DIM)
|
||||
swa2d = s["swa_cache"].view(s["swa_cache"].shape[0], -1)
|
||||
q_roped = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q, kv, swa2d, s["slot_mapping"], s["positions"], cos_sin, H,
|
||||
cfg.rms_norm_eps, SWA_BLOCK, False, False, True, False,
|
||||
) # fmt: skip
|
||||
attn = torch.empty(M, H, HEAD_DIM, dtype=bf, device=device)
|
||||
rocm_aiter_ops.triton_sparse_mla_fwd(
|
||||
q_roped, s["swa_cache"], attn, HEAD_DIM**-0.5, swa_ptr, swa_ind,
|
||||
attn_sink=sink, extra_kv_buffer=s["comp_cache"],
|
||||
extra_kv_indptr=top_ptr, extra_kv_indices=top_ind,
|
||||
) # fmt: skip
|
||||
a8 = torch.empty(M, H * HEAD_DIM, dtype=torch.float8_e4m3fn, device=device)
|
||||
a_s = torch.empty(M, H * HEAD_DIM // 32, dtype=torch.uint8, device=device)
|
||||
rocm_inverse_rope_mxfp8_rows(attn, s["positions"], cos_sin, ROPE, a8, a_s)
|
||||
z = torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(a8, a_s, wo_a, wo_a_s, G, O_RANK)
|
||||
z8, zs = mxfp8_e4m3_quantize(z)
|
||||
part = rocm_mxfp8_block32_gemm(z8, zs, wo_b, wo_b_s, bf).contiguous()
|
||||
a = tensor_model_parallel_all_reduce(part.clone())
|
||||
res2, post2, comb2, xf, ffn_pre = seam("ffn", res, a, post, comb, attn_pre)
|
||||
out = moe_out(xf)
|
||||
|
||||
# ---- the whole layer: K1 + K2
|
||||
o = runner.forward(
|
||||
mw, x0, res0, post0, comb0, pre0, s["positions"], s["slot_mapping"],
|
||||
s["swa_cache"], s["swa_indices"], s["swa_lens"], s["token_to_req"],
|
||||
topk_indices=s["topk"], comp_cache=s["comp_cache"],
|
||||
comp_block_table=s["comp_block_table"],
|
||||
) # fmt: skip
|
||||
off = scratch_layout(M, tp)["normed"][0]
|
||||
normed = runner.scratch(M)[off : off + M * HIDDEN * 2].view(bf).view(M, HIDDEN)
|
||||
tag = f"ratio {ratio}, {R} x {T}"
|
||||
assert _cos(runner.res_mid[:M], res) > 0.99999, tag
|
||||
for got, want in (
|
||||
(runner.post_a, post),
|
||||
(runner.comb_a, comb),
|
||||
(runner.pre_a, attn_pre),
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
got[:M].view_as(want), want, rtol=1e-4, atol=1e-5
|
||||
)
|
||||
same_in = moe_out(normed)
|
||||
# relative errors about 3x what the kernels show (0.3-0.5%)
|
||||
assert _cos(o[1], res2) > 0.9999 and _rel(o[1], res2) < 0.01, tag
|
||||
assert _cos(normed, xf) > 0.999 and _rel(normed, xf) < 0.012, tag
|
||||
# TODO: back to cos > 0.9999, rel < 0.015 once
|
||||
# https://github.com/ROCm/aiter/issues/6548 is fixed: aiter's fused-FP8
|
||||
# stage-1 MoE kernels use another MXFP8 scale rule (1.53% at TP4)
|
||||
assert _cos(o[0], same_in) > 0.9998 and _rel(o[0], same_in) < 0.02, tag
|
||||
# per token, where both FFN inputs pick the same experts
|
||||
same = (top6(xf) == top6(normed)).all(1)
|
||||
assert same.float().mean() >= 0.5, tag
|
||||
tok = torch.nn.functional.cosine_similarity(o[0].float(), out.float(), dim=1)
|
||||
assert tok[same].min() > 0.99, tag
|
||||
|
||||
# ---- the FFN launch, from the same unreduced wo_b output
|
||||
f = runner.ffn(mw, part, res, post, comb, attn_pre)
|
||||
torch.testing.assert_close(f[1], res2, rtol=2**-8, atol=1e-6)
|
||||
torch.testing.assert_close(normed, xf, rtol=2**-7, atol=1e-6)
|
||||
# TODO: back to 0.9999 / 0.015 with the one above (ROCm/aiter#6548)
|
||||
assert _cos(f[0], out) > 0.9998 and _rel(f[0], out) < 0.02, tag
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
whole, ffn = runner._outs(M, res0), runner._outs(M, res)
|
||||
args = (s["positions"], s["slot_mapping"], s["swa_cache"], s["swa_indices"])
|
||||
meta = dict(
|
||||
topk_indices=s["topk"], comp_cache=s["comp_cache"],
|
||||
comp_block_table=s["comp_block_table"],
|
||||
) # fmt: skip
|
||||
|
||||
def step(mw=mw, x0=x0, res0=res0, post0=post0, comb0=comb0, pre0=pre0,
|
||||
args=args, s=s, meta=meta, whole=whole, part=part, res=res,
|
||||
post=post, comb=comb, attn_pre=attn_pre, ffn=ffn): # fmt: skip
|
||||
runner.forward(
|
||||
mw, x0, res0, post0, comb0, pre0, *args, s["swa_lens"],
|
||||
s["token_to_req"], **meta, outs=whole,
|
||||
) # fmt: skip
|
||||
runner.ffn(mw, part, res, post, comb, attn_pre, outs=ffn)
|
||||
return [*whole, *ffn]
|
||||
|
||||
steps.append(step)
|
||||
|
||||
# ---- the hand-off protocol under back-to-back launches: both step widths
|
||||
# alternating in one graph, every replay bit for bit the first's
|
||||
outs = [t for step in steps for t in step()]
|
||||
torch.accelerator.synchronize()
|
||||
ref = [t.clone() for t in outs]
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
for step in steps:
|
||||
step()
|
||||
for i in range(REPLAYS):
|
||||
graph.replay()
|
||||
if i % 50 == 49 or i == REPLAYS - 1:
|
||||
torch.accelerator.synchronize()
|
||||
bad = [
|
||||
j for j, (t, r) in enumerate(zip(outs, ref)) if not torch.equal(t, r)
|
||||
]
|
||||
assert not bad, f"replay {i}: outputs {bad} differ from the first run"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _on_cdna4(), reason="the mono kernels target CDNA4")
|
||||
@pytest.mark.parametrize("tp", [2, 4])
|
||||
def test_mono_decode_layer_matches_vllm(monkeypatch: pytest.MonkeyPatch, tp: int):
|
||||
if torch.accelerator.device_count() < tp:
|
||||
pytest.skip(f"needs {tp} GPUs")
|
||||
pytest.importorskip("flydsl")
|
||||
multi_process_parallel(monkeypatch, tp, 1, _mono_numerics)
|
||||
@@ -14,10 +14,8 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_configs,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_configs
|
||||
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
|
||||
from vllm.v1.engine.core import EngineCore as V1EngineCore
|
||||
|
||||
from ..utils import (
|
||||
|
||||
@@ -12,9 +12,13 @@ import gc
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.models.interfaces import _language_model_by_module
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsMultiModal,
|
||||
_language_model_by_module,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -57,3 +61,42 @@ def test_cached_entry_is_dropped_with_its_key():
|
||||
gc.collect()
|
||||
|
||||
assert not any(isinstance(k, _MultiModalModel) for k in _language_model_by_module)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marked", [False, True])
|
||||
@pytest.mark.parametrize("text_model", ["clip", "siglip"])
|
||||
def test_text_tower_without_loader_hook_is_found(text_model, marked, caplog):
|
||||
from vllm.model_executor.models.clip import CLIPTextTransformer
|
||||
from vllm.model_executor.models.siglip import SiglipTextTransformer
|
||||
|
||||
cls = CLIPTextTransformer if text_model == "clip" else SiglipTextTransformer
|
||||
# Only token embeddings are needed to exercise text tower discovery.
|
||||
tower = cls.__new__(cls)
|
||||
nn.Module.__init__(tower)
|
||||
tower.embeddings = nn.Module()
|
||||
tower.embeddings.token_embedding = nn.Embedding(4, 2)
|
||||
model = nn.Module()
|
||||
model.text_model = tower
|
||||
model._language_model_names = ["text_model"] if marked else []
|
||||
|
||||
caplog.clear()
|
||||
language_model = SupportsMultiModal.get_language_model(model)
|
||||
|
||||
assert language_model is tower
|
||||
input_ids = torch.tensor([0, 1])
|
||||
torch.testing.assert_close(
|
||||
language_model.embed_input_ids(input_ids),
|
||||
tower.embeddings.token_embedding.weight[:2],
|
||||
)
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
@pytest.mark.parametrize("marked", [False, True])
|
||||
@pytest.mark.parametrize("method", ["embed_input_ids", "forward"])
|
||||
def test_language_model_discovery_rejects_noncallable_methods(method, marked):
|
||||
model = _MultiModalModel()
|
||||
model._language_model_names = ["language_model"] if marked else []
|
||||
setattr(model.language_model, method, None)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="No language model found"):
|
||||
SupportsMultiModal.get_language_model(model)
|
||||
|
||||
@@ -113,9 +113,6 @@ def test_registry_model_property(model_arch, is_mm, score_type):
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,is_pp",
|
||||
[
|
||||
# TODO(woosuk): Re-enable this once the MLP Speculator is supported
|
||||
# in V1.
|
||||
# ("MLPSpeculatorPreTrainedModel", False),
|
||||
("DeepseekV2ForCausalLM", True),
|
||||
("Qwen2VLForConditionalGeneration", True),
|
||||
],
|
||||
@@ -127,6 +124,22 @@ def test_registry_is_pp(model_arch, is_pp):
|
||||
assert model_info.supports_pp is is_pp
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", not current_platform.is_rocm()),
|
||||
("Qwen4ExpForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Qwen4ExpMTP", not current_platform.is_rocm()),
|
||||
("LlamaForCausalLM", False),
|
||||
],
|
||||
)
|
||||
def test_registry_supports_hc_sp(model_arch, supported):
|
||||
"""HC SP is opt-in for each model implementation, including MTP."""
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
assert model_info.supports_hc_sp is supported
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch",
|
||||
|
||||
@@ -29,6 +29,42 @@ class TopKRouter(nn.Module):
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class SelectionScoredRouter(TopKRouter):
|
||||
"""Scores the selected logits only (GraniteMoE/Aria style), in fp32."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight).float()
|
||||
value, index = torch.topk(logits, self.top_k, dim=-1)
|
||||
value = (
|
||||
torch.sigmoid(value) if self.sigmoid else F.softmax(value, dim=-1)
|
||||
).type_as(hidden_states)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class EpsilonRenormRouter(TopKRouter):
|
||||
"""Renormalizes via an epsilon-offset denominator (DeepSeek-V3 style)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = torch.sigmoid(logits)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
denominator = value.sum(dim=-1, keepdim=True) + 1e-20
|
||||
value /= denominator
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class UntraceableRouter(TopKRouter):
|
||||
"""Scores and selects, then loses the trace before the renormalization."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
scores = F.softmax(logits, dim=-1)
|
||||
value, index = torch.topk(scores, self.top_k, dim=-1)
|
||||
if value.sum() > 0: # data-dependent branch: tracing stops here
|
||||
value = value / value.sum(dim=-1, keepdim=True)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ScaledRouter(TopKRouter):
|
||||
"""Greedy router scaling its top-k weights (DeepSeek `routed_scaling_factor`)."""
|
||||
|
||||
@@ -108,6 +144,16 @@ class DisconnectedRouter(TopKRouter):
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ExtraInputRouter(TopKRouter):
|
||||
"""Router taking a block-owned correction bias as a second input (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states, bias=0.0):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
|
||||
value = logits.sigmoid().gather(1, index)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class MoEExperts(nn.Module):
|
||||
"""Packed experts (3D weights); only its name (`experts`) matters here."""
|
||||
|
||||
@@ -130,17 +176,19 @@ class MoEBlock(nn.Module):
|
||||
|
||||
def _shared(self, x, logits):
|
||||
"""The term added to the experts' output (none for a plain block)."""
|
||||
return 0
|
||||
return None
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
logits, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights) + self._shared(x, logits)
|
||||
out = self.experts(x, index, weights)
|
||||
if (shared := self._shared(x, logits)) is not None:
|
||||
out = out + shared
|
||||
return out.reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class MoEBlockNoShared(MoEBlock):
|
||||
"""No shared-expert child but a gate-derived add -> trace skipped, still fuses."""
|
||||
"""No shared-expert child but a gate-derived add -> declined, not misread."""
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return logits.sum()
|
||||
@@ -257,6 +305,115 @@ class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
|
||||
self.extra = BufferScale()
|
||||
|
||||
|
||||
class MoEBlockExpertsKeywords(MoEBlock):
|
||||
"""Passes the experts' routing by keyword -> still fuses."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, weights=weights, index=index)
|
||||
|
||||
|
||||
class ScaledExperts(MoEExperts):
|
||||
def forward(self, hidden_states, index, weights, scale=1.0):
|
||||
return hidden_states * scale
|
||||
|
||||
|
||||
class MoEBlockExpertsExtraArg(MoEBlock):
|
||||
"""Passes the experts an extra argument the fused forward would drop."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = ScaledExperts()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights, scale=2.0)
|
||||
|
||||
|
||||
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
|
||||
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
_, weights, index = self.gate(hidden_states)
|
||||
x = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
out = self.experts(x, index, weights).view(*hidden_states.shape)
|
||||
return out + self.shared_expert(hidden_states)
|
||||
|
||||
|
||||
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
|
||||
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return super().forward(hidden_states) / 2
|
||||
|
||||
|
||||
class MoEBlockScaled(MoEBlockSharedNoGate):
|
||||
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
|
||||
"""An unrecognized router with a scaled experts' output."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
|
||||
"""An unrecognized router with a shared expert."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockGateExtraInput(MoEBlock):
|
||||
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(ExtraInputRouter)
|
||||
self.register_buffer("e_score_correction_bias", torch.zeros(8))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x, self.e_score_correction_bias)
|
||||
return self.experts(x, index, weights)
|
||||
|
||||
|
||||
class MoEBlockCastCombine(MoEBlockSharedNoGate):
|
||||
"""Combines the outputs in fp32 (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
|
||||
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fc1_latent_proj = nn.Identity()
|
||||
self.fc2_latent_proj = nn.Identity()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.fc2_latent_proj(
|
||||
self.experts(self.fc1_latent_proj(x), index, weights)
|
||||
)
|
||||
return out + self.shared_expert(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_detects_router(sigmoid):
|
||||
with torch.device("meta"):
|
||||
@@ -268,6 +425,19 @@ def test_moe_fuser_detects_router(sigmoid):
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_matches_router_scoring_the_selection(sigmoid):
|
||||
"""Scoring the selection rather than all logits still allows the fuser to match."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(lambda: SelectionScoredRouter(sigmoid=sigmoid))
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.gate_name == "gate"
|
||||
assert fuser.scoring_func == ("sigmoid" if sigmoid else "softmax")
|
||||
assert fuser.renormalize is (not sigmoid)
|
||||
assert fuser.router_dtype == torch.float32
|
||||
|
||||
|
||||
def test_moe_fuser_matches_scaled_router():
|
||||
"""Weight scaling after the top-k (DeepSeek style) does not break matching."""
|
||||
with torch.device("meta"):
|
||||
@@ -275,6 +445,27 @@ def test_moe_fuser_matches_scaled_router():
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_cls,expected",
|
||||
[
|
||||
(TopKRouter, True), # divides by the sum of the selection
|
||||
(EpsilonRenormRouter, True), # ... with an epsilon in the denominator
|
||||
(ScaledRouter, False), # scales the weights instead of renormalizing
|
||||
(UntraceableRouter, None), # trace stops before it could say
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_reads_renormalize_from_the_gate(router_cls, expected):
|
||||
"""The router's graph, not the config, decides whether vLLM renormalizes: it
|
||||
is the code being replaced, and it already bakes in `norm_topk_prob` because
|
||||
the flag is a Python bool that fx resolves at trace time. Absence of the
|
||||
division only means "no" on a complete trace."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(router_cls)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.renormalize is expected
|
||||
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
@@ -315,16 +506,6 @@ def test_moe_fuser_detects_shared_experts():
|
||||
assert fuser.shared_gate_name == "shared_expert_gate"
|
||||
|
||||
|
||||
def test_moe_fuser_skips_shared_detection_without_extra_children():
|
||||
"""With only experts and gate, shared-expert detection (and its block trace)
|
||||
is skipped, so a gate-derived add is not misread as a shared expert."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNoShared()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
def test_moe_fuser_shared_without_gate():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedNoGate()
|
||||
@@ -352,6 +533,10 @@ def test_moe_fuser_detects_non_glu_shared_expert():
|
||||
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
|
||||
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
|
||||
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
|
||||
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
|
||||
MoEBlockNoShared, # gate-derived term added to the experts' output
|
||||
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
|
||||
MoEBlockGateExtraInput, # gate input the fused forward drops
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_declines_unsupported(block_cls):
|
||||
@@ -374,3 +559,53 @@ def test_moe_fuser_router_requires_connected_dataflow():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(DisconnectedRouter)
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_unrecognized_router_moves_shared_expert():
|
||||
"""Without vLLM routing, the shared expert still moves to vLLM."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedBiasedRouter()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert fuser.scoring_func is None
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block_cls", "scoring_func"),
|
||||
[
|
||||
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
|
||||
(MoEBlock, "softmax"), # no shared output to divide
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
|
||||
assert fuser.scoring_func == scoring_func
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_needs_vllm_routing():
|
||||
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockScaledBiasedRouter()
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_matches_experts_called_by_keyword():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockExpertsKeywords()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_with_vllm_routing():
|
||||
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
|
||||
assert fuser.scoring_func == "softmax"
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
|
||||
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts")
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for skipping dynamic rope's device sync where it can never rescale."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import HunYuanDenseV1Config, LlamaConfig
|
||||
from transformers.models.hunyuan_v1_dense.modeling_hunyuan_v1_dense import (
|
||||
HunYuanDenseV1RotaryEmbedding,
|
||||
)
|
||||
from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding
|
||||
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
)
|
||||
|
||||
MAX_POSITION_EMBEDDINGS = 128
|
||||
CONFIG_KWARGS = dict(
|
||||
hidden_size=64,
|
||||
num_attention_heads=4,
|
||||
num_hidden_layers=1,
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
)
|
||||
DYNAMIC = {"rope_type": "dynamic", "factor": 4.0, "rope_theta": 1e4}
|
||||
|
||||
|
||||
def make_vllm_config(hf_config, max_model_len: int) -> SimpleNamespace:
|
||||
model_config = SimpleNamespace(hf_config=hf_config, max_model_len=max_model_len)
|
||||
return SimpleNamespace(model_config=model_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rope_parameters,max_model_len,compilable",
|
||||
[
|
||||
(DYNAMIC, MAX_POSITION_EMBEDDINGS, True),
|
||||
(DYNAMIC, MAX_POSITION_EMBEDDINGS + 1, False),
|
||||
# The longrope layer type still syncs
|
||||
(
|
||||
{"full_attention": DYNAMIC, "sliding_attention": {"rope_type": "longrope"}},
|
||||
MAX_POSITION_EMBEDDINGS,
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_can_enable_torch_compile(rope_parameters, max_model_len, compilable):
|
||||
"""Dynamic rope only blocks compilation when it can still rescale."""
|
||||
config = SimpleNamespace(
|
||||
max_position_embeddings=MAX_POSITION_EMBEDDINGS,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
config.get_text_config = lambda: config
|
||||
vllm_config = make_vllm_config(config, max_model_len)
|
||||
assert can_enable_torch_compile(vllm_config) is compilable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config,rotary_cls",
|
||||
[
|
||||
(LlamaConfig(**CONFIG_KWARGS, rope_parameters=DYNAMIC), LlamaRotaryEmbedding),
|
||||
# Hunyuan's dynamic init also applies `alpha`, which must be kept
|
||||
(
|
||||
HunYuanDenseV1Config(
|
||||
**CONFIG_KWARGS, head_dim=16, rope_parameters=DYNAMIC | {"alpha": 1e3}
|
||||
),
|
||||
HunYuanDenseV1RotaryEmbedding,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_disable_noop_dynamic_rope(monkeypatch, config, rotary_cls):
|
||||
"""The update no longer syncs and the frequencies are unchanged."""
|
||||
inputs = torch.zeros(1, 8, 64), torch.arange(8)[None, :]
|
||||
rotary_emb = rotary_cls(config)
|
||||
expected = rotary_emb(*inputs)
|
||||
|
||||
vllm_config = make_vllm_config(config, MAX_POSITION_EMBEDDINGS)
|
||||
disable_noop_dynamic_rope(rotary_emb, vllm_config)
|
||||
|
||||
def no_sync(*args, **kwargs):
|
||||
raise AssertionError("dynamic rope update still runs")
|
||||
|
||||
monkeypatch.setattr(torch, "max", no_sync)
|
||||
actual = rotary_emb(*inputs)
|
||||
assert all(torch.equal(e, a) for e, a in zip(expected, actual))
|
||||
Reference in New Issue
Block a user