Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-16 13:41:12 +09:00
517 changed files with 21831 additions and 6611 deletions
+34
View File
@@ -11,6 +11,7 @@ from vllm.compilation.passes.ir.lowering_pass import (
)
from vllm.config import get_current_vllm_config
from vllm.ir import ops
from vllm.model_executor.layers.activation import GeluAndMulSparse
from vllm.platforms import current_platform
from ...backend import TestBackend
@@ -34,6 +35,15 @@ class Model(nn.Module):
return x6 + 3.0
class SparseActivationModel(nn.Module):
def __init__(self):
super().__init__()
self.activation = GeluAndMulSparse(0.95, "tanh")
def forward(self, x):
return self.activation(x)
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
def test_lowering_rms_norm(rms_provider, default_vllm_config):
torch.set_default_device(current_platform.device_type)
@@ -67,3 +77,27 @@ def test_lowering_rms_norm(rms_provider, default_vllm_config):
torch.testing.assert_close(output_unlowered, output)
torch.testing.assert_close(output_unlowered, output2)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
torch.set_default_device("cuda")
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
backend = TestBackend(lowering_pass)
model = SparseActivationModel()
x = torch.randn(8, 32768, dtype=torch.bfloat16)
with (
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
ir.enable_torch_wrap(True),
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x)
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
assert selected == {"gelu_and_mul_sparse": "triton"}
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
x, 1.6448536269514722, "tanh"
)
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.fused_act_quant import (
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
@@ -45,7 +46,7 @@ from vllm.platforms import current_platform
# Mock linear layer for testing fusion paths that don't have real kernel support
class MockLinearForFusion(torch.nn.Module):
"""Mock linear layer that exposes input_quant_key for fusion testing."""
"""Mock linear layer that exposes an input key for fusion testing."""
def __init__(
self,
@@ -55,7 +56,7 @@ class MockLinearForFusion(torch.nn.Module):
input_global_scale_inv=None,
):
super().__init__()
self.input_quant_key = quant_key
self._input_quant_key = quant_key
if input_scale is not None:
self.input_scale = input_scale
if input_global_scale is not None:
@@ -118,7 +119,7 @@ def test_manual_fusion_fp8_static_with_linear(
# Enable fusion
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
if not hasattr(fp8_linear, "input_quant_key"):
if get_input_quant_key(fp8_linear) is None:
pytest.skip(
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
)
@@ -315,7 +316,7 @@ def test_manual_fusion_nvfp4_dynamic(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
"""Test that maybe_fused_act_quant falls back when no input_quant_key."""
"""Test that maybe_fused_act_quant falls back without an input key."""
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
@@ -327,7 +328,7 @@ def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Linear without input_quant_key attribute
# Linear without an input quantization key
mock_linear = torch.nn.Linear(128, 128)
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
+65
View File
@@ -13,6 +13,8 @@ import pytest
import torch
import vllm.envs as envs
import vllm.kernels # noqa: F401 to register kernels
from vllm import ir
from vllm.compilation.backends import VllmBackend
from vllm.compilation.caching import (
StandaloneCompiledArtifacts,
@@ -28,6 +30,9 @@ from vllm.config import (
)
from vllm.envs import disable_envs_cache
from vllm.forward_context import set_forward_context
from vllm.ir import ops
from vllm.model_executor.layers.activation import GeluAndMulSparse
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from ..utils import create_new_process_for_each_test
@@ -77,6 +82,18 @@ class CompiledModTuple(torch.nn.Module):
return reference_fn_tuple(x)
@support_torch_compile(dynamic_arg_dims={"x": 0})
class SparseActivationAotModel(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
self.up_proj = torch.nn.Linear(32, 256, bias=False)
self.activation = GeluAndMulSparse(0.95, "tanh")
self.down_proj = torch.nn.Linear(128, 16, bias=False)
def forward(self, x: torch.Tensor):
return self.down_proj(self.activation(self.up_proj(x)))
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
@@ -175,6 +192,54 @@ def test_save_and_load(monkeypatch: pytest.MonkeyPatch):
assert torch.allclose(ret, expected)
@pytest.mark.skipif(
not current_platform.is_cuda() or not is_torch_equal_or_newer("2.10.0"),
reason="requires CUDA and torch 2.10",
)
def test_sparse_activation_standalone_aot_preserves_dependencies(
monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path
):
def native_reference(model: SparseActivationAotModel, x: torch.Tensor):
projected = torch.nn.functional.linear(x, model.up_proj.weight)
activated = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
projected, model.activation.std_multiplier, "tanh"
)
return torch.nn.functional.linear(activated, model.down_proj.weight)
with monkeypatch.context() as m:
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
m.setenv("VLLM_CACHE_ROOT", str(vllm_tmp_cache / "vllm_cache"))
vllm_config = make_vllm_config()
torch.manual_seed(0)
inputs = [
torch.randn(4, 32, device="cuda", dtype=torch.bfloat16),
torch.randn(7, 32, device="cuda", dtype=torch.bfloat16),
]
with (
use_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
ir.enable_torch_wrap(True),
):
compiled = SparseActivationAotModel(vllm_config=vllm_config).to(
device="cuda", dtype=torch.bfloat16
)
compiled.requires_grad_(False)
expected = [native_reference(compiled, x) for x in inputs]
actual = [compiled(x) for x in inputs]
for result, reference in zip(actual, expected):
assert torch.isfinite(result).all()
torch.testing.assert_close(
result,
reference,
**ops.gelu_and_mul_sparse.get_tolerance(result.dtype),
)
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
from torch._subclasses import FakeTensorMode
+10 -7
View File
@@ -39,9 +39,9 @@ from vllm.distributed.parallel_state import (
initialize_model_parallel,
)
from vllm.forward_context import set_forward_context
from vllm.models.deepseek_v4_1.common.engram import EngramLayout
from vllm.models.deepseek_v4_1.nvidia import engram as engram_ops
from vllm.models.deepseek_v4_1.nvidia.engram import (
from vllm.models.deepseek_v41.common.engram import EngramLayout
from vllm.models.deepseek_v41.nvidia import engram as engram_ops
from vllm.models.deepseek_v41.nvidia.engram import (
Engram,
ParallelEngramEmbedding,
engram_head_shard_rank,
@@ -473,8 +473,8 @@ def _check_dummy_hash_model_forward(
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
):
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
from vllm.models.deepseek_v4_1.common.engram import NgramHashState
from vllm.models.deepseek_v4_1.nvidia import model as model_ops
from vllm.models.deepseek_v41.common.engram import NgramHashState
from vllm.models.deepseek_v41.nvidia import model as model_ops
dp_rank = vllm_config.parallel_config.data_parallel_rank
if dp_shared_memory and dp_rank != 1:
@@ -491,7 +491,9 @@ def _check_dummy_hash_model_forward(
state.forward = lambda *args: ids.unsqueeze(1)
class Decoder(SimpleNamespace):
def __call__(self, hidden, positions, input_ids, *args):
# Engram hashes and the mask stay the last positional arguments;
# **kwargs absorbs the decoder's keyword-only flags.
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
hashes, keep = args[-2:]
if dp_shared_memory and dp_rank == 1:
assert hashes is None and keep is None
@@ -504,7 +506,8 @@ def _check_dummy_hash_model_forward(
if dp_rank == 1:
assert torch.all(hashes == engram_ops.DEAD_ID)
output = engram.embed(hashes[:, 0])
return output, None, None, None, None
# Trailing None is previous_aux; this stub captures no aux states.
return output, None, None, None, None, None
model = SimpleNamespace(
use_mega_moe=False,
@@ -20,17 +20,23 @@ covered by the GPU tests in `test_sharded_rdt_trainer.py`.
import contextlib
import gc
import os
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
import vllm.distributed.weight_transfer.sharded_rdt_trainer as trainer_mod
from vllm.distributed.nixl_utils import alias_nixl_for_ray
from vllm.distributed.weight_transfer.sharded_rdt_common import (
buffer_alloc_bytes,
register_nixl_memory,
)
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
DEFAULT_GATHER_LOOKAHEAD,
@@ -45,6 +51,114 @@ GROUP_A = ("model.layers.0.w", "model.layers.0.b")
GROUP_B = ("model.layers.1.w",)
@pytest.fixture
def ray_nixl_transport(monkeypatch):
import ray
from ray.experimental.rdt import util
from ray.experimental.rdt.nixl_tensor_transport import NixlTensorTransport
from vllm.distributed import nixl_utils
from vllm.platforms import current_platform
transport = NixlTensorTransport()
package = ModuleType("nixl_rocm")
api = ModuleType("nixl_rocm._api")
api.__dict__.update(nixl_agent=Mock(), nixl_agent_config=Mock())
package.__dict__["_api"] = api
modules = {package.__name__: package, api.__name__: api}
imported_rcache_limits = []
def import_nixl(name):
imported_rcache_limits.append(os.environ.get("UCX_RCACHE_MAX_UNRELEASED"))
monkeypatch.setitem(sys.modules, name, modules[name])
return modules[name]
nixl_names = ("nixl", "nixl._api", *modules)
for name in nixl_names:
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.delenv("UCX_RCACHE_MAX_UNRELEASED", raising=False)
importer = Mock(side_effect=import_nixl)
monkeypatch.setattr(
nixl_utils, "importlib", SimpleNamespace(import_module=importer)
)
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: True)
monkeypatch.setattr(util, "transport_managers", {"NIXL": transport})
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr(
ray,
"get_runtime_context",
lambda: SimpleNamespace(get_actor_id=lambda: "rdt-actor"),
)
try:
yield transport, api, importer, imported_rcache_limits
finally:
# The helper inserts aliases directly; remove them before monkeypatch
# restores any modules and environment values present before this test.
for name in nixl_names:
sys.modules.pop(name, None)
os.environ.pop("UCX_RCACHE_MAX_UNRELEASED", None)
def test_ray_registers_memory_through_rocm_alias(ray_nixl_transport):
"""Ray's registration must reach the ROCm agent through the alias."""
_, api, importer, _ = ray_nixl_transport
register_nixl_memory(torch.empty(4, dtype=torch.uint8))
api.nixl_agent.return_value.register_memory.assert_called()
assert sys.modules["nixl"] is sys.modules["nixl_rocm"]
assert sys.modules["nixl._api"] is api
importer.reset_mock()
alias_nixl_for_ray()
importer.assert_not_called()
@pytest.mark.parametrize("rcache_limit", [None, "4096"])
def test_ray_nixl_sets_ucx_limit_before_import(
monkeypatch, ray_nixl_transport, rcache_limit
):
"""UCX must see the default or the user's override during the first import."""
_, _, _, imported_rcache_limits = ray_nixl_transport
if rcache_limit is not None:
monkeypatch.setenv("UCX_RCACHE_MAX_UNRELEASED", rcache_limit)
alias_nixl_for_ray()
assert imported_rcache_limits == [rcache_limit or "1024"] * 2
@pytest.mark.parametrize("existing_api", [False, True])
def test_ray_nixl_preserves_non_rocm_or_existing_imports(
monkeypatch, ray_nixl_transport, existing_api
):
from vllm.platforms import current_platform
_, _, importer, _ = ray_nixl_transport
if existing_api:
api = ModuleType("nixl._api")
monkeypatch.setitem(sys.modules, "nixl._api", api)
else:
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
alias_nixl_for_ray()
importer.assert_not_called()
assert "UCX_RCACHE_MAX_UNRELEASED" not in os.environ
if existing_api:
assert sys.modules["nixl._api"] is api
else:
assert "nixl._api" not in sys.modules
def test_ray_nixl_alias_reports_missing_rocm_package(monkeypatch, ray_nixl_transport):
from vllm.distributed import nixl_utils
transport, api, importer, _ = ray_nixl_transport
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: False)
with pytest.raises(ImportError, match="nixl_rocm"):
register_nixl_memory(torch.empty(1))
assert transport._nixl_agent is None
api.nixl_agent.assert_not_called()
importer.assert_not_called()
assert "nixl" not in sys.modules
assert "nixl._api" not in sys.modules
@pytest.fixture
def server_factory(monkeypatch):
"""Build real `_RDTProducerServer` instances without a GPU.
+42
View File
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the ``vllm serve`` CLI subcommand."""
import argparse
from unittest.mock import MagicMock, patch
import pytest
from vllm.entrypoints.cli.serve import run_headless
class _StopAfterEngineConfig(Exception):
pass
def test_headless_imports_reasoning_parser_plugin_before_engine_config():
plugin_path = "/tmp/custom_reasoning_parser.py"
args = argparse.Namespace(
api_server_count=0,
reasoning_parser_plugin=plugin_path,
)
engine_args = MagicMock()
with (
patch(
"vllm.entrypoints.cli.serve.vllm.AsyncEngineArgs.from_cli_args",
return_value=engine_args,
),
patch(
"vllm.reasoning.ReasoningParserManager.import_reasoning_parser"
) as import_plugin,
):
def create_engine_config(**kwargs):
import_plugin.assert_called_once_with(plugin_path)
raise _StopAfterEngineConfig
engine_args.create_engine_config.side_effect = create_engine_config
with pytest.raises(_StopAfterEngineConfig):
run_headless(args)
+13 -10
View File
@@ -58,7 +58,6 @@ def _enable_cohere_api(monkeypatch):
flag inside the test body.
"""
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
# ----------------------------------------------------------------------
@@ -128,15 +127,19 @@ def _generate_request() -> GenerateRequest:
)
def _build_app(handler: _Handler | None) -> FastAPI:
def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
return app
def _build_render_app(chat_handler, render_handler) -> FastAPI:
def _build_render_app(
chat_handler, render_handler, *, enable_scale_out: bool = True
) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
attach_router(app)
app.state.cohere_serving_chat_v2 = chat_handler
app.state.serving_render = render_handler
@@ -151,11 +154,13 @@ def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
into the ``CohereError`` wire shape.
"""
app = FastAPI()
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
# ``validation_exception_handler`` reads ``req.app.state.args``; the
# real cli builds this via argparse.
app.state.args = Namespace(log_error_stack=False)
app.state.args = Namespace(
log_error_stack=False, enable_scale_out=True, tokens_only=False
)
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
app.exception_handler(RequestValidationError)(validation_exception_handler)
app.exception_handler(HTTPException)(http_exception_handler)
app.exception_handler(VLLMError)(vllm_error_handler)
@@ -402,10 +407,8 @@ class TestRenderEndpoint:
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" in paths
def test_route_not_registered_when_flag_unset(self, monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
app = FastAPI()
attach_router(app)
def test_route_not_registered_when_flag_unset(self):
app = _build_app(handler=None, enable_scale_out=False)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" not in paths
assert "/cohere/v2/chat" in paths
+2 -1
View File
@@ -57,6 +57,8 @@ def server():
# the conversation to surface a thinking block (SmolLM2 doesn't
# emit Cohere-style reasoning tokens out of the box).
"--no-cohere-is-reasoning-model",
# Registers /cohere/v2/chat/render alongside /cohere/v2/chat.
"--enable-scale-out",
]
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
@@ -71,7 +73,6 @@ def server():
env_dict = {
"VLLM_CPU_KVCACHE_SPACE": "1",
"VLLM_ENABLE_COHERE_API": "1",
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
@@ -5,6 +5,7 @@ import multiprocessing
import socket
import threading
import time
from types import SimpleNamespace
from unittest.mock import patch
import pytest
@@ -14,6 +15,7 @@ from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
exit_before_report_worker,
)
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.utils import (
APIServerProcessManager,
get_engine_client_zmq_addr,
@@ -32,6 +34,18 @@ def mock_run_api_server_worker(listen_address, sock, args, client_config=None):
print("Mock worker completed successfully")
def update_admission_stats_worker(listen_address, sock, args, client_config):
stats = SharedAdmissionStats(
client_config,
client_config["client_count"],
client_config["client_index"],
)
value = client_config["client_index"] + 1
stats.set_num_requests(value)
args.barrier.wait()
args.results.put(stats.get_num_requests())
# Module-level stub for the gather_actual_addresses test. Must be
# importable by `multiprocessing.spawn` (no closures, no nesting).
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
@@ -131,6 +145,31 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
assert not proc.is_alive()
@pytest.mark.skip_global_cleanup
def test_api_server_processes_share_admission_stats(api_server_args):
args = api_server_args.copy()
spawn_context = multiprocessing.get_context("spawn")
worker_args = SimpleNamespace(
barrier=spawn_context.Barrier(args["num_servers"]),
results=spawn_context.Queue(),
max_num_queued_reqs=10,
)
args["args"] = worker_args
args["target_server_fn"] = update_admission_stats_worker
manager = APIServerProcessManager(**args)
try:
for process in manager.processes:
process.join(timeout=30)
assert process.exitcode == 0
assert [worker_args.results.get(timeout=5) for _ in range(3)] == [6] * 3
finally:
manager.shutdown()
worker_args.results.close()
worker_args.results.join_thread()
@patch("vllm.v1.utils.run_api_server_worker_proc", mock_run_api_server_worker)
def test_wait_for_completion_or_failure(api_server_args):
"""Test that wait_for_completion_or_failure works with failures."""
@@ -34,6 +34,7 @@ ARGS = [
"hermes",
"--reasoning-parser",
"deepseek_r1",
"--enable-scale-out",
]
TOOLS = [
@@ -54,11 +55,7 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
@pytest.fixture(scope="module")
def server():
with RemoteOpenAIServer(
MODEL,
ARGS,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
yield remote_server
@@ -181,8 +181,9 @@ run_scale_out_ec() {
# Encode-only EC producer instance. It runs the vision encoder and
# publishes embeddings through the ECExampleConnector shared storage.
echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_E" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
env CUDA_VISIBLE_DEVICES="$GPU_E" \
vllm serve "$MODEL" \
--enable-scale-out \
--port "$ENCODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
@@ -204,8 +205,9 @@ run_scale_out_ec() {
# features plus ec_transfer_params and loads the embeddings that the
# encode instance published.
echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_PD" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
env CUDA_VISIBLE_DEVICES="$GPU_PD" \
vllm serve "$MODEL" \
--enable-scale-out \
--port "$PREFILL_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
@@ -27,15 +27,10 @@ def vision_server():
"1",
"--limit-mm-per-prompt.video",
"0",
"--enable-scale-out",
]
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
with RemoteOpenAIServer(
VISION_MODEL_NAME,
args,
env_dict=env_overrides,
) as remote_server:
with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server:
yield remote_server
+16 -37
View File
@@ -3,7 +3,6 @@
from argparse import Namespace
import pytest
from fastapi import FastAPI
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
@@ -22,69 +21,49 @@ SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
def registered_paths(
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
supported_tasks: tuple[SupportedTask, ...],
*,
tokens_only: bool = False,
enable_scale_out: bool = False,
) -> set[str]:
app = FastAPI()
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
args = Namespace(
tokens_only=tokens_only,
enable_scale_out=enable_scale_out,
enable_fault_tolerance=False,
)
app.state.args = args
register_api_routers(args, app, supported_tasks)
return {route.path for route in app.routes}
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_scale_out_routes_are_disabled_by_default():
paths = registered_paths(("generate",))
assert SCALE_OUT_PATHS.isdisjoint(paths)
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_disabled_scale_out_routes_are_logged(caplog):
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert any(
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
for record in caplog.records
)
assert any("--enable-scale-out" in record.message for record in caplog.records)
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
paths = registered_paths(("generate",))
def test_scale_out_routes_can_be_enabled_for_generate_server():
paths = registered_paths(("generate",), enable_scale_out=True)
assert paths >= SCALE_OUT_PATHS
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_render_routes_remain_enabled_for_render_server():
paths = registered_paths(("render",))
assert paths >= RENDER_PATHS
assert GENERATE_PATH not in paths
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
registered_paths(("render",))
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, "/abort_requests"}
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
with pytest.raises(ValueError, match="--tokens-only"):
registered_paths(("generate",), tokens_only=True)
@@ -34,12 +34,9 @@ def server():
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
"--enable-scale-out",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -37,13 +37,10 @@ def server():
"4096",
"--enforce-eager",
"--no-enable-prefix-caching",
"--enable-scale-out",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -69,6 +69,7 @@ def server(request):
# Option B: deterministic prefix caching
# "--enable-prefix-caching",
# "--deterministic-prefix-caching",
"--enable-scale-out",
]
extra_args = getattr(request, "param", None)
@@ -79,11 +80,7 @@ def server(request):
else [str(extra_args)]
)
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -0,0 +1,20 @@
model_name: "nvidia/GLM-5.2-NVFP4"
accuracy_threshold: 0.90
tolerance: 0.0
num_questions: 1319
num_fewshot: 5
max_tokens: 256
max_concurrency: 32
request_timeout_seconds: 1800
startup_max_wait_seconds: 1200
server_args: >-
--tensor-parallel-size 4
--max-model-len 8192
--max-num-seqs 32
--max-num-batched-tokens 4096
--enforce-eager
--kv-cache-dtype bfloat16
--attention-config '{"hisparse_config":{}}'
--kv-transfer-config '{"kv_connector":"HiSparseConnector","kv_role":"kv_both","kv_connector_extra_config":{"host_pool_gib":32}}'
env:
VLLM_USE_V2_MODEL_RUNNER: "1"
@@ -3,14 +3,18 @@ accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
max_concurrency: 100
# Preserve KV-cache headroom after CUDA graph memory profiling and bound the
# DCP-aware FlashInfer workspace to 2.39 GiB at the 4,096-token scheduler cap.
server_args: >-
--compilation-config '{"cudagraph_mode":"PIECEWISE"}'
--gpu-memory-utilization 0.96
--max-model-len 4096
--max-num-batched-tokens 32768
--max-num-batched-tokens 4096
--safetensors-load-strategy prefetch
--moe-backend flashinfer_cutlass
--prefill-context-parallel-size 4
--decode-context-parallel-size 4
--dcp-comm-backend ag_rs
--enable-expert-parallel
--kv-cache-dtype fp8
env:
@@ -3,8 +3,10 @@ accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
max_concurrency: 100
# Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation.
server_args: >-
--compilation-config '{"cudagraph_mode":"NONE"}'
--gpu-memory-utilization 0.85
--max-model-len 4096
--max-num-batched-tokens 32768
--safetensors-load-strategy prefetch
@@ -0,0 +1,22 @@
model_name: "Qwen/Qwen3.5-4B"
# full GSM8K set is 1319 questions
num_questions: 1319
num_fewshot: 5
max_concurrency: 128
max_tokens: 1024
use_chat_completions: true
temperature: 0.7
seed: 42
# Calibrated by running non-watermaked 13 times
# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations)
accuracy_threshold: 0.885
tolerance: 0.015
server_args: >-
--language-model-only
--max-model-len 8192
--max-num-seqs 128
--gpu-memory-utilization 0.4
--default-chat-template-kwargs '{"enable_thinking": false}'
--watermark-config '{"key": 42, "context_width": 4}'
@@ -0,0 +1 @@
GLM-5.2-NVFP4-HiSparse.yaml
@@ -0,0 +1 @@
Qwen3.5-4B-watermark.yaml
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
as_quantized_activation,
expose_input_quant_key,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
@@ -118,13 +119,13 @@ def test_bridge_marks_supporting_and_skips_others():
supported = _probe(FlashInferCutlassNvFp4LinearKernel)
layer = torch.nn.Module()
expose_input_quant_key(layer, supported)
assert layer.input_quant_key == kNvfp4Dynamic
assert get_input_quant_key(layer) == kNvfp4Dynamic
unsupported = _probe(FlashInferTrtllmNvFp4LinearKernel)
assert unsupported.input_quant_key() is None
layer = torch.nn.Module()
expose_input_quant_key(layer, unsupported)
assert not hasattr(layer, "input_quant_key")
assert get_input_quant_key(layer) is None
def test_as_quantized_activation_validates_key():
@@ -0,0 +1,226 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm correctness tests for the strided DSA candidate mask.
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
``apply_candidate_mask``. It differs in two ways that need pinning down:
* it launches a fixed number of programs that stride the column axis, rather
than one program per 1024-column tile, so at ``max_model_len`` widths the
grid no longer scales with a workspace that is mostly padding;
* it stops at each row's ``end`` instead of sanitizing the whole width.
The second is only sound because every consumer bounds its scan by the same
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
contract is agreement with the shared kernel over ``[0, end)`` and nothing
beyond it, which is what these tests assert rather than comparing whole rows.
"""
import pytest
import torch
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
BLOCK_SIZE = 8
TOPK_BLOCKS = 64
# Above 128 tiles the grid saturates and programs start striding; below it the
# min() clamp applies instead. Both regimes need covering, so the widths here
# deliberately straddle 128 * 1024.
NARROW = 8192
WIDE = 262144
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
"""Logits, bounds and candidate blocks shared by both kernels."""
torch.manual_seed(0)
# A column stride > 1 keeps the kernels honest about stride_col; the decode
# caller slices a workspace, so contiguous rows are not guaranteed.
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
logits[0, : min(16, width)] = 0.0
if rows > 2:
logits[2, 5] = float("nan")
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
nblocks = (width + block_size - 1) // block_size
candidates = torch.randint(
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
)
# -1 is the "no block" sentinel the selector emits for short rows.
candidates[:, ::7] = -1
return logits, row_ks, row_ke, candidates
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_apply_candidate_mask_strided,
)
ref = logits.clone()
got = logits.clone()
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
_apply_candidate_mask_strided(
got, row_ks, row_ke, candidates, block_size, row_repeat
)
return ref, got
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
"""Compare only the region a consumer can read."""
for row in range(ref.shape[0]):
end = int(row_ke[row // row_repeat])
if end <= 0:
continue
torch.testing.assert_close(
got[row, :end],
ref[row, :end],
rtol=0,
atol=0,
equal_nan=True,
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
)
@pytest.mark.parametrize("width", [NARROW, WIDE])
@pytest.mark.parametrize("col_stride", [1, 2])
def test_matches_shared_kernel_ragged_ends(width, col_stride):
"""Ends on and around tile boundaries, plus the empty and full rows."""
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
width,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=col_stride,
dtype=torch.int32,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
_assert_agrees_within_ends(ref, got, row_ke, 1)
@pytest.mark.parametrize("block_size", [8, 16])
def test_matches_shared_kernel_with_starts(block_size):
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
ends = [0, 64, 4096, 8192]
starts = [0, 7, 1024, 3]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
NARROW,
ends,
starts,
block_size=block_size,
col_stride=1,
dtype=torch.int64,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
_assert_agrees_within_ends(ref, got, row_ke, 1)
def test_matches_shared_kernel_row_repeat():
"""Speculative decode: next_n logit rows share one bound."""
row_repeat = 3
ends = [0, 1025, WIDE]
logits, row_ks, row_ke, candidates = _inputs(
len(ends) * row_repeat,
WIDE,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
def test_leaves_columns_past_end_untouched():
"""The deliberate difference from the shared kernel, pinned explicitly.
Skipping the tail is where the speedup comes from, so a change that
quietly restores full-width sanitizing should fail here rather than just
get slower. Work stops at the tile boundary containing ``end``, not at
``end`` itself, so that is the bound asserted.
"""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_MASK_TILE,
_apply_candidate_mask_strided,
)
ends = [0, 1, 1025, WIDE // 2]
logits, row_ks, row_ke, candidates = _inputs(
len(ends),
WIDE,
ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
sentinel = 1.5
logits.fill_(sentinel)
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
for row, end in enumerate(ends):
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
tail = logits[row, touched:]
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
f"row {row} (end {end}) was written past column {touched}"
)
# And the bound is tight: a row that ends early must not have cost a
# full-width pass, which is the whole point of the kernel.
assert touched <= end + _MASK_TILE
def test_cudagraph_replay_tracks_changing_ends():
"""The grid is static so a FULL capture stays valid when ends change.
Capturing bakes the grid, so a data-dependent program count would freeze
whatever lengths were resident at capture. Replaying against longer and
shorter contexts than the captured ones is the property that matters.
"""
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_apply_candidate_mask_strided,
)
rows = 4
captured_ends = [1024, 4096, 8192, 2048]
logits, row_ks, row_ke, candidates = _inputs(
rows,
WIDE,
captured_ends,
None,
block_size=BLOCK_SIZE,
col_stride=1,
dtype=torch.int32,
)
base = logits.clone()
# Warm up outside the capture: Triton compiles on first call, and autotune
# launches would otherwise happen inside the graph.
_apply_candidate_mask_strided(
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
logits.copy_(base)
graph.replay()
torch.accelerator.synchronize()
ref = base.clone()
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
_assert_agrees_within_ends(ref, logits, row_ke, 1)
@@ -30,7 +30,10 @@ def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
def _install_fake_aiter_modules(
monkeypatch: pytest.MonkeyPatch, *, supports_fp8: bool
monkeypatch: pytest.MonkeyPatch,
*,
supports_fp8: bool,
supports_causal: bool = False,
) -> None:
aiter_mod: Any = types.ModuleType("aiter")
mla_mod: Any = types.ModuleType("aiter.mla")
@@ -65,6 +68,14 @@ def _install_fake_aiter_modules(
mla_decode_fwd = mla_decode_fwd_without_fp8
if supports_causal:
inner = mla_decode_fwd
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
return inner(*args, **kwargs)
mla_decode_fwd = mla_decode_fwd_with_causal
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod.mla = mla_mod
@@ -135,3 +146,107 @@ def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
assert _check_aiter_mla_fp8_support() is True
assert _check_aiter_mla_fp8_support() is True
assert signature_mock.call_count == 1
def test_missing_causal_arg_fails_closed(monkeypatch):
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
with pytest.raises(RuntimeError, match="causal-only"):
rocm_aiter_ops.mla_decode_supports_non_causal()
with pytest.raises(RuntimeError, match="causal-only"):
AiterMLABackend.supports_non_causal()
@pytest.mark.parametrize("has_kernels", [False, True])
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
"""The Python argument alone does not guarantee an arch has the kernel."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_non_causal_asm_kernels",
lambda: has_kernels,
)
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
assert AiterMLABackend.supports_non_causal() is has_kernels
@pytest.mark.parametrize("causal", [True, False])
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
"""The decode op must pass the block mask through; omitting it is causal-only."""
seen: list = []
def mla_decode_fwd(*args, causal=None, **kwargs):
seen.append(causal)
mla_mod: Any = types.ModuleType("aiter.mla")
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod: Any = types.ModuleType("aiter")
aiter_mod.mla = mla_mod
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
import torch
import vllm._aiter_ops as aiter_ops
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
torch.zeros(2, 1, 8),
torch.zeros(2, 1, 1, 8),
torch.zeros(2, 1, 8),
torch.zeros(3, dtype=torch.int32),
1,
causal=causal,
)
assert seen == [causal]
def _aiter_mla_validate(
monkeypatch,
*,
dtype,
kv_cache_dtype,
use_non_causal,
):
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
)
return AiterMLABackend.validate_configuration(
head_size=576,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
block_size=1,
use_mla=True,
has_sink=False,
use_sparse=False,
use_mm_prefix=False,
use_per_head_quant_scales=False,
device_capability=DeviceCapability(9, 5),
attn_type=AttentionType.DECODER,
use_non_causal=use_non_causal,
)
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
import torch
reasons = _aiter_mla_validate(
monkeypatch,
dtype=torch.float16,
kv_cache_dtype="auto",
use_non_causal=True,
)
assert any("non-causal fp16" in reason for reason in reasons)
@@ -125,7 +125,7 @@ def test_fused_q_kv_rmsnorm_quant_matches_separate(num_tokens, q_size, dtype):
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.models.deepseek_v4_1.common.ops.query_quant import (
from vllm.models.deepseek_v41.common.ops.query_quant import (
fused_q_kv_rmsnorm_quant,
)
@@ -198,15 +198,15 @@ def test_shared_query_quant_preserves_projection(
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
from vllm.models.deepseek_v4_1.attention import (
from vllm.models.deepseek_v41.attention import (
DeepseekV4Attention,
DeepseekV4Indexer,
)
from vllm.models.deepseek_v4_1.common.ops.query_quant import (
from vllm.models.deepseek_v41.common.ops.query_quant import (
can_fuse_query_quant,
fused_q_kv_rmsnorm_quant,
)
from vllm.models.deepseek_v4_1.quant_config import DeepseekV4FP8Config
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
cls: (
type[FlashInferCutedslMxfp8LinearKernel]
+183
View File
@@ -0,0 +1,183 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.kernels # noqa: F401
from tests.ir.ir_test_utils import assert_close, clone_args
from vllm import ir
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
gelu_and_mul_sparse_native = ir.ops.gelu_and_mul_sparse.impls["native"].impl_fn
def test_gelu_and_mul_sparse_registration():
expected = {
"native": True,
"triton": HAS_TRITON and current_platform.is_cuda(),
}
actual = {
provider: impl.supported
for provider, impl in ir.ops.gelu_and_mul_sparse.impls.items()
}
assert actual == expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("num_tokens", [1, 8, 64])
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 8192, 13824, 16384, 32768])
def test_gelu_and_mul_sparse_triton(
dtype: torch.dtype, num_tokens: int, intermediate_size: int
):
torch.manual_seed(0)
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=num_tokens,
hidden_size=intermediate_size,
dtype=dtype,
)
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
assert impl.supports_args(*args)
expected = gelu_and_mul_sparse_native(*clone_args(args))
actual = impl.impl_fn(*clone_args(args))
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
mask_mismatch = torch.count_nonzero((actual == 0) != (expected == 0))
assert mask_mismatch / actual.numel() < 1e-3
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_second_seed():
torch.manual_seed(1)
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=32,
hidden_size=8192,
dtype=torch.bfloat16,
)
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
expected = gelu_and_mul_sparse_native(*clone_args(args))
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("intermediate_size", [1, 31, 128])
def test_gelu_and_mul_sparse_triton_small_sizes(
dtype: torch.dtype, intermediate_size: int
):
torch.manual_seed(0)
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=5,
hidden_size=intermediate_size,
dtype=dtype,
)
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
expected = gelu_and_mul_sparse_native(*clone_args(args))
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
assert torch.equal(actual == 0, expected == 0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_nd_and_empty():
torch.set_default_device("cuda")
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
x = torch.randn(2, 3, 16384, dtype=torch.bfloat16)
args = (x, 1.6448536269514722, "tanh")
actual = impl.impl_fn(*args)
expected = gelu_and_mul_sparse_native(*args)
assert actual.shape == (2, 3, 8192)
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
empty = torch.empty(0, 16384, dtype=torch.bfloat16)
empty_out = impl.impl_fn(empty, 1.6448536269514722, "tanh")
assert empty_out.shape == (0, 8192)
assert empty_out.dtype == empty.dtype
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
def test_gelu_and_mul_sparse_triton_constant_rows(dtype: torch.dtype):
torch.set_default_device("cuda")
gate = torch.full((4, 8192), 3.0, dtype=dtype)
up = torch.randn_like(gate)
x = torch.cat((gate, up), dim=-1)
args = (x, 1.6448536269514722, "tanh")
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
expected = gelu_and_mul_sparse_native(*args)
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
assert torch.count_nonzero(actual) == 0
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("nonfinite", [float("nan"), float("inf")])
def test_gelu_and_mul_sparse_triton_nonfinite_gate(nonfinite: float):
torch.set_default_device("cuda")
x = torch.randn(2, 16384, dtype=torch.bfloat16)
x[0, 0] = nonfinite
args = (x, 1.6448536269514722, "tanh")
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
expected = gelu_and_mul_sparse_native(*args)
torch.testing.assert_close(actual, expected, equal_nan=True)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_fallback_conditions():
torch.set_default_device("cuda")
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
x = torch.randn(4, 16384, dtype=torch.bfloat16)
assert impl.supports_args(x, 1.0, "tanh")
assert not impl.supports_args(x, 1.0, "none")
assert not impl.supports_args(x[:, ::2], 1.0, "tanh")
assert not impl.supports_args(x[:, :-1], 1.0, "tanh")
assert not impl.supports_args(
torch.randn(1, 65538, dtype=torch.bfloat16), 1.0, "tanh"
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_dispatch_and_opcheck():
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=8,
hidden_size=4096,
dtype=torch.bfloat16,
)
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
with ir.ops.gelu_and_mul_sparse.set_priority(["triton", "native"]):
dispatched = ir.ops.gelu_and_mul_sparse(*args)
torch.library.opcheck(torch.ops.vllm_ir.gelu_and_mul_sparse, args)
direct = impl.impl_fn(*args)
torch.testing.assert_close(dispatched, direct, rtol=0.0, atol=0.0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_cudagraph():
torch.set_default_device("cuda")
x = torch.randn(16, 32768, dtype=torch.bfloat16)
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
for _ in range(3):
expected = impl.impl_fn(x, 1.6448536269514722, "tanh")
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = impl.impl_fn(x, 1.6448536269514722, "tanh")
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Contract tests for DeepEPV2PrepareAndFinalize.finalize_async.
Hermetic: fake buffer/event, single process, no GPU collectives — only
requires deep_ep to be importable.
T1 finalize_async -> combine is issued with async_with_compute_stream=True
and the join is deferred to the receiver, which waits
on the combine event (device-side) before the copy.
T2 DBO active -> combine falls back to async_with_compute_stream=False;
the receiver still produces the correct output.
T3 finalize (sync) -> combine is issued synchronously and the output is
copied immediately.
"""
import pytest
import torch
from vllm.utils.import_utils import has_deep_ep_v2
requires_deep_ep_v2 = pytest.mark.skipif(
not has_deep_ep_v2(),
reason="Requires DeepEP v2 (ElasticBuffer)",
)
if has_deep_ep_v2():
import vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 as _dv2
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
DeepEPV2PrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceContiguous,
)
class _FakeEvent:
def __init__(self, has_event: bool):
self.event = object() if has_event else None
self.waited = False
def current_stream_wait(self):
assert self.event is not None
self.waited = True
class _FakeBuffer:
def __init__(self, out: torch.Tensor):
self.out = out
self.calls: list[dict] = []
self.last_event: _FakeEvent | None = None
def combine(
self,
x,
handle,
topk_weights,
async_with_compute_stream,
allocate_on_comm_stream,
):
self.calls.append({"async_with_compute_stream": async_with_compute_stream})
# DeepEP returns a completion event only for an async combine.
self.last_event = _FakeEvent(has_event=async_with_compute_stream)
return self.out, None, self.last_event
def _make_pf(out: torch.Tensor):
pf = DeepEPV2PrepareAndFinalize(
buffer=_FakeBuffer(out),
num_dispatchers=1,
dp_size=1,
rank_expert_offset=0,
num_experts=8,
num_topk=2,
)
pf.handles[0] = object()
return pf
def _run(pf, output: torch.Tensor, do_async: bool):
empty = torch.empty(0, dtype=torch.bfloat16)
weights = torch.empty(0, 2)
ids = torch.empty(0, 2, dtype=torch.int64)
fn = pf.finalize_async if do_async else pf.finalize
return fn(output, empty, weights, ids, False, TopKWeightAndReduceContiguous())
@requires_deep_ep_v2
def test_finalize_async_defers_join_to_receiver():
out_src = torch.full((4, 8), 7.0, dtype=torch.bfloat16)
pf = _make_pf(out_src)
dst = torch.zeros_like(out_src)
recv = _run(pf, dst, do_async=True)
assert pf.buffer.calls[0]["async_with_compute_stream"] is True
assert not torch.equal(dst, out_src), "join must not happen before receiver"
recv()
assert pf.buffer.last_event.waited, "receiver must join via event wait"
assert torch.equal(dst, out_src)
@requires_deep_ep_v2
def test_finalize_async_under_dbo_uses_sync_combine(monkeypatch):
monkeypatch.setattr(_dv2, "dbo_enabled", lambda: True)
out_src = torch.full((4, 8), 7.0, dtype=torch.bfloat16)
pf = _make_pf(out_src)
dst = torch.zeros_like(out_src)
recv = _run(pf, dst, do_async=True)
assert pf.buffer.calls[0]["async_with_compute_stream"] is False
recv()
assert not pf.buffer.last_event.waited, "no event to wait on for sync combine"
assert torch.equal(dst, out_src)
@requires_deep_ep_v2
def test_finalize_sync():
out_src = torch.full((4, 8), 7.0, dtype=torch.bfloat16)
pf = _make_pf(out_src)
dst = torch.zeros_like(out_src)
ret = _run(pf, dst, do_async=False)
assert ret is None
assert pf.buffer.calls[0]["async_with_compute_stream"] is False
assert torch.equal(dst, out_src)
+68
View File
@@ -565,6 +565,74 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
monkeypatch: pytest.MonkeyPatch,
):
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
monkeypatch: pytest.MonkeyPatch,
):
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
monkeypatch: pytest.MonkeyPatch,
):
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert "AITER_SITUV2_A4W4" not in os.environ
@pytest.mark.parametrize("moe_padding", [True, False])
def test_aiter_moe_padding_env_var(
moe_padding: bool,
+6 -1
View File
@@ -27,6 +27,7 @@ from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
fp8_gemm_nt,
get_tma_aligned_size,
is_deep_gemm_e8m0_used,
per_block_cast_to_fp8,
should_use_deepgemm_for_fp8_linear,
)
@@ -291,7 +292,11 @@ def test_w8a8_block_fp8_deep_gemm_matmul(M, N, K, block_size, out_dtype, seed):
A_fp8, As_fp8 = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=True, tma_aligned_scales=True
)
B_fp8, Bs_fp8 = per_block_cast_to_fp8(B_fp32, block_size=block_size)
B_fp8, Bs_fp8 = per_block_cast_to_fp8(
B_fp32,
block_size=block_size,
use_ue8m0=is_deep_gemm_e8m0_used(),
)
As = As_fp8.to(torch.float32)
Bs = Bs_fp8.to(torch.float32)
@@ -37,6 +37,20 @@ MAX_SKINNY_BATCH_SIZE = hybrid_module.MAX_SKINNY_BATCH_SIZE
# ---------------------------------------------------------------------------
def _pack_zp_rows_for_kernel(zp_nkg: torch.Tensor) -> torch.Tensor:
"""Pack raw uint4 zero points along N: [N, G] int32 -> [N//8, G] int32.
Row n's nibble lands in word[n//8] at bits 4*(n%8).
"""
assert zp_nkg.dtype == torch.int32
N, G = zp_nkg.shape
assert N % 8 == 0
shifts = (torch.arange(8, device=zp_nkg.device, dtype=torch.int32) * 4)[:, None]
return torch.sum(
(zp_nkg.view(N // 8, 8, G) & 0xF) << shifts, dim=1, dtype=torch.int32
).contiguous()
def _rdna_hybrid_w4a16_reference(
x_mk: torch.Tensor,
w_int4_nk: torch.Tensor,
@@ -50,8 +64,8 @@ def _rdna_hybrid_w4a16_reference(
x_mk: [M, K] fp16/bf16
w_int4_nk: [N, K] int32 with raw uint4 values in [0, 15]
scales_nkg: [N, K//G] fp16/bf16
zp_nkg: [N, K//G] fp16/bf16 raw zero points (already in act dtype),
or None for symmetric (uint4b8, dequant subtracts 8)
zp_nkg: [N, K//G] int32 raw zero points in [0, 15], or None for
symmetric (uint4b8, dequant subtracts 8)
"""
G = group_size
N, K = w_int4_nk.shape
@@ -110,13 +124,16 @@ def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M)
0.05 * torch.rand((N, K // group_size), device=device, dtype=torch.float32)
).to(dtype)
# Optional raw zero points [N, K//G] in act dtype.
# Optional raw zero points, [N, K//G] int32 for the reference and packed
# along N for the op.
if has_zp:
zp_nkg = torch.randint(
0, 16, (N, K // group_size), device=device, dtype=torch.int32
).to(dtype)
)
w_zp = _pack_zp_rows_for_kernel(zp_nkg)
else:
zp_nkg = None
w_zp = None
from vllm.utils.platform_utils import num_compute_units
@@ -124,7 +141,7 @@ def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M)
x_mk,
w_q,
scales_nkg,
zp_nkg,
w_zp,
None, # bias
num_compute_units(),
group_size,
@@ -332,7 +349,7 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_init):
"""uint4 (asymmetric): zero points unpacked to raw values in act dtype."""
"""uint4 (asymmetric): zero points are packed [N//8, K//G] int32."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
@@ -374,10 +391,11 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
)
kernel.process_weights_after_loading(layer)
# Zero-points: unpacked to [N, K//G], cast to act dtype, raw values [0..15].
assert layer.weight_zero_point.dtype == torch.float16
assert tuple(layer.weight_zero_point.shape) == (N, K // G)
expected_zp = zeros_int4_gn.t().to(torch.float16) # [N, K//G]
# Zero-points: [N//8, K//G] int32, row n's nibble at word[n//8]
# bits 4*(n%8).
assert layer.weight_zero_point.dtype == torch.int32
assert tuple(layer.weight_zero_point.shape) == (N // 8, K // G)
expected_zp = _pack_zp_rows_for_kernel(zeros_int4_gn.t().contiguous())
torch.testing.assert_close(layer.weight_zero_point, expected_zp)
# Quantized weights match symmetric path's layout regardless of zp.
@@ -518,3 +536,30 @@ def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.skipif(
not hasattr(torch.ops, "_rocm_C")
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
reason="wvSplitK_int4_g not built",
)
def test_wvsplitk_int4_g_rejects_unpacked_zero_points():
"""Act-dtype zero points raise instead of being read as packed words.
Both layouts are 2D with compatible extents, so only the dtype separates
them; misreading one as the other returns wrong numbers silently.
"""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
K, N, G, M = 256, 64, 128, 1
a = torch.randn((M, K), device=device, dtype=torch.float16)
w = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
torch.int8
)
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
zp_unpacked = torch.zeros((N, K // G), device=device, dtype=torch.float16)
with pytest.raises(RuntimeError, match="Zero points must be int32 or uint32"):
ops.wvSplitK_int4_g(w, a, scales, num_compute_units(), G, zp_unpacked, None)
@@ -12,20 +12,90 @@ from unittest.mock import patch
import pytest
import torch
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.kernels.linear import (
AiterInt8ScaledMMLinearKernel,
CPUInt8ScaledMMLinearKernel,
HummingFP8ScaledMMLinearKernel,
Int8ScaledMMLinearKernel,
Int8ScaledMMLinearLayerConfig,
ScaledMMLinearKernel,
_get_linear_backend,
_resolve_backend_kernels,
init_fp8_linear_kernel,
init_int8_linear_kernel,
register_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import PlatformEnum
pytestmark = pytest.mark.cpu_test
def test_auto_linear_backend_is_unchanged_without_overrides():
kernels = [object]
quantizations = (
"fp8_block_w8a8",
"fp8_w8a8",
"int8_w8a8",
"mixed_precision",
"mxfp8",
"mxfp4",
"mxfp6",
"w8a16_fp8",
"nvfp4_w4a4",
"nvfp4_w4a16",
)
config = VllmConfig(kernel_config=KernelConfig(linear_backend="auto"))
with set_current_vllm_config(config):
for quantization in quantizations:
assert _get_linear_backend(quantization=quantization) == "auto"
assert (
_resolve_backend_kernels(kernels, "test", quantization=quantization)
is kernels
)
def test_linear_backend_override_is_quantization_specific():
config = VllmConfig(
kernel_config=KernelConfig(
linear_backend="cutlass",
linear_backend_per_quant={"nvfp4_w4a16": "humming"},
)
)
with set_current_vllm_config(config):
assert _get_linear_backend(quantization="nvfp4_w4a16") == "humming"
assert _get_linear_backend(quantization="nvfp4_w4a4") == "cutlass"
assert _get_linear_backend(quantization="fp8_w8a8") == "cutlass"
@patch.object(HummingFP8ScaledMMLinearKernel, "is_supported", return_value=(True, None))
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_fp8_linear_backend_override(platform_mock, _):
platform_mock._enum = PlatformEnum.CUDA
config = VllmConfig(
kernel_config=KernelConfig(
linear_backend="cutlass",
linear_backend_per_quant={"fp8_w8a8": "humming"},
)
)
with set_current_vllm_config(config):
kernel = init_fp8_linear_kernel(
activation_quant_key=kFp8StaticTensorSym,
weight_quant_key=kFp8StaticTensorSym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
weight_shape=(128, 128),
)
assert type(kernel) is HummingFP8ScaledMMLinearKernel
def test_is_supported_is_abstract():
"""Test that is_supported() is properly defined as abstract."""
assert issubclass(ScaledMMLinearKernel, ABC)
+251 -5
View File
@@ -42,6 +42,122 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize(
"cache_dtype,kv_mxfp8",
[
(torch.uint8, False),
(torch.uint8, True),
(torch.bfloat16, False),
(torch.float8_e4m3fn, False),
],
)
@pytest.mark.parametrize("num_tokens", [1, 17, 1023, 1024])
@pytest.mark.parametrize("use_graph", [False, True])
def test_dspark_context_kv_matches_query_insert(
cache_dtype, kv_mxfp8, num_tokens, use_graph
):
"""KV-only insertion must preserve every cache byte, including graph replay."""
from vllm.models.deepseek_v41.nvidia.dspark import _insert_context_kv
torch.manual_seed(42)
block_size = 256
num_blocks = math.ceil((num_tokens + 7) / block_size)
row_size = (528 if kv_mxfp8 else 584) if cache_dtype == torch.uint8 else 512
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
backing = torch.full((num_blocks, page_stride), 3, device="cuda").to(cache_dtype)
cache = backing.as_strided(
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
)
reference_backing = backing.clone()
reference = reference_backing.as_strided(cache.shape, cache.stride())
kv = torch.randn(num_tokens + 3, 512, device="cuda", dtype=torch.bfloat16)
positions = torch.arange(num_tokens + 3, device="cuda") + 7
angles = torch.randn(num_tokens + 16, 32, device="cuda")
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_tokens]
slots[::5] = -1
scale = torch.tensor([0.7], device="cuda")
# Only what _insert_context_kv actually reads: the record width comes off
# the cache tensor, so no per-record width attribute is stubbed here.
attn = SimpleNamespace(
swa_cache_layer=SimpleNamespace(kv_cache=cache, block_size=block_size),
kv_mxfp8=kv_mxfp8,
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
_flashinfer_fp8_kv_scale=scale,
)
def legacy_insert():
q = torch.zeros(kv.shape[0], 8, 512, dtype=kv.dtype, device="cuda")
if cache_dtype == torch.uint8:
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
reference.view(num_blocks, -1),
slots,
positions,
cos_sin,
8,
1e-20,
block_size,
True,
kv_mxfp8,
)
elif cache_dtype == torch.bfloat16:
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
q,
kv,
reference,
slots,
positions,
cos_sin,
1e-20,
block_size,
)
else:
q_fp8 = torch.empty_like(q, dtype=cache_dtype)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
q,
kv,
q_fp8,
reference,
slots,
positions,
cos_sin,
scale,
scale,
1e-20,
block_size,
)
def insert():
_insert_context_kv(attn, kv, positions, slots)
insert()
graph = None
if use_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
insert()
for iteration in range(3):
kv.normal_()
positions.add_(1)
if iteration == 2:
slots.fill_(-1)
backing.view(torch.uint8).fill_(165)
reference_backing.copy_(backing)
before = kv.clone()
legacy_insert()
insert() if graph is None else graph.replay()
torch.testing.assert_close(
backing.view(torch.uint8),
reference_backing.view(torch.uint8),
rtol=0,
atol=0,
)
torch.testing.assert_close(kv, before, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="graph capture coverage"
)
@@ -65,7 +181,7 @@ def test_v41_fused_save_compress_and_insert(
a Torch reference with BF16 rounding before RoPE. Replays change both
inputs and history so stale scratch/cache values cannot satisfy the test.
"""
from vllm.models.deepseek_v4_1.common.ops.fused_compress_quant_cache import (
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
fused_save_compress_norm,
rope_quant_insert,
)
@@ -215,6 +331,136 @@ def test_v41_fused_save_compress_and_insert(
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
def _mxfp8_record_reference(
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Reference V4.1 record for one token: 512 fp8 bytes + 16 UE8M0 scales.
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
"""
row = latent_row.float()
c, s = cos_sin_row.chunk(2)
rotated = row.clone()
even, odd = row[448::2], row[449::2]
rotated[448::2] = even * c - odd * s
rotated[449::2] = odd * c + even * s
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
"""The 528-byte V4.1 record quantizes the RoPE dims instead of keeping them
in bf16, so a page is [block_size x 512 fp8][block_size x 16 UE8M0].
Byte-exact against the reference quantizer, including the rows and page
padding the kernel must leave alone.
"""
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
torch.manual_seed(7)
device = "cuda"
cache_block = 64
cache_stride = math.ceil(cache_block * 528 / 512) * 512
num_tokens = 12
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
angles = torch.randn(64, 32, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
cache = cache_backing.as_strided((2, cache_block, 528), (cache_stride, 528, 1))
# Leave one token unmapped so the kernel's negative-slot skip is covered.
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
slots[3] = -1
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
expected = torch.full_like(cache_backing, 165)
for t in range(num_tokens):
slot = slots[t].item()
pos = positions[t].item()
if slot < 0 or (pos + 1) % compress_ratio:
continue
page, row = divmod(slot, cache_block)
values, scales = _mxfp8_record_reference(
latent[t], cos_sin[pos // compress_ratio * compress_ratio]
)
expected[page, row * 512 : (row + 1) * 512] = values
scale_offset = cache_block * 512 + row * 16
expected[page, scale_offset : scale_offset + 16] = scales
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_v41_mxfp8_cache_round_trip():
"""quantize_and_insert -> dequantize_and_gather recovers the V4.1 record.
The RoPE dims now go through fp8, so they carry quantization error too --
the check is that every dim survives within one MXFP8 tile step, and that
the bytes on the way match the reference quantizer.
"""
from vllm.models.deepseek_v41.common.ops import (
dequantize_and_gather_k_cache,
quantize_and_insert_k_cache,
)
torch.manual_seed(11)
device = "cuda"
block_size = 64
num_tokens = 70
num_blocks = 4
page_bytes = math.ceil(block_size * 528 / 512) * 512
k = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
cache = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
quantize_and_insert_k_cache(
k, cache, slot_mapping, block_size=block_size, bytes_per_token=528
)
for t in (0, 1, block_size, num_tokens - 1):
values, scales = _ue8m0_reference(k[t].float(), 32, 448.0)
page, row = divmod(t, block_size)
torch.testing.assert_close(
cache[page, row * 512 : (row + 1) * 512],
values.view(torch.uint8),
rtol=0,
atol=0,
)
scale_offset = block_size * 512 + row * 16
torch.testing.assert_close(
cache[page, scale_offset : scale_offset + 16],
(scales.log2() + 127).to(torch.uint8),
rtol=0,
atol=0,
)
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
dequantize_and_gather_k_cache(
out,
cache.view(num_blocks, block_size, 528),
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
gather_lens=None,
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
1, -1
),
block_size=block_size,
offset=0,
)
# Half an e4m3 ULP at the top of the range (16 units) bounds the error.
tile_amax = k.float().abs().view(num_tokens, 16, 32).amax(-1).clamp(min=1e-4)
tolerance = 16.0 * (tile_amax / 448.0).log2().ceil().exp2()
error = (out[0].float() - k.float()).abs()
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
@pytest.mark.parametrize("store_fp8", [False, True])
@@ -226,7 +472,7 @@ def test_v41_rope_insert_plain_row(compress_ratio: int, store_fp8: bool):
Non-boundary positions, negative slots and untouched rows must keep their
prior contents.
"""
from vllm.models.deepseek_v4_1.common.ops.fused_compress_quant_cache import (
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
@@ -301,7 +547,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
capacity`` and keep padding tokens at PAD."""
from unittest.mock import MagicMock
from vllm.models.deepseek_v4_1.compressor import CompressorMetadataBuilder
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import CircularBufferSpec
@@ -351,11 +597,11 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph):
"""Both reused-event joins must publish this forward's states and cache rows."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v4_1.attention import (
from vllm.models.deepseek_v41.attention import (
DeepseekV4Attention,
DeepseekV4Indexer,
)
from vllm.models.deepseek_v4_1.compressor import DeepseekCompressor
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
torch.manual_seed(43)
raw = torch.randn(19, 1024, device="cuda")
+6 -6
View File
@@ -7,15 +7,15 @@ from types import SimpleNamespace
import pytest
import torch
from vllm.models.deepseek_v4_1.common import engram as engram_ops
from vllm.models.deepseek_v4_1.common.engram import (
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram as CommonEngram,
)
from vllm.models.deepseek_v4_1.common.engram import (
from vllm.models.deepseek_v41.common.engram import (
NgramHashState,
)
from vllm.models.deepseek_v4_1.nvidia import engram as nvidia_engram_ops
from vllm.models.deepseek_v4_1.nvidia.engram import Engram, ParallelEngramEmbedding
from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops
from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding
from vllm.platforms import current_platform
@@ -503,7 +503,7 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
def test_v2_model_state_gathers_lookback_window():
"""The window is gathered on device from the runner's token history."""
from vllm.models.deepseek_v4_1.nvidia.model_state import DeepseekV41ModelState
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
depth, max_num_reqs, max_model_len = 3, 4, 16
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
@@ -573,6 +573,126 @@ def test_fused_norm_rope_ds_mla(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def quantize_to_e2m1(x: torch.Tensor) -> torch.Tensor:
"""Round to nearest e2m1, saturating to +-6 like cvt.rn.satfinite.e2m1x2.f32."""
mags = torch.tensor(E2M1_MAGNITUDES, dtype=torch.float32, device=x.device)
a = x.float().abs().clamp_max(6.0)
mids = (mags[:-1] + mags[1:]) / 2
code = torch.bucketize(a, mids, right=True)
on_tie = (a.unsqueeze(-1) == mids).any(dim=-1)
tie_code = torch.bucketize(a, mids, right=False)
code = torch.where(on_tie, tie_code + (tie_code & 1), code)
return (torch.signbit(x).to(torch.uint8) << 3) | code.to(torch.uint8)
def nvfp4_sf_byte(s: torch.Tensor) -> torch.Tensor:
"""Scale-factor byte permutation shared with FlashMLA: 8*(s&3) + (s>>2)."""
return 8 * (s % 4) + (s // 4)
@pytest.mark.skipif(
not current_platform.is_device_capability(100),
reason="nvfp4_ds_mla requires SM100 (Blackwell)",
)
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int):
"""nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only).
Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble
= even element) | 64 B unscaled e4m3 RoPE | 32 B byte-permuted e4m3 tile
scales, one per 16 NoPE elements.
"""
torch.manual_seed(5)
dev = "cuda"
max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
# Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3
# subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale
# BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a
# future change to the scale rule cannot silently stop covering this.
# amax must land in (6, 9) * 2^-9 after rms_norm; 7.5 sits mid-band, with
# margin for the per-token RMS to vary.
kv_c[:, :16] = (
7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16)
)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.ones(KV_LORA, device=dev, dtype=torch.bfloat16)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
bs = max_pos
mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8)
slot = torch.arange(num_tokens, device=dev, dtype=torch.int64)
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
q_out = K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
mla_cos_sin,
None,
None,
None,
EPS,
None,
topk,
slot_mapping=slot,
indexer_k_cache=None,
mla_kv_cache=mla_cache,
mla_kv_cache_dtype="nvfp4_ds_mla",
mla_k_scale=None,
has_indexer=False,
index_rope_interleave=False,
)
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (nvfp4_ds_mla)")
kv_ref = rms_norm(kv_c, kvw) # [N, 512] fp32
kpe_ref = rope(k_pe.float(), pos, mla_cos_sin, interleave=True) # [N, 64]
tiles = kv_ref.view(num_tokens, 32, 16)
amax = tiles.abs().amax(dim=-1)
scale_target = torch.clamp_min(amax / 6.0, 2.0**-9)
ref_scale = scale_target.to(FP8) # round-to-nearest
# The crafted tile must actually exercise the saturating path.
assert (ref_scale[:, 0].float() < scale_target[:, 0]).all(), (
"tile 0 should round its scale DOWN; adjust the crafted magnitude"
)
assert (amax[:, 0] / ref_scale[:, 0].float() > 6.0).all(), (
"tile 0 should saturate e2m1; adjust the crafted magnitude"
)
ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1))
ref_codes = ref_codes.reshape(num_tokens, KV_LORA)
cache = mla_cache[0, :num_tokens] # [N, 352] uint8
packed = cache[:, : KV_LORA // 2]
got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1)
got_codes = got_codes.reshape(num_tokens, KV_LORA) # low nibble = even elem
torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0)
got_rope = cache[:, KV_LORA // 2 : KV_LORA // 2 + ROPE_DIM].view(FP8)
assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3")
perm = nvfp4_sf_byte(torch.arange(32, device=dev))
got_scale = cache[:, KV_LORA // 2 + ROPE_DIM :].view(FP8)[:, perm]
torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0)
# No indexer on this call: top-k buffer must be untouched.
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
def test_fused_norm_rope_supports_large_token_count():
"""Keep the token count off CUDA grid-y at its 65,536-block boundary."""
num_tokens = 65536
@@ -424,6 +424,104 @@ def test_kv_path_matches_reference(num_tokens: int, block_size: int):
)
# ── Test 2c: DeepSeek V4.1 MXFP8 record (kv_mxfp8=True) ──────────────────────
V41_HEAD_BYTES = HEAD_DIM + HEAD_DIM // 32 # 512 fp8 + 16 UE8M0 scales = 528
@pytest.mark.parametrize("num_tokens", [1, 17, 64])
@pytest.mark.parametrize("block_size", [32, 64])
def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int):
"""``kv_mxfp8=True`` writes DeepSeek's V4.1 record: all 512 dims as fp8 e4m3
with one UE8M0 scale per 32 dims, no bf16 RoPE tail.
Both sides quantize the same bf16-rounded rotated row, so the bytes must
match the Triton encoder exactly apart from the documented 1-ULP bf16
rounding split on the rotation itself.
"""
from vllm.models.deepseek_v41.common.ops import (
dequantize_and_gather_k_cache as v41_dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v41.common.ops import (
quantize_and_insert_k_cache as v41_quantize_and_insert_k_cache,
)
torch.manual_seed(3)
device = "cuda"
dtype = torch.bfloat16
max_pos = 4096
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
k_cache_ref = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
v41_quantize_and_insert_k_cache(
apply_rope_gptj_last_k(kv, positions, cos_sin_cache),
k_cache_ref,
slot_mapping,
block_size=block_size,
use_fnuz=USE_FNUZ,
bytes_per_token=V41_HEAD_BYTES,
)
k_cache_fused = torch.zeros_like(k_cache_ref)
q_dummy = torch.zeros(num_tokens, 1, HEAD_DIM, dtype=dtype, device=device)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_dummy,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
64,
1e-6,
block_size,
True,
True, # kv_mxfp8
)
# A pre-quantization bf16 tie can land the two encoders on adjacent e4m3
# codes, so compare after dequantizing and allow one step at the tile scale.
def _dequant(cache_2d):
out = torch.zeros(1, num_tokens, HEAD_DIM, dtype=dtype, device=device)
v41_dequantize_and_gather_k_cache(
out,
cache_2d.view(num_blocks, block_size, V41_HEAD_BYTES),
torch.tensor([num_tokens], dtype=torch.int32, device=device),
None,
torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(0),
block_size,
offset=0,
use_fnuz=USE_FNUZ,
)
return out[0]
rec_ref = _dequant(k_cache_ref).float()
rec_fused = _dequant(k_cache_fused).float()
tile_scale = (
(rec_ref.abs().view(num_tokens, 16, 32).amax(-1).clamp(min=1e-4) / FP8_MAX)
.log2()
.ceil()
.exp2()
)
# One e4m3 ULP at the top of the range is 32 quantized units.
step = 32.0 * tile_scale.repeat_interleave(32, dim=-1)
assert ((rec_fused - rec_ref).abs() <= step).all()
# The NoPE dims see no rotation at all, so they must be byte-identical.
nope_bytes = slice(0, block_size * HEAD_DIM)
ref_rows = k_cache_ref[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
fused_rows = k_cache_fused[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
torch.testing.assert_close(
fused_rows[..., :NOPE_DIM], ref_rows[..., :NOPE_DIM], rtol=0, atol=0
)
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
@@ -359,7 +359,7 @@ def test_indexer_k_inserts_only_valid_groups_into_padded_pages(
num_tokens, compress_ratio, use_fp4, cache_dtype
):
"""Insert group ends and preserve skipped slots, graph rows, and page padding."""
from vllm.models.deepseek_v4_1.common.ops.indexer_k_store import (
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
indexer_k_norm_rope_store,
)
@@ -442,7 +442,7 @@ def test_indexer_k_inserts_only_valid_groups_into_padded_pages(
@torch.inference_mode()
def test_indexer_k_cuda_graph_replay_reads_current_projection(use_fp4):
"""Replay must consume updated keys and slot mapping through the captured API."""
from vllm.models.deepseek_v4_1.common.ops.indexer_k_store import (
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
indexer_k_norm_rope_store,
)
@@ -501,7 +501,7 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
``block_size > 1``, so a row-major store would feed the indexer permuted
key bytes and randomize its top-k. Assert the write/read pair is exact.
"""
from vllm.models.deepseek_v4_1.common.ops.indexer_k_store import (
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
indexer_k_norm_rope_store,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
@@ -560,3 +560,55 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
torch.testing.assert_close(
k_scale[emitted], expected_scale[emitted], rtol=0, atol=0
)
@pytest.mark.parametrize("use_cutedsl", [False, True])
@pytest.mark.skipif(
not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
),
reason="MXFP4 indexer cache requires an SM100-family GPU",
)
@torch.inference_mode()
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
result rounded once, so the scoring kernel needs no cast."""
if use_cutedsl and not has_cutedsl():
pytest.skip("cutedsl (cutlass) not installed")
device = "cuda"
torch.manual_seed(0)
num_tokens, n_head = 257, 32
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
positions = torch.randint(
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
)
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
softmax_scale, head_scale = HEAD_DIM**-0.5, 0.125 # pow2 head_scale, see above
_, weights_ref = _reference(
positions, q, cos_sin_cache, weights, softmax_scale, head_scale, n_head, True
)
cutedsl_patch = (
mock.patch(
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
return_value=False,
)
if not use_cutedsl
else contextlib.nullcontext()
)
with cutedsl_patch:
_, weights_fused = fused_indexer_q_rope_quant(
positions,
q.clone(),
cos_sin_cache,
weights,
softmax_scale,
head_scale,
use_fp4=True,
weights_out_dtype=torch.bfloat16,
)
assert weights_fused.dtype == torch.bfloat16
assert torch.equal(weights_fused, weights_ref.to(torch.bfloat16))
+26 -4
View File
@@ -91,7 +91,7 @@ class MockLinearFp8Static(torch.nn.Module):
def __init__(self, input_scale: torch.Tensor):
super().__init__()
self.input_quant_key = kFp8StaticTensorSym
self._input_quant_key = kFp8StaticTensorSym
self.input_scale = input_scale
@@ -100,15 +100,21 @@ class MockLinearFp8Dynamic128(torch.nn.Module):
def __init__(self):
super().__init__()
self.input_quant_key = kFp8Dynamic128Sym
self._input_quant_key = kFp8Dynamic128Sym
class MockLinearNoQuant(torch.nn.Module):
"""Mock linear layer with no input_quant_key (no fusion)."""
"""Mock linear layer with no input quantization key."""
pass
class MockLinearRequiresUnquantized(MockLinearFp8Static):
"""Mock consumer that needs the original activation for another branch."""
requires_unquantized_input = True
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@@ -199,7 +205,7 @@ def test_maybe_fused_act_quant_fallback(
hidden_size: int,
dtype: torch.dtype,
) -> None:
"""Test maybe_fused_act_quant falls back when no input_quant_key."""
"""Test maybe_fused_act_quant falls back without an input quantization key."""
device = "cuda:0"
torch.set_default_device(device)
@@ -216,3 +222,19 @@ def test_maybe_fused_act_quant_fallback(
ref_out = act_fn(x)
torch.testing.assert_close(result, ref_out)
@torch.inference_mode()
def test_maybe_fused_act_quant_preserves_required_unquantized_input(
default_vllm_config,
) -> None:
device = "cuda:0"
act_fn = SiluAndMul()
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
linear = MockLinearRequiresUnquantized(scale)
x = torch.randn(17, 256, dtype=torch.bfloat16, device=device)
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, torch.Tensor)
torch.testing.assert_close(result, act_fn(x))
+39 -10
View File
@@ -17,6 +17,8 @@ from vllm.model_executor.kernels.mhc.tilelang_kernels import (
MhcFusedTileLangKernel,
MhcPostTileLangKernel,
MhcPreBigFuseTileLangKernel,
mhc_fused_post_pre_split_config,
require_fused_post_pre_config,
)
from vllm.model_executor.warmup import jit_warmup_tilelang_helper
@@ -113,17 +115,32 @@ def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config(
@pytest.mark.parametrize(
("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"),
[(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)],
("num_tokens", "hidden_size", "expected_tile_n", "expected_n_splits"),
[
(1, 4096, 2, 8),
(4, 8192, 2, 8),
(8, 5120, 2, 8),
(16, 5120, 6, 8),
(32, 7168, 6, 8),
],
)
def test_mhc_fused_dispatch_matches_legacy_runtime_config(
def test_mhc_fused_dispatch_matches_the_launch_config(
num_tokens: int,
hidden_size: int,
expected_n_splits: int,
expected_tile_n: int,
expected_n_splits: int,
) -> None:
kernel = MhcFusedTileLangKernel()
"""The compile key must be the config the launch will actually use.
Both read mhc_fused_post_pre_split_config, so this pins the tuned bands
and guards against the two drifting apart again.
"""
kernel = MhcFusedTileLangKernel()
tile_n, n_splits, n_thr = require_fused_post_pre_config(
num_tokens, hidden_size, hc_mult=4
)
assert (tile_n, n_splits) == (expected_tile_n, expected_n_splits)
assert kernel.dispatch(
num_tokens=num_tokens,
hidden_size=hidden_size,
@@ -131,11 +148,20 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
) == kernel.CompileKey(
hidden_size=hidden_size,
hc_mult=4,
n_splits=expected_n_splits,
tile_n=expected_tile_n,
n_splits=n_splits,
tile_n=tile_n,
n_thr=n_thr,
)
def test_mhc_fused_declines_shapes_it_cannot_tile() -> None:
"""Above the token cutoff, and for a hidden size the block cannot split."""
assert mhc_fused_post_pre_split_config(33, 5120, 4) is None
assert mhc_fused_post_pre_split_config(1, 5137, 4) is None
with pytest.raises(ValueError, match="does not cover num_tokens"):
require_fused_post_pre_config(33, 5120, 4)
@pytest.mark.parametrize(
("kernel", "compile_key"),
[
@@ -235,8 +261,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
MhcFusedTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=4,
tile_n=3,
n_splits=8,
tile_n=6,
n_thr=128,
),
),
(
@@ -246,6 +273,7 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
hc_mult=4,
n_splits=8,
tile_n=2,
n_thr=128,
),
),
(
@@ -253,8 +281,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
MhcFusedTileLangKernel.CompileKey(
hidden_size=8192,
hc_mult=4,
n_splits=4,
n_splits=8,
tile_n=2,
n_thr=128,
),
),
(
+447 -9
View File
@@ -11,12 +11,17 @@ import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.layers.mhc as mhc_layers
from vllm.model_executor.kernels.mhc.tilelang import (
_torch_hc_prenorm_gemm,
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
mhc_fused_post_pre_split_config,
)
from vllm.model_executor.kernels.mhc.torch import (
mhc_post_torch,
mhc_pre_delayed_torch,
)
from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch
from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
@@ -32,9 +37,13 @@ from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
)
from vllm.models.deepseek_v4_1.nvidia.model import (
from vllm.models.deepseek_v41.nvidia.model import (
DeepseekV4DecoderLayer as DeepseekV41DecoderLayer,
)
from vllm.models.deepseek_v41.nvidia.ops.mega_mhc import (
is_mega_mhc_supported,
mhc_shifted_post_pre_deep_gemm,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -104,7 +113,7 @@ def test_hc_collapse_custom_op_supports_compile():
@pytest.mark.parametrize("num_tokens", [1, 7])
def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
"""Match reference DSparkBlock.forward_head's hc_pre before its RMSNorm."""
from vllm.models.deepseek_v4_1.nvidia import dspark
from vllm.models.deepseek_v41.nvidia import dspark
set_random_seed(0)
hidden_size, hc_mult = 5120, 4
@@ -115,9 +124,9 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
carried_mixes = []
def make_layer(mix):
def forward(hidden, positions, ids, pre, post, res, residual):
def forward(hidden, positions, ids, pre, post, res, residual, **kwargs):
carried_mixes.append(pre)
return hidden, None, None, None, mix
return hidden, None, None, None, mix, None
return forward
@@ -143,6 +152,27 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize("num_tokens", [1, 7, 32])
@pytest.mark.parametrize("tp_size", [4, 8])
def test_hc_collapse_before_gather_is_bitwise_equal(num_tokens, tp_size):
"""Simulate SP on one GPU, including padding and ranks without valid tokens."""
set_random_seed(0)
padded_tokens = num_tokens + (-num_tokens) % tp_size
streams = torch.randn(padded_tokens, 4, 5120, dtype=torch.bfloat16, device=DEVICE)
pre_mix = torch.rand(padded_tokens, 4, device=DEVICE)
stream_shards = streams.chunk(tp_size)
mix_shards = pre_mix.chunk(tp_size)
before = hc_collapse_triton(
torch.cat(stream_shards)[:num_tokens], torch.cat(mix_shards)[:num_tokens]
)
after = torch.cat(
[hc_collapse_triton(x, pre) for x, pre in zip(stream_shards, mix_shards)]
)[:num_tokens]
assert after.shape == (num_tokens, 5120)
torch.testing.assert_close(after, before, atol=0, rtol=0)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize(
"num_tokens,sinkhorn_repeat,norm_eps",
@@ -235,6 +265,171 @@ def test_deepseek_v41_mhc_pre_delayed(
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
def mhc_fused_post_pre_delayed_ref(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
*mix_args,
pre_mix: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Project the post result before it is rounded to BF16.
The fused kernel keeps the updated residual streams in registers, so its
projection sees the FP32 post result while the streams it stores are the
BF16 rounding of it.
"""
post = torch.einsum("tij,tih->tjh", comb_res_mix, residual.float())
post = post + post_layer_mix * x.float().unsqueeze(-2)
return mhc_pre_delayed_torch(
post.bfloat16(), *mix_args, pre_mix=pre_mix, x=post.flatten(1)
)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 128])
@pytest.mark.parametrize("hidden_size", [4096, 7168])
@pytest.mark.parametrize("carried", [False, True])
def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carried):
"""Fold the post block into the delayed pre without changing its outputs."""
set_random_seed(0)
hc_mult = 4
mix_size = hc_mult * (hc_mult + 2)
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
residual = torch.randn(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
post_layer_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
comb_res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
fn = torch.randn(mix_size, hc_mult * hidden_size, device=DEVICE) * 0.02
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
base = torch.randn(mix_size, device=DEVICE)
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE) if carried else None
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE)
weight.uniform_(0.5, 1.5)
mix_args = (fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
residual_cur, post, comb, layer_input, next_pre, aux = (
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
)
)
assert aux.shape == (0, hidden_size)
assert [
residual_cur.shape,
post.shape,
comb.shape,
layer_input.shape,
next_pre.shape,
] == [
(num_tokens, hc_mult, hidden_size),
(num_tokens, hc_mult, 1),
(num_tokens, hc_mult, hc_mult),
(num_tokens, hidden_size),
(num_tokens, hc_mult),
]
if num_tokens == 0:
return
# The post mapping and the collapse are what the unfused kernels produce.
residual_ref = torch.ops.vllm.mhc_post_tilelang(
x, residual, post_layer_mix, comb_res_mix
)
layer_input_ref = mhc_pre_delayed_tilelang(
residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
)[2]
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0)
torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0)
post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix=pre_mix
)
# Above the fused kernel's token count the projection falls back to a
# split-k TF32 GEMM over the stored BF16 streams.
tol = (
1e-5
if mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) is not None
else 5e-3
)
for actual, expected in (
(post, post_ref),
(comb, comb_ref),
(next_pre, next_pre_ref),
):
torch.testing.assert_close(actual, expected, atol=tol, rtol=tol)
# Folding the stream mean into the collapse must not change it, or anything
# else the op returns.
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_layer_mix,
comb_res_mix,
*mix_args,
pre_mix,
weight,
1e-6,
True,
)
torch.testing.assert_close(captured[5], residual_cur.mean(dim=1), atol=0, rtol=0)
unchanged = (residual_cur, post, comb, layer_input, next_pre)
for with_aux, without in zip(captured[:5], unchanged, strict=True):
torch.testing.assert_close(with_aux, without, atol=0, rtol=0)
# The unnormalized epilogue has its own fused aux path.
bare = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, None, 1e-6, True
)
torch.testing.assert_close(bare[5], bare[0].mean(dim=1), atol=0, rtol=0)
# Decode replays this op from a captured graph.
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
)
graph.replay()
eager = (residual_cur, post, comb, layer_input, next_pre, aux)
for replayed, expected in zip(captured, eager, strict=True):
torch.testing.assert_close(replayed, expected, atol=0, rtol=0)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("carried", [False, True])
@pytest.mark.parametrize("capture_aux", [False, True])
def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_aux):
set_random_seed(0)
x = torch.randn(2, 5120, dtype=torch.bfloat16, device=DEVICE)
residual = torch.randn(2, 4, 5120, dtype=torch.bfloat16, device=DEVICE)
post_layer_mix = torch.rand(2, 4, 1, device=DEVICE)
comb_res_mix = torch.rand(2, 4, 4, device=DEVICE)
fn = torch.randn(24, 20480, device=DEVICE) * 0.02
pre_mix = torch.rand(2, 4, device=DEVICE) if carried else None
scale = torch.ones(3, device=DEVICE)
base = torch.zeros(24, device=DEVICE)
torch.library.opcheck(
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang.default,
(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
scale,
base,
1e-20,
1e-6,
1e-6,
2.0,
20,
pre_mix,
None,
1e-6,
capture_aux,
),
)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
def test_deepseek_v41_decoder_mixes_match_torch(
@@ -295,15 +490,86 @@ def test_deepseek_v41_decoder_mixes_match_torch(
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
return post, res, decoder.attn_norm(collapsed), pre
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
residual = mhc_post_torch(x, residual, post_mix, res_mix)
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
return residual, *reference(residual, *args, **kw), aux
monkeypatch.setattr(
"vllm.models.deepseek_v4_1.nvidia.model.mhc_pre_delayed_tilelang",
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
reference,
)
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre",
fused_reference,
)
expected = decoder(x, positions, None, **kwargs)
for result, ref in zip(actual, expected, strict=True):
assert actual[-1] is None and expected[-1] is None
for result, ref in zip(actual[:-1], expected[:-1], strict=True):
torch.testing.assert_close(result, ref, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("entry", ["fused", "engram"])
def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_config):
"""Read the aux hidden state back out of the seam that already computed it.
The draft model's input is the mean over hc streams of the previous
layer's post. That post now runs inside this layer's fused pre, so the
captured value must equal a standalone ``mhc_post_tilelang`` on the same
inputs -- and Engram must not be folded into it, since the injection
happens after the aux consumers read the stream.
"""
set_random_seed(0)
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
nn.Module.__init__(decoder)
decoder.hc_mult = 4
decoder.hc_sinkhorn_iters = 20
decoder.hc_eps = 1e-6
decoder.rms_norm_eps = 1e-6
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.engram = None
from vllm.model_executor.layers.layernorm import RMSNorm
decoder.attn_norm = decoder.ffn_norm = RMSNorm(5120, 1e-6).to(
device=DEVICE, dtype=torch.bfloat16
)
decoder.attn = lambda positions, x, _: x * 0.5
decoder.ffn = lambda x, input_ids: x * 0.25
with torch.device(DEVICE):
decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02
decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
decoder.hc_attn_base = torch.randn(24)
decoder.hc_ffn_base = torch.randn(24)
x = torch.randn(3, 5120, dtype=torch.bfloat16)
kwargs = dict(
pre_mix=torch.rand(3, 4),
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
post_mix=torch.rand(3, 4, 1),
res_mix=torch.rand(3, 4, 4),
)
if entry == "engram":
class FakeEngram(nn.Module):
layer_hash_index = 0
def forward(self, residual, hashes, mask):
return residual + 0.125
decoder.engram = FakeEngram()
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
positions = torch.arange(3)
expected = torch.ops.vllm.mhc_post_tilelang(
x, kwargs["residual"], kwargs["post_mix"], kwargs["res_mix"]
).mean(dim=1)
previous_aux = decoder(x, positions, None, **kwargs, capture_previous_aux=True)[-1]
torch.testing.assert_close(previous_aux, expected, atol=0, rtol=0)
assert decoder(x, positions, None, **kwargs)[-1] is None
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("carried", [False, True])
def test_mhc_pre_delayed_custom_op_supports_compile(carried):
@@ -319,6 +585,104 @@ def test_mhc_pre_delayed_custom_op_supports_compile(carried):
)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="DeepGEMM Mega mHC requires SM100-family CUDA",
)
@pytest.mark.parametrize("num_tokens", [0, 1, 17, 128, 1024])
@pytest.mark.parametrize("hidden_size", [5120, 7168])
def test_deep_gemm_mega_mhc_correctness(num_tokens, hidden_size):
set_random_seed(0)
hc_mult = 4
if not is_mega_mhc_supported(hidden_size, 4):
pytest.skip("DeepGEMM Mega mHC is not available")
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
residual = torch.randn(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
previous_mix = torch.rand(num_tokens, hc_mult, device=DEVICE)
post_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
fn = torch.randn(24, hc_mult * hidden_size, device=DEVICE) * 0.01
scale = torch.randn(3, device=DEVICE) * 0.1
base = torch.randn(24, device=DEVICE) * 0.1
norm_weight = torch.empty(
hidden_size, dtype=torch.bfloat16, device=DEVICE
).uniform_(0.9, 1.1)
expected_residual = (
mhc_post_tilelang(x, residual, post_mix, res_mix) if num_tokens else residual
)
(
expected_post_mix,
expected_res_mix,
expected_y_bf16,
expected_previous_mix,
) = mhc_pre_delayed_tilelang(
expected_residual,
fn,
scale,
base,
2e-5,
3e-4,
2e-6,
1.25,
10,
pre_mix=previous_mix,
norm_weight=norm_weight,
norm_eps=7e-6,
)
args = (
x,
residual,
previous_mix,
post_mix,
res_mix,
fn,
scale,
base,
2e-5,
3e-4,
1.25,
2e-6,
10,
norm_weight,
7e-6,
)
actual = mhc_shifted_post_pre_deep_gemm(*args)
expected = (
expected_residual,
expected_post_mix,
expected_res_mix,
expected_y_bf16,
expected_previous_mix,
)
for i, (result, ref) in enumerate(zip(actual, expected, strict=True)):
# Match the existing mHC BF16 tolerance: post rounding differences
# can be amplified by cancellation in the carried collapse.
atol, rtol = (1.6e-2, 1e-2) if i in (0, 3) else (1e-6, 1e-3)
torch.testing.assert_close(result, ref, atol=atol, rtol=rtol)
if num_tokens:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
# DeepGEMM initializes barriers per stream before capture.
mhc_shifted_post_pre_deep_gemm(*args)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
captured = mhc_shifted_post_pre_deep_gemm(*args)
for multiplier in (0.5, -1.0):
x.mul_(multiplier)
graph.replay()
eager = mhc_shifted_post_pre_deep_gemm(*args)
for result, ref in zip(captured, eager, strict=True):
torch.testing.assert_close(result, ref, atol=0, rtol=0)
torch.cuda.current_stream().wait_stream(stream)
def sinkhorn_normalize_ref(x: torch.Tensor, repeat: int, eps: float) -> torch.Tensor:
x = x.softmax(-1) + eps
x = x / (x.sum(-2, keepdim=True) + eps)
@@ -743,8 +1107,12 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
)
expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix)
actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix)
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
*args, pre_mix=pre_mix
)
# No post was requested, so the residual comes straight back.
assert residual_out is residual
for i in (0, 1, 3):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
# The collapse is the same FP32 multiply-and-sum in both paths.
@@ -786,7 +1154,12 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x)
actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x)
# forward_hip leads with the residual; no post was requested here, so it is
# the caller's own tensor and the reference's four outputs follow it.
residual_out, *actual = op.forward_hip(
broadcast_residual, broadcast_fn, *args[2:], x=x
)
assert residual_out is broadcast_residual
assert not took_aiter, "the broadcast seam must not reach AITER"
for i in range(4):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
@@ -1028,3 +1401,68 @@ def test_deepseek_v4_mhc_broadcast_refit_refreshes_in_place(monkeypatch):
assert layer.hc_attn_fn_broadcast is buffer
expected = layer.hc_attn_fn.detach().view(-1, 2, 8).sum(dim=1)
assert torch.equal(layer.hc_attn_fn_broadcast, expected)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128])
@pytest.mark.parametrize("carried", [False, True])
def test_mhc_fused_post_pre_delayed_rocm_aiter(
num_tokens, carried, default_vllm_config
):
"""Folding the post into the pre projection must match doing them apart.
The residual is bfloat16, so the two accumulation orders are allowed to
land a rounding step apart; the tolerance here is one bfloat16 ULP.
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs(
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
)
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16, device=DEVICE)
post_layer_mix = (
torch.rand(num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE) + 0.5
)
comb_res_mix = (
torch.rand(num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=DEVICE)
+ 0.2
)
pre_mix = (
torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
if carried
else None
)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6
pre_args = (fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, 1.0, 20)
next_residual = mhc_post_torch(x, residual, post_layer_mix, comb_res_mix)
expected = (
next_residual,
*mhc_pre_delayed_torch(next_residual, *pre_args, pre_mix=pre_mix),
)
actual = MHCPreDelayedOp().forward_hip(
residual,
*pre_args,
pre_mix=pre_mix,
sublayer_out=x,
post_layer_mix=post_layer_mix,
comb_res_mix=comb_res_mix,
)
for got, want in zip(actual, expected, strict=True):
torch.testing.assert_close(got, want.reshape(got.shape), atol=2e-2, rtol=8e-3)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
def test_mhc_fused_post_pre_delayed_falls_back_for_large_batches():
"""AITER's heuristic hands large batches back to the unfused path."""
from vllm._aiter_ops import rocm_aiter_ops
assert not rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1)
assert rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1 << 20)
+85 -2
View File
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa:
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
gemma_fused_add_rmsnorm,
gemma_rmsnorm,
swiglu_oai_quantize_mxfp8,
swiglu_oai_split,
)
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
@@ -167,6 +168,25 @@ def test_swiglu_oai_split(m, inter, limit, dtype):
assert _relerr(got, ref) < 5e-3
@torch.inference_mode()
def test_swiglu_oai_quantize_mxfp8_uses_e4m3_range_for_scale():
# Keep a small value in two MX blocks next to their maxima. The scale must
# use E4M3's finite range (448), otherwise that value underflows to zero.
# Keep the third block empty to cover the reference's finite tiny clamp.
m, inter = 3, 96
gate = torch.full((m, inter), 2.0, device=DEVICE, dtype=torch.float16)
up = torch.full((m, inter), 2**-10, device=DEVICE, dtype=torch.float16)
up[:, :64:32] = 1.0
up[:, 64:] = 0.0
gate_up = torch.cat((gate, up), dim=-1)
got_q, got_s = swiglu_oai_quantize_mxfp8(gate_up, alpha=0.0, beta=0.0, limit=None)
ref_q, ref_s = _mxfp8_e4m3_quantize_torch(up, is_sf_swizzled_layout=False)
assert torch.equal(got_s, ref_s)
assert torch.equal(got_q, ref_q)
# --------------------------------------------------------------------------- #
# Fused MXFP8 activation quant (Triton vs torch reference)
# --------------------------------------------------------------------------- #
@@ -179,8 +199,8 @@ def test_mxfp8_quant_triton_matches_torch(shape, dtype):
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
# E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
# most a 1-step difference at exact powers of two.
# Both paths use the E4M3-aware scale calculation; allow a 1-step
# difference at exact powers of two due to floating-point rounding.
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
# Dequantized values agree to fp8 granularity.
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
@@ -424,6 +444,69 @@ def test_mxfp8_linear_emulation_bf16_at_load(
assert _relerr(out.float(), out_ref.float()) < 2e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 linear: the K % 128 != 0 weight is dequantized once at load
# --------------------------------------------------------------------------- #
@requires_gfx950
# (5120, 576) and (5120, 288): DSv4.1 shared-expert down_proj at TP4 and TP8.
@pytest.mark.parametrize("shape", [(5120, 576), (5120, 288), (256, 2048)])
@torch.inference_mode()
def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape):
"""``dot_scaled`` tiles K by 128, so an unaligned weight must not reach it.
``process_weights_after_loading`` converts those to BF16 once; a weight that
never went through it has to fall back per step rather than take the
invalid path, which is what keying the dispatch on the weight dtype alone
would do.
"""
from vllm.model_executor.kernels.linear.mxfp8 import rocm_native
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
N, K = shape
aligned = K % 128 == 0
torch.manual_seed(0)
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16) * 0.1
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
x = torch.randn(7, K, device=DEVICE, dtype=torch.bfloat16) * 0.5
out_ref = torch.nn.functional.linear(x, dequant_mxfp8_to_bf16(w_fp8, w_scale))
# dot_scaled re-quantizes the activation, the dequant paths do not.
tol = 5e-2 if aligned else 2e-3
def make_layer():
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
return layer
kernel = rocm_native.RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig())
layer = make_layer()
kernel.process_weights_after_loading(layer)
assert (layer.weight.element_size() >= 2) is not aligned
assert _relerr(kernel.apply_weights(layer, x).float(), out_ref.float()) < tol
# An unaligned weight that is still FP8 must not reach dot_scaled.
unprocessed = make_layer()
unprocessed.weight_scale = torch.nn.Parameter(
w_scale[:N, : K // 32].contiguous(), requires_grad=False
)
calls = []
real = rocm_native._mxfp8_dot_scaled_linear
def _spy(*a, **k):
calls.append(1)
return real(*a, **k)
rocm_native._mxfp8_dot_scaled_linear = _spy
try:
out = kernel.apply_weights(unprocessed, x)
finally:
rocm_native._mxfp8_dot_scaled_linear = real
assert bool(calls) is aligned
assert _relerr(out.float(), out_ref.float()) < tol
# ── EP expert_mask handling for the FlyDSL (AITER_MXFP8) MoE ────────────────
# The map/mask choice lives in ``RoutedExperts.expert_map``: it hands AITER
# experts (``consumes_expert_mask``) the precomputed 0/1 ``expert_mask`` and
+229
View File
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.config.vllm import OptimizationLevel
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
requires_sm90 = pytest.mark.skipif(
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
reason="This test requires SM90 or newer",
)
class _StaticFp8Linear(torch.nn.Module):
def __init__(self, input_scale: torch.Tensor) -> None:
super().__init__()
self._input_quant_key = kFp8StaticTensorSym
self.input_scale = input_scale
def _assert_fp8_bitwise_equal(actual: torch.Tensor, expected: torch.Tensor) -> None:
assert actual.dtype == expected.dtype == current_platform.fp8_dtype()
torch.testing.assert_close(
actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0
)
def _assert_quantized_activation(
result: QuantizedActivation, x: torch.Tensor, scale: torch.Tensor
) -> None:
assert result.scale is scale
assert result.orig_dtype == x.dtype
assert result.orig_shape == x.shape
assert result.quant_key == kFp8StaticTensorSym
@pytest.fixture(scope="module")
def o2_relu2_fp8_ops():
config = VllmConfig(
optimization_level=OptimizationLevel.O2,
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
yield (
ReLUSquaredActivation(),
QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR),
)
@requires_sm90
@pytest.mark.parametrize("shape", [(1, 1), (1, 16), (17, 5120), (128, 10240)])
@torch.inference_mode()
def test_relu2_static_fp8_quant(o2_relu2_fp8_ops, shape) -> None:
torch.manual_seed(0)
x = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.1495535671710968, device="cuda", dtype=torch.float32)
relu2, quant_fp8 = o2_relu2_fp8_ops
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@pytest.mark.parametrize(
"enabled_ops",
[(), ("relu2",), ("quant_fp8",), ("relu2", "quant_fp8")],
)
@torch.inference_mode()
def test_relu2_static_fp8_quant_ignores_custom_op_dispatch(enabled_ops) -> None:
custom_ops = ["none", *(f"+{op}" for op in enabled_ops)]
config = VllmConfig(
optimization_level=OptimizationLevel.O2,
compilation_config=CompilationConfig(custom_ops=custom_ops),
)
with set_current_vllm_config(config):
torch.manual_seed(0)
relu2 = ReLUSquaredActivation()
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@torch.inference_mode()
def test_relu2_static_fp8_quant_without_compilation() -> None:
config = VllmConfig(optimization_level=OptimizationLevel.O0)
with set_current_vllm_config(config):
torch.manual_seed(0)
relu2 = ReLUSquaredActivation()
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@pytest.mark.parametrize(
"scale_value",
[
0.00435965,
0.007149832788854837,
0.03125,
0.0714285746216774,
0.1495535671710968,
0.1517857164144516,
6.03571415,
],
)
@torch.inference_mode()
def test_relu2_static_fp8_quant_all_bf16_values(o2_relu2_fp8_ops, scale_value) -> None:
bits = torch.arange(1 << 16, dtype=torch.int32).to(torch.uint16)
x = bits.view(torch.bfloat16).reshape(1, -1).to("cuda")
scale = torch.tensor(scale_value, device="cuda", dtype=torch.float32)
relu2, quant_fp8 = o2_relu2_fp8_ops
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@torch.inference_mode()
def test_relu2_static_fp8_quant_empty(o2_relu2_fp8_ops) -> None:
x = torch.empty(0, device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
relu2, _ = o2_relu2_fp8_ops
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
assert result.data.shape == x.shape
assert result.data.dtype == current_platform.fp8_dtype()
@requires_sm90
@pytest.mark.parametrize("unsupported", ["dtype", "layout", "scale"])
@torch.inference_mode()
def test_relu2_static_fp8_quant_falls_back(o2_relu2_fp8_ops, unsupported: str) -> None:
x = torch.randn((17, 32), device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
if unsupported == "dtype":
x = x.to(torch.float16)
elif unsupported == "layout":
x = x.T
else:
scale = scale.to(torch.float16)
relu2, _ = o2_relu2_fp8_ops
expected = relu2(x)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, torch.Tensor)
assert not isinstance(result, QuantizedActivation)
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@requires_sm90
@torch.inference_mode()
def test_relu2_static_fp8_quant_torch_compile_dynamic(
o2_relu2_fp8_ops,
) -> None:
torch.manual_seed(1)
scale = torch.tensor(0.00435965, device="cuda", dtype=torch.float32)
relu2, quant_fp8 = o2_relu2_fp8_ops
linear = _StaticFp8Linear(scale)
def fused(x: torch.Tensor) -> torch.Tensor:
result = maybe_fused_act_quant(relu2, x, linear)
assert isinstance(result, QuantizedActivation)
return result.data
compile_count = 0
def counting_inductor_backend(gm, example_inputs):
nonlocal compile_count
compile_count += 1
return torch._inductor.compile(gm, example_inputs)
torch._dynamo.reset()
try:
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
torch._dynamo.mark_dynamic(x, 0)
compiled_fused = torch.compile(
fused, backend=counting_inductor_backend, fullgraph=True
)
for candidate in (
x,
torch.randn((31, 5120), device="cuda", dtype=torch.bfloat16),
torch.randn((63, 5120), device="cuda", dtype=torch.bfloat16),
):
expected, _ = quant_fp8(relu2(candidate), scale)
_assert_fp8_bitwise_equal(compiled_fused(candidate), expected)
assert compile_count == 1
finally:
torch._dynamo.reset()
+289
View File
@@ -22,11 +22,22 @@ def _has_device_capability(major: int) -> bool:
return current_platform.is_cuda() and current_platform.has_device_capability(major)
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
# DeepSelect is compiled for sm_100a/sm_103a only.
requires_sm100 = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="DeepSelect requires SM100a/SM103a",
)
requires_gfx950 = pytest.mark.skipif(
not _on_gfx950(), reason="This test exercises the gfx950 launch configuration"
)
COOPERATIVE_TOPK_BACKEND = pytest.param(
@@ -487,6 +498,150 @@ def test_top_k_per_row_decode_gfx950_long_c4a_1d_seq_lens() -> None:
)
def _assert_exact_topk(
logits: torch.Tensor, indices: torch.Tensor, row_ends: torch.Tensor
) -> None:
ends = row_ends.reshape(-1, 1).clamp_min(0)
valid = (indices >= 0) & (indices < ends)
assert torch.equal(valid.sum(1), ends.clamp_max(indices.shape[1]).flatten())
assert torch.all(valid | (indices == -1))
ordered = indices.sort(1).values
assert not ((ordered[:, 1:] == ordered[:, :-1]) & (ordered[:, 1:] >= 0)).any()
width = int(ends.max())
visible = logits[:, :width].clone()
columns = torch.arange(width, device=logits.device)
visible.masked_fill_(columns[None, :] >= ends, float("-inf"))
expected = visible.topk(min(indices.shape[1], width), dim=1).values
selected = logits.gather(1, indices.clamp_min(0).long())
selected.masked_fill_(~valid, float("-inf"))
selected = selected.sort(1, descending=True).values
assert torch.equal(selected[:, : expected.shape[1]], expected)
@requires_gfx950
@torch.inference_mode()
def test_top_k_per_row_decode_gfx950_k512_1d_seq_lens() -> None:
"""Honor causal offsets and exact compressed lengths, including short rows."""
next_n, width, top_k = 6, 16_384, 512
lengths = torch.tensor(
[0, 1, 511, 512, 513, 1023, 1024, 16_000],
dtype=torch.int32,
device="cuda",
)
offsets = torch.arange(next_n, dtype=torch.int32, device="cuda")
row_ends = (lengths[:, None] - next_n + offsets + 1).clamp_min(0).flatten()
rows = row_ends.numel()
storage = torch.full((rows, width + 8), 1e20, dtype=torch.float32, device="cuda")
logits = storage[:, 1 : width + 1]
logits[:, :16_000] = torch.arange(16_000, dtype=torch.float32, device="cuda")
indices = torch.full((rows, top_k), -777, dtype=torch.int32, device="cuda")
_run_topk_backend(
"top_k_per_row_decode", logits, lengths, indices, top_k, width, next_n
)
_assert_exact_topk(logits, indices, row_ends)
@pytest.mark.parametrize(
("rows", "row_length"),
[
pytest.param(128, 65_536, id="single-before-64k"),
pytest.param(128, 65_537, id="four-splits-after-64k"),
pytest.param(129, 262_144, id="single-before-three-split-length"),
pytest.param(129, 262_145, id="three-splits-after-256k"),
pytest.param(160, 262_145, id="three-splits-upper-row-bound"),
pytest.param(161, 262_144, id="single-before-two-split-length"),
pytest.param(161, 262_145, id="two-splits-after-256k"),
pytest.param(256, 262_145, id="two-splits-upper-row-bound"),
pytest.param(257, 262_145, id="large-row-single-block"),
pytest.param(384, 262_145, id="optimized-row-limit"),
pytest.param(385, 262_145, id="native-fallback"),
],
)
@requires_gfx950
@torch.inference_mode()
def test_top_k_per_row_decode_gfx950_k512_split_policy_boundaries(
rows: int, row_length: int
) -> None:
width, top_k = 262_145, 512
row = torch.arange(width, dtype=torch.float32, device="cuda")
logits = row.expand(rows, -1)
lengths = torch.full((rows,), row_length, dtype=torch.int32, device="cuda")
indices = torch.empty((rows, top_k), dtype=torch.int32, device="cuda")
_run_topk_backend("top_k_per_row_decode", logits, lengths, indices, top_k, width)
expected = torch.arange(
row_length - top_k, row_length, dtype=torch.int32, device="cuda"
).expand_as(indices)
assert torch.equal(indices.sort(1).values, expected)
@requires_gfx950
@torch.inference_mode()
def test_top_k_per_row_decode_gfx950_k512_masked_graph_replay() -> None:
"""Preserve visible top-k scores when masks and lengths change on replay."""
rows, next_n, width, top_k = 24, 6, 262_145, 512
logits = torch.full((rows, width), 1e20, dtype=torch.float32, device="cuda")
lengths = torch.full(
(rows // next_n, next_n), 70_000, dtype=torch.int32, device="cuda"
)
indices = torch.full((rows, top_k), -777, dtype=torch.int32, device="cuda")
logits[:, :70_000] = float("-inf")
bin_sizes = [128, 129, 1024, 1025]
for row in range(rows):
pattern = row % 9
if pattern < 4:
count = [0, 1, 511, 513][pattern]
logits[row, :count] = torch.arange(
count, dtype=torch.float32, device="cuda"
)
elif pattern == 4:
logits[row, :511] = 0
logits[row, 511] = torch.finfo(torch.float32).min
else:
count = bin_sizes[pattern - 5]
higher = max(0, top_k - count // 2)
logits[row, :higher] = 2
logits[row, higher : higher + count] = 1
logits[row, higher] = 1 + 2**-23
def run() -> None:
_run_topk_backend(
"top_k_per_row_decode", logits, lengths, indices, top_k, width, next_n
)
run()
_assert_exact_topk(logits, indices, lengths)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
indices.fill_(-777)
graph.replay()
_assert_exact_topk(logits, indices, lengths)
bounds = [
0,
511,
512,
513,
16_384,
16_385,
131_072,
131_073,
262_144,
262_145,
]
row_bounds = torch.tensor(bounds, dtype=torch.int32, device="cuda")
repeats = (rows + len(bounds) - 1) // len(bounds)
lengths.flatten().copy_(row_bounds.repeat(repeats)[:rows])
logits[:] = torch.arange(width, dtype=torch.float32, device="cuda")
indices.fill_(-777)
graph.replay()
_assert_exact_topk(logits, indices, lengths)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
@@ -620,6 +775,31 @@ def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
compress_ratio=4,
num_rows=257,
max_valid_seq_len=250_000,
num_columns=524_288,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=2,
num_rows=385,
max_valid_seq_len=250_000,
num_columns=524_288,
topk_tokens=512,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=2,
num_rows=384,
max_valid_seq_len=50_000,
num_columns=1_048_577,
topk_tokens=512,
on_gfx950=True,
),
decode_kernel,
@@ -638,6 +818,15 @@ def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
)
ineligible_cases = [
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
dict(
is_prefill=False,
compress_ratio=2,
num_rows=384,
max_valid_seq_len=1_048_576,
num_columns=1_048_576,
topk_tokens=512,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
@@ -743,6 +932,106 @@ def test_deepseek_workspace_topk(
)
@pytest.mark.skipif(not _has_device_capability(80), reason="Requires SM80+")
@pytest.mark.parametrize("rows", [65, 128])
@pytest.mark.parametrize("top_k", [512, 1024, 2048])
@pytest.mark.parametrize(
"distribution", ["random", "10LSBits", "ascending", "constant", "sampled_peaks"]
)
@torch.inference_mode()
def test_persistent_topk_sampled_graph(
rows: int, top_k: int, distribution: str
) -> None:
"""Sampling and both fallbacks preserve exact values across graph replays."""
if torch.cuda.get_device_properties(0).shared_memory_per_block_optin < 144 * 1024:
pytest.skip("Sampled top-k requires at least 144 KiB of shared memory")
set_random_seed(42)
min_sampled_length = 98304 if top_k == 512 else 65536
width = min_sampled_length + 3
lengths = torch.full((rows,), width, dtype=torch.int32, device="cuda")
logits = create_random_logits(
torch.zeros_like(lengths),
lengths,
torch.float32,
42,
False,
"10LSBits" if distribution == "10LSBits" else "random",
)
if distribution == "ascending":
logits.copy_(torch.arange(width, device="cuda", dtype=torch.float32))
elif distribution == "constant":
logits.fill_(1.0)
elif distribution == "sampled_peaks":
# A biased sample leaves fewer than k survivors and must fall back.
logits.zero_()
sample = torch.arange(4096, device="cuda")
positions = (sample // 32) * (width // 128) + sample % 32
logits[:, positions] = 1 + sample.float() / 4096
original = logits.clone()
indices = torch.empty((rows, top_k), dtype=torch.int32, device="cuda")
workspace = torch.empty(RADIX_TOPK_WORKSPACE_SIZE, dtype=torch.uint8, device="cuda")
def run() -> None:
torch.ops._C.persistent_topk(
logits,
lengths.view(-1, 4 if rows == 128 else 1),
indices,
workspace,
top_k,
width,
)
run()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
positions = torch.arange(width, device="cuda")
slots = torch.arange(top_k, device="cuda")
for step in range(3):
bounds = torch.tensor(
[
-1,
0,
1,
top_k - 1,
top_k,
32769,
min_sampled_length - 1,
min_sampled_length,
width,
width + 17,
],
dtype=torch.int32,
device="cuda",
)
lengths.copy_(
bounds[(torch.arange(rows, device="cuda") + step) % bounds.numel()]
)
logits.copy_(original if step != 1 else original.flip(1))
logits.masked_fill_(positions[None] >= lengths[:, None], float("nan"))
indices.fill_(-2)
graph.replay()
valid = slots[None] < lengths.clamp(max=top_k)[:, None]
assert torch.all(indices[~valid] == -1)
assert torch.all(((indices >= 0) & (indices < lengths[:, None])) == valid)
ordered_indices = indices.sort(dim=1).values
assert torch.all(
(ordered_indices[:, 1:] != ordered_indices[:, :-1])
| (ordered_indices[:, 1:] == -1)
)
selected = logits.gather(1, indices.clamp_min(0).long())
selected.masked_fill_(~valid, -float("inf"))
expected = logits.masked_fill(
positions[None] >= lengths[:, None], -float("inf")
)
torch.testing.assert_close(
selected.sort(dim=1, descending=True).values,
expected.topk(top_k, dim=1).values,
atol=0,
rtol=0,
)
def run_large_context_topk_test(
batch_size: int,
seq_lens: list[int],
+10
View File
@@ -31,6 +31,9 @@ from vllm.lora.layers import (
)
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
from vllm.lora.punica_wrapper import get_punica_wrapper
from vllm.model_executor.layers.fusion.quant_activation import (
get_input_quant_key,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
@@ -56,6 +59,13 @@ TOLERANCES = {
torch.bfloat16: (3e-2, 2e-2),
}
def test_lora_linear_requires_unquantized_input() -> None:
layer = RowParallelLinearWithLoRA.__new__(RowParallelLinearWithLoRA)
layer._input_quant_key = object()
assert get_input_quant_key(layer) is None
pytestmark = [
pytest.mark.skipif(
not (
@@ -115,7 +115,9 @@ def test_b12x_backend_registration_priority_and_selection(
assert names.index(before) < names.index(kernel_cls.__name__) < names.index(after)
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
monkeypatch.setattr(linear_mod, "_get_linear_backend", lambda: "b12x")
monkeypatch.setattr(
linear_mod, "_get_linear_backend", lambda *, quantization: "b12x"
)
monkeypatch.setattr(
kernel_cls,
"is_supported",
@@ -784,7 +786,9 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
import vllm.model_executor.kernels.linear as linear_mod
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
monkeypatch.setattr(linear_mod, "_get_linear_backend", lambda: "b12x")
monkeypatch.setattr(
linear_mod, "_get_linear_backend", lambda *, quantization: "b12x"
)
monkeypatch.setitem(
linear_mod._POSSIBLE_NVFP4_KERNELS,
PlatformEnum.CUDA,
@@ -450,3 +450,547 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
keep = valid & (block >= 0) & ((cols - ks[:, None]) // block_size == block)
reference = torch.where(keep, 3.0, -torch.inf)
torch.testing.assert_close(logits, reference, rtol=0, atol=0)
# --- DeepGEMM sparse-MQA indexer path (V4.1 two-level candidate filtering) ---
def _make_candidates(lens: torch.Tensor, num_candidates: int, cbk: int):
"""Production-faithful candidates: top min(avail, K) blocks per row —
full coverage for lens <= K*cbk, a fixed-size subset beyond that."""
avail = ((lens.cpu() + cbk - 1) // cbk).tolist()
rows = []
for a in avail:
n = min(a, num_candidates)
picks = torch.randperm(a)[:n].sort().values[:num_candidates].tolist()
rows.append(picks + [-1] * (num_candidates - len(picks)))
return torch.tensor(rows, dtype=torch.int32)
def _candidate_token_mask(
candidates: torch.Tensor, cbk: int, total: int, ks: torch.Tensor | None = None
):
"""[rows, total] bool mask of the candidate-covered token positions.
``ks`` converts request-local candidate blocks to absolute positions
(prefill's packed workspace); pass None when already request-local.
"""
rows = candidates.shape[0]
device = candidates.device
# Sentinel block keeps -1 padding from scattering onto real columns.
width = max(total, (candidates.shape[1] + 1) * cbk)
if ks is not None:
width += int(ks.max()) + cbk
in_cand = torch.zeros(rows, width, dtype=torch.bool, device=device)
block_id = torch.where(candidates >= 0, candidates.long(), candidates.shape[1])
tok = block_id.unsqueeze(2) * cbk + torch.arange(cbk, device=device)
if ks is not None:
tok = tok + ks.unsqueeze(1).unsqueeze(2)
keep = (candidates >= 0).unsqueeze(2).expand(-1, -1, cbk)
in_cand.scatter_(1, tok.flatten(1), keep.flatten(1))
return in_cand[:, :total]
def _reference_topk(masked: torch.Tensor, topk: int) -> torch.Tensor:
"""Plain ``torch.topk`` over -inf-masked logits; slots past a row's valid
count are -1 (the indexer's "no token" sentinel)."""
values, indices = masked.topk(topk, dim=1)
return torch.where(values > float("-inf"), indices.int(), -1)
def _assert_same_selection(
logits: torch.Tensor,
got: torch.Tensor,
expected: torch.Tensor,
row_starts: torch.Tensor | None = None,
) -> None:
"""Both top-k index sets must pick the same multiset of logit values per
row (-1 padding must match too). Indices themselves may differ: the
top-k kernels break ties among equal values in nondeterministic order."""
for r in range(logits.shape[0]):
base = int(row_starts[r]) if row_starts is not None else 0
g, e = got[r][got[r] >= 0], expected[r][expected[r] >= 0]
assert g.numel() == e.numel(), f"row {r}: {g.numel()} vs {e.numel()} selected"
vg = logits[r][(base + g).long()].sort().values
ve = logits[r][(base + e).long()].sort().values
assert torch.equal(vg, ve), f"row {r}: selected values differ"
def _sparse_scratch(rows: int, num_sparse: int, topk: int) -> dict:
"""Caller-owned buffers the sparse path writes into (the metadata builder
owns these in production)."""
i32 = dict(dtype=torch.int32, device="cuda")
return dict(
sparse_indices=torch.zeros(rows, num_sparse, **i32),
end=torch.zeros(rows, **i32),
col_indices=torch.zeros(rows, topk, **i32),
)
def _expand_candidates_reference(
candidates: torch.Tensor, ks: torch.Tensor, ke: torch.Tensor, cbk: int, sbk: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""PyTorch reference for `candidate_blocks_to_sparse_indices`: expand
each candidate block into ``cbk // sbk`` sparse blocks, keep those inside
``[0, ke - ks)``, sort ascending, anchor at ``ks // sbk``, pad by
repeating the last valid block, and count the valid sparse columns."""
ratio = cbk // sbk
rows, num_candidates = candidates.shape
num_sparse = num_candidates * ratio
big = torch.iinfo(torch.int32).max
offsets = torch.arange(ratio, device=candidates.device, dtype=torch.int32)
blocks = (candidates.unsqueeze(-1) * ratio + offsets).flatten(1)
max_block = ((ke - ks).clamp(min=0) + sbk - 1) // sbk
valid = (blocks >= 0) & (blocks < max_block.unsqueeze(1))
sorted_blocks, _ = torch.where(valid, blocks, big).sort(dim=1)
n_valid = (sorted_blocks != big).sum(dim=1, dtype=torch.int32)
base = (ks // sbk).unsqueeze(1)
kernel_blocks = torch.where(sorted_blocks != big, sorted_blocks, 0) + base
last_slot = (n_valid - 1).clamp(min=0).unsqueeze(1).long()
last_block = kernel_blocks.gather(1, last_slot)
pad = torch.arange(num_sparse, device=candidates.device) >= n_valid.unsqueeze(1)
sparse_indices = torch.where(pad, last_block, kernel_blocks)
last_start = last_block.squeeze(1) * sbk + ks % sbk
last_fill = (ke - last_start).clamp(min=1, max=sbk)
end = torch.where(n_valid > 0, (n_valid - 1) * sbk + last_fill, 0)
return sparse_indices.int(), end.int()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
def test_candidate_blocks_to_sparse_indices_math():
"""Candidate->sparse-block expansion: ks anchoring, range filtering,
repeat-last padding, and the per-row valid column count (``end``)."""
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
candidate_blocks_to_sparse_indices,
)
# Rows cover: exactly one block, an unaligned ks with a partial tail
# block, an empty row, duplicates / -1 padding / out-of-range garbage
# (warmup runs touch uninitialized buffers), and int32-overflow junk.
ks = torch.tensor([64, 13, 100, 8, 0], dtype=torch.int32, device="cuda")
ke = torch.tensor([72, 30, 100, 40, 1000], dtype=torch.int32, device="cuda")
candidates = torch.tensor(
[
[0, -1, -1, -1],
[2, 0, -1, -1],
[3, -1, -1, -1],
[1, 1, 7, 0],
[2**30, -(2**30), 0, -1],
],
dtype=torch.int32,
device="cuda",
)
indices, end = candidate_blocks_to_sparse_indices(
candidates, ks, ke, candidate_block_size=8, sparse_block_kv=8
)
assert indices.tolist() == [
[8, 8, 8, 8],
[1, 3, 3, 3],
[12, 12, 12, 12],
[1, 2, 2, 2],
[0, 0, 0, 0],
]
# end: row 1's last block covers [29, 37) but ke = 30 -> one valid token;
# row 3 keeps the duplicated block (production candidates are unique).
assert end.tolist() == [8, 9, 0, 24, 8]
# Wider candidate blocks expand into multiple sparse blocks.
idx2, end2 = candidate_blocks_to_sparse_indices(
torch.tensor([[1, 0]], dtype=torch.int32, device="cuda"),
torch.tensor([0], dtype=torch.int32, device="cuda"),
torch.tensor([48], dtype=torch.int32, device="cuda"),
candidate_block_size=16,
sparse_block_kv=8,
)
assert idx2.tolist() == [[0, 1, 2, 3]]
assert end2.tolist() == [32]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
@pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)])
def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int):
"""The Triton expansion equals the PyTorch reference on production-shaped
random inputs, including strided (column-sliced) candidate rows and
strided output buffers."""
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
candidate_blocks_to_sparse_indices,
)
torch.manual_seed(0)
rows, num_candidates = 41, 2048
ratio = cbk // sbk
lens = torch.randint(1, 40000, (rows,), dtype=torch.int32)
lens[0], lens[1] = 1, 3
ks = torch.zeros(rows, dtype=torch.int32)
ks[1:] = lens.cumsum(0)[:-1]
ks, ke = ks.cuda(), (ks + lens).cuda()
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
# Row 2 gets out-of-range garbage past the padding, as a CUDA-graph
# warm-up on uninitialized buffers would.
candidates[2, -3:] = torch.tensor([2**30, -(2**29), 10**6], dtype=torch.int32)
ref_indices, ref_end = _expand_candidates_reference(candidates, ks, ke, cbk, sbk)
cand_buf = torch.full((rows, num_candidates + 64), -7, dtype=torch.int32).cuda()
cand_buf[:, :num_candidates] = candidates
si_buf = torch.zeros(rows, num_candidates * ratio + 32, dtype=torch.int32).cuda()
end_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
ks_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
ks_buf[::2] = ks
got_indices, got_end = candidate_blocks_to_sparse_indices(
cand_buf[:, :num_candidates],
ks_buf[::2],
ke,
cbk,
sbk,
out=(si_buf[:, : num_candidates * ratio], end_buf[::2]),
)
assert torch.equal(got_indices, ref_indices)
assert torch.equal(got_end, ref_end)
assert torch.equal(
cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7)
)
assert not si_buf[:, num_candidates * ratio :].any()
assert not end_buf[1::2].any()
def _skip_unless_sm100_sparse_kernels():
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
pytest.skip("SM100 required")
deep_gemm = pytest.importorskip("deep_gemm")
if not hasattr(deep_gemm, "fp8_fp4_sparse_mqa_logits"):
pytest.skip("DeepGEMM >= 2.8 required for sparse MQA logits")
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
has_deep_select,
)
if not has_deep_select():
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
return deep_gemm
def _quant_fp4(x: torch.Tensor):
from deep_gemm.utils import per_token_cast_to_fp4
return per_token_cast_to_fp4(x, use_ue8m0=True, gran_k=32, use_packed_ue8m0=True)
@pytest.mark.parametrize("capture", [False, True])
def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
"""The sparse-MQA candidate path selects exactly the same logit values as
``torch.topk`` over dense bf16 logits masked to the candidates (the two
DeepGEMM kernels agree bitwise on candidate columns), incl. under CUDA
graph capture, and a later layer reusing the step's expansion/schedule
selects the same values."""
deep_gemm = _skip_unless_sm100_sparse_kernels()
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
sparse_mqa_logits_prefill_chunk,
)
torch.manual_seed(0)
rows, num_heads, head_dim = 37, 32, 128
cbk, topk, num_candidates = 8, 512, 2048 # V4.1-Flash production values
lens = torch.randint(1, 30000, (rows,), dtype=torch.int32)
lens[0], lens[1], lens[2] = 1, 5, 2048 # single token, sub-block, long
ks = torch.zeros(rows, dtype=torch.int32)
ks[1:] = lens.cumsum(0)[:-1]
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
total = int(lens.sum())
q_fp, q_sf = _quant_fp4(
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
)
k_fp, k_sf = _quant_fp4(
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
)
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
dense = deep_gemm.fp8_fp4_mqa_logits(
(
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
q_sf.view(rows, num_heads),
),
(k_fp.view(torch.int8), k_sf.view(total)),
weights,
ks,
ke,
clean_logits=False,
logits_dtype=torch.bfloat16,
)
cols = torch.arange(total, device="cuda")
in_range = (cols - ks.unsqueeze(1) >= 0) & (
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
)
masked = dense.masked_fill(
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
float("-inf"),
).float()
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
scratch = _sparse_scratch(rows, num_candidates, topk)
def run_sparse(kernel_metadata=None):
return sparse_mqa_logits_prefill_chunk(
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
q_sf.view(rows, num_heads),
k_fp.view(torch.int8),
k_sf.view(total),
weights,
ks,
ke,
candidates,
cbk,
cbk,
topk,
out,
kernel_metadata=kernel_metadata,
**scratch,
)
if capture:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
run_sparse() # warm up JIT + the DeepGEMM workspace pre-capture
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
run_sparse()
out.fill_(-1)
graph.replay()
else:
kernel_metadata = run_sparse()
_assert_same_selection(dense, out, ref, row_starts=ks)
out.fill_(-1)
run_sparse(kernel_metadata)
_assert_same_selection(dense, out, ref, row_starts=ks)
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
"""MXFP4 paged cache in DeepGEMM's fused layout: per page, all tokens'
fp4 data first, then their packed UE8M0 scales."""
device = req_lens.device
pages_per = (req_lens + page_kv - 1) // page_kv
num_pages, max_pages = int(pages_per.sum()), int(pages_per.max())
kv_cache = torch.zeros(
num_pages, page_kv, 1, head_dim // 2 + 4, dtype=torch.uint8, device=device
)
block_table = torch.zeros(
len(req_lens), max_pages, dtype=torch.int32, device=device
)
page_pool = torch.randperm(num_pages, device=device)
offset = 0
for b in range(len(req_lens)):
n = int(pages_per[b])
block_table[b, :n] = page_pool[offset : offset + n].to(torch.int32)
offset += n
k_fp, k_sf = _quant_fp4(
torch.randn(int(req_lens[b]), head_dim, device=device, dtype=torch.bfloat16)
)
rows68 = torch.zeros(n * page_kv, 68, dtype=torch.uint8, device=device)
rows68[: int(req_lens[b]), : head_dim // 2] = k_fp.view(torch.uint8)
rows68[: int(req_lens[b]), head_dim // 2 :] = k_sf.view(torch.uint8).view(-1, 4)
by_page = rows68.view(n, page_kv, 68)
split = torch.cat(
[by_page[..., :64].reshape(n, -1), by_page[..., 64:].reshape(n, -1)], dim=1
).view(n, page_kv, 1, head_dim // 2 + 4)
kv_cache[block_table[b, :n].long()] = split
return kv_cache, block_table
@pytest.mark.parametrize("next_n", [1, 2])
@pytest.mark.parametrize("capture", [False, True])
def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool):
"""Paged sparse-MQA decode selects exactly the values ``torch.topk`` picks
from the dense bf16 masked logits, incl. MTP rows (next_n > 1 flattened to
one query row each), CUDA graph capture/replay and schedule reuse by a
later layer."""
deep_gemm = _skip_unless_sm100_sparse_kernels()
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
sparse_mqa_logits_paged_decode,
)
torch.manual_seed(0)
batch, num_heads, head_dim, page_kv = 9, 32, 128, 128
cbk, topk, num_candidates = 8, 512, 2048
rows = batch * next_n
req_lens = torch.randint(1, 30000, (batch,), dtype=torch.int32, device="cuda")
req_lens[0], req_lens[1] = 1, 30000 # tiny row + a filtered (subset) row
# Row (b, j) sees L_b - next_n + j + 1 tokens (native MTP layout).
row_lens = (
(req_lens.unsqueeze(1) - next_n + torch.arange(1, next_n + 1, device="cuda"))
.clamp(min=0)
.to(torch.int32)
)
kv_cache, block_table = _build_fp4_paged_cache(req_lens, page_kv, head_dim)
assert kv_cache.stride(0) % 512 == 0 # the paged sparse kernel requires it
q_fp, q_sf = _quant_fp4(
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
)
q = q_fp.view(torch.int8).view(batch, next_n, num_heads, head_dim // 2)
q_scale = q_sf.view(batch, next_n, num_heads)
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
def reference_topk():
context_lens_2d = row_lens.view(batch, next_n)
sched = deep_gemm.get_paged_mqa_logits_metadata(
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
)
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
(q, q_scale),
kv_cache,
weights,
context_lens_2d,
block_table,
sched,
int(req_lens.max()),
False,
torch.bfloat16,
None,
)
lens_flat = row_lens.flatten()
cols = torch.arange(logits.shape[1], device="cuda")
masked = logits.masked_fill(
~(
(cols.unsqueeze(0) < lens_flat.unsqueeze(1))
& _candidate_token_mask(candidates, cbk, logits.shape[1])
),
float("-inf"),
).float()
return _reference_topk(masked, topk), logits
expected, dense_logits = reference_topk()
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
# The metadata builder flattens every query to one row: per-row context
# lengths, an expanded block table and a row -> request map.
context_lens = row_lens.flatten().contiguous()
row_block_table = block_table.repeat_interleave(next_n, dim=0)
row_indices = torch.arange(batch, dtype=torch.int32, device="cuda")
row_indices = row_indices.repeat_interleave(next_n)
row_ks = torch.zeros(rows, dtype=torch.int32, device="cuda")
scratch = _sparse_scratch(rows, num_candidates, topk)
def run_sparse(o: torch.Tensor, kernel_metadata=None):
return sparse_mqa_logits_paged_decode(
q.view(rows, 1, num_heads, head_dim // 2),
q_scale.view(rows, 1, num_heads),
kv_cache,
weights,
context_lens,
row_block_table,
row_indices,
candidates,
cbk,
cbk,
topk,
o,
row_ks=row_ks,
kernel_metadata=kernel_metadata,
**scratch,
)
if capture:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
run_sparse(out)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
run_sparse(out)
out.fill_(-1)
graph.replay()
else:
kernel_metadata = run_sparse(out)
_assert_same_selection(dense_logits, out, expected)
out.fill_(-1)
run_sparse(out, kernel_metadata)
_assert_same_selection(dense_logits, out, expected)
def test_sparse_mqa_logits_accuracy_vs_torch():
"""Sparse-kernel numerics vs a PyTorch fp32 reference on the dequantized
MXFP4 inputs."""
_skip_unless_sm100_sparse_kernels()
from deep_gemm.utils import cast_back_from_fp4
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
candidate_blocks_to_sparse_indices,
)
from vllm.utils.deep_gemm import (
fp8_fp4_sparse_mqa_logits,
get_sparse_mqa_logits_metadata,
)
torch.manual_seed(0)
rows, num_heads, head_dim = 17, 32, 128
sbk, num_candidates = 8, 2048
lens = torch.randint(1, 1500, (rows,), dtype=torch.int32)
lens[0] = 1
ks = torch.zeros(rows, dtype=torch.int32)
ks[1:] = lens.cumsum(0)[:-1]
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
total = int(lens.sum())
q_fp, q_sf = _quant_fp4(
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
)
k_fp, k_sf = _quant_fp4(
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
)
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
candidates = _make_candidates(lens, num_candidates, sbk).cuda()
sparse_idx, end = candidate_blocks_to_sparse_indices(candidates, ks, ke, sbk, sbk)
metadata = get_sparse_mqa_logits_metadata(
ks, ke, total, sparse_idx, torch.int8, sbk
)
logits = fp8_fp4_sparse_mqa_logits(
(
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
q_sf.view(rows, num_heads),
),
(k_fp.view(torch.int8), k_sf.view(total)),
weights,
metadata,
sparse_idx.shape[1],
sbk,
)
# PyTorch fp32 reference on the dequantized inputs.
q_deq = cast_back_from_fp4(q_fp, q_sf, gran_k=32, use_packed_ue8m0=True)
k_deq = cast_back_from_fp4(k_fp, k_sf, gran_k=32, use_packed_ue8m0=True)
dots = torch.einsum(
"rhd,td->rht",
q_deq.view(rows, num_heads, head_dim).float(),
k_deq.float(),
)
ref = (dots.relu() * weights.float().unsqueeze(-1)).sum(1) # [rows, total]
# Compare on valid sparse columns (absolute packed-workspace positions).
width = sparse_idx.shape[1] * sbk
cols = torch.arange(width, device="cuda")
valid = cols.unsqueeze(0) < end.unsqueeze(1).long()
pos = (
sparse_idx.repeat_interleave(sbk, dim=1).long() * sbk
+ (ks % sbk).unsqueeze(1)
+ (cols % sbk).unsqueeze(0)
).clamp(min=0, max=total - 1)
abs_err = (logits.float() - ref.gather(1, pos)).abs()[valid]
row_scale = ref.abs().amax(dim=1)
norm_err = ((logits.float() - ref.gather(1, pos)).abs() / row_scale.unsqueeze(1))[
valid
]
print(
f"sparse vs torch fp32: max_abs={abs_err.max().item():.4f}; "
f"rowmax-normalized p50={norm_err.median().item():.5f} "
f"p99={norm_err.quantile(0.99).item():.5f} max={norm_err.max().item():.5f}"
)
# bf16-level numerics: far below the value gaps that change top-k selection.
assert norm_err.quantile(0.99).item() < 1e-2
assert norm_err.max().item() < 2e-2
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import importlib.machinery
import inspect
import sys
import types
from unittest.mock import Mock
from weakref import WeakKeyDictionary, ref
@@ -817,6 +820,110 @@ def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch):
assert "0.scale" not in model.state_dict()
@pytest.fixture
def hpc_rope_norm(monkeypatch, default_vllm_config):
"""Import HpcRopeNorm with the external ``hpc`` package stubbed out."""
if "hpc" not in sys.modules:
stub = types.ModuleType("hpc")
stub.__spec__ = importlib.machinery.ModuleSpec("hpc", loader=None)
stub.QuantType = types.SimpleNamespace( # type: ignore[attr-defined]
QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR=types.SimpleNamespace(value=0)
)
monkeypatch.setitem(sys.modules, "hpc", stub)
from vllm.model_executor.layers.hpc import rope_norm
monkeypatch.setattr(rope_norm, "_hpc_rope_norm_instances", {})
return rope_norm
def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm):
"""The fused HPC kernel is handed the live QK-norm weights after a refit.
Drives the production ``_forward_impl`` with a recording ``hpc`` stub. The
Q/K norm weights it receives must be the model's own float32 parameters,
so a layerwise reload that rewrites them in place (same storage) is what
the kernel sees. Previously the kernel read separate mirrors that no
reload path refreshed.
"""
from vllm.model_executor.layers.layernorm import RMSNorm
head_dim, num_heads, num_kv_heads, block_size = 128, 8, 1, 4
layer = torch.nn.Module()
layer.q_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
layer.k_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
layer.hpc_rope_norm = hpc_rope_norm.HpcRopeNorm(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
cos_sin_cache=torch.ones(16, head_dim),
use_qk_norm=True,
fallback_qnorm=layer.q_norm,
fallback_knorm=layer.k_norm,
kv_cache_dtype="auto",
layer_name="hpc_test_layer",
)
model = torch.nn.Sequential(layer)
rnorm = layer.hpc_rope_norm
calls: list[dict] = []
def record(*args, **kwargs):
calls.append(kwargs)
monkeypatch.setattr(sys.modules["hpc"], "rope_norm_store_kv", record, raising=False)
def kernel_norm_weights():
q_size, kv_size = num_heads * head_dim, num_kv_heads * head_dim
qkv = torch.zeros(1, q_size + 2 * kv_size, dtype=torch.bfloat16)
kv_cache = torch.zeros(
2, num_kv_heads, block_size, 2 * head_dim, dtype=torch.bfloat16
)
attn_layer = types.SimpleNamespace(
_k_scale=torch.ones(1), _v_scale=torch.ones(1)
)
attn_metadata = types.SimpleNamespace(
num_actual_tokens=1,
num_decodes=1,
num_decode_tokens=1,
num_prefills=0,
num_prefill_tokens=0,
max_query_len=1,
decode_query_len=1,
qo_indptr=None,
qo_indptr_decode=None,
slot_mapping=torch.tensor([4]),
seq_lens=torch.tensor([1]),
block_table_tensor=torch.tensor([[1]]),
hpc_kv_written=False,
)
output = torch.zeros(1, q_size, dtype=torch.bfloat16)
rnorm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output)
return calls[-1]["q_norm_weight"], calls[-1]["k_norm_weight"]
def loaded(value):
return torch.full((head_dim,), value, dtype=torch.bfloat16)
default_weight_loader(layer.q_norm.weight, loaded(0.5))
default_weight_loader(layer.k_norm.weight, loaded(0.25))
q, k = kernel_norm_weights()
assert torch.equal(q, loaded(0.5).float())
assert torch.equal(k, loaded(0.25).float())
q_ptr, k_ptr = q.data_ptr(), k.data_ptr()
record_metadata_for_reloading(model)
initialize_layerwise_reload(model)
layer.q_norm.weight.weight_loader(layer.q_norm.weight, loaded(2.0))
layer.k_norm.weight.weight_loader(layer.k_norm.weight, loaded(3.0))
finalize_layerwise_reload(model, model_config=None)
q, k = kernel_norm_weights()
assert q.dtype == k.dtype == torch.float32
assert torch.equal(q, loaded(2.0).float())
assert torch.equal(k, loaded(3.0).float())
assert (q.data_ptr(), k.data_ptr()) == (q_ptr, k_ptr)
assert not hasattr(rnorm, "qnorm_weight")
@pytest.mark.parametrize(
"tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
)
@@ -7,6 +7,7 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
identical outputs.
"""
import glob
import os
import shutil
import subprocess
@@ -21,7 +22,6 @@ import pytest
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.model_executor.model_loader.weight_cache.protocol import get_socket_path
from vllm.platforms import current_platform
DAEMON_TIMEOUT_S = 600
@@ -79,9 +79,7 @@ class WeightCacheDaemon:
# Poll for the socket files rather than a log line: model loading can
# pull in JIT compilers that swap the process's stderr and swallow
# everything logged afterwards, making log-based readiness flaky.
expected = [
get_socket_path(gpu_id, self.socket_dir) for gpu_id in range(self.tp_size)
]
pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock")
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if self._proc.poll() is not None:
@@ -89,7 +87,7 @@ class WeightCacheDaemon:
f"Weight cache daemon exited with {self._proc.returncode}:\n"
f"{self._logs()}"
)
if all(os.path.exists(path) for path in expected):
if len(glob.glob(pattern)) >= self.tp_size:
return
time.sleep(1.0)
raise TimeoutError(
@@ -3,6 +3,8 @@
from unittest.mock import Mock, patch
import torch
def test_nemotron_h_lm_head_receives_quant_config():
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
@@ -32,3 +34,28 @@ def test_nemotron_h_lm_head_receives_quant_config():
MockLMHead.assert_called_once()
call_kwargs = MockLMHead.call_args.kwargs
assert call_kwargs["quant_config"] is mock_quant_config
def test_relu2_fp8_fusion_uses_registry():
from vllm.model_executor.models.nemotron_h import NemotronHMLP
projected = torch.empty((1, 1), dtype=torch.bfloat16)
fused = Mock()
act_fn = Mock()
down_proj = Mock(side_effect=lambda x: (x, None))
mlp = NemotronHMLP.__new__(NemotronHMLP)
torch.nn.Module.__init__(mlp)
mlp.up_proj = Mock(return_value=(projected, None))
mlp.down_proj = down_proj
mlp.act_fn = act_fn
with patch(
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
return_value=fused,
) as maybe_fused:
result = mlp(Mock())
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
act_fn.assert_not_called()
assert result is fused
+35 -14
View File
@@ -10,6 +10,7 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm.config import ParallelConfig
from vllm.multimodal.processing import InputProcessingContext
@@ -293,6 +294,25 @@ def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
def _dspark_vocab_stub(input_vocab_size: int, draft_vocab_size: int):
"""A Qwen3 DSpark model with only what `load_weights` reads."""
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
nn.Module.__init__(model)
object.__setattr__(
model,
"config",
SimpleNamespace(
vocab_size=input_vocab_size,
draft_vocab_size=draft_vocab_size,
),
)
object.__setattr__(model, "target_vocab_size", 100)
object.__setattr__(model, "model", Mock(confidence_head=None))
return model
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
("input_vocab_size", "draft_vocab_size", "weights", "error"),
@@ -311,24 +331,25 @@ def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
input_vocab_size, draft_vocab_size, weights, error
):
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
nn.Module.__init__(model)
object.__setattr__(
model,
"config",
SimpleNamespace(
vocab_size=input_vocab_size,
draft_vocab_size=draft_vocab_size,
),
)
object.__setattr__(model, "target_vocab_size", 100)
model = _dspark_vocab_stub(input_vocab_size, draft_vocab_size)
with pytest.raises(ValueError, match=error):
model.load_weights(weights)
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("draft_vocab_size", [100, 101])
def test_qwen3_dspark_accepts_draft_vocab_at_least_target(draft_vocab_size):
"""A draft vocab >= the target's maps identically, so needs no lm_head/d2t.
Guards the padded case (draft > target): the draft's unembedding is padded
to its physical size, which is not a reduced vocabulary.
"""
model = _dspark_vocab_stub(100, draft_vocab_size)
model.load_weights([])
@pytest.mark.skip_global_cleanup
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
@@ -351,7 +372,7 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
attention_backend=None,
kv_cache_dtype=None,
),
parallel_config=SimpleNamespace(tensor_parallel_size=1),
parallel_config=ParallelConfig(),
attention_config=SimpleNamespace(backend=None),
cache_config=SimpleNamespace(),
load_config=SimpleNamespace(),
@@ -14,12 +14,14 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
from vllm import LLM, SamplingParams
from vllm.assets.audio import AudioAsset
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.kv_cache_interface import SlidingWindowSpec
from ....utils import ROCM_ENGINE_KWARGS
from ....utils import ROCM_ENGINE_KWARGS, create_new_process_for_each_test
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn"
@@ -130,6 +132,45 @@ async def async_engine():
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def _build_inputs(
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
) -> tuple[list[dict], list[SamplingParams]]:
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
inputs = []
sampling_params = []
for audio_asset in audio_assets:
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
req = TranscriptionRequest(
audio=audio.to_base64(audio.format),
streaming=StreamingMode.OFFLINE,
language=None,
)
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
tokens = tokenized.tokens
audio_array = tokenized.audios[0].audio_array
num_samples = audio_array.shape[0]
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
inputs.append(
{
"multi_modal_data": {"audio": [(audio_array, None)]},
"prompt_token_ids": tokens,
}
)
return inputs, sampling_params
def _assert_expected_text(outputs) -> None:
texts = _normalize([out.outputs[0].text for out in outputs])
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
assert got == expected, (
f"Output mismatch at index {i}:\n"
f" got: {got!r}\n"
f" expected: {expected!r}"
)
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -138,51 +179,39 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypa
with vllm_runner(**vllm_kwargs) as vllm_model:
assert_encoder_kv_cache_spec(vllm_model.llm)
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
def from_file(file_path: str):
audio = Audio.from_file(file_path, strict=False)
req = TranscriptionRequest(
audio=audio.to_base64(audio.format),
streaming=StreamingMode.OFFLINE,
language=None,
)
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
_assert_expected_text(outputs)
return (tokenized.tokens, tokenized.audios[0].audio_array)
tokenized_list = [
from_file(audio_asset.get_local_path()) for audio_asset in audio_assets
]
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
)
@pytest.mark.parametrize(
"cudagraph_mode",
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
)
@create_new_process_for_each_test(method="spawn")
def test_voxtral_realtime_cudagraph(
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
inputs = []
sampling_params = []
vllm_kwargs = {**ENGINE_CONFIG}
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
vllm_kwargs["enforce_eager"] = False
vllm_kwargs["compilation_config"] = {
"cudagraph_mode": cudagraph_mode,
# max_num_seqs is 4, so this is the only decode size worth capturing.
"cudagraph_capture_sizes": [4],
}
for tokens, audio_array in tokenized_list:
num_samples = audio_array.shape[0]
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
sampling_params.append(
SamplingParams(temperature=0.0, max_tokens=max_tokens)
)
with vllm_runner(**vllm_kwargs) as vllm_model:
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
input_dict = {
"multi_modal_data": {"audio": [(audio_array, None)]},
"prompt_token_ids": tokens,
}
inputs.append(input_dict)
outputs = vllm_model.llm.generate(
inputs,
sampling_params=sampling_params,
)
texts = _normalize([out.outputs[0].text for out in outputs])
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
assert got == expected, (
f"Output mismatch at index {i}:\n"
f" got: {got!r}\n"
f" expected: {expected!r}"
)
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
_assert_expected_text(outputs)
@pytest.mark.asyncio
@@ -6,9 +6,13 @@ from collections.abc import Mapping
import pytest
import torch
from PIL import Image as PILImage
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
from vllm.model_executor.models.gemma4_mm import (
Gemma4ImagePixelInputs,
_dummy_audio_num_samples,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
@@ -204,3 +208,41 @@ def test_limit_mm_per_prompt(
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
"""The unified audio feature extractor does not define ``fft_length``.
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
unconditionally, so memory profiling raised ``AttributeError`` for every
unified Gemma 4 deployment in which audio is the modality with the largest
per-item token count -- which is the case as soon as ``audio_seq_length``
is raised above the video budget.
"""
class _UnifiedFeatureExtractor:
sampling_rate = 16000
class _UnifiedProcessor:
feature_extractor = _UnifiedFeatureExtractor()
audio_seq_length = 3000
audio_ms_per_token = 40
# 3000 tokens * 40 ms = 120 s at 16 kHz
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
"""The mel/tower extractor keeps its existing behaviour."""
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
self.fft_length = 512
self.sampling_rate = 16000
class _TowerProcessor:
feature_extractor = _TowerFeatureExtractor()
audio_seq_length = 750
audio_ms_per_token = 40
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
@@ -14,7 +14,7 @@ from ...utils import build_model_context
class _StubModel:
"""Carries only the two attributes the token helpers read from
"""Carries only the attributes the token-count method reads from
`self`, so the real methods can be exercised without constructing
the full `nn.Module` (vision tower, language model, etc.)."""
@@ -22,18 +22,15 @@ class _StubModel:
pooled_grid_length: int
get_num_mm_encoder_tokens = (
LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
)
get_num_mm_connector_tokens = (
LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
get_mm_lora_token_counts = (
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
)
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
def test_num_mm_tokens_match_real_config(model_id):
"""The stored grid lengths must match what `__init__` derives from
the real HF config, and the two helpers must invert each other's
the real HF config, and the LoRA token counts must preserve the
frame-level scaling exactly."""
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
config = ctx.model_config.hf_config
@@ -49,10 +46,13 @@ def test_num_mm_tokens_match_real_config(model_id):
for num_frames in (1, 2, 8, 16, 32):
num_video_tokens = num_frames * pooled_grid_length**2
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=num_video_tokens,
)
assert encoder_tokens == num_frames * patch_grid_length**2
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
assert connector_tokens == num_video_tokens
@@ -72,10 +72,13 @@ def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_fram
num_video_tokens = num_frames * pooled_grid_length**2
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=num_video_tokens,
)
assert encoder_tokens == num_frames * patch_grid_length**2
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
assert connector_tokens == num_video_tokens
@@ -84,5 +87,9 @@ def test_num_mm_tokens_zero():
stub.patch_grid_length = 24
stub.pooled_grid_length = 12
assert get_num_mm_encoder_tokens(stub, 0) == 0
assert get_num_mm_connector_tokens(stub, 0) == 0
assert get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=0,
) == (0, 0)
+12 -3
View File
@@ -1610,6 +1610,12 @@ def test_fused_conv_correctness(
dtype=torch.bfloat16,
generator=rng,
)
outer_residual = torch.randn(
inputs.shape,
device=device,
dtype=torch.bfloat16,
generator=rng,
)
null_state = conv_state[NULL_BLOCK_ID].clone()
residual_kernel = residual.clone()
residual_reference = residual.clone()
@@ -1617,6 +1623,7 @@ def test_fused_conv_correctness(
module._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual_kernel,
outer_residual=outer_residual,
metadata=metadata,
conv_state=conv_state,
conv_weights=weights,
@@ -1630,15 +1637,17 @@ def test_fused_conv_correctness(
conv_state_len=module.conv_state_len,
dilation=module.short_conv_dilation,
)
residual_reference = outer_residual + residual_reference
torch.testing.assert_close(
residual_kernel.float(), residual_reference.float(), atol=3e-2, rtol=3e-2
assert torch.equal(
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
)
assert torch.equal(conv_state, state_reference)
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
if case.graph_padding:
assert torch.equal(
residual_kernel[num_real_tokens:], residual[num_real_tokens:]
residual_kernel[num_real_tokens:],
(outer_residual + residual)[num_real_tokens:],
)
@@ -944,6 +944,7 @@ def test_qsa_sparse_paged_attention_correctness(
q = torch.randn(
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
output_gate = torch.randn_like(q)
kv_cache = torch.randn(
num_cache_blocks,
page_size,
@@ -1023,6 +1024,7 @@ def test_qsa_sparse_paged_attention_correctness(
block_table,
token_to_req,
use_prefill_config=use_prefill_config,
output_gate=output_gate,
)
expected = _qsa_sparse_paged_attention_reference(
q,
@@ -1033,6 +1035,7 @@ def test_qsa_sparse_paged_attention_correctness(
token_to_req,
scale,
)
expected = expected * torch.sigmoid(output_gate)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
+4
View File
@@ -461,6 +461,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
trust_remote_code=True,
is_available_online=False,
),
"NanbeigeForCausalLM": _HfExamplesInfo(
"Nanbeige/Nanbeige4.2-3B",
trust_remote_code=True,
),
"OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
"Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
"Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
+2 -2
View File
@@ -18,8 +18,8 @@ from vllm.models.deepseek_v4.nvidia.model import (
)
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.models.deepseek_v4_1.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
+1 -1
View File
@@ -216,7 +216,7 @@ def test_v41_dspark_loads_linear_scales(
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
from vllm.models.deepseek_v4_1.nvidia import dspark
from vllm.models.deepseek_v41.nvidia import dspark
mxfp8 = scale_dtype != torch.float32
scale_name = "weight_scale" if mxfp8 else "weight_scale_inv"
+1 -1
View File
@@ -574,7 +574,7 @@ class TestImplicitReasoningEnd:
# the text lexer.
assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID}
assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1
assert thinking_parser.find_reasoning_end_offset([7, 8]) is None
assert thinking_parser.find_reasoning_end_offset([7, 8]) == 2
def test_streaming_reasoning_implicit_end(self, thinking_parser):
chunks = [
@@ -1616,3 +1616,36 @@ class TestCommaInStringValueRegression:
assert result.tools_called is True
args = json.loads(result.tool_calls[0].function.arguments)
assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214"
class TestGemma4IsReasoningEnd:
@staticmethod
def _parser(thinking: bool):
return Gemma4Parser(
_make_tokenizer(_PLAIN_ANSWER_TOKENS),
chat_template_kwargs={"enable_thinking": thinking},
)
@pytest.mark.parametrize(
("ids", "thinking", "ended"),
[
([CHANNEL_START_ID, 3000, CHANNEL_END_ID], True, True),
([CHANNEL_START_ID, 3000, TOOL_CALL_START_ID], True, True),
([CHANNEL_START_ID, 3000], True, False),
([NEW_TURN_ID, 9100], True, False),
([CHANNEL_END_ID, NEW_TURN_ID, 9100], True, False),
([NEW_TURN_ID, 9100], False, True),
# The bundled template's thinking-off prompt: closed channel.
([NEW_TURN_ID, 9100, CHANNEL_START_ID, 3000, CHANNEL_END_ID], False, True),
# A marker still wins with thinking off: the grammar honours it.
([CHANNEL_START_ID, 3000], False, False),
# A tool-call opener after the channel closed: still ended.
(
[CHANNEL_START_ID, 3000, CHANNEL_END_ID, TOOL_CALL_START_ID, 9200],
True,
True,
),
],
)
def test_is_reasoning_end(self, ids, thinking, ended):
assert self._parser(thinking).is_reasoning_end(ids) is ended
+2 -2
View File
@@ -185,8 +185,8 @@ class TestReasoningEndTokenIds:
def test_find_reasoning_end_offset_returns_first_match(self):
engine = _make_engine()
assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1
assert engine.find_reasoning_end_offset([5, 6]) is None
assert engine.find_reasoning_end_offset([]) is None
assert engine.find_reasoning_end_offset([5, 6]) == 2
assert engine.find_reasoning_end_offset([]) == 0
# Rejected-draft placeholders never match.
assert engine.find_reasoning_end_offset([-1, 201]) == 1
+39 -3
View File
@@ -180,9 +180,8 @@ class TestIsReasoningEnd:
"""Tool examples before the generation <think> must not end reasoning."""
assert not parser.is_reasoning_end([_TOOL_CALL_ID, _TEXT_ID, _THINK_START_ID])
def test_paired_tool_call_not_end(self, parser):
"""Paired <tool_call>...</tool_call> (from template) is NOT end."""
assert not parser.is_reasoning_end(
def test_completed_tool_call_after_open_think_is_end(self, parser):
assert parser.is_reasoning_end(
[_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
)
@@ -196,6 +195,43 @@ class TestIsReasoningEnd:
assert not parser.is_reasoning_end([])
class TestIsReasoningEndDerivedFromGrammar:
@pytest.mark.parametrize(
("ids", "thinking", "ended"),
[
# Explicit close, and the implicit close on <tool_call>.
([_THINK_START_ID, 1, _THINK_END_ID], True, True),
([_THINK_START_ID, 1, _TOOL_CALL_ID], True, True),
# Still inside the block.
([_THINK_START_ID, 1], True, False),
([_THINK_END_ID, _THINK_START_ID, 1], True, False),
# Uncommitted: a bare turn, an empty sequence. Thinking on starts
# in REASONING, so reasoning is expected.
([_IM_END_ID, _IM_START_ID, 1], True, False),
([], True, False),
# Thinking off starts in CONTENT with no way into REASONING, so
# a <think> the template left behind is inert, and a silent tail
# is ended because no block is expected.
([_THINK_START_ID, 1], False, True),
([_THINK_END_ID, _THINK_START_ID, 1], False, True),
([_IM_END_ID, _IM_START_ID, 1], False, True),
([], False, True),
],
)
def test_is_reasoning_end(self, mock_tokenizer, ids, thinking, ended):
parser = Qwen3Parser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": thinking}
)
assert parser.is_reasoning_end(ids) is ended
@pytest.mark.parametrize("thinking", [True, False])
def test_wait_for_reasoning_follows_thinking(self, mock_tokenizer, thinking):
parser = Qwen3Parser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": thinking}
)
assert parser.parser_engine_config.wait_for_reasoning is thinking
class TestIsReasoningEndTurnBoundaries:
"""Reasoning markers from earlier turns must not mark the new turn's
reasoning as finished (think-marker injection / interleaved-thinking
+6 -6
View File
@@ -77,8 +77,8 @@ def test_deepseek_v41_mxfp8_scale_loading(
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.models.deepseek_v4_1 import quant_config as quant_module
from vllm.models.deepseek_v4_1.nvidia import model as model_module
from vllm.models.deepseek_v41 import quant_config as quant_module
from vllm.models.deepseek_v41.nvidia import model as model_module
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
quant_config = quant_module.DeepseekV4FP8Config(
@@ -214,8 +214,8 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
"""The VL wrapper must map ``.scale`` keys to the parameter the linear
quant method registers, as the text model does. A hardcoded
``weight_scale_inv`` raised KeyError for native MXFP8 checkpoints."""
from vllm.models.deepseek_v4_1.nvidia import model as model_module
from vllm.models.deepseek_v4_1.nvidia import vl_model as vl_module
from vllm.models.deepseek_v41.nvidia import model as model_module
from vllm.models.deepseek_v41.nvidia import vl_model as vl_module
vllm_config = SimpleNamespace(
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
@@ -318,8 +318,8 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4Model
from vllm.models.deepseek_v4_1.quant_config import DeepseekV4FP8Config
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4Model
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
if not is_deep_gemm_supported() or not current_platform.is_device_capability_family(
+145
View File
@@ -1668,3 +1668,148 @@ def test_suffix_match_at_module_boundary(prefix, ignored_layer, expected):
)
is expected
)
_GLM5_MXFP4_WEIGHT = {
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
}
_GLM5_BLOCK_FP8_WEIGHT = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
_GLM5_BLOCK_FP8_INPUT = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
}
_GLM5_GATE_UP = "language_model.model.layers.0.mlp.gate_up_proj"
def _glm5_mixed_precision_config() -> QuarkConfig:
"""Global MXFP4 with the dense-MLP gate/up shards marked block-FP8."""
fp8 = {"weight": _GLM5_BLOCK_FP8_WEIGHT, "input_tensors": _GLM5_BLOCK_FP8_INPUT}
return QuarkConfig(
{
"global_quant_config": {
"weight": _GLM5_MXFP4_WEIGHT,
"input_tensors": None,
},
"layer_type_quant_config": {},
"layer_quant_config": {"*mlp.gate_proj": fp8, "*mlp.up_proj": fp8},
"exclude": [],
}
)
class _GLM5RecordingParam:
"""Stand-in for a BF16 target param that records what gets loaded."""
def __init__(self):
self.loaded_weight: torch.Tensor | None = None
self.loaded_shard: object = "unset"
def weight_loader(self, param, loaded_weight, shard_id=None):
self.loaded_weight = loaded_weight
self.loaded_shard = shard_id
def _glm5_block_fp8(out_dim: int, in_dim: int, block: int = 128):
"""Return a (fp8 weight, f32 per-block scale) pair."""
weight = (torch.randn(out_dim, in_dim) * 0.1).to(torch.float8_e4m3fn)
scale = torch.rand(out_dim // block, in_dim // block, dtype=torch.float32) + 0.5
return weight, scale
def test_glm5next_gate_up_proj_mapping_is_genuine_fusion():
from vllm.models.glm5next.common.model import Glm5NextForConditionalGeneration
mapping = Glm5NextForConditionalGeneration.packed_modules_mapping
assert mapping["gate_up_proj"] == ["gate_proj", "up_proj"]
def test_glm5next_genuine_fusion_resolves_gate_up_to_block_fp8():
# The fused module must expand to its real shard names so each resolves to
# the per-layer block-FP8 entry (rather than the global MXFP4 scheme).
config = _glm5_mixed_precision_config()
config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
_, _, scheme_cls = config.get_scheme_cls(LinearBase, _GLM5_GATE_UP)
assert scheme_cls is QuarkW8A8Fp8PerBlock
def test_glm5next_attn_loader_accepts_quark_weight_scale():
# Regression for KeyError on '...kv_a_proj_with_mqa.weight_scale': the fused
# q_a/kv_a projection is kept BF16 and dequantized on load; the Quark scale
# name must be recognized and routed to the correct fused shard.
from vllm.models.glm5next.common.model import (
_dequant_fp8_block,
_try_load_fp8_attn_proj,
)
prefix = "layers.3.self_attn"
target = _GLM5RecordingParam()
params_dict = {f"{prefix}.fused_qkv_a_proj.weight": target}
buf: dict = {}
loaded: set = set()
weight, scale = _glm5_block_fp8(256, 256)
# fp8 weight arrives first -> buffered, nothing loaded yet.
assert (
_try_load_fp8_attn_proj(
f"{prefix}.kv_a_proj_with_mqa.weight", weight, buf, params_dict, loaded, 0
)
is True
)
assert target.loaded_weight is None
# Quark-named scale completes the pair -> dequantize + load.
assert (
_try_load_fp8_attn_proj(
f"{prefix}.kv_a_proj_with_mqa.weight_scale",
scale,
buf,
params_dict,
loaded,
0,
)
is True
)
assert target.loaded_weight is not None
assert target.loaded_weight.dtype == torch.bfloat16
assert target.loaded_shard == 1 # kv_a is shard 1 of fused_qkv_a_proj
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
assert f"{prefix}.fused_qkv_a_proj.weight" in loaded
def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv():
# DeepSeek scale name keeps working unchanged.
from vllm.models.glm5next.common.model import (
_dequant_fp8_block,
_try_load_fp8_attn_proj,
)
prefix = "layers.7.self_attn"
target = _GLM5RecordingParam()
params_dict = {f"{prefix}.o_proj.weight": target}
buf: dict = {}
loaded: set = set()
weight, scale = _glm5_block_fp8(128, 256)
_try_load_fp8_attn_proj(
f"{prefix}.o_proj.weight", weight, buf, params_dict, loaded, 0
)
assert (
_try_load_fp8_attn_proj(
f"{prefix}.o_proj.weight_scale_inv", scale, buf, params_dict, loaded, 0
)
is True
)
assert target.loaded_shard is None # o_proj is a direct (non-fused) proj
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
+85
View File
@@ -6,13 +6,17 @@ Run `pytest tests/samplers/test_beam_search.py`.
"""
import json
from types import SimpleNamespace
from unittest.mock import Mock
import jsonschema
import pytest
from transformers import AutoModelForSeq2SeqLM
from vllm import CompletionOutput, RequestOutput
from vllm.assets.audio import AudioAsset
from vllm.entrypoints.llm import LLM
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
@@ -45,6 +49,87 @@ MM_BEAM_WIDTHS = [2]
MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
monkeypatch,
abort_after: int,
prompt_token: int,
terminal_logprobs: SampleLogprobs | None,
) -> None:
"""Return tokens and scores from the last completed step for aborted prompts.
Other prompts in the batch continue generating.
"""
def run_requests(prompts, **kwargs):
results = []
for prompt in prompts:
tokens = prompt["prompt_token_ids"]
if tokens[0] == prompt_token:
assert len(tokens) <= abort_after + 1, "Continued after abort"
aborted = (
tokens[0] == prompt_token
and len(tokens) == abort_after + 1
and tokens[-1] != 12
)
results.append(
RequestOutput(
request_id="inner",
prompt=None,
prompt_token_ids=tokens,
prompt_logprobs=None,
finished=True,
outputs=[
CompletionOutput(
index=0,
text="",
token_ids=[] if aborted else [11],
cumulative_logprob=None,
logprobs=terminal_logprobs
if aborted
else [{11: Logprob(-0.1), 12: Logprob(-0.2)}],
finish_reason="abort" if aborted else "length",
)
],
)
)
return results
llm = LLM.__new__(LLM)
llm.llm_engine = Mock()
tokenizer = SimpleNamespace(
eos_token_id=0,
decode=lambda tokens, skip_special_tokens=False: " ".join(
str(token) for token in tokens if not skip_special_tokens or token != 0
),
)
llm.renderer = Mock(get_tokenizer=Mock(return_value=tokenizer))
monkeypatch.setattr(llm, "_preprocess_cmpl", lambda prompts: prompts)
monkeypatch.setattr(llm, "_render_and_run_requests", run_requests)
prompts = [
{"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2]
]
params = BeamSearchParams(beam_width=2, max_tokens=3)
outputs = llm.beam_search(prompts, params)
aborted, normal = outputs
expected_tokens = (
[[prompt_token, 11], [prompt_token, 12]] if abort_after else [[prompt_token]]
)
assert [beam.tokens for beam in aborted.sequences] == expected_tokens
for beam in aborted.sequences:
assert beam.finish_reason == "abort"
assert beam.text == tokenizer.decode(
beam.tokens, skip_special_tokens=params.skip_special_tokens
)
assert len(beam.logprobs) == abort_after
assert aborted.sequences[0].cum_logprob == pytest.approx(-0.1 * abort_after)
assert len(normal.sequences) == 2
assert all(len(beam.tokens) == 4 for beam in normal.sequences)
assert all(beam.finish_reason != "abort" for beam in normal.sequences)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", MAX_TOKENS)
+90 -2
View File
@@ -5,14 +5,19 @@ import pytest
from vllm import CompletionOutput, RequestOutput
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
from vllm.logprobs import Logprob
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.sampling_params import BeamSearchParams
class _Tokenizer:
eos_token_id = 0
special_token_id = eos_token_id
def decode(self, token_ids: list[int]) -> str:
def decode(self, token_ids: list[int], skip_special_tokens: bool = False) -> str:
if skip_special_tokens:
token_ids = [
token_id for token_id in token_ids if token_id != self.special_token_id
]
return " ".join(str(token_id) for token_id in token_ids)
@@ -72,3 +77,86 @@ async def test_beam_search_handles_extra_logprob_candidates() -> None:
assert outputs[0].outputs[0].finish_reason == "stop"
assert outputs[0].outputs[0].token_ids == []
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("skip_special_tokens", "expected_text"),
[
pytest.param(True, "", id="skip"),
pytest.param(False, "0", id="keep"),
],
)
async def test_beam_search_respects_skip_special_tokens(
skip_special_tokens: bool, expected_text: str
) -> None:
prompt = {
"type": "token",
"prompt": "prompt",
"prompt_token_ids": [1],
}
params = BeamSearchParams(
beam_width=1,
max_tokens=1,
ignore_eos=True,
skip_special_tokens=skip_special_tokens,
)
outputs = [
output async for output in _Serving().beam_search(prompt, "request", params)
]
assert outputs[0].outputs[0].text == expected_text
assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id]
@pytest.mark.asyncio
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
async def test_beam_search_abort_returns_partial_outputs(
monkeypatch,
abort_after: int,
prompt_token: int,
terminal_logprobs: SampleLogprobs | None,
) -> None:
"""Abort returns each beam's tokens and scores from the last completed step."""
calls = 0
async def generate(prompt, *args, **kwargs):
nonlocal calls
calls += 1
depth = len(prompt["prompt_token_ids"]) - 1
assert depth <= abort_after, "Beam search continued after abort"
result = await anext(_EngineClient().generate(prompt, *args, **kwargs))
output = result.outputs[0]
output.token_ids = [11]
output.logprobs = [{11: Logprob(-0.1), 12: Logprob(-0.2)}]
output.finish_reason = "length"
if depth == abort_after and prompt["prompt_token_ids"][-1] != 12:
output.token_ids = []
output.logprobs = terminal_logprobs
output.finish_reason = "abort"
yield result
serving = _Serving()
monkeypatch.setattr(serving.engine_client, "generate", generate)
prompt = {"type": "token", "prompt": "prompt", "prompt_token_ids": [prompt_token]}
outputs = [
output
async for output in serving.beam_search(
prompt, "request", BeamSearchParams(beam_width=2, max_tokens=4)
)
]
assert calls == 1 + 2 * abort_after
assert len(outputs) == 1
assert outputs[0].finished
assert outputs[0].request_id == "request"
expected_tokens = [[11], [12]] if abort_after else [[]]
assert [output.token_ids for output in outputs[0].outputs] == expected_tokens
for output in outputs[0].outputs:
assert output.finish_reason == "abort"
assert output.text == _Tokenizer().decode(list(output.token_ids))
assert output.logprobs is not None
assert len(output.logprobs) == abort_after
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1 * abort_after)
+118
View File
@@ -337,6 +337,108 @@ def test_hisparse_rejects_non_cuda(monkeypatch):
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
@pytest.mark.parametrize(
"kv_transfer_config",
[
KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 128},
),
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_both",
kv_connector_extra_config={
"connectors": [
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
},
{
"kv_connector": "HiSparseConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"host_pool_gib": 128},
},
]
},
),
KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_consumer",
kv_connector_extra_config={
"connectors": [
{"kv_connector": "NixlConnector", "kv_role": "kv_consumer"},
{
"kv_connector": "HiSparseConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"host_pool_gib": 128},
},
]
},
),
],
ids=["standalone", "multi-connector", "pd-decode"],
)
def test_hisparse_connector_implies_attention_config(monkeypatch, kv_transfer_config):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
config = VllmConfig(
kv_transfer_config=kv_transfer_config,
# Skip the HMA auto-detect block: it imports the connector class,
# which pulls the platform's compiled attention extensions.
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=False,
),
)
assert isinstance(config.attention_config.hisparse_config, HiSparseConfig)
def test_hisparse_connector_preserves_explicit_attention_config(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
config = VllmConfig(
attention_config=AttentionConfig(
hisparse_config=HiSparseConfig(device_buffer_size=512)
),
kv_transfer_config=KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 128},
),
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=False,
),
)
assert config.attention_config.hisparse_config.device_buffer_size == 512
def test_hisparse_connector_without_cuda_still_rejected(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
VllmConfig(
kv_transfer_config=KVTransferConfig(
kv_connector="HiSparseConnector",
kv_role="kv_both",
kv_connector_extra_config={"host_pool_gib": 128},
)
)
def test_no_hisparse_connector_keeps_attention_config_unset(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
config = VllmConfig(
kv_transfer_config=KVTransferConfig(
kv_connector="OffloadingConnector", kv_role="kv_both"
)
)
assert config.attention_config.hisparse_config is None
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
MRV1 paths and off breakable cudagraphs by default."""
@@ -3322,6 +3424,22 @@ def test_ir_op_priority_default():
assert priority_config.fused_add_rms_norm == ["native"]
@pytest.mark.parametrize("mode", [CompilationMode.NONE, CompilationMode.VLLM_COMPILE])
@pytest.mark.parametrize("backend", ["inductor", "eager"])
def test_ir_op_platform_defaults_support_sparse_gelu(mode, backend):
"""Worker initialization must not select an unregistered sparse GELU provider."""
from vllm import ir
config = SimpleNamespace(
compilation_config=CompilationConfig(mode=mode, backend=backend)
)
priority = current_platform.get_default_ir_op_priority(config)
expected = ["triton", "native"] if current_platform.is_cuda() else ["native"]
with priority.set_priority():
assert ir.ops.gelu_and_mul_sparse.get_priority() == expected
def test_ir_op_priority_str():
"""Test that passing a comma-delimited string works"""
from vllm.config.kernel import IrOpPriorityConfig
-35
View File
@@ -43,41 +43,6 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
assert "VLLM_API_KEY" not in envs.compile_factors()
def test_scale_out_endpoints_flag_is_runtime_only(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is True
assert "VLLM_ENABLE_SCALE_OUT_ENDPOINTS" not in envs.compile_factors()
def test_scale_out_endpoints_flag_distinguishes_unset_from_disabled(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is False
def test_scale_out_endpoints_flag_treats_empty_as_unset(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "")
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None
@pytest.mark.parametrize("value", ["-1", "2", "01", "+1", "invalid", " "])
def test_scale_out_endpoints_flag_rejects_values_other_than_zero_or_one(
monkeypatch: pytest.MonkeyPatch, value: str
):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", value)
with pytest.raises(ValueError, match="must be 0 or 1"):
_ = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS
def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False)
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False)
+30
View File
@@ -12,6 +12,7 @@ import subprocess
import sys
from collections.abc import Callable
from typing import Protocol
from unittest.mock import Mock
import pytest
from pydantic import ValidationError
@@ -27,10 +28,39 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import BeamSearchParams
from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding
from vllm.v1.engine.input_processor import InputProcessor
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
@pytest.mark.parametrize("offset", [-1, 0, 2, 3, 4, 2**127])
def test_routed_experts_offset_validated_before_engine_submission(offset):
"""Reject invalid offsets even when routed-expert output is disabled."""
processor = Mock(spec=InputProcessor)
processor.tokenizer = None
processor.generation_config_fields = {}
processor.renderer = Mock()
processor.renderer.get_eos_token_id.return_value = None
processor.vllm_config = Mock()
params = SamplingParams(routed_experts_prompt_start=offset)
prompt = {"type": "token", "prompt_token_ids": [1, 2, 3]}
if not 0 <= offset <= 3:
with pytest.raises(VLLMValidationError, match="routed_experts_prompt_start"):
InputProcessor.process_inputs(
processor, "invalid", prompt, params, ("generate",)
)
params.routed_experts_prompt_start = 0
request = InputProcessor.process_inputs(
processor, "valid", prompt, params, ("generate",)
)
assert (
request.sampling_params.routed_experts_prompt_start
== params.routed_experts_prompt_start
)
# --- Stop strings: public requests cap the number of stop strings ---------
@@ -897,3 +897,111 @@ class TestStreamingExtraction:
}
assert args_text.count("replace_all") == 1
class TestBareAndMalformedOpeners:
"""Tests for bare and malformed tool call openers.
See https://github.com/vllm-project/vllm/issues/53431.
"""
def test_bare_opener_non_streaming(self, parser, mock_request):
model_output = '<|tool_call>:get_weather{location:<|"|>London<|"|>}<tool_call|>'
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "London"}
def test_documented_form_still_works(self, parser, mock_request):
model_output = (
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}<tool_call|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
def test_bare_opener_multiple_args(self, parser, mock_request):
model_output = (
"<|tool_call>:get_weather{"
'location:<|"|>Tokyo<|"|>,unit:<|"|>celsius<|"|>}'
"<tool_call|>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "Tokyo", "unit": "celsius"}
def test_malformed_no_brace_bounded(self, parser, mock_request):
model_output = "<|tool_call>call:bad_func no brace<tool_call|>"
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
name = result.tool_calls[0].function.name
assert "<tool_call|>" not in name
def test_malformed_then_well_formed(self, parser, mock_request):
model_output = (
"<|tool_call>call:bad_func no brace<tool_call|>"
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}'
"<tool_call|>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "bad_func no brace"
assert result.tool_calls[1].function.name == "get_weather"
assert "London" in result.tool_calls[1].function.arguments
def test_bare_malformed_then_well_formed(self, parser, mock_request):
model_output = (
"<|tool_call>:bad_func no brace<tool_call|>"
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}'
"<tool_call|>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "bad_func no brace"
assert result.tool_calls[1].function.name == "get_weather"
def test_three_mixed_calls(self, parser, mock_request):
model_output = (
"<|tool_call>:bad no brace<tool_call|>"
'<|tool_call>call:search{input:<|"|>hello<|"|>}<tool_call|>'
"<|tool_call>:set{flag:true,count:5}<tool_call|>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 3
assert result.tool_calls[0].function.name == "bad no brace"
assert result.tool_calls[1].function.name == "search"
assert result.tool_calls[2].function.name == "set"
def test_colon_in_content_not_broken(self, parser, mock_request):
model_output = (
"Hello: world"
'<|tool_call>:get_weather{location:<|"|>London<|"|>}<tool_call|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert result.content is not None
assert "Hello" in result.content
def test_bare_opener_empty_args(self, parser, mock_request):
model_output = "<|tool_call>:set_status{}<tool_call|>"
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert result.tool_calls[0].function.name == "set_status"
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from xgrammar import Grammar, StructuralTag
from xgrammar import Grammar, StructuralTag, builtin_structural_tag
from xgrammar.testing import _is_grammar_accept_string
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -81,10 +81,14 @@ def test_supported_structural_tag_models_include_vllm_builtins():
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
def test_deepseek_v41_format_constraints_ignore_parameter_schema(
def test_deepseek_v41_older_xgrammar_keeps_format_constraints(
choice,
sample_tools_strict,
monkeypatch,
):
monkeypatch.setattr(
builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False
)
tool_choice = (
ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name=choice)
@@ -128,7 +132,11 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
reasoning=False,
)
grammar = Grammar.from_structural_tag(tag)
call = '<|DSML| invoke name="get_weather">\n</|DSML| invoke>\n'
call = (
'<|DSML| invoke name="get_weather">\n'
'<|DSML| parameter name="city" string="true">Paris</|DSML| parameter>\n'
"</|DSML| invoke>\n"
)
assert _is_grammar_accept_string(
grammar, "\n\n<|DSML| calls>\n" + call + "</|DSML| calls>"
)
@@ -141,6 +149,72 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
)
@pytest.mark.skipif(
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
reason="Requires XGrammar's DeepSeek V4.1 builtin",
)
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
@pytest.mark.parametrize("reasoning", [False, True])
def test_deepseek_v41_constrains_parameters_after_reasoning(
sample_tools_strict, choice, reasoning
):
"""Strict calls must enforce the schema without requiring another think close."""
tool_choice = (
ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name=choice)
)
if choice == "get_weather"
else choice
)
sample_tools_strict[0].function.parameters["additionalProperties"] = False
tag = get_model_structural_tag(
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning
)
grammar = Grammar.from_structural_tag(tag)
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
end = "</|DSML| invoke>\n</|DSML| calls>"
parameter = (
'<|DSML| parameter name="city" string="true">Paris</|DSML| parameter>\n'
)
assert _is_grammar_accept_string(grammar, begin + parameter + end)
for invalid in (
"", # missing required city
parameter * 2,
parameter.replace('name="city"', 'name="unknown"'),
parameter.replace('string="true">Paris', 'string="false">42'),
parameter.replace("| parameter", "|parameter"),
'{"city":"Paris"}', # the encoder emits DSML parameters, not a JSON body
):
assert not _is_grammar_accept_string(grammar, begin + invalid + end)
assert not _is_grammar_accept_string(
grammar, "reason</think>" + begin + parameter + end
)
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
@pytest.mark.skipif(
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
reason="Requires XGrammar's DeepSeek V4.1 builtin",
)
def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools):
sample_tools[0].function.strict = False
tag = get_model_structural_tag(
"deepseek_v41", sample_tools, "required", reasoning=False
)
grammar = Grammar.from_structural_tag(tag)
call = (
'<|DSML| invoke name="get_weather">\n'
'<|DSML| parameter name="extra" string="false">'
'{"nested":[true,null,1.5]}</|DSML| parameter>\n'
"</|DSML| invoke>\n"
)
output = "\n\n<|DSML| calls>\n" + call * 2 + "</|DSML| calls>"
assert _is_grammar_accept_string(grammar, output)
assert not _is_grammar_accept_string(
grammar, output.replace('{"nested":[true,null,1.5]}', "invalid")
)
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
def test_get_model_structural_tag_supports_all_xgrammar_builtins(
model: str,
+18
View File
@@ -123,6 +123,24 @@ def test_suppressing_works_while_compiling(monkeypatch):
with_gpu_sync_check(suppressed)()
@create_new_process_for_each_test()
def test_copy_checker_is_traceable_while_compiling(monkeypatch):
"""The copy wrapper must bypass ContextVars while Dynamo is tracing."""
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
def convert_like(x, like):
return x.to(device=like.device, dtype=like.dtype)
compiled = torch.compile(convert_like, fullgraph=True)
result = compiled(
torch.ones(4, device="cuda", dtype=torch.float64),
torch.ones(4, device="cuda", dtype=torch.float32),
)
assert result.dtype == torch.float32
@create_new_process_for_each_test()
def test_sync_debug_mode_restored_after_checked_call(monkeypatch):
"""The mode is armed only for the duration of a checked call. Leaving it
+25
View File
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from vllm.utils.torch_utils import (
OMP_NUM_THREADS_SET_BY_VLLM,
async_tensor_h2d,
available_cpu_count,
common_broadcastable_dtype,
current_stream,
@@ -192,3 +194,26 @@ def test_runtime_threads_override_vllm_set_omp_num_threads(
torch.set_num_threads(3)
set_torch_threads_for_runtime()
assert torch.get_num_threads() == 1
@pytest.mark.parametrize(
"device", ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]
)
def test_async_tensor_h2d_staging(device):
"""Staging must preserve values while normalizing dtype and layout."""
# Non-contiguous source with a dtype conversion: would previously sync.
src = torch.arange(12, dtype=torch.int64).reshape(3, 4).T
result = async_tensor_h2d(src, device=device, dtype=torch.int32)
assert result.dtype == torch.int32
assert result.is_contiguous()
assert torch.equal(result.cpu(), src.to(torch.int32))
# numpy and list sources still work.
assert torch.equal(
async_tensor_h2d(np.arange(4, dtype=np.int64), device=device),
torch.arange(4, device=device),
)
assert torch.equal(
async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32),
torch.tensor([1, 2, 3], dtype=torch.int32, device=device),
)
+15 -1
View File
@@ -13,7 +13,7 @@ from vllm.v1.attention.ops.dcp import (
mask_dcp_empty_shards_,
)
pytestmark = pytest.mark.skipif(
requires_accelerator = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="needs a CUDA or ROCm device for the Triton pack kernel",
)
@@ -44,6 +44,7 @@ def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc)
@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)])
@pytest.mark.parametrize("tokens_per_req", [1, 2, 3])
@pytest.mark.parametrize("num_pad_rows", [0, 2])
@requires_accelerator
@pytest.mark.parametrize(
"seq_lens_list",
[
@@ -97,6 +98,7 @@ def test_fused_mask_matches_eager(
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("seed", range(8))
@requires_accelerator
def test_fused_mask_matches_eager_ragged(dtype, seed):
"""Non-uniform per-request query lengths.
@@ -137,6 +139,7 @@ def test_fused_mask_matches_eager_ragged(dtype, seed):
)
@requires_accelerator
def test_mask_disabled_is_unmasked():
"""Passing no seq_lens/query_start_loc must leave the LSE untouched."""
torch.manual_seed(0)
@@ -149,3 +152,14 @@ def test_mask_disabled_is_unmasked():
packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None)
assert torch.isfinite(packed.float()).all()
def test_eager_mask_handles_rank_with_no_local_sequences():
"""Padded graph rows are all empty when a DCP rank has no sequences."""
lse = torch.randn(8, 4, dtype=torch.float32)
seq_lens = torch.empty(0, dtype=torch.int32)
query_start_loc = torch.tensor([0], dtype=torch.int32)
mask_dcp_empty_shards_(lse, seq_lens, query_start_loc)
assert torch.isneginf(lse).all()
@@ -151,32 +151,44 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
assert kwargs == {}
def test_builder_attaches_its_state(monkeypatch):
@pytest.mark.parametrize("use_mha", [False, True])
@pytest.mark.parametrize("num_decodes", [0, 1])
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
"""MHA prefill rows must not make the MQA kernel read beyond its query."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder.state = FakeState(TOPK)
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None
metadata.num_prefills = 1
metadata.num_decode_tokens = num_decodes
monkeypatch.setattr(
sm90_mod.FlashInferMLASparseMetadataBuilder,
"build",
lambda *_args, **_kwargs: metadata,
)
cam = SimpleNamespace(
num_reqs=1,
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
seq_lens=torch.tensor([1], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
num_reqs=num_decodes + 1,
query_start_loc_cpu=torch.tensor(
[0, 1, 6] if num_decodes else [0, 5], dtype=torch.int32
),
seq_lens_cpu_upper_bound=torch.tensor(
[1402, 5] if num_decodes else [5], dtype=torch.int32
),
positions=None,
)
result = builder.build(0, cam)
assert result.state is builder.state
assert builder.state.plan_calls[0][0] == 1
assert builder.state.plan_calls[0][1].tolist() == [1]
expected_lens = [1402] if num_decodes else []
if not use_mha:
expected_lens += [1, 2, 3, 4, 5]
assert builder.state.plan_calls[0][0] == len(expected_lens)
assert builder.state.plan_calls[0][1].tolist() == expected_lens
def test_plan_uses_state_params(monkeypatch):
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU-only tests for FlashInfer sparse MLA backend constraints."""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
_DEFAULT_WORKSPACE_BUFFER_SIZE,
FlashInferMLASparseTRTLLMBackend,
_required_workspace_bytes,
compute_trtllm_sparse_mla_workspace_bytes,
)
# Reporter config from vllm#50781: GLM-5.2 (64 q heads), TP=8, DCP=8,
# max_num_batched_tokens=16384, FlashInfer 0.6.14.
_REPORTER_HEADS_PER_RANK = 64 // 8
_REPORTER_DCP = 8
_REPORTER_MAX_TOKENS = 16384
# From the reported crash: trtllm-gen requested exactly this many bytes for
# trtllm_gen_softmax_workspace after an 8 MiB counter carve had left
# 404,750,336 of the 413,138,944-byte default buffer.
_REPORTER_OBSERVED_SOFTMAX_BYTES = 1_611_661_312
_REPORTER_OBSERVED_PRECARVE_BYTES = 8_388_608
def test_reporter_softmax_carve_is_reproduced_byte_exactly():
# The crashing step scheduled 12288 tokens (a single 12K-token request);
# the softmax slab formula must reproduce the kernel's request exactly:
# 8 * (8 heads/rank * 8 dcp) * 12288 tokens * 256 + 1 MiB guard.
softmax_bytes = compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes=0,
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=12288,
)
assert softmax_bytes == _REPORTER_OBSERVED_SOFTMAX_BYTES
def test_reporter_config_covers_observed_overflow():
computed = compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert computed >= (
_REPORTER_OBSERVED_SOFTMAX_BYTES + _REPORTER_OBSERVED_PRECARVE_BYTES
)
def test_non_dcp_size_is_unchanged():
computed = compute_trtllm_sparse_mla_workspace_bytes(
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
dcp_world_size=1,
num_heads_per_rank=128,
max_num_batched_tokens=65536,
)
assert computed == _DEFAULT_WORKSPACE_BUFFER_SIZE
def test_combined_pcp_dcp_rejects_flashinfer_sparse():
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=4,
decode_context_parallel_size=4,
),
model_config=None,
)
with patch("vllm.config.get_current_vllm_config", return_value=config):
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
device_capability=SimpleNamespace(major=10),
)
assert reason is not None
assert "use FLASHMLA_SPARSE" in reason
@pytest.mark.parametrize("pcp_size,dcp_size", [(4, 1), (1, 4)])
def test_single_context_parallel_mode_allows_flashinfer_sparse(
pcp_size: int, dcp_size: int
):
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=pcp_size,
decode_context_parallel_size=dcp_size,
),
model_config=None,
)
with patch("vllm.config.get_current_vllm_config", return_value=config):
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
device_capability=SimpleNamespace(major=10),
)
assert reason is None
def test_default_constant_matches_envs_default(monkeypatch):
from vllm import envs
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
assert _DEFAULT_WORKSPACE_BUFFER_SIZE == envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
def test_env_unset_returns_computed(monkeypatch):
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
required = _required_workspace_bytes(
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert required == compute_trtllm_sparse_mla_workspace_bytes(
_DEFAULT_WORKSPACE_BUFFER_SIZE,
_REPORTER_DCP,
_REPORTER_HEADS_PER_RANK,
_REPORTER_MAX_TOKENS,
)
def test_env_override_below_computed_is_respected_with_warning(monkeypatch):
override = 100 * 1024 * 1024
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
with patch(
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
) as mock_logger:
required = _required_workspace_bytes(
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert required == override
mock_logger.warning_once.assert_called_once()
def test_env_override_above_computed_is_respected_without_warning(monkeypatch):
override = 8 * 1024 * 1024 * 1024
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
with patch(
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
) as mock_logger:
required = _required_workspace_bytes(
dcp_world_size=_REPORTER_DCP,
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
)
assert required == override
mock_logger.warning_once.assert_not_called()
@@ -307,6 +307,23 @@ def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape():
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_reuses_device_rank_tensor(monkeypatch):
seq_lens = torch.tensor([0, 1, 7, 8, 17], dtype=torch.int32)
world = 4
rank = 2
rank_tensor = torch.tensor(rank, dtype=torch.int32)
expected = get_dcp_local_seq_lens(seq_lens, world, rank, 2)
def fail_tensor_construction(*args, **kwargs):
raise AssertionError("rank tensor must be reused")
with monkeypatch.context() as patch:
patch.setattr(torch, "tensor", fail_tensor_construction)
actual = get_dcp_local_seq_lens(seq_lens, world, rank_tensor, 2)
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_must_run_after_decode_expansion():
world = 2
rank = 1
@@ -979,7 +996,7 @@ def test_pcp_plan_deinterleave_restores_global_order(
monkeypatch.setattr("vllm.v1.attention.backends.mla.indexer.PIN_MEMORY", False)
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.indexer.async_copy_to_gpu",
"vllm.v1.attention.backends.mla.indexer.async_tensor_h2d",
lambda x, device: x.to(device),
)
scheduled = np.array(req_lens, dtype=np.int64)
@@ -8,7 +8,7 @@ import torch
from tests.v1.attention.utils import create_vllm_config
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
from vllm.models.deepseek_v4_1.sparse_mla import (
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
@@ -179,6 +179,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
prefill_context_parallel_size=1,
cp_kv_cache_interleave_size=1,
),
)
@@ -193,6 +194,37 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
}
def test_indexer_warmup_includes_pcp_normalized_dcp_key(monkeypatch):
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.indexer.get_dcp_group",
lambda: SimpleNamespace(rank_in_group=2),
)
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
model_config=SimpleNamespace(
hf_text_config=SimpleNamespace(compress_ratios=[32], index_kpool=32)
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=4,
prefill_context_parallel_size=4,
cp_kv_cache_interleave_size=1,
),
)
keys = BuildPrefillChunkMetadataKernel().get_warmup_keys(config)
# Triton's compile key normalizes generic i32 values to 2 and divisible
# i32 values (including zero) to 16.
assert {(key.dcp_rank, key.dcp_world) for key in keys} == {(2, 2), (16, 1)}
assert {
(
key.input_variant.is_aligned("uncompressed_seq_lens"),
key.input_variant.is_aligned("cu_compressed_seq_lens"),
)
for key in keys
} == {(False, False), (False, True), (True, False), (True, True)}
def test_compressed_slot_mapping_warmup_includes_index_kpool():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=256),
+20
View File
@@ -25,6 +25,7 @@ from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention import mla_attention as mla_attention_module
from vllm.model_executor.layers.attention.mla_attention import (
MLAAttention,
MLACommonBaseImpl,
QueryLenSupport,
_DecodeConcatQuantFP8,
_use_masked_mha,
@@ -166,6 +167,25 @@ def test_glm5_flashinfer_masked_mha_routing(
)
@pytest.mark.parametrize("qk_rope_head_dim", [64, 0], ids=["rope", "nope"])
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
"""The K concat used by the MLA prefill context loop must equal torch.cat of
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
instead of allocating and copying."""
torch.manual_seed(0)
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, 1, qk_rope_head_dim, dtype=torch.bfloat16)
impl = SimpleNamespace(_use_flashinfer_concat_mla_k=False)
k = MLACommonBaseImpl._concat_k_nope_k_pe(impl, k_nope, k_pe)
expected = torch.cat([k_nope, k_pe.expand(-1, num_heads, -1)], dim=-1)
assert k.shape == (num_tokens, num_heads, qk_nope_head_dim + qk_rope_head_dim)
torch.testing.assert_close(k, expected, rtol=0, atol=0)
assert (k.data_ptr() == k_nope.data_ptr()) == (qk_rope_head_dim == 0)
def test_masked_mha_routing_is_dimension_specific():
assert _use_masked_mha(
backend_name="FLASHMLA_SPARSE",
@@ -313,6 +313,40 @@ class TestBackendValidation:
)
assert invalid_reasons == []
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
RoPE-carrying 320-wide query does not."""
try:
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
FlashAttnPrefillBackend,
)
except ImportError:
pytest.skip("MLA prefill backend not available")
return
capability = DeviceCapability(major=10, minor=0)
def validate(qk_rope_head_dim: int) -> list[str]:
selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=256,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=256,
),
)
with patch.object(
FlashAttnPrefillBackend, "is_available", return_value=True
):
return FlashAttnPrefillBackend.validate_configuration(
capability, selector_config
)
assert validate(qk_rope_head_dim=0) == []
invalid_reasons = validate(qk_rope_head_dim=64)
assert len(invalid_reasons) == 1
assert "supported MLA dimensions" in invalid_reasons[0]
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
@@ -176,3 +176,8 @@ def test_pad_unpad_round_trip_preserves_head_order(num_heads):
assert unpadded.shape == q.shape
torch.testing.assert_close(unpadded, q)
def test_a_non_causal_block_never_routes_to_gluon(gluon_available):
"""A small-head block that normally uses Gluon must use ASM when non-causal."""
assert not AiterMLAHelper.use_gluon_verify(12, 8, "auto", causal=False)
@@ -130,6 +130,7 @@ def _builder(
_mla_q_dtype=torch.bfloat16,
_mla_kv_dtype=torch.bfloat16,
decode_attn_out_dtype=torch.bfloat16,
_decode_causal=True,
)
# Bound to the stub rather than faked: the verify flatten's per-row view is
# part of what _build_decode is being tested for.
@@ -155,6 +156,25 @@ def test_backend_declares_uniform_batch_support():
)
@pytest.mark.parametrize("causal", [True, False])
def test_build_hands_common_causal_flag_to_aiter_builder(monkeypatch, causal):
seen = []
def fake_common_build(self, *args):
seen.append(self._decode_causal)
return SimpleNamespace(decode=None, prefill=None)
monkeypatch.setattr(
rocm_aiter_mla.MLACommonMetadataBuilder, "build", fake_common_build
)
builder = object.__new__(AiterMLAMetadataBuilder)
builder._fp8_prefill_enabled = False
builder.build(0, SimpleNamespace(causal=causal))
assert seen == [causal]
def test_dcp_verify_row_view_is_causal_per_row():
"""Row t must cover this rank's share of positions [0, seq_len - qlen + t].
@@ -949,3 +969,81 @@ def test_persistent_metadata_gate_without_gluon_build(
assert metadata.has_persistent_metadata is expect_persistent
assert get_mla_metadata_v1.called is expect_persistent
def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen):
"""Drive _build_decode with causal=False and hand back the aiter mock."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
builder = _builder(
mtp_decode_qlen=mtp_qlen,
kv_cache_dtype=kv_cache_dtype,
num_heads=num_heads,
)
builder._decode_causal = False
metadata = AiterMLAMetadataBuilder._build_decode(
builder,
block_table_tensor=torch.arange(16, dtype=torch.int32).view(2, 8),
seq_lens_device=torch.tensor([7, 5], dtype=torch.int32),
max_seq_len=7,
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
num_decode_tokens=2 * qlen,
dcp_tot_seq_lens_device=None,
)
return metadata, get_mla_metadata_v1
def test_non_causal_build_hands_the_mask_to_aiter(monkeypatch):
"""The schedule carries the mask, so is_causal must follow the block."""
metadata, get_mla_metadata_v1 = _build_non_causal(
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=8, mtp_qlen=8
)
assert metadata.has_persistent_metadata
# 6th positional arg of get_mla_metadata_v1 is is_causal.
assert get_mla_metadata_v1.call_args.args[5] is False
@pytest.mark.parametrize("num_heads", [12, 16, 48])
def test_a_two_token_fp8_block_is_refused_without_a_fold(monkeypatch, num_heads):
"""16-head (and padded-to-16) qlen-2 fp8 has no non-causal kernel.
48/80/112 (and 33 padded to 48) fold to H16 while keeping Q2, so they
must be refused the same way as native 16-head qlen 2.
"""
with pytest.raises(ValueError, match="2-token"):
_build_non_causal(
monkeypatch,
num_heads=num_heads,
kv_cache_dtype="fp8",
qlen=2,
mtp_qlen=8,
)
def test_a_two_token_fp8_block_is_allowed_when_folded(monkeypatch):
"""32 heads at qlen 2 folds onto the 16-head / 4-token kernel."""
metadata, get_mla_metadata_v1 = _build_non_causal(
monkeypatch,
num_heads=32,
kv_cache_dtype="fp8",
qlen=2,
mtp_qlen=8,
)
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.called
def test_a_two_token_bf16_block_is_allowed(monkeypatch):
"""The bf16 fold has no such hole, so the guard must not fire there."""
metadata, _ = _build_non_causal(
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
)
assert metadata.has_persistent_metadata
+106 -1
View File
@@ -22,16 +22,24 @@ from tests.v1.attention.utils import (
create_vllm_config,
)
from vllm import _custom_ops as ops
from vllm.config import HiSparseConfig, SpeculativeConfig, set_current_vllm_config
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
from vllm.config import (
CUDAGraphMode,
HiSparseConfig,
SpeculativeConfig,
set_current_vllm_config,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import mla_attention
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
from vllm.model_executor.layers.attention.sparse_mla_attention import (
GLOBAL_TOPK_MASK_MAX_BYTES,
SparseMLACommonImpl,
SparseMLAPrefillMetadata,
_is_masked_mha_available,
_masked_mha_workspace_fits,
_topk_mask_shape,
_use_dense_mha_prefill,
@@ -896,6 +904,75 @@ def test_triton_convert_req_index_to_global_index_decode_only(
torch.testing.assert_close(result, reference_result, rtol=0, atol=0)
def test_index_group_convert_during_piecewise_capture():
"""In piecewise cudagraph mode the indexer is captured while the convert
runs in the following eager break. The convert's side stream must not
wait on ``logical_topk_ready`` there: an event recorded inside a captured
segment is graph-local, and an eager wait on it raises
cudaErrorInvalidValue."""
device = torch.device(DEVICE_TYPE)
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
logical_topk_indices = torch.randint(
0,
block_size * blocks_per_req,
(num_tokens, num_topk),
dtype=torch.int32,
device=device,
)
builder = SparseMLAIndexGroupBuilder(logical_topk_indices)
group, layer_index = builder.register_layer(is_index_producing_layer=True)
assert layer_index == 0 and group.has_indexer
req_id = torch.arange(num_tokens, dtype=torch.int32, device=device) % num_requests
block_table = torch.arange(
num_requests * blocks_per_req, dtype=torch.int32, device=device
).view(num_requests, blocks_per_req)
attn_metadata = SimpleNamespace(
req_id_per_token=req_id, block_table=block_table, block_size=block_size
)
expected = triton_convert_req_index_to_global_index(
req_id,
block_table,
logical_topk_indices,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk,
)
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
marker = torch.zeros(1, device=device)
capture_stream = torch.cuda.Stream(device=device)
# torch.cuda.stream routes through vLLM's patched torch.cuda.set_stream,
# so current_stream() tracks the capture stream as in production.
with (
torch.cuda.stream(capture_stream),
set_forward_context(
None, vllm_config, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
),
):
capture = BreakableCUDAGraphCapture()
with capture:
marker.add_(1)
# Record inside a captured segment, convert in the eager
# break — mirrors mla.py -> unified_mla_attention_with_output.
group.set_logical_topk_ready(layer_index)
capture.add_eager(
lambda: group.convert_logical_to_physical_topk(
layer_index,
logical_topk_indices,
attn_metadata,
block_stride_rows=None,
return_valid_counts=False,
)
)
marker.add_(1)
capture.replay()
capture_stream.synchronize()
result = group.physical_topk_indices[:num_tokens]
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
@@ -1165,6 +1242,33 @@ def test_masked_mha_workspace_guards_long_routing_policy(
)
@pytest.mark.parametrize(
("model_dims", "kv_cache_dtype", "fa_version", "expected"),
[
pytest.param((128, 512, 128, 64, 128), "auto", 4, True, id="deepseek_v32"),
pytest.param((64, 512, 192, 64, 256), "auto", 4, True, id="glm5"),
pytest.param((64, 512, 256, 0, 256), "auto", 4, True, id="glm53_flash_nope"),
pytest.param((64, 512, 256, 0, 256), "fp8", 4, False, id="quantized_kv"),
pytest.param((64, 512, 256, 0, 256), "auto", 3, False, id="no_fa4"),
pytest.param((64, 512, 256, 64, 256), "auto", 4, False, id="rope_320"),
pytest.param((128, 512, 256, 0, 256), "auto", 4, False, id="wrong_heads"),
],
)
def test_is_masked_mha_available_model_dims(
monkeypatch, model_dims, kv_cache_dtype, fa_version, expected
):
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
unquantized KV cache, nothing else."""
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
monkeypatch.setattr(
mod.current_platform, "is_device_capability_family", lambda family: True
)
monkeypatch.setattr(mod, "get_flash_attn_version", lambda **kwargs: fa_version)
assert _is_masked_mha_available(*model_dims, kv_cache_dtype) is expected
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
"""Request count and context chunk length are independent multipliers."""
base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
@@ -1198,6 +1302,7 @@ PREFILL_BATCH_SPECS = {
[
pytest.param(128, 128, 64, 128, id="deepseek_hd192_v128"),
pytest.param(64, 192, 64, 256, id="glm5_hd256_v256"),
pytest.param(64, 256, 0, 256, id="glm53_flash_nope_hd256_v256"),
],
)
def test_sparse_backend_prefill_correctness(
@@ -299,7 +299,14 @@ def test_a_junction_past_the_prompt_falls_back_and_registers_nothing():
running = request.num_computed_tokens
assert not mamba.req_to_blocks["r"][running // block_size].is_null
request.shared_prefix_boundary = running
assert mamba._cache_partial_tail_block(request, running) is None
assert (
mamba._cache_partial_tail_block(
request,
running,
retention_interval=manager.coordinator.retention_interval,
)
is None
)
@pytest.mark.parametrize("eagle_group", [None, 0])
@@ -615,6 +615,77 @@ def test_internal_checkpoint_uses_partial_hash_lifecycle():
assert num_computed == 112
@pytest.mark.parametrize(
"retention_interval,transient_published",
[(None, True), (0, False)],
)
def test_internal_checkpoint_publication_respects_retention(
retention_interval, transient_published
):
"""With retention_interval=0, a mid-prompt internal checkpoint is
request-local and must not enter the prefix cache; the prompt-end
checkpoint keeps the existing publication behavior. Dense retention
(None) is unchanged.
"""
hash_block_size = 16
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=32,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = retention_interval
request = make_request("producer", list(range(240)), hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
# Mid-prompt chunk: exports a transient checkpoint at 112.
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
transient_hash = request.block_hashes[112 // hash_block_size - 1]
transient_hit = manager.block_pool.get_cached_block(transient_hash, [1])
assert (transient_hit is not None) == transient_published
request.num_computed_tokens = 128
manager.new_step_starts()
# Final chunk: the prompt-end checkpoint at 224 stays published.
assert manager.allocate_slots(request, 112) is not None
end_hash = request.block_hashes[224 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
def test_transient_checkpoint_evicts_retained_boundary_hash():
"""With retention_interval=0, the checkpoint slot can coincide with a
reachable boundary block that this step's full-block pass just hashed.
The checkpoint state overwrites the slot, so the stale boundary hash
must be evicted rather than left pointing at the wrong state.
"""
hash_block_size = 16
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=32,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = 0
request = make_request("producer", list(range(240)), hash_block_size, sha256)
# Block 2 ends at the shared-prefix junction 96, so the full-block pass
# hashes it as a retained boundary.
request.shared_prefix_boundary = 96
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
# The same slot is the transient checkpoint block (state@112): both the
# boundary hash and the checkpoint itself must stay unpublished.
boundary_hash = request.block_hashes[96 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(boundary_hash, [1]) is None
checkpoint_hash = request.block_hashes[112 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
hash_block_size = mamba_block_size = 32
manager = make_full_mamba_manager(
+1 -5
View File
@@ -334,7 +334,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
request = create_requests(num_requests=1, num_tokens=1)[0]
request.structured_output_request = Mock()
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
request.structured_output_request.grammar.accept_tokens.return_value = False
request.status = RequestStatus.RUNNING
request.num_computed_tokens = request.num_tokens
request.num_output_placeholders = 1
@@ -343,10 +342,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
scheduler.connector = None
scheduler.ec_connector = None
scheduler.structured_output_manager = Mock()
scheduler.structured_output_manager.should_advance.return_value = True
scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
lambda request, new_token_ids: new_token_ids
)
scheduler.structured_output_manager.accept_tokens.return_value = False
scheduler.requests = {request.request_id: request}
scheduler.running = [request]
scheduler.waiting = Mock()
+56 -4
View File
@@ -3208,6 +3208,45 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
assert "attention compute is unchanged" in caplog_vllm.text
def test_hidden_states_with_tp_scales_page_size():
"""When TP shrinks KV pages below the hidden-state per-token cost,
get_kv_cache_groups must scale up target block sizes so that the
common page accommodates the unsharded hidden states."""
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
# = 16 * 2 * 64 * 2 = 4096 bytes.
kv_spec = new_kv_cache_spec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.bfloat16,
)
# Hidden-state per-token cost = num_hidden_states(6) * hidden_size(512)
# * dtype(2) = 6144 bytes, which exceeds the 4096-byte KV page.
hs_spec = HiddenStateCacheSpec(
block_size=16,
num_kv_heads=6,
head_size=512,
dtype=torch.bfloat16,
)
specs = {
"target.0.attn": kv_spec,
"target.1.attn": kv_spec,
"cache_only_layers.48": hs_spec,
}
groups = get_kv_cache_groups(_grouping_config(), specs)
# The hidden-state layer should be present and no assertion should fire.
all_layers = {name for g in groups for name in g.layer_names}
assert "cache_only_layers.48" in all_layers
# The target group block sizes must have been scaled up.
for g in groups:
if "cache_only_layers.48" not in g.layer_names:
assert g.kv_cache_spec.block_size > kv_spec.block_size
def test_get_kv_cache_spec_kind_prefers_specific_attention_subclasses():
assert get_kv_cache_spec_kind(new_mla_spec()) == KVCacheSpecKind.MLA_ATTENTION
@@ -4135,9 +4174,9 @@ def test_draft_group_not_annotated_without_spec_decode():
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
# No group carries the draft marker, so every consumer falls back to
# flagging all groups -- including Mamba ones, which then can never report
# a hit. That is silent today; it must at least be visible.
# No group carries the draft marker, so consumers fall back to
# conservative behavior that silently breaks reuse for Mamba groups.
# That must at least be visible.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
)
@@ -4146,7 +4185,20 @@ def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
assert "no KV cache group could be identified as the draft model's" in (
caplog_vllm.text
)
assert "Mamba groups" in caplog_vllm.text
def test_unidentifiable_draft_without_mamba_does_not_warn(caplog_vllm):
# Pure-attention models degrade gracefully under the consumers'
# conservative fallback (a one-block hit drop at most), so the warning
# stays silent to avoid noise on every unannotated EAGLE deployment.
specs = {
"target.attn.0": new_mla_spec(block_size=64),
"target.attn.1": new_mla_spec(block_size=64),
}
groups = get_kv_cache_groups(_spec_decode_grouping_config(), specs)
assert not any(g.is_eagle_group for g in groups)
assert "could be identified as the draft model's" not in caplog_vllm.text
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
+50 -17
View File
@@ -51,6 +51,7 @@ from vllm.v1.outputs import (
ECConnectorOutput,
KVConnectorOutput,
ModelRunnerOutput,
RoutedExpertsLists,
SamplingMaskLists,
make_empty_encoder_model_runner_output,
)
@@ -131,6 +132,44 @@ def test_add_requests():
assert len(scheduler.waiting) == i + 1
def test_routed_experts_prompt_start_at_prompt_end():
"""A prompt-end offset should return empty routing data, not crash."""
scheduler = create_scheduler()
(request,) = create_requests(num_requests=1)
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
manager = Mock()
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
scheduler.enable_return_routed_experts = True
scheduler.routed_experts_mgr = manager
scheduler._re_block_ids = {request.request_id: []}
outputs = scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[0]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
routed_experts=RoutedExpertsLists(
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
slot_mapping=np.arange(num_tokens),
),
),
)
manager.get.assert_called_once_with(
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
)
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
def test_finish_request():
scheduler = create_scheduler()
requests = create_requests(num_requests=10)
@@ -3635,7 +3674,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
)
request.structured_output_request = Mock()
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
request.structured_output_request.grammar.accept_tokens.return_value = False
request.status = RequestStatus.RUNNING
request.num_computed_tokens = request.num_tokens
@@ -3643,10 +3681,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
scheduler.connector = None
scheduler.ec_connector = None
scheduler.structured_output_manager = Mock()
scheduler.structured_output_manager.should_advance.return_value = True
scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
lambda request, new_token_ids: new_token_ids
)
scheduler.structured_output_manager.accept_tokens.return_value = False
scheduler.requests = {request.request_id: request}
scheduler.running = [request]
scheduler.waiting = Mock()
@@ -3695,8 +3730,8 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
)
engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
request.structured_output_request.grammar.accept_tokens.assert_called_once_with(
request.request_id, [123]
scheduler.structured_output_manager.accept_tokens.assert_called_once_with(
request, [123]
)
assert request.resumable is False
assert request.status == RequestStatus.FINISHED_ERROR
@@ -6534,16 +6569,15 @@ def _decode_ready_request(scheduler):
return request
def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
"""ngram_gpu pads unfilled draft slots with -1; update_draft_token_ids must
def test_update_draft_token_ids_strips_ngram_padding():
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
strip them before grammar.validate_tokens (which otherwise raises)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
grammar = _RecordingGrammar()
request.structured_output_request = SimpleNamespace(grammar=grammar)
monkeypatch.setattr(
scheduler.structured_output_manager, "should_advance", lambda req: True
request.structured_output_request = SimpleNamespace(
grammar=grammar, reasoning_ended=True
)
scheduler.update_draft_token_ids(
@@ -6554,16 +6588,15 @@ def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
assert request.spec_token_ids == [10, 11]
def test_update_draft_token_ids_in_output_strips_padding(monkeypatch):
def test_update_draft_token_ids_in_output_strips_padding():
"""Same guard on the output path; the -1 pad-back for the rejected count
is preserved (only the input to validate_tokens is stripped)."""
is preserved (only the input to manager validation is stripped)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
grammar = _RecordingGrammar()
request.structured_output_request = SimpleNamespace(grammar=grammar)
monkeypatch.setattr(
scheduler.structured_output_manager, "should_advance", lambda req: True
request.structured_output_request = SimpleNamespace(
grammar=grammar, reasoning_ended=True
)
scheduler_output = SimpleNamespace(
@@ -43,6 +43,7 @@ def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
manager._capture_descs = {CUDAGraphMode.PIECEWISE: [desc]}
manager._graphs_captured = False
manager.use_breakable_cg = True
manager.ubatch_runner = None
create_calls = []
forward_calls = []
@@ -31,9 +31,8 @@ from vllm.v1.request import Request
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.worker import mamba_utils
from vllm.v1.worker.gpu_input_batch import CachedRequestState
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch
from vllm.v1.worker.mamba_utils import get_mamba_groups
@@ -315,7 +314,7 @@ def get_fake_process_mamba_fn(
action: tuple[int, int],
kv_cache_config: KVCacheConfig,
forward_context: dict[str, Any],
input_batch: GPUInputBatch,
input_batch: InputBatch,
):
assert copy_info is not None
if action == (-1, -1):
@@ -371,7 +370,7 @@ def get_fake_process_mamba_fn(
kv_cache_config: KVCacheConfig,
cache_config: CacheConfig,
mamba_state_idx: dict[str, int],
input_batch: GPUInputBatch,
input_batch: InputBatch,
requests: dict[str, CachedRequestState],
forward_context: dict[str, Any],
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
+167
View File
@@ -0,0 +1,167 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU speculative-decoding end-to-end correctness."""
import os
import pytest
from tests.models.utils import check_logprobs_close
from tests.v1.e2e.spec_decode.utils import compute_acceptance_len
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
pytestmark = pytest.mark.cpu_model
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
# Bound the KV cache so the run does not scale with host memory.
os.environ.setdefault("VLLM_CPU_KVCACHE_SPACE", "1")
MODEL = "Qwen/Qwen3-0.6B"
NUM_LOGPROBS = 5
SP = SamplingParams(max_tokens=48, temperature=0, logprobs=NUM_LOGPROBS)
# Repetitive prompts so the ngram proposer finds matches and actually drafts;
# without hits the run is indistinguishable from plain decoding.
PROMPTS = [
"The capital of France is Paris. The capital of Italy is Rome. "
"The capital of Spain is Madrid. The capital of France is",
"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - b\n\n"
"def mul(a, b):\n return",
"one two three four five one two three four five one two three four",
]
def _make_llm(**overrides) -> LLM:
base = dict(
model=MODEL,
dtype="bfloat16",
max_model_len=1024,
disable_log_stats=False,
)
base.update(overrides)
return LLM(**base)
def _tuples(outputs) -> list[tuple[list[int], str, object]]:
return [
(list(o.outputs[0].token_ids), o.outputs[0].text, o.outputs[0].logprobs)
for o in outputs
]
@pytest.fixture(scope="module")
def baseline_refs():
llm = _make_llm()
refs = _tuples(llm.generate(PROMPTS, SP))
del llm
return refs
def _spec_metric(llm: LLM, name: str) -> float:
for metric in llm.get_metrics():
if metric.name == name:
return float(getattr(metric, "value", 0.0))
return 0.0
def test_ngram_spec_decode_matches_baseline(baseline_refs):
"""Greedy output must be unchanged by speculative decoding.
The drafted-token check keeps the test from passing vacuously when the
proposer silently produces nothing.
"""
llm = _make_llm(
speculative_config={
"method": "ngram",
"prompt_lookup_max": 5,
"prompt_lookup_min": 3,
"num_speculative_tokens": 3,
},
)
got = _tuples(llm.generate(PROMPTS, SP))
drafted = _spec_metric(llm, "vllm:spec_decode_num_draft_tokens")
accepted = _spec_metric(llm, "vllm:spec_decode_num_accepted_tokens")
del llm
assert drafted > 0, "ngram proposer drafted no tokens; the test would be vacuous"
assert accepted > 0, f"no draft tokens accepted out of {drafted} drafted"
check_logprobs_close(
outputs_0_lst=baseline_refs,
outputs_1_lst=got,
name_0="no_spec_decode",
name_1="ngram_spec_decode",
)
SMOKE_PROMPTS = [
"The capital of France is",
"2 + 2 equals",
"In one word, the color of the sky is",
"Q: If a train travels 60 miles in 1.5 hours, what is its average speed?\nA:",
]
SMOKE_SP = SamplingParams(temperature=0.0, max_tokens=32, ignore_eos=True)
# Methods routed to the V2 runner, using the same target/draft pairs as the GPU
# suite. MTP needs no separate draft checkpoint: its speculator layer is
# trained jointly with, and shipped inside, the target model's own weights.
SMOKE_CONFIGS = [
pytest.param(
"meta-llama/Llama-3.2-1B-Instruct",
{
"method": "eagle3",
"model": "nm-testing/Llama3_2_1B_speculator.eagle3",
"num_speculative_tokens": 3,
},
id="eagle3",
),
pytest.param(
"Qwen/Qwen3-4B",
{
"method": "dflash",
"model": "z-lab/Qwen3-4B-DFlash-b16",
# The checkpoint allows 16, but accepted tokens plateau around 64
# regardless, so a larger draft only burns CPU on rejected tokens.
"num_speculative_tokens": 4,
},
id="dflash",
),
pytest.param(
"Qwen/Qwen3.5-0.8B-Base",
{
"method": "mtp",
"num_speculative_tokens": 3,
},
id="mtp",
),
]
# Measured 1.85 (eagle3), 1.99 (dflash), 2.29 (mtp) on CPU; this floor leaves
# room for numeric drift while still failing if speculation stops paying off.
MIN_ACCEPTANCE_LEN = 1.5
@pytest.mark.skipif(
not HAS_TRITON, reason="the V2 runner needs triton-cpu, which CI builds"
)
@pytest.mark.parametrize("model,speculative_config", SMOKE_CONFIGS)
def test_v2_speculator_smoke(model: str, speculative_config: dict):
"""Gates that drafts are accepted often enough for speculation to pay off."""
llm = _make_llm(model=model, speculative_config=speculative_config)
try:
assert llm.llm_engine.vllm_config.use_v2_model_runner, (
f"{speculative_config['method']} fell back to the V1 runner; "
"this test is meant to cover the V2 path"
)
llm.generate(SMOKE_PROMPTS, SMOKE_SP)
acceptance = compute_acceptance_len(llm.get_metrics())
assert acceptance >= MIN_ACCEPTANCE_LEN, (
f"acceptance length {acceptance:.3f} below {MIN_ACCEPTANCE_LEN}; "
"drafts are being rejected far more than expected"
)
finally:
del llm
@@ -116,7 +116,9 @@ def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
reported = first["ec_transfer_params"]
assert [handle] == [value for key, value in reported.items() if key != "ec_items"]
assert {key: value for key, value in reported.items() if key != "ec_items"} == {
"encoder-side-hash": handle
}
assert "ec_transfer_params" not in body
# Attempt 2's encode reports nothing: decode must be told nothing.
@@ -154,10 +156,22 @@ def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("prefill", [False, True])
@pytest.mark.parametrize(
"request_options",
[
{},
{
"mm_processor_kwargs": {"max_pixels": 262144},
"media_io_kwargs": {"image": {"image_mode": "RGB"}},
"priority": -3,
"session_id": "session-123",
},
],
)
async def test_http_roundtrip_preserves_payload_and_response_bytes(
proxy, monkeypatch, stream, prefill
proxy, monkeypatch, stream, prefill, request_options
):
"""Exercise real HTTP hops, rewrite and a decode retry without model servers."""
"""Preserve media and scheduling context across HTTP hops and retries."""
seen: dict[str, list[dict]] = {"encode": [], "prefill": [], "decode": []}
prefix = b'data: {"content":"'
# Split a multibyte character at the old 1024-byte forwarding boundary.
@@ -174,10 +188,13 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
seen[stage].append(body)
if stage == "encode":
item = body["messages"][0]["content"][0]
mm_hash = item["uuid"]
if body.get("mm_processor_kwargs"):
mm_hash += "-processed"
return web.json_response(
{
"ec_transfer_params": {
item["uuid"]: {
mm_hash: {
"metadata": {"image_grid_thw": [[1, 2, 2]]},
"peer_port": len(seen[stage]),
}
@@ -230,6 +247,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
"max_tokens": 32,
"seed": 42,
"structured_outputs": {"choice": ["A", "B"]},
**request_options,
}
await proxy.on_startup()
try:
@@ -247,6 +265,15 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
assert len(seen["encode"]) == 4
assert len(seen["decode"]) == 2
for requests in seen.values():
for forwarded in requests:
for key in (
"mm_processor_kwargs",
"media_io_kwargs",
"priority",
"session_id",
):
assert forwarded.get(key) == body.get(key)
final = seen["decode"][-1]
for key in [
"model",
@@ -265,7 +292,11 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
"image_embeds": {"image_grid_thw": [1, 2, 2]},
"uuid": proxy.content_uuid(item),
}
assert final["ec_transfer_params"][proxy.content_uuid(item)]["peer_port"] in (3, 4)
mm_hash = proxy.content_uuid(item)
if request_options:
mm_hash += "-processed"
assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash}
assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4)
if prefill:
assert final["kv_transfer_params"] == {"remote_block_ids": [2]}
+39
View File
@@ -14,6 +14,7 @@ These tests cover:
import argparse
import asyncio
import multiprocessing
from http import HTTPStatus
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -34,6 +35,7 @@ from vllm.exceptions import (
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.utils.argparse_utils import human_readable_int
from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.output_processor import OutputProcessor
@@ -66,6 +68,7 @@ def _make_async_llm(
llm.output_processor = MagicMock()
llm.output_processor.get_num_unfinished_requests.return_value = num_unfinished
llm.output_processor.get_num_queued_tokens.return_value = num_queued_tokens
llm.admission_stats = None
return llm
@@ -84,6 +87,42 @@ def _make_scheduler_config(**kwargs) -> SchedulerConfig:
)
def _make_shared_stats(client_count: int = 2) -> list[SharedAdmissionStats]:
counters = multiprocessing.RawArray(
"q", SharedAdmissionStats.num_counters(client_count)
)
return [
SharedAdmissionStats({"mp_admission_counters": counters}, client_count, index)
for index in range(client_count)
]
@pytest.mark.skip_global_cleanup
def test_shared_admission_stats_aggregate_api_servers():
server_0, server_1 = _make_shared_stats()
server_0.set_num_requests(2)
assert server_0.get_num_requests() == 2
server_1.set_num_requests(3)
assert server_0.get_num_requests() == server_1.get_num_requests() == 5
server_0.set_num_requests(1)
assert server_0.get_num_requests() == 4
@pytest.mark.skip_global_cleanup
def test_shared_request_snapshot_enforces_global_limit():
server_0, server_1 = _make_shared_stats()
server_0.set_num_requests(2)
llm = _make_async_llm(max_num_queued_reqs=2)
llm.admission_stats = server_1
with pytest.raises(QueueOverflowError):
llm.check_admission()
assert server_0.get_num_requests() == 2
# ---------------------------------------------------------------------------
# Exception classes
# ---------------------------------------------------------------------------
+27
View File
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
@pytest.fixture(autouse=True)
def _mock_cpu_backend_block_sizes(monkeypatch):
"""Mock the CPU attention backend's kernel block-size declaration.
Connector unit tests exercise transfer logic, never the attention
kernel, and hard-code block_size=16. On CPU-only machines
`CPUAttentionBackend` declares `MultipleOf(32)` (#54042), so every
worker construction would fail in `select_common_block_size` before
reaching the code under test.
"""
if not current_platform.is_cpu():
return
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend
monkeypatch.setattr(
CPUAttentionBackend,
"get_supported_kernel_block_sizes",
staticmethod(lambda: [MultipleOf(16)]),
)
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock, call
import pytest
@@ -74,7 +75,7 @@ from vllm.v1.kv_offload.base import (
make_offload_key,
)
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import RequestStatus
@@ -195,15 +196,17 @@ def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
def _make_partial_tail_request(
scheduler: OffloadingConnectorScheduler,
kv_transfer_params: dict[str, Any] | None = None,
) -> MagicMock:
request = MagicMock()
request.request_id = "req"
request.kv_transfer_params = None
request.kv_transfer_params = kv_transfer_params
request.num_prompt_tokens = 30
request.num_tokens = 30
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
request.all_token_ids = list(range(30))
request.lora_request = None
request.skip_reading_prefix_cache = False
request.is_finished.return_value = False
scheduler.on_new_request(request)
return request
@@ -427,6 +430,53 @@ def test_lookup_cap_stops_at_authoritative_prefix_boundary():
assert scheduler._req_status["req"].partial_tail_boundary == 20
def test_max_load_tokens_zero_skips_partial_tail_lookup():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 0}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 16) == (0, False)
scheduler.manager.lookup.assert_not_called()
assert all(
state.num_hit_chunks == 1 for state in scheduler._req_status["req"].group_states
)
def test_max_load_tokens_caps_tokens_beyond_gpu_prefix():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 8}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 16) == (8, True)
assert scheduler._req_status["req"].partial_tail_boundary == 24
def test_max_load_tokens_rounds_down_without_partial_tail():
scheduler = _make_partial_tail_scheduler()
scheduler.config = scheduler.config._replace(supports_partial_tail=False)
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 20}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 0) == (16, True)
@pytest.mark.parametrize("value", ["8", 8.5, -1, True])
def test_invalid_max_load_tokens_is_ignored(value):
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": value}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 0) == (28, True)
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
"""A recurrent group may hold unhashed blocks inside the computed prefix.
@@ -1708,6 +1758,15 @@ class TestMaximalPrefixLookup:
class TestSlidingWindowLookup:
def test_pending_chunk_behind_gap_does_not_defer(self):
"""A pending suffix too short for the window cannot improve the hit."""
sched = _make_scheduler_with_lookup(
{1: LookupResult.HIT, 2: LookupResult.HIT, 4: LookupResult.HIT_PENDING}
)
assert (
sched._sliding_window_lookup(to_keys([1, 2, 3, 4]), 2, _EMPTY_REQ_CTX) == 2
)
def test_all_hit_exact_window(self):
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
assert sched._sliding_window_lookup(to_keys([1, 2]), 2, _EMPTY_REQ_CTX) == 2
@@ -1826,6 +1885,73 @@ class TestSlidingWindowLookup:
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_swa_load_does_not_wait_for_unusable_pending_suffix(
request_runner, async_scheduling
):
"""A CPU store beyond a missing SWA chunk must not block a ready prefix."""
groups = [
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=4, num_kv_heads=1, head_size=1, dtype=torch.float32
),
),
KVCacheGroupSpec(
["swa"],
SlidingWindowSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=8,
),
),
]
runner = request_runner(
block_size=4,
num_gpu_blocks=32,
async_scheduling=async_scheduling,
kv_cache_groups=groups,
)
sched = runner.connector_scheduler
manager = CPUOffloadingManager(num_chunks=16)
sched.manager = manager
runner.new_request(token_ids=list(range(17)))
state = sched._req_status["0"]
state.update_offload_keys()
full, swa = (g.offload_keys for g in state.group_states)
ready = manager.prepare_store(full[:4] + swa[:2], state.req_context)
assert ready is not None
manager.complete_store(ready.keys_to_store, state.req_context)
pending = manager.prepare_store(swa[3:4], state.req_context)
assert pending is not None
# The last SWA chunk is still being written; the preceding one is missing.
output = runner.scheduler.schedule()
metadata = output.kv_connector_metadata
assert isinstance(metadata, OffloadingConnectorMetadata)
[(job_id, load)] = metadata.load_jobs.items()
assert isinstance(load.dst_spec, GPULoadStoreSpec)
assert load.dst_spec.group_sizes == [2, 2]
assert manager.lookup(swa[3], state.req_context) is LookupResult.HIT_PENDING
runner.scheduler.update_from_output(
output,
ModelRunnerOutput.with_kv_conn_output_only(
KVConnectorOutput(
finished_recving={"0"},
kv_connector_worker_meta=OffloadingWorkerMetadata(
completed_jobs={job_id: 1}
),
)
),
)
resumed = runner.scheduler.schedule()
assert resumed.num_scheduled_tokens == {"0": 9}
assert manager.lookup(swa[3], state.req_context) is LookupResult.HIT_PENDING
@pytest.mark.parametrize("alignment_chunk_count", [4, 8, 64])
@pytest.mark.parametrize("sliding_window_chunks", [1, 2, 3])
@pytest.mark.parametrize("is_eagle_group", [False, True])
@@ -2888,6 +3014,47 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
runner.manager.lookup.assert_not_called()
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_max_load_tokens_limits_external_load(request_runner, async_scheduling: bool):
"""The load cap limits external reads without changing the store path."""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
blocks_per_chunk=blocks_per_chunk,
)
runner.new_request(token_ids=[0] * tokens_per_chunk)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=(0, 1, 2),
)
runner.scheduler.reset_prefix_cache()
runner.manager.lookup.reset_mock()
runner.new_request(
token_ids=[0] * tokens_per_chunk,
kv_transfer_params={"max_load_tokens": 0},
)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_loaded=(),
expected_stored=(0, 1, 2),
)
runner.manager.lookup.assert_not_called()
# ---------------------------------------------------------------------------
# Eagle/MTP test class
# ---------------------------------------------------------------------------
@@ -39,10 +39,12 @@ from vllm.utils.network_utils import (
make_zmq_path,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
SlidingWindowSpec,
compute_layer_kv_cache_shape_bytes,
)
@@ -338,7 +340,7 @@ def test_write_mode_saves_local_block_ids():
req_meta = kv_connector_metadata.reqs_to_save[request_id]
for block_id, block in zip(
req_meta.local_block_ids,
req_meta.local_block_ids[0],
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -393,7 +395,7 @@ def test_write_mode_with_chunked_prefill_saves_local_block_ids():
req_meta = kv_connector_metadata.reqs_to_save[request_id]
for block_id, block in zip(
req_meta.local_block_ids,
req_meta.local_block_ids[0],
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -432,7 +434,7 @@ def test_read_mode_loads_remote_block_ids():
].req_to_blocks[request_id]
# Set remote block ids to be fetched.
request.kv_transfer_params["remote_block_ids"] = block_list
request.kv_transfer_params["remote_block_ids"] = [block_list]
# Remote Prefill, triggers MoRIIOConnectorMetadata.
@@ -457,7 +459,7 @@ def test_read_mode_loads_remote_block_ids():
req_meta = kv_connector_metadata.reqs_to_recv[request_id]
for block_id, block in zip(
req_meta.local_block_ids,
req_meta.local_block_ids[0],
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -699,3 +701,225 @@ def test_resolve_host_ip_prefers_extra_config():
fallback = get_ip()
assert resolve_host_ip({}) == fallback
assert resolve_host_ip({"host_ip": ""}) == fallback
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
"""One full-attention group + one sliding-window group (Gemma-like)."""
full_spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
)
sw_spec = SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
sliding_window=32,
)
num_blocks = 2
page = full_spec.page_size_bytes
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=2 * num_blocks * page,
layers=["full0", "sw0"],
layer_stride=num_blocks * page,
block_stride=page,
)
],
kv_cache_groups=[
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
KVCacheGroupSpec(layer_names=["sw0"], kv_cache_spec=sw_spec),
],
)
def _read_scheduler(
kv_cache_config: KVCacheConfig, disable_hma: bool = False
) -> MoRIIOConnectorScheduler:
vllm_config = create_vllm_config(role="kv_producer", read_mode=True)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
with set_current_vllm_config(vllm_config):
connector = MoRIIOConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
assert connector.connector_scheduler is not None
return connector.connector_scheduler
def test_hma_blocks_per_sw_two_groups():
"""A Full + SlidingWindow config runs with HMA and computes block budgets
correctly"""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
assert scheduler._is_hma_required is True
# cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention.
assert scheduler.blocks_per_sw == [0, 3]
@pytest.mark.parametrize(
"swa_enabled, disable_hma, expected_is_hma",
[
(True, False, True), # sliding-window group present, HMA enabled
(True, True, False), # sliding-window group present but HMA disabled
(False, False, False), # full-attention only, HMA not needed
],
)
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""_is_hma_required tracks both the KV cache groups and the
--disable-hybrid-kv-cache-manager flag. When HMA is off,
get_exchange_clipped_blocks must be a no-op."""
config = (
_make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config()
)
scheduler = _read_scheduler(config, disable_hma=disable_hma)
assert scheduler._is_hma_required is expected_is_hma
if not expected_is_hma:
blocks = [[1, 2, 3, 4, 5]]
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
def test_non_sliding_window_hybrid_is_rejected():
"""A hybrid group that is not sliding-window (e.g. chunked-local
attention) must fail closed rather than be silently mistransferred."""
full_spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
)
local_spec = ChunkedLocalAttentionSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
attention_chunk_size=32,
)
num_blocks = 2
page = full_spec.page_size_bytes
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=2 * num_blocks * page,
layers=["full0", "local0"],
layer_stride=num_blocks * page,
block_stride=page,
)
],
kv_cache_groups=[
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
KVCacheGroupSpec(layer_names=["local0"], kv_cache_spec=local_spec),
],
)
with pytest.raises(NotImplementedError, match="sliding-window hybrid"):
_read_scheduler(config)
def test_token_count_basis_uses_full_attention_group():
"""Chunked-prefill token counting must use an unclipped full-attention
group (blocks_per_sw == 0), not a clipped sliding-window group."""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
# Group 0 is the full-attention group
assert scheduler._full_attn_group_idx == 0
assert scheduler._full_attn_block_size == 16
def test_get_exchange_clipped_blocks_clips_only_sw_group():
"""get_exchange_clipped_blocks keeps the full attn group intact and clips the
sliding-window group to its window tail."""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
full = [10, 11, 12, 13, 14]
sw = [20, 21, 22, 23, 24]
clipped = scheduler.get_exchange_clipped_blocks([full, sw])
assert clipped[0] == full
assert clipped[1] == [22, 23, 24]
def test_metadata_hma_block_ids_preserved_per_group():
"""add_new_req stores per-group (BlockIds) block lists unchanged for both
read and write, so the hybrid group structure is retained."""
metadata = MoRIIOConnectorMetadata()
# Assume:
# - Full-attention group (6 blocks) +
# - sliding-window group already clipped to its window tail (3 blocks).
fa_blocks = [0, 1, 2, 3, 4, 5]
sw_blocks = [10, 11, 12]
local_block_ids = [fa_blocks, sw_blocks]
remote_block_ids = [[100, 101, 102, 103, 104, 105], [200, 201, 202]]
base_params = {
"remote_engine_id": "remote-engine",
"remote_host": "127.0.0.1",
"remote_handshake_port": 6301,
"remote_notify_port": 61005,
"remote_block_ids": remote_block_ids,
}
# Read mode: both local and remote ids stay per-group.
metadata.add_new_req(
request_id="recv-req",
local_block_ids=local_block_ids,
kv_transfer_params={**base_params, "transfer_id": "recv-req"},
)
recv_meta = metadata.reqs_to_recv["recv-req"]
assert recv_meta.local_block_ids == [fa_blocks, sw_blocks]
assert recv_meta.remote_block_ids == remote_block_ids
# Write mode: the decode peer allocates its own blocks, so #remote_block_ids may
# be empty, Local group structure is kept.
metadata.add_new_req(
request_id="save-req",
local_block_ids=local_block_ids,
kv_transfer_params={
**base_params,
"transfer_id": "save-req",
"remote_block_ids": [],
},
write_mode=True,
)
save_meta = metadata.reqs_to_save["save-req"]
assert save_meta.local_block_ids == [fa_blocks, sw_blocks]
assert save_meta.remote_block_ids == []
def test_single_group_path_unchanged():
"""A full attn config does not enable HMA and never clips blocks."""
scheduler = _read_scheduler(_make_test_kv_cache_config())
assert scheduler._is_hma_required is False
assert scheduler.blocks_per_sw == [0]
blocks = [[1, 2, 3, 4, 5]]
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
def test_hybrid_write_mode_rejected():
"""Hybrid KV cache groups are unsupported in WRITE mode and fail closed."""
vllm_config = create_vllm_config(role="kv_producer", read_mode=False)
with (
set_current_vllm_config(vllm_config),
pytest.raises(NotImplementedError),
):
MoRIIOConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
_make_hybrid_kv_cache_config(),
)
def test_worker_layer_to_group_routing(mock_parallel_groups):
"""The worker maps every layer to its KV cache group correctly."""
vllm_config = create_vllm_config(role="kv_consumer", read_mode=True)
# Building the worker directly bypasses MoRIIOConnector._set_port_defaults,
# so provide the ports manually.
vllm_config.kv_transfer_config.kv_connector_extra_config.update(
{
"http_port": 12346,
"handshake_port": 12347,
"notify_port": 12348,
}
)
with set_current_vllm_config(vllm_config):
worker = FakeMoRIIOConnectorWorker(
vllm_config,
"engine0",
hand_shake_latency=0,
kv_cache_config=_make_hybrid_kv_cache_config(),
)
assert worker.layer_to_group == {"full0": 0, "sw0": 1}
@@ -20,12 +20,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import
from vllm.v1.request import RequestStatus
_request_finished = MoRIIOConnectorScheduler.request_finished
_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
return SimpleNamespace(
s = SimpleNamespace(
is_producer=True,
_is_hma_required=False,
blocks_per_sw=[0],
_global_dp_rank=global_dp_rank,
engine_id="engine-abc",
host_ip="10.0.0.1",
@@ -39,6 +42,8 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
)
),
)
s.get_exchange_clipped_blocks = _clip_blocks.__get__(s)
return s
def _req(
+149 -84
View File
@@ -4,6 +4,7 @@
import contextlib
import inspect
import os
import queue
import tempfile
import textwrap
import time
@@ -341,6 +342,10 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
assert req_meta.remote.request_id == f"prefill-{42}"
# do_remote_prefill is consumed by request_finished to prevent re-issuing.
assert request.kv_transfer_params["do_remote_prefill"] is False
# The scheduler is not waiting on this recv -- the request is already gone
# from self.requests, so reporting it would trip `assert req_id in
# self.requests` in _update_from_kv_xfer_finished.
assert req_meta.awaiting_kvs is False
@patch(
@@ -1948,17 +1953,43 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
assert memory_types == ["DRAM", "VRAM"]
def test_mixed_memory_read_notifies_after_both_transfers_finish():
@pytest.fixture
def recv_worker():
"""Receive lifecycle state without distributed or device initialization."""
worker = object.__new__(NixlConnectorWorker)
worker.transfer_topo = MagicMock()
worker.transfer_topo.block_size_ratio.return_value = 1
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_block_size=16, remote_physical_blocks_per_logical=1
)
worker._physical_blocks_per_logical_kv_block = 1
worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))}
worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set()
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
worker._reqs_to_send = {}
worker._replicated_pcp_done_sending = set()
worker._is_hma_required = False
worker._has_mamba = False
worker.use_host_buffer = False
worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False
worker.tp_rank = 0
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = NixlKVConnectorStats()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry()
return worker
def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker):
worker = recv_worker
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
worker._dram_src_handles_by_block_size = {16: 10}
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = defaultdict(list)
worker._pending_recv_notifs = {}
worker._failed_inflight_recvs = set()
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
@@ -1985,27 +2016,20 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
np.testing.assert_array_equal(device_read.args[4], [7])
worker.nixl_wrapper.send_notif.assert_not_called()
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
assert worker.get_finished() == (set(), {"request"})
worker.nixl_wrapper.send_notif.assert_called_once_with(
"prefill", notif_msg=b"request:1"
)
def test_mixed_memory_read_failure_does_not_notify_producer():
def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
"""A failed half of a split READ must suppress its success notification."""
worker = object.__new__(NixlConnectorWorker)
worker._recving_metadata = {"request": MagicMock()}
worker = recv_worker
worker._recving_transfers = {"request": [101, 102]}
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker._failed_inflight_recvs = set()
worker._is_hma_required = True
worker._failed_recv_reqs = MagicMock()
worker._log_failure = MagicMock() # type: ignore[method-assign]
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
worker._pop_done_transfers(worker._recving_transfers)
assert worker.get_finished() == (set(), {"request"})
assert "request" not in worker._pending_recv_notifs
worker.nixl_wrapper.send_notif.assert_not_called()
@@ -2695,6 +2719,74 @@ class FailingNixlWrapper(FakeNixlWrapper):
return super().check_xfer_state(handle)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
)
@pytest.mark.parametrize("awaiting_kvs", [True, False])
@pytest.mark.parametrize("notification_fails", [False, True])
def test_empty_recv_is_reported_only_when_awaited(
default_vllm_config, dist_init, awaiting_kvs, notification_fails
):
"""An empty recv is reported iff the scheduler parked the request on it.
_read_blocks returns early when the local block list is empty -- D already
holds the KV and only the producer needs telling. Two kinds of caller reach
that path, and they need opposite handling:
Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the
request is named in neither _recving_transfers nor _failed_recv_reqs and
never reaches finished_recving. The scheduler has no other way to release a
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
skipped_waiting holding its blocks for the life of the process.
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
free P's blocks for a request aborted before it was scheduled, or a readback
on a request that stays RUNNING. Reporting either one crashes the engine
core -- _update_from_kv_xfer_finished asserts the id is still in
self.requests and that the request is parked or finished.
A failed notification changes neither: the KV is local either way, and the
producer frees its own blocks on a timeout.
"""
vllm_config = create_vllm_config()
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
connector.engine_id,
hand_shake_latency=0.0,
kv_cache_config=connector._kv_cache_config,
)
connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails
request_id = f"test_empty_recv_awaited_{awaiting_kvs}"
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=(), # empty: the whole prompt is already cached locally
kv_transfer_params={
"remote_block_ids": [[20, 21, 22]],
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
},
awaiting_kvs=awaiting_kvs,
)
connector.bind_connector_metadata(metadata)
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
connector.start_load_kv(dummy_ctx)
connector.bind_connector_metadata(NixlConnectorMetadata())
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
_, done_recving = connector.get_finished(finished_req_ids=set())
assert (request_id in done_recving) is awaiting_kvs
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
@@ -2897,13 +2989,10 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
# Wait for handshake to fail
time.sleep(0.3)
# Check that blocks were marked invalid
invalid_blocks = connector.get_block_ids_with_load_errors()
assert invalid_blocks == {1, 2, 3}
# Check that request appears in get_finished
assert connector.get_block_ids_with_load_errors() == set()
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id in done_recving
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
@patch(
@@ -2954,6 +3043,7 @@ def test_transfer_setup_failure_returns_finished(
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
assert connector.get_block_ids_with_load_errors() == set()
results = connector.get_transfer_results(finished_req_ids=set())
assert request_id in results.finished_recving
assert results.failed_recving == {request_id}
@@ -3176,74 +3266,49 @@ def test_failed_request_skips_kv_postprocessing(
assert invalid_blocks == {1, 2, 3}
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
)
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
default_vllm_config, dist_init
):
"""Failures across polls report once, after the last writer is terminal."""
vllm_config = create_vllm_config()
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
)
worker = connector.connector_worker
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state):
"""Failure must not release blocks while a sibling can still write to them."""
worker = recv_worker
worker._recving_transfers["request"] = [101, 102]
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
request_id = "test_two_handles_failing_in_separate_polls"
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=([1, 2, 3],),
kv_transfer_params={
"remote_block_ids": ([4, 5, 6],),
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
},
)
connector.bind_connector_metadata(metadata)
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
connector.start_load_kv(dummy_ctx)
connector.bind_connector_metadata(NixlConnectorMetadata())
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
assert "request" in worker._recving_metadata
worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101)
# Give the request a second handle, as a second remote rank would.
handles = worker._recving_transfers[request_id]
assert len(handles) == 1
first, second = handles[0], object()
worker._recving_transfers[request_id] = [first, second]
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == {"request"}
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
worker.nixl_wrapper.send_notif.assert_not_called()
# Poll 1: the first handle has failed, the second is still running.
def first_failed(handle):
return "ERR" if handle is first else "PROC"
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
with patch.object(
worker.nixl_wrapper, "check_xfer_state", side_effect=first_failed
):
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id not in done_recving
assert request_id in worker._recving_metadata
assert worker._recving_transfers[request_id] == [second]
assert connector.get_block_ids_with_load_errors() == set()
def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker):
"""A polling exception and failed release do not prove that DMA has stopped."""
worker = recv_worker
worker._recving_transfers["request"] = [101]
worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"]
worker.nixl_wrapper.release_xfer_handle.side_effect = [
RuntimeError("release"),
None,
]
# Only the last terminal handle permits the scheduler to reuse the blocks.
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
_, done_recving = connector.get_finished(finished_req_ids=set())
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == set()
assert worker.get_block_ids_with_load_errors() == set()
assert "request" in worker._recving_metadata
assert done_recving == {request_id}
assert request_id not in worker._recving_transfers
assert request_id not in worker._recving_metadata
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == set()
results = worker.get_transfer_results()
assert results.finished_recving == results.failed_recving == {"request"}
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
assert worker.nixl_wrapper.release_xfer_handle.call_count == 2
def _set_test_speculative_config(
@@ -149,8 +149,49 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
_, _, local_num_computed_blocks, awaiting_kvs = scheduler._reqs_need_recv[
request.request_id
]
assert local_num_computed_blocks == (2, 1)
# There are external tokens to pull, so the scheduler parks the request and
# is waiting to see it in finished_recving.
assert awaiting_kvs is True
@pytest.mark.cpu_test
def test_full_local_hit_is_not_awaited_by_the_scheduler():
"""A recv with num_external_tokens == 0 must not be flagged as awaited.
The request keeps running -- there is nothing to pull, only a notification
owed to the producer -- so the worker must not report it in
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
request that is parked or already done."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import KVCacheBlock
scheduler = object.__new__(NixlPullConnectorScheduler)
scheduler._reqs_in_batch = set()
scheduler._reqs_need_save = {}
scheduler._reqs_need_recv = {}
scheduler.use_host_buffer = False
scheduler.is_bidirectional_kv_xfer_enabled = False
scheduler._is_hma_required = False
scheduler.kv_cache_config = MagicMock(
select_transfer_block_ids=lambda block_ids: block_ids
)
blocks = KVCacheBlocks(blocks=([KVCacheBlock(block_id=0, _block_hash=object())],))
request = create_request(do_remote_prefill=True)
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=0)
_, local_block_ids, _, awaiting_kvs = scheduler._reqs_need_recv[request.request_id]
assert not local_block_ids # full local hit: nothing to pull
assert awaiting_kvs is False
@pytest.mark.cpu_test
@@ -877,6 +918,7 @@ def _make_mock_worker_for_desc_ids(
worker = MagicMock(spec=NixlConnectorWorker)
worker.num_regions = num_regions
worker._uses_region_group_mapping = False
worker._transfer_layer_group_ids = ()
worker._has_mamba = has_mamba
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
@@ -12,6 +12,7 @@ mid-decode (silent corruption of an unrelated request).
"""
from collections import defaultdict
from threading import Event, Lock
from unittest.mock import MagicMock, patch
import numpy as np
@@ -112,6 +113,7 @@ def test_local_descriptors_follow_each_region_pool_capacity():
worker.block_len_per_layer = [16, 16]
worker.block_stride_per_layer = [16, 16]
worker.region_num_blocks = [2, 3]
worker._transfer_layer_region_indices = ()
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
@@ -119,7 +121,8 @@ def test_local_descriptors_follow_each_region_pool_capacity():
@pytest.mark.cpu_test
def test_overlaid_transfer_groups_share_region_geometry():
@pytest.mark.parametrize("push_pp", [False, True])
def test_overlaid_transfer_groups_share_region_geometry(push_pp):
"""Groups overlaid on one allocation share its transfer region."""
import msgspec
@@ -127,6 +130,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
@@ -153,7 +159,14 @@ def test_overlaid_transfer_groups_share_region_geometry():
}
groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches]
worker = object.__new__(NixlConnectorWorker)
worker_cls = NixlPushConnectorWorker if push_pp else NixlConnectorWorker
worker = object.__new__(worker_cls)
if push_pp:
worker._push_writer_stop = Event()
worker._push_writer_wake = Event()
worker._push_writer_thread = MagicMock()
worker._sending_transfers_lock = Lock()
worker._sending_transfers = defaultdict(list)
worker.tp_rank = 0
worker.world_size = 1
worker.transfer_tp_rank = 0
@@ -198,6 +211,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
worker._desc_is_dram_by_block_size = {}
worker._desc_pos_by_block_size = {}
worker._dram_src_handles_by_block_size = {}
worker._transfer_layer_names = ()
worker._transfer_layer_region_indices = ()
worker._transfer_layer_group_ids = ()
worker._region_is_mla = []
worker.block_len_per_layer = []
worker.block_stride_per_layer = []
@@ -205,7 +221,8 @@ def test_overlaid_transfer_groups_share_region_geometry():
worker.use_host_buffer = False
worker.host_xfer_buffers = {}
worker.device_kv_caches = {}
worker.pp_size = 1
worker.pp_size = 2 if push_pp else 1
worker._is_hma_required = True
worker.dcp_size = 1
worker.pcp_size = 1
worker.kv_buffer_device = "cuda"
@@ -240,7 +257,13 @@ def test_overlaid_transfer_groups_share_region_geometry():
expected_addrs = [
backing.data_ptr() + block * block_stride for block in range(num_blocks)
]
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
num_desc_regions = 2 if push_pp else 1
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions
assert worker.num_descs == num_blocks * num_desc_regions
assert (
worker.dst_region_num_blocks[worker.engine_id]
== [num_blocks] * num_desc_regions
)
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes,
@@ -248,6 +271,7 @@ def test_overlaid_transfer_groups_share_region_geometry():
)
assert metadata.region_group_ids == [-1]
assert metadata.region_num_blocks == [num_blocks]
assert metadata.region_members == ([["layer.0", "layer.1"]] if push_pp else [])
assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]]

Some files were not shown because too many files have changed in this diff Show More