Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.compilation.passes.ir.lowering_pass import (
|
||||
)
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.ir import ops
|
||||
from vllm.model_executor.layers.activation import GeluAndMulSparse
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...backend import TestBackend
|
||||
@@ -34,6 +35,15 @@ class Model(nn.Module):
|
||||
return x6 + 3.0
|
||||
|
||||
|
||||
class SparseActivationModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.activation = GeluAndMulSparse(0.95, "tanh")
|
||||
|
||||
def forward(self, x):
|
||||
return self.activation(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
|
||||
def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
@@ -67,3 +77,27 @@ def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
|
||||
torch.testing.assert_close(output_unlowered, output)
|
||||
torch.testing.assert_close(output_unlowered, output2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
|
||||
torch.set_default_device("cuda")
|
||||
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
|
||||
backend = TestBackend(lowering_pass)
|
||||
model = SparseActivationModel()
|
||||
x = torch.randn(8, 32768, dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x)
|
||||
|
||||
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
|
||||
assert selected == {"gelu_and_mul_sparse": "triton"}
|
||||
|
||||
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
|
||||
x, 1.6448536269514722, "tanh"
|
||||
)
|
||||
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.fused_act_quant import (
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
expose_input_quant_key,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
@@ -45,7 +46,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
# Mock linear layer for testing fusion paths that don't have real kernel support
|
||||
class MockLinearForFusion(torch.nn.Module):
|
||||
"""Mock linear layer that exposes input_quant_key for fusion testing."""
|
||||
"""Mock linear layer that exposes an input key for fusion testing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -55,7 +56,7 @@ class MockLinearForFusion(torch.nn.Module):
|
||||
input_global_scale_inv=None,
|
||||
):
|
||||
super().__init__()
|
||||
self.input_quant_key = quant_key
|
||||
self._input_quant_key = quant_key
|
||||
if input_scale is not None:
|
||||
self.input_scale = input_scale
|
||||
if input_global_scale is not None:
|
||||
@@ -118,7 +119,7 @@ def test_manual_fusion_fp8_static_with_linear(
|
||||
# Enable fusion
|
||||
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
|
||||
|
||||
if not hasattr(fp8_linear, "input_quant_key"):
|
||||
if get_input_quant_key(fp8_linear) is None:
|
||||
pytest.skip(
|
||||
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
|
||||
)
|
||||
@@ -315,7 +316,7 @@ def test_manual_fusion_nvfp4_dynamic(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
|
||||
"""Test that maybe_fused_act_quant falls back when no input_quant_key."""
|
||||
"""Test that maybe_fused_act_quant falls back without an input key."""
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
@@ -327,7 +328,7 @@ def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
silu_and_mul = SiluAndMul()
|
||||
# Linear without input_quant_key attribute
|
||||
# Linear without an input quantization key
|
||||
mock_linear = torch.nn.Linear(128, 128)
|
||||
|
||||
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
|
||||
|
||||
@@ -13,6 +13,8 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.kernels # noqa: F401 to register kernels
|
||||
from vllm import ir
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.compilation.caching import (
|
||||
StandaloneCompiledArtifacts,
|
||||
@@ -28,6 +30,9 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.ir import ops
|
||||
from vllm.model_executor.layers.activation import GeluAndMulSparse
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
@@ -77,6 +82,18 @@ class CompiledModTuple(torch.nn.Module):
|
||||
return reference_fn_tuple(x)
|
||||
|
||||
|
||||
@support_torch_compile(dynamic_arg_dims={"x": 0})
|
||||
class SparseActivationAotModel(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.up_proj = torch.nn.Linear(32, 256, bias=False)
|
||||
self.activation = GeluAndMulSparse(0.95, "tanh")
|
||||
self.down_proj = torch.nn.Linear(128, 16, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
return self.down_proj(self.activation(self.up_proj(x)))
|
||||
|
||||
|
||||
def make_vllm_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
@@ -175,6 +192,54 @@ def test_save_and_load(monkeypatch: pytest.MonkeyPatch):
|
||||
assert torch.allclose(ret, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not is_torch_equal_or_newer("2.10.0"),
|
||||
reason="requires CUDA and torch 2.10",
|
||||
)
|
||||
def test_sparse_activation_standalone_aot_preserves_dependencies(
|
||||
monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path
|
||||
):
|
||||
def native_reference(model: SparseActivationAotModel, x: torch.Tensor):
|
||||
projected = torch.nn.functional.linear(x, model.up_proj.weight)
|
||||
activated = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
|
||||
projected, model.activation.std_multiplier, "tanh"
|
||||
)
|
||||
return torch.nn.functional.linear(activated, model.down_proj.weight)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_AOT_COMPILE", "1")
|
||||
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
|
||||
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
|
||||
m.setenv("VLLM_CACHE_ROOT", str(vllm_tmp_cache / "vllm_cache"))
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
torch.manual_seed(0)
|
||||
inputs = [
|
||||
torch.randn(4, 32, device="cuda", dtype=torch.bfloat16),
|
||||
torch.randn(7, 32, device="cuda", dtype=torch.bfloat16),
|
||||
]
|
||||
|
||||
with (
|
||||
use_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled = SparseActivationAotModel(vllm_config=vllm_config).to(
|
||||
device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
compiled.requires_grad_(False)
|
||||
expected = [native_reference(compiled, x) for x in inputs]
|
||||
actual = [compiled(x) for x in inputs]
|
||||
|
||||
for result, reference in zip(actual, expected):
|
||||
assert torch.isfinite(result).all()
|
||||
torch.testing.assert_close(
|
||||
result,
|
||||
reference,
|
||||
**ops.gelu_and_mul_sparse.get_tolerance(result.dtype),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
|
||||
from torch._subclasses import FakeTensorMode
|
||||
|
||||
@@ -39,9 +39,9 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.models.deepseek_v4_1.common.engram import EngramLayout
|
||||
from vllm.models.deepseek_v4_1.nvidia import engram as engram_ops
|
||||
from vllm.models.deepseek_v4_1.nvidia.engram import (
|
||||
from vllm.models.deepseek_v41.common.engram import EngramLayout
|
||||
from vllm.models.deepseek_v41.nvidia import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.nvidia.engram import (
|
||||
Engram,
|
||||
ParallelEngramEmbedding,
|
||||
engram_head_shard_rank,
|
||||
@@ -473,8 +473,8 @@ def _check_dummy_hash_model_forward(
|
||||
vllm_config, engram, head_sizes, weight, scales, dp_shared_memory
|
||||
):
|
||||
"""A metadata-less replica joins DP sharding but skips a shared lookup."""
|
||||
from vllm.models.deepseek_v4_1.common.engram import NgramHashState
|
||||
from vllm.models.deepseek_v4_1.nvidia import model as model_ops
|
||||
from vllm.models.deepseek_v41.common.engram import NgramHashState
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_ops
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
if dp_shared_memory and dp_rank != 1:
|
||||
@@ -491,7 +491,9 @@ def _check_dummy_hash_model_forward(
|
||||
state.forward = lambda *args: ids.unsqueeze(1)
|
||||
|
||||
class Decoder(SimpleNamespace):
|
||||
def __call__(self, hidden, positions, input_ids, *args):
|
||||
# Engram hashes and the mask stay the last positional arguments;
|
||||
# **kwargs absorbs the decoder's keyword-only flags.
|
||||
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
|
||||
hashes, keep = args[-2:]
|
||||
if dp_shared_memory and dp_rank == 1:
|
||||
assert hashes is None and keep is None
|
||||
@@ -504,7 +506,8 @@ def _check_dummy_hash_model_forward(
|
||||
if dp_rank == 1:
|
||||
assert torch.all(hashes == engram_ops.DEAD_ID)
|
||||
output = engram.embed(hashes[:, 0])
|
||||
return output, None, None, None, None
|
||||
# Trailing None is previous_aux; this stub captures no aux states.
|
||||
return output, None, None, None, None, None
|
||||
|
||||
model = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
|
||||
@@ -20,17 +20,23 @@ covered by the GPU tests in `test_sharded_rdt_trainer.py`.
|
||||
|
||||
import contextlib
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeout
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as trainer_mod
|
||||
from vllm.distributed.nixl_utils import alias_nixl_for_ray
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_common import (
|
||||
buffer_alloc_bytes,
|
||||
register_nixl_memory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
DEFAULT_GATHER_LOOKAHEAD,
|
||||
@@ -45,6 +51,114 @@ GROUP_A = ("model.layers.0.w", "model.layers.0.b")
|
||||
GROUP_B = ("model.layers.1.w",)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ray_nixl_transport(monkeypatch):
|
||||
import ray
|
||||
from ray.experimental.rdt import util
|
||||
from ray.experimental.rdt.nixl_tensor_transport import NixlTensorTransport
|
||||
|
||||
from vllm.distributed import nixl_utils
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
transport = NixlTensorTransport()
|
||||
package = ModuleType("nixl_rocm")
|
||||
api = ModuleType("nixl_rocm._api")
|
||||
api.__dict__.update(nixl_agent=Mock(), nixl_agent_config=Mock())
|
||||
package.__dict__["_api"] = api
|
||||
modules = {package.__name__: package, api.__name__: api}
|
||||
imported_rcache_limits = []
|
||||
|
||||
def import_nixl(name):
|
||||
imported_rcache_limits.append(os.environ.get("UCX_RCACHE_MAX_UNRELEASED"))
|
||||
monkeypatch.setitem(sys.modules, name, modules[name])
|
||||
return modules[name]
|
||||
|
||||
nixl_names = ("nixl", "nixl._api", *modules)
|
||||
for name in nixl_names:
|
||||
monkeypatch.delitem(sys.modules, name, raising=False)
|
||||
monkeypatch.delenv("UCX_RCACHE_MAX_UNRELEASED", raising=False)
|
||||
importer = Mock(side_effect=import_nixl)
|
||||
monkeypatch.setattr(
|
||||
nixl_utils, "importlib", SimpleNamespace(import_module=importer)
|
||||
)
|
||||
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: True)
|
||||
monkeypatch.setattr(util, "transport_managers", {"NIXL": transport})
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
ray,
|
||||
"get_runtime_context",
|
||||
lambda: SimpleNamespace(get_actor_id=lambda: "rdt-actor"),
|
||||
)
|
||||
try:
|
||||
yield transport, api, importer, imported_rcache_limits
|
||||
finally:
|
||||
# The helper inserts aliases directly; remove them before monkeypatch
|
||||
# restores any modules and environment values present before this test.
|
||||
for name in nixl_names:
|
||||
sys.modules.pop(name, None)
|
||||
os.environ.pop("UCX_RCACHE_MAX_UNRELEASED", None)
|
||||
|
||||
|
||||
def test_ray_registers_memory_through_rocm_alias(ray_nixl_transport):
|
||||
"""Ray's registration must reach the ROCm agent through the alias."""
|
||||
_, api, importer, _ = ray_nixl_transport
|
||||
register_nixl_memory(torch.empty(4, dtype=torch.uint8))
|
||||
api.nixl_agent.return_value.register_memory.assert_called()
|
||||
assert sys.modules["nixl"] is sys.modules["nixl_rocm"]
|
||||
assert sys.modules["nixl._api"] is api
|
||||
|
||||
importer.reset_mock()
|
||||
alias_nixl_for_ray()
|
||||
importer.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rcache_limit", [None, "4096"])
|
||||
def test_ray_nixl_sets_ucx_limit_before_import(
|
||||
monkeypatch, ray_nixl_transport, rcache_limit
|
||||
):
|
||||
"""UCX must see the default or the user's override during the first import."""
|
||||
_, _, _, imported_rcache_limits = ray_nixl_transport
|
||||
if rcache_limit is not None:
|
||||
monkeypatch.setenv("UCX_RCACHE_MAX_UNRELEASED", rcache_limit)
|
||||
alias_nixl_for_ray()
|
||||
assert imported_rcache_limits == [rcache_limit or "1024"] * 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("existing_api", [False, True])
|
||||
def test_ray_nixl_preserves_non_rocm_or_existing_imports(
|
||||
monkeypatch, ray_nixl_transport, existing_api
|
||||
):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_, _, importer, _ = ray_nixl_transport
|
||||
if existing_api:
|
||||
api = ModuleType("nixl._api")
|
||||
monkeypatch.setitem(sys.modules, "nixl._api", api)
|
||||
else:
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
alias_nixl_for_ray()
|
||||
importer.assert_not_called()
|
||||
assert "UCX_RCACHE_MAX_UNRELEASED" not in os.environ
|
||||
if existing_api:
|
||||
assert sys.modules["nixl._api"] is api
|
||||
else:
|
||||
assert "nixl._api" not in sys.modules
|
||||
|
||||
|
||||
def test_ray_nixl_alias_reports_missing_rocm_package(monkeypatch, ray_nixl_transport):
|
||||
from vllm.distributed import nixl_utils
|
||||
|
||||
transport, api, importer, _ = ray_nixl_transport
|
||||
monkeypatch.setattr(nixl_utils, "is_nixl_available", lambda: False)
|
||||
with pytest.raises(ImportError, match="nixl_rocm"):
|
||||
register_nixl_memory(torch.empty(1))
|
||||
assert transport._nixl_agent is None
|
||||
api.nixl_agent.assert_not_called()
|
||||
importer.assert_not_called()
|
||||
assert "nixl" not in sys.modules
|
||||
assert "nixl._api" not in sys.modules
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server_factory(monkeypatch):
|
||||
"""Build real `_RDTProducerServer` instances without a GPU.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the ``vllm serve`` CLI subcommand."""
|
||||
|
||||
import argparse
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.cli.serve import run_headless
|
||||
|
||||
|
||||
class _StopAfterEngineConfig(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def test_headless_imports_reasoning_parser_plugin_before_engine_config():
|
||||
plugin_path = "/tmp/custom_reasoning_parser.py"
|
||||
args = argparse.Namespace(
|
||||
api_server_count=0,
|
||||
reasoning_parser_plugin=plugin_path,
|
||||
)
|
||||
engine_args = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.entrypoints.cli.serve.vllm.AsyncEngineArgs.from_cli_args",
|
||||
return_value=engine_args,
|
||||
),
|
||||
patch(
|
||||
"vllm.reasoning.ReasoningParserManager.import_reasoning_parser"
|
||||
) as import_plugin,
|
||||
):
|
||||
|
||||
def create_engine_config(**kwargs):
|
||||
import_plugin.assert_called_once_with(plugin_path)
|
||||
raise _StopAfterEngineConfig
|
||||
|
||||
engine_args.create_engine_config.side_effect = create_engine_config
|
||||
|
||||
with pytest.raises(_StopAfterEngineConfig):
|
||||
run_headless(args)
|
||||
@@ -58,7 +58,6 @@ def _enable_cohere_api(monkeypatch):
|
||||
flag inside the test body.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
@@ -128,15 +127,19 @@ def _generate_request() -> GenerateRequest:
|
||||
)
|
||||
|
||||
|
||||
def _build_app(handler: _Handler | None) -> FastAPI:
|
||||
def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
return app
|
||||
|
||||
|
||||
def _build_render_app(chat_handler, render_handler) -> FastAPI:
|
||||
def _build_render_app(
|
||||
chat_handler, render_handler, *, enable_scale_out: bool = True
|
||||
) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = chat_handler
|
||||
app.state.serving_render = render_handler
|
||||
@@ -151,11 +154,13 @@ def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
|
||||
into the ``CohereError`` wire shape.
|
||||
"""
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
# ``validation_exception_handler`` reads ``req.app.state.args``; the
|
||||
# real cli builds this via argparse.
|
||||
app.state.args = Namespace(log_error_stack=False)
|
||||
app.state.args = Namespace(
|
||||
log_error_stack=False, enable_scale_out=True, tokens_only=False
|
||||
)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
app.exception_handler(RequestValidationError)(validation_exception_handler)
|
||||
app.exception_handler(HTTPException)(http_exception_handler)
|
||||
app.exception_handler(VLLMError)(vllm_error_handler)
|
||||
@@ -402,10 +407,8 @@ class TestRenderEndpoint:
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" in paths
|
||||
|
||||
def test_route_not_registered_when_flag_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
def test_route_not_registered_when_flag_unset(self):
|
||||
app = _build_app(handler=None, enable_scale_out=False)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" not in paths
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
@@ -57,6 +57,8 @@ def server():
|
||||
# the conversation to surface a thinking block (SmolLM2 doesn't
|
||||
# emit Cohere-style reasoning tokens out of the box).
|
||||
"--no-cohere-is-reasoning-model",
|
||||
# Registers /cohere/v2/chat/render alongside /cohere/v2/chat.
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
|
||||
@@ -71,7 +73,6 @@ def server():
|
||||
env_dict = {
|
||||
"VLLM_CPU_KVCACHE_SPACE": "1",
|
||||
"VLLM_ENABLE_COHERE_API": "1",
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
|
||||
@@ -5,6 +5,7 @@ import multiprocessing
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -14,6 +15,7 @@ from tests.entrypoints.launchers.api_server._api_server_spawn_workers import (
|
||||
exit_before_report_worker,
|
||||
)
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
get_engine_client_zmq_addr,
|
||||
@@ -32,6 +34,18 @@ def mock_run_api_server_worker(listen_address, sock, args, client_config=None):
|
||||
print("Mock worker completed successfully")
|
||||
|
||||
|
||||
def update_admission_stats_worker(listen_address, sock, args, client_config):
|
||||
stats = SharedAdmissionStats(
|
||||
client_config,
|
||||
client_config["client_count"],
|
||||
client_config["client_index"],
|
||||
)
|
||||
value = client_config["client_index"] + 1
|
||||
stats.set_num_requests(value)
|
||||
args.barrier.wait()
|
||||
args.results.put(stats.get_num_requests())
|
||||
|
||||
|
||||
# Module-level stub for the gather_actual_addresses test. Must be
|
||||
# importable by `multiprocessing.spawn` (no closures, no nesting).
|
||||
def defer_addresses_stub_worker(listen_address, sock, args, client_config):
|
||||
@@ -131,6 +145,31 @@ def test_api_server_process_manager_init(api_server_args, with_stats_update):
|
||||
assert not proc.is_alive()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_api_server_processes_share_admission_stats(api_server_args):
|
||||
args = api_server_args.copy()
|
||||
spawn_context = multiprocessing.get_context("spawn")
|
||||
worker_args = SimpleNamespace(
|
||||
barrier=spawn_context.Barrier(args["num_servers"]),
|
||||
results=spawn_context.Queue(),
|
||||
max_num_queued_reqs=10,
|
||||
)
|
||||
args["args"] = worker_args
|
||||
args["target_server_fn"] = update_admission_stats_worker
|
||||
manager = APIServerProcessManager(**args)
|
||||
|
||||
try:
|
||||
for process in manager.processes:
|
||||
process.join(timeout=30)
|
||||
assert process.exitcode == 0
|
||||
|
||||
assert [worker_args.results.get(timeout=5) for _ in range(3)] == [6] * 3
|
||||
finally:
|
||||
manager.shutdown()
|
||||
worker_args.results.close()
|
||||
worker_args.results.join_thread()
|
||||
|
||||
|
||||
@patch("vllm.v1.utils.run_api_server_worker_proc", mock_run_api_server_worker)
|
||||
def test_wait_for_completion_or_failure(api_server_args):
|
||||
"""Test that wait_for_completion_or_failure works with failures."""
|
||||
|
||||
@@ -34,6 +34,7 @@ ARGS = [
|
||||
"hermes",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
TOOLS = [
|
||||
@@ -54,11 +55,7 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteOpenAIServer(
|
||||
MODEL,
|
||||
ARGS,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -181,8 +181,9 @@ run_scale_out_ec() {
|
||||
# Encode-only EC producer instance. It runs the vision encoder and
|
||||
# publishes embeddings through the ECExampleConnector shared storage.
|
||||
echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_E" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_E" \
|
||||
vllm serve "$MODEL" \
|
||||
--enable-scale-out \
|
||||
--port "$ENCODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
@@ -204,8 +205,9 @@ run_scale_out_ec() {
|
||||
# features plus ec_transfer_params and loads the embeddings that the
|
||||
# encode instance published.
|
||||
echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_PD" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_PD" \
|
||||
vllm serve "$MODEL" \
|
||||
--enable-scale-out \
|
||||
--port "$PREFILL_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
|
||||
@@ -27,15 +27,10 @@ def vision_server():
|
||||
"1",
|
||||
"--limit-mm-per-prompt.video",
|
||||
"0",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
VISION_MODEL_NAME,
|
||||
args,
|
||||
env_dict=env_overrides,
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
@@ -22,69 +21,49 @@ SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
*,
|
||||
tokens_only: bool = False,
|
||||
enable_scale_out: bool = False,
|
||||
) -> set[str]:
|
||||
app = FastAPI()
|
||||
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
|
||||
args = Namespace(
|
||||
tokens_only=tokens_only,
|
||||
enable_scale_out=enable_scale_out,
|
||||
enable_fault_tolerance=False,
|
||||
)
|
||||
app.state.args = args
|
||||
register_api_routers(args, app, supported_tasks)
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default():
|
||||
paths = registered_paths(("generate",))
|
||||
|
||||
assert SCALE_OUT_PATHS.isdisjoint(paths)
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(caplog):
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert any(
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
|
||||
for record in caplog.records
|
||||
)
|
||||
assert any("--enable-scale-out" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
paths = registered_paths(("generate",))
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server():
|
||||
paths = registered_paths(("generate",), enable_scale_out=True)
|
||||
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server():
|
||||
paths = registered_paths(("render",))
|
||||
|
||||
assert paths >= RENDER_PATHS
|
||||
assert GENERATE_PATH not in paths
|
||||
|
||||
|
||||
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
|
||||
registered_paths(("render",))
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, "/abort_requests"}
|
||||
|
||||
|
||||
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="--tokens-only"):
|
||||
registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
@@ -34,12 +34,9 @@ def server():
|
||||
"--enable-return-routed-experts",
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
"--enable-scale-out",
|
||||
]
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -37,13 +37,10 @@ def server():
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ def server(request):
|
||||
# Option B: deterministic prefix caching
|
||||
# "--enable-prefix-caching",
|
||||
# "--deterministic-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
extra_args = getattr(request, "param", None)
|
||||
@@ -79,11 +80,7 @@ def server(request):
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
model_name: "nvidia/GLM-5.2-NVFP4"
|
||||
accuracy_threshold: 0.90
|
||||
tolerance: 0.0
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_tokens: 256
|
||||
max_concurrency: 32
|
||||
request_timeout_seconds: 1800
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--tensor-parallel-size 4
|
||||
--max-model-len 8192
|
||||
--max-num-seqs 32
|
||||
--max-num-batched-tokens 4096
|
||||
--enforce-eager
|
||||
--kv-cache-dtype bfloat16
|
||||
--attention-config '{"hisparse_config":{}}'
|
||||
--kv-transfer-config '{"kv_connector":"HiSparseConnector","kv_role":"kv_both","kv_connector_extra_config":{"host_pool_gib":32}}'
|
||||
env:
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
@@ -3,14 +3,18 @@ accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
# Preserve KV-cache headroom after CUDA graph memory profiling and bound the
|
||||
# DCP-aware FlashInfer workspace to 2.39 GiB at the 4,096-token scheduler cap.
|
||||
server_args: >-
|
||||
--compilation-config '{"cudagraph_mode":"PIECEWISE"}'
|
||||
--gpu-memory-utilization 0.96
|
||||
--max-model-len 4096
|
||||
--max-num-batched-tokens 32768
|
||||
--max-num-batched-tokens 4096
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--prefill-context-parallel-size 4
|
||||
--decode-context-parallel-size 4
|
||||
--dcp-comm-backend ag_rs
|
||||
--enable-expert-parallel
|
||||
--kv-cache-dtype fp8
|
||||
env:
|
||||
|
||||
@@ -3,8 +3,10 @@ accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
# Leave headroom for FlashInfer CUTLASS autotuning after KV-cache allocation.
|
||||
server_args: >-
|
||||
--compilation-config '{"cudagraph_mode":"NONE"}'
|
||||
--gpu-memory-utilization 0.85
|
||||
--max-model-len 4096
|
||||
--max-num-batched-tokens 32768
|
||||
--safetensors-load-strategy prefetch
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
model_name: "Qwen/Qwen3.5-4B"
|
||||
# full GSM8K set is 1319 questions
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 128
|
||||
max_tokens: 1024
|
||||
use_chat_completions: true
|
||||
temperature: 0.7
|
||||
seed: 42
|
||||
|
||||
# Calibrated by running non-watermaked 13 times
|
||||
# the threshold is the average accuracy, and the tolerance is set to 3 sigma (standard deviations)
|
||||
accuracy_threshold: 0.885
|
||||
tolerance: 0.015
|
||||
|
||||
server_args: >-
|
||||
--language-model-only
|
||||
--max-model-len 8192
|
||||
--max-num-seqs 128
|
||||
--gpu-memory-utilization 0.4
|
||||
--default-chat-template-kwargs '{"enable_thinking": false}'
|
||||
--watermark-config '{"key": 42, "context_width": 4}'
|
||||
@@ -0,0 +1 @@
|
||||
GLM-5.2-NVFP4-HiSparse.yaml
|
||||
@@ -0,0 +1 @@
|
||||
Qwen3.5-4B-watermark.yaml
|
||||
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
as_quantized_activation,
|
||||
expose_input_quant_key,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
@@ -118,13 +119,13 @@ def test_bridge_marks_supporting_and_skips_others():
|
||||
supported = _probe(FlashInferCutlassNvFp4LinearKernel)
|
||||
layer = torch.nn.Module()
|
||||
expose_input_quant_key(layer, supported)
|
||||
assert layer.input_quant_key == kNvfp4Dynamic
|
||||
assert get_input_quant_key(layer) == kNvfp4Dynamic
|
||||
|
||||
unsupported = _probe(FlashInferTrtllmNvFp4LinearKernel)
|
||||
assert unsupported.input_quant_key() is None
|
||||
layer = torch.nn.Module()
|
||||
expose_input_quant_key(layer, unsupported)
|
||||
assert not hasattr(layer, "input_quant_key")
|
||||
assert get_input_quant_key(layer) is None
|
||||
|
||||
|
||||
def test_as_quantized_activation_validates_key():
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm correctness tests for the strided DSA candidate mask.
|
||||
|
||||
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
|
||||
``apply_candidate_mask``. It differs in two ways that need pinning down:
|
||||
|
||||
* it launches a fixed number of programs that stride the column axis, rather
|
||||
than one program per 1024-column tile, so at ``max_model_len`` widths the
|
||||
grid no longer scales with a workspace that is mostly padding;
|
||||
* it stops at each row's ``end`` instead of sanitizing the whole width.
|
||||
|
||||
The second is only sound because every consumer bounds its scan by the same
|
||||
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
|
||||
contract is agreement with the shared kernel over ``[0, end)`` and nothing
|
||||
beyond it, which is what these tests assert rather than comparing whole rows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 8
|
||||
TOPK_BLOCKS = 64
|
||||
# Above 128 tiles the grid saturates and programs start striding; below it the
|
||||
# min() clamp applies instead. Both regimes need covering, so the widths here
|
||||
# deliberately straddle 128 * 1024.
|
||||
NARROW = 8192
|
||||
WIDE = 262144
|
||||
|
||||
|
||||
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
|
||||
"""Logits, bounds and candidate blocks shared by both kernels."""
|
||||
torch.manual_seed(0)
|
||||
# A column stride > 1 keeps the kernels honest about stride_col; the decode
|
||||
# caller slices a workspace, so contiguous rows are not guaranteed.
|
||||
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
|
||||
logits[0, : min(16, width)] = 0.0
|
||||
if rows > 2:
|
||||
logits[2, 5] = float("nan")
|
||||
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
|
||||
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
|
||||
nblocks = (width + block_size - 1) // block_size
|
||||
candidates = torch.randint(
|
||||
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
|
||||
)
|
||||
# -1 is the "no block" sentinel the selector emits for short rows.
|
||||
candidates[:, ::7] = -1
|
||||
return logits, row_ks, row_ke, candidates
|
||||
|
||||
|
||||
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ref = logits.clone()
|
||||
got = logits.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
|
||||
_apply_candidate_mask_strided(
|
||||
got, row_ks, row_ke, candidates, block_size, row_repeat
|
||||
)
|
||||
return ref, got
|
||||
|
||||
|
||||
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
|
||||
"""Compare only the region a consumer can read."""
|
||||
for row in range(ref.shape[0]):
|
||||
end = int(row_ke[row // row_repeat])
|
||||
if end <= 0:
|
||||
continue
|
||||
torch.testing.assert_close(
|
||||
got[row, :end],
|
||||
ref[row, :end],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
equal_nan=True,
|
||||
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [NARROW, WIDE])
|
||||
@pytest.mark.parametrize("col_stride", [1, 2])
|
||||
def test_matches_shared_kernel_ragged_ends(width, col_stride):
|
||||
"""Ends on and around tile boundaries, plus the empty and full rows."""
|
||||
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
width,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=col_stride,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [8, 16])
|
||||
def test_matches_shared_kernel_with_starts(block_size):
|
||||
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
|
||||
ends = [0, 64, 4096, 8192]
|
||||
starts = [0, 7, 1024, 3]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
NARROW,
|
||||
ends,
|
||||
starts,
|
||||
block_size=block_size,
|
||||
col_stride=1,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
def test_matches_shared_kernel_row_repeat():
|
||||
"""Speculative decode: next_n logit rows share one bound."""
|
||||
row_repeat = 3
|
||||
ends = [0, 1025, WIDE]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends) * row_repeat,
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
|
||||
|
||||
|
||||
def test_leaves_columns_past_end_untouched():
|
||||
"""The deliberate difference from the shared kernel, pinned explicitly.
|
||||
|
||||
Skipping the tail is where the speedup comes from, so a change that
|
||||
quietly restores full-width sanitizing should fail here rather than just
|
||||
get slower. Work stops at the tile boundary containing ``end``, not at
|
||||
``end`` itself, so that is the bound asserted.
|
||||
"""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_MASK_TILE,
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ends = [0, 1, 1025, WIDE // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
sentinel = 1.5
|
||||
logits.fill_(sentinel)
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
for row, end in enumerate(ends):
|
||||
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
|
||||
tail = logits[row, touched:]
|
||||
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
|
||||
f"row {row} (end {end}) was written past column {touched}"
|
||||
)
|
||||
# And the bound is tight: a row that ends early must not have cost a
|
||||
# full-width pass, which is the whole point of the kernel.
|
||||
assert touched <= end + _MASK_TILE
|
||||
|
||||
|
||||
def test_cudagraph_replay_tracks_changing_ends():
|
||||
"""The grid is static so a FULL capture stays valid when ends change.
|
||||
|
||||
Capturing bakes the grid, so a data-dependent program count would freeze
|
||||
whatever lengths were resident at capture. Replaying against longer and
|
||||
shorter contexts than the captured ones is the property that matters.
|
||||
"""
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
rows = 4
|
||||
captured_ends = [1024, 4096, 8192, 2048]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
rows,
|
||||
WIDE,
|
||||
captured_ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
base = logits.clone()
|
||||
|
||||
# Warm up outside the capture: Triton compiles on first call, and autotune
|
||||
# launches would otherwise happen inside the graph.
|
||||
_apply_candidate_mask_strided(
|
||||
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
|
||||
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
|
||||
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
|
||||
logits.copy_(base)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
ref = base.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, logits, row_ke, 1)
|
||||
@@ -30,7 +30,10 @@ def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
|
||||
def _install_fake_aiter_modules(
|
||||
monkeypatch: pytest.MonkeyPatch, *, supports_fp8: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
supports_fp8: bool,
|
||||
supports_causal: bool = False,
|
||||
) -> None:
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
@@ -65,6 +68,14 @@ def _install_fake_aiter_modules(
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_without_fp8
|
||||
|
||||
if supports_causal:
|
||||
inner = mla_decode_fwd
|
||||
|
||||
def mla_decode_fwd_with_causal(*args, causal=True, **kwargs):
|
||||
return inner(*args, **kwargs)
|
||||
|
||||
mla_decode_fwd = mla_decode_fwd_with_causal
|
||||
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod.mla = mla_mod
|
||||
|
||||
@@ -135,3 +146,107 @@ def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert _check_aiter_mla_fp8_support() is True
|
||||
assert signature_mock.call_count == 1
|
||||
|
||||
|
||||
def test_missing_causal_arg_fails_closed(monkeypatch):
|
||||
"""Old aiter wheels have no causal=; do not report that as a valid draft path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=False)
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
rocm_aiter_ops.mla_decode_supports_non_causal()
|
||||
with pytest.raises(RuntimeError, match="causal-only"):
|
||||
AiterMLABackend.supports_non_causal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_kernels", [False, True])
|
||||
def test_supports_non_causal_requires_arch_kernels(monkeypatch, has_kernels):
|
||||
"""The Python argument alone does not guarantee an arch has the kernel."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_non_causal_asm_kernels",
|
||||
lambda: has_kernels,
|
||||
)
|
||||
assert rocm_aiter_ops.mla_decode_supports_non_causal() is True
|
||||
assert AiterMLABackend.supports_non_causal() is has_kernels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_decode_fwd_forwards_causal_to_aiter(monkeypatch, causal):
|
||||
"""The decode op must pass the block mask through; omitting it is causal-only."""
|
||||
seen: list = []
|
||||
|
||||
def mla_decode_fwd(*args, causal=None, **kwargs):
|
||||
seen.append(causal)
|
||||
|
||||
mla_mod: Any = types.ModuleType("aiter.mla")
|
||||
mla_mod.mla_decode_fwd = mla_decode_fwd
|
||||
aiter_mod: Any = types.ModuleType("aiter")
|
||||
aiter_mod.mla = mla_mod
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
|
||||
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
|
||||
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
|
||||
aiter_ops._rocm_aiter_mla_decode_fwd_impl(
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(2, 1, 1, 8),
|
||||
torch.zeros(2, 1, 8),
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
1,
|
||||
causal=causal,
|
||||
)
|
||||
assert seen == [causal]
|
||||
|
||||
|
||||
def _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
*,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
use_non_causal,
|
||||
):
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLABackend
|
||||
|
||||
_install_fake_aiter_modules(monkeypatch, supports_fp8=True, supports_causal=True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_non_causal_asm_kernels", lambda: True
|
||||
)
|
||||
return AiterMLABackend.validate_configuration(
|
||||
head_size=576,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
block_size=1,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=False,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=DeviceCapability(9, 5),
|
||||
attn_type=AttentionType.DECODER,
|
||||
use_non_causal=use_non_causal,
|
||||
)
|
||||
|
||||
|
||||
def test_non_causal_fp16_auto_cache_is_rejected(monkeypatch):
|
||||
"""Pinned AITER aborts fp16 Q; auto-select must fall through to Triton."""
|
||||
import torch
|
||||
|
||||
reasons = _aiter_mla_validate(
|
||||
monkeypatch,
|
||||
dtype=torch.float16,
|
||||
kv_cache_dtype="auto",
|
||||
use_non_causal=True,
|
||||
)
|
||||
assert any("non-causal fp16" in reason for reason in reasons)
|
||||
|
||||
@@ -125,7 +125,7 @@ def test_fused_q_kv_rmsnorm_quant_matches_separate(num_tokens, q_size, dtype):
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.common.ops.query_quant import (
|
||||
from vllm.models.deepseek_v41.common.ops.query_quant import (
|
||||
fused_q_kv_rmsnorm_quant,
|
||||
)
|
||||
|
||||
@@ -198,15 +198,15 @@ def test_shared_query_quant_preserves_projection(
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
|
||||
from vllm.models.deepseek_v4_1.attention import (
|
||||
from vllm.models.deepseek_v41.attention import (
|
||||
DeepseekV4Attention,
|
||||
DeepseekV4Indexer,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.common.ops.query_quant import (
|
||||
from vllm.models.deepseek_v41.common.ops.query_quant import (
|
||||
can_fuse_query_quant,
|
||||
fused_q_kv_rmsnorm_quant,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.quant_config import DeepseekV4FP8Config
|
||||
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
|
||||
|
||||
cls: (
|
||||
type[FlashInferCutedslMxfp8LinearKernel]
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.kernels # noqa: F401
|
||||
from tests.ir.ir_test_utils import assert_close, clone_args
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
gelu_and_mul_sparse_native = ir.ops.gelu_and_mul_sparse.impls["native"].impl_fn
|
||||
|
||||
|
||||
def test_gelu_and_mul_sparse_registration():
|
||||
expected = {
|
||||
"native": True,
|
||||
"triton": HAS_TRITON and current_platform.is_cuda(),
|
||||
}
|
||||
actual = {
|
||||
provider: impl.supported
|
||||
for provider, impl in ir.ops.gelu_and_mul_sparse.impls.items()
|
||||
}
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 64])
|
||||
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 8192, 13824, 16384, 32768])
|
||||
def test_gelu_and_mul_sparse_triton(
|
||||
dtype: torch.dtype, num_tokens: int, intermediate_size: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=intermediate_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
assert impl.supports_args(*args)
|
||||
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
actual = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
mask_mismatch = torch.count_nonzero((actual == 0) != (expected == 0))
|
||||
assert mask_mismatch / actual.numel() < 1e-3
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_second_seed():
|
||||
torch.manual_seed(1)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=32,
|
||||
hidden_size=8192,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("intermediate_size", [1, 31, 128])
|
||||
def test_gelu_and_mul_sparse_triton_small_sizes(
|
||||
dtype: torch.dtype, intermediate_size: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=5,
|
||||
hidden_size=intermediate_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
assert torch.equal(actual == 0, expected == 0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_nd_and_empty():
|
||||
torch.set_default_device("cuda")
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
x = torch.randn(2, 3, 16384, dtype=torch.bfloat16)
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
actual = impl.impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
assert actual.shape == (2, 3, 8192)
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
empty = torch.empty(0, 16384, dtype=torch.bfloat16)
|
||||
empty_out = impl.impl_fn(empty, 1.6448536269514722, "tanh")
|
||||
assert empty_out.shape == (0, 8192)
|
||||
assert empty_out.dtype == empty.dtype
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_gelu_and_mul_sparse_triton_constant_rows(dtype: torch.dtype):
|
||||
torch.set_default_device("cuda")
|
||||
gate = torch.full((4, 8192), 3.0, dtype=dtype)
|
||||
up = torch.randn_like(gate)
|
||||
x = torch.cat((gate, up), dim=-1)
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
|
||||
assert torch.count_nonzero(actual) == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("nonfinite", [float("nan"), float("inf")])
|
||||
def test_gelu_and_mul_sparse_triton_nonfinite_gate(nonfinite: float):
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(2, 16384, dtype=torch.bfloat16)
|
||||
x[0, 0] = nonfinite
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
torch.testing.assert_close(actual, expected, equal_nan=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_fallback_conditions():
|
||||
torch.set_default_device("cuda")
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
x = torch.randn(4, 16384, dtype=torch.bfloat16)
|
||||
|
||||
assert impl.supports_args(x, 1.0, "tanh")
|
||||
assert not impl.supports_args(x, 1.0, "none")
|
||||
assert not impl.supports_args(x[:, ::2], 1.0, "tanh")
|
||||
assert not impl.supports_args(x[:, :-1], 1.0, "tanh")
|
||||
assert not impl.supports_args(
|
||||
torch.randn(1, 65538, dtype=torch.bfloat16), 1.0, "tanh"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_dispatch_and_opcheck():
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=8,
|
||||
hidden_size=4096,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
with ir.ops.gelu_and_mul_sparse.set_priority(["triton", "native"]):
|
||||
dispatched = ir.ops.gelu_and_mul_sparse(*args)
|
||||
torch.library.opcheck(torch.ops.vllm_ir.gelu_and_mul_sparse, args)
|
||||
|
||||
direct = impl.impl_fn(*args)
|
||||
torch.testing.assert_close(dispatched, direct, rtol=0.0, atol=0.0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_cudagraph():
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(16, 32768, dtype=torch.bfloat16)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
for _ in range(3):
|
||||
expected = impl.impl_fn(x, 1.6448536269514722, "tanh")
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = impl.impl_fn(x, 1.6448536269514722, "tanh")
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Contract tests for DeepEPV2PrepareAndFinalize.finalize_async.
|
||||
|
||||
Hermetic: fake buffer/event, single process, no GPU collectives — only
|
||||
requires deep_ep to be importable.
|
||||
|
||||
T1 finalize_async -> combine is issued with async_with_compute_stream=True
|
||||
and the join is deferred to the receiver, which waits
|
||||
on the combine event (device-side) before the copy.
|
||||
T2 DBO active -> combine falls back to async_with_compute_stream=False;
|
||||
the receiver still produces the correct output.
|
||||
T3 finalize (sync) -> combine is issued synchronously and the output is
|
||||
copied immediately.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.import_utils import has_deep_ep_v2
|
||||
|
||||
requires_deep_ep_v2 = pytest.mark.skipif(
|
||||
not has_deep_ep_v2(),
|
||||
reason="Requires DeepEP v2 (ElasticBuffer)",
|
||||
)
|
||||
|
||||
if has_deep_ep_v2():
|
||||
import vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 as _dv2
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
|
||||
DeepEPV2PrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceContiguous,
|
||||
)
|
||||
|
||||
|
||||
class _FakeEvent:
|
||||
def __init__(self, has_event: bool):
|
||||
self.event = object() if has_event else None
|
||||
self.waited = False
|
||||
|
||||
def current_stream_wait(self):
|
||||
assert self.event is not None
|
||||
self.waited = True
|
||||
|
||||
|
||||
class _FakeBuffer:
|
||||
def __init__(self, out: torch.Tensor):
|
||||
self.out = out
|
||||
self.calls: list[dict] = []
|
||||
self.last_event: _FakeEvent | None = None
|
||||
|
||||
def combine(
|
||||
self,
|
||||
x,
|
||||
handle,
|
||||
topk_weights,
|
||||
async_with_compute_stream,
|
||||
allocate_on_comm_stream,
|
||||
):
|
||||
self.calls.append({"async_with_compute_stream": async_with_compute_stream})
|
||||
# DeepEP returns a completion event only for an async combine.
|
||||
self.last_event = _FakeEvent(has_event=async_with_compute_stream)
|
||||
return self.out, None, self.last_event
|
||||
|
||||
|
||||
def _make_pf(out: torch.Tensor):
|
||||
pf = DeepEPV2PrepareAndFinalize(
|
||||
buffer=_FakeBuffer(out),
|
||||
num_dispatchers=1,
|
||||
dp_size=1,
|
||||
rank_expert_offset=0,
|
||||
num_experts=8,
|
||||
num_topk=2,
|
||||
)
|
||||
pf.handles[0] = object()
|
||||
return pf
|
||||
|
||||
|
||||
def _run(pf, output: torch.Tensor, do_async: bool):
|
||||
empty = torch.empty(0, dtype=torch.bfloat16)
|
||||
weights = torch.empty(0, 2)
|
||||
ids = torch.empty(0, 2, dtype=torch.int64)
|
||||
fn = pf.finalize_async if do_async else pf.finalize
|
||||
return fn(output, empty, weights, ids, False, TopKWeightAndReduceContiguous())
|
||||
|
||||
|
||||
@requires_deep_ep_v2
|
||||
def test_finalize_async_defers_join_to_receiver():
|
||||
out_src = torch.full((4, 8), 7.0, dtype=torch.bfloat16)
|
||||
pf = _make_pf(out_src)
|
||||
dst = torch.zeros_like(out_src)
|
||||
recv = _run(pf, dst, do_async=True)
|
||||
assert pf.buffer.calls[0]["async_with_compute_stream"] is True
|
||||
assert not torch.equal(dst, out_src), "join must not happen before receiver"
|
||||
recv()
|
||||
assert pf.buffer.last_event.waited, "receiver must join via event wait"
|
||||
assert torch.equal(dst, out_src)
|
||||
|
||||
|
||||
@requires_deep_ep_v2
|
||||
def test_finalize_async_under_dbo_uses_sync_combine(monkeypatch):
|
||||
monkeypatch.setattr(_dv2, "dbo_enabled", lambda: True)
|
||||
out_src = torch.full((4, 8), 7.0, dtype=torch.bfloat16)
|
||||
pf = _make_pf(out_src)
|
||||
dst = torch.zeros_like(out_src)
|
||||
recv = _run(pf, dst, do_async=True)
|
||||
assert pf.buffer.calls[0]["async_with_compute_stream"] is False
|
||||
recv()
|
||||
assert not pf.buffer.last_event.waited, "no event to wait on for sync combine"
|
||||
assert torch.equal(dst, out_src)
|
||||
|
||||
|
||||
@requires_deep_ep_v2
|
||||
def test_finalize_sync():
|
||||
out_src = torch.full((4, 8), 7.0, dtype=torch.bfloat16)
|
||||
pf = _make_pf(out_src)
|
||||
dst = torch.zeros_like(out_src)
|
||||
ret = _run(pf, dst, do_async=False)
|
||||
assert ret is None
|
||||
assert pf.buffer.calls[0]["async_with_compute_stream"] is False
|
||||
assert torch.equal(dst, out_src)
|
||||
@@ -565,6 +565,74 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("AITER_SITUV2_A4W4", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert "AITER_SITUV2_A4W4" not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_padding", [True, False])
|
||||
def test_aiter_moe_padding_env_var(
|
||||
moe_padding: bool,
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_gemm_nt,
|
||||
get_tma_aligned_size,
|
||||
is_deep_gemm_e8m0_used,
|
||||
per_block_cast_to_fp8,
|
||||
should_use_deepgemm_for_fp8_linear,
|
||||
)
|
||||
@@ -291,7 +292,11 @@ def test_w8a8_block_fp8_deep_gemm_matmul(M, N, K, block_size, out_dtype, seed):
|
||||
A_fp8, As_fp8 = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=True, tma_aligned_scales=True
|
||||
)
|
||||
B_fp8, Bs_fp8 = per_block_cast_to_fp8(B_fp32, block_size=block_size)
|
||||
B_fp8, Bs_fp8 = per_block_cast_to_fp8(
|
||||
B_fp32,
|
||||
block_size=block_size,
|
||||
use_ue8m0=is_deep_gemm_e8m0_used(),
|
||||
)
|
||||
|
||||
As = As_fp8.to(torch.float32)
|
||||
Bs = Bs_fp8.to(torch.float32)
|
||||
|
||||
@@ -37,6 +37,20 @@ MAX_SKINNY_BATCH_SIZE = hybrid_module.MAX_SKINNY_BATCH_SIZE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pack_zp_rows_for_kernel(zp_nkg: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack raw uint4 zero points along N: [N, G] int32 -> [N//8, G] int32.
|
||||
|
||||
Row n's nibble lands in word[n//8] at bits 4*(n%8).
|
||||
"""
|
||||
assert zp_nkg.dtype == torch.int32
|
||||
N, G = zp_nkg.shape
|
||||
assert N % 8 == 0
|
||||
shifts = (torch.arange(8, device=zp_nkg.device, dtype=torch.int32) * 4)[:, None]
|
||||
return torch.sum(
|
||||
(zp_nkg.view(N // 8, 8, G) & 0xF) << shifts, dim=1, dtype=torch.int32
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _rdna_hybrid_w4a16_reference(
|
||||
x_mk: torch.Tensor,
|
||||
w_int4_nk: torch.Tensor,
|
||||
@@ -50,8 +64,8 @@ def _rdna_hybrid_w4a16_reference(
|
||||
x_mk: [M, K] fp16/bf16
|
||||
w_int4_nk: [N, K] int32 with raw uint4 values in [0, 15]
|
||||
scales_nkg: [N, K//G] fp16/bf16
|
||||
zp_nkg: [N, K//G] fp16/bf16 raw zero points (already in act dtype),
|
||||
or None for symmetric (uint4b8, dequant subtracts 8)
|
||||
zp_nkg: [N, K//G] int32 raw zero points in [0, 15], or None for
|
||||
symmetric (uint4b8, dequant subtracts 8)
|
||||
"""
|
||||
G = group_size
|
||||
N, K = w_int4_nk.shape
|
||||
@@ -110,13 +124,16 @@ def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M)
|
||||
0.05 * torch.rand((N, K // group_size), device=device, dtype=torch.float32)
|
||||
).to(dtype)
|
||||
|
||||
# Optional raw zero points [N, K//G] in act dtype.
|
||||
# Optional raw zero points, [N, K//G] int32 for the reference and packed
|
||||
# along N for the op.
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(
|
||||
0, 16, (N, K // group_size), device=device, dtype=torch.int32
|
||||
).to(dtype)
|
||||
)
|
||||
w_zp = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
else:
|
||||
zp_nkg = None
|
||||
w_zp = None
|
||||
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
@@ -124,7 +141,7 @@ def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M)
|
||||
x_mk,
|
||||
w_q,
|
||||
scales_nkg,
|
||||
zp_nkg,
|
||||
w_zp,
|
||||
None, # bias
|
||||
num_compute_units(),
|
||||
group_size,
|
||||
@@ -332,7 +349,7 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
|
||||
|
||||
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
|
||||
def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_init):
|
||||
"""uint4 (asymmetric): zero points unpacked to raw values in act dtype."""
|
||||
"""uint4 (asymmetric): zero points are packed [N//8, K//G] int32."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
@@ -374,10 +391,11 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
# Zero-points: unpacked to [N, K//G], cast to act dtype, raw values [0..15].
|
||||
assert layer.weight_zero_point.dtype == torch.float16
|
||||
assert tuple(layer.weight_zero_point.shape) == (N, K // G)
|
||||
expected_zp = zeros_int4_gn.t().to(torch.float16) # [N, K//G]
|
||||
# Zero-points: [N//8, K//G] int32, row n's nibble at word[n//8]
|
||||
# bits 4*(n%8).
|
||||
assert layer.weight_zero_point.dtype == torch.int32
|
||||
assert tuple(layer.weight_zero_point.shape) == (N // 8, K // G)
|
||||
expected_zp = _pack_zp_rows_for_kernel(zeros_int4_gn.t().contiguous())
|
||||
torch.testing.assert_close(layer.weight_zero_point, expected_zp)
|
||||
|
||||
# Quantized weights match symmetric path's layout regardless of zp.
|
||||
@@ -518,3 +536,30 @@ def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(torch.ops, "_rocm_C")
|
||||
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
|
||||
reason="wvSplitK_int4_g not built",
|
||||
)
|
||||
def test_wvsplitk_int4_g_rejects_unpacked_zero_points():
|
||||
"""Act-dtype zero points raise instead of being read as packed words.
|
||||
|
||||
Both layouts are 2D with compatible extents, so only the dtype separates
|
||||
them; misreading one as the other returns wrong numbers silently.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
K, N, G, M = 256, 64, 128, 1
|
||||
a = torch.randn((M, K), device=device, dtype=torch.float16)
|
||||
w = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
|
||||
torch.int8
|
||||
)
|
||||
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
zp_unpacked = torch.zeros((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Zero points must be int32 or uint32"):
|
||||
ops.wvSplitK_int4_g(w, a, scales, num_compute_units(), G, zp_unpacked, None)
|
||||
|
||||
@@ -12,20 +12,90 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
ScaledMMLinearKernel,
|
||||
_get_linear_backend,
|
||||
_resolve_backend_kernels,
|
||||
init_fp8_linear_kernel,
|
||||
init_int8_linear_kernel,
|
||||
register_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import PlatformEnum
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def test_auto_linear_backend_is_unchanged_without_overrides():
|
||||
kernels = [object]
|
||||
quantizations = (
|
||||
"fp8_block_w8a8",
|
||||
"fp8_w8a8",
|
||||
"int8_w8a8",
|
||||
"mixed_precision",
|
||||
"mxfp8",
|
||||
"mxfp4",
|
||||
"mxfp6",
|
||||
"w8a16_fp8",
|
||||
"nvfp4_w4a4",
|
||||
"nvfp4_w4a16",
|
||||
)
|
||||
config = VllmConfig(kernel_config=KernelConfig(linear_backend="auto"))
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
for quantization in quantizations:
|
||||
assert _get_linear_backend(quantization=quantization) == "auto"
|
||||
assert (
|
||||
_resolve_backend_kernels(kernels, "test", quantization=quantization)
|
||||
is kernels
|
||||
)
|
||||
|
||||
|
||||
def test_linear_backend_override_is_quantization_specific():
|
||||
config = VllmConfig(
|
||||
kernel_config=KernelConfig(
|
||||
linear_backend="cutlass",
|
||||
linear_backend_per_quant={"nvfp4_w4a16": "humming"},
|
||||
)
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
assert _get_linear_backend(quantization="nvfp4_w4a16") == "humming"
|
||||
assert _get_linear_backend(quantization="nvfp4_w4a4") == "cutlass"
|
||||
assert _get_linear_backend(quantization="fp8_w8a8") == "cutlass"
|
||||
|
||||
|
||||
@patch.object(HummingFP8ScaledMMLinearKernel, "is_supported", return_value=(True, None))
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_fp8_linear_backend_override(platform_mock, _):
|
||||
platform_mock._enum = PlatformEnum.CUDA
|
||||
config = VllmConfig(
|
||||
kernel_config=KernelConfig(
|
||||
linear_backend="cutlass",
|
||||
linear_backend_per_quant={"fp8_w8a8": "humming"},
|
||||
)
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
kernel = init_fp8_linear_kernel(
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
weight_shape=(128, 128),
|
||||
)
|
||||
|
||||
assert type(kernel) is HummingFP8ScaledMMLinearKernel
|
||||
|
||||
|
||||
def test_is_supported_is_abstract():
|
||||
"""Test that is_supported() is properly defined as abstract."""
|
||||
assert issubclass(ScaledMMLinearKernel, ABC)
|
||||
|
||||
@@ -42,6 +42,122 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize(
|
||||
"cache_dtype,kv_mxfp8",
|
||||
[
|
||||
(torch.uint8, False),
|
||||
(torch.uint8, True),
|
||||
(torch.bfloat16, False),
|
||||
(torch.float8_e4m3fn, False),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 1023, 1024])
|
||||
@pytest.mark.parametrize("use_graph", [False, True])
|
||||
def test_dspark_context_kv_matches_query_insert(
|
||||
cache_dtype, kv_mxfp8, num_tokens, use_graph
|
||||
):
|
||||
"""KV-only insertion must preserve every cache byte, including graph replay."""
|
||||
from vllm.models.deepseek_v41.nvidia.dspark import _insert_context_kv
|
||||
|
||||
torch.manual_seed(42)
|
||||
block_size = 256
|
||||
num_blocks = math.ceil((num_tokens + 7) / block_size)
|
||||
row_size = (528 if kv_mxfp8 else 584) if cache_dtype == torch.uint8 else 512
|
||||
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
|
||||
backing = torch.full((num_blocks, page_stride), 3, device="cuda").to(cache_dtype)
|
||||
cache = backing.as_strided(
|
||||
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
|
||||
)
|
||||
reference_backing = backing.clone()
|
||||
reference = reference_backing.as_strided(cache.shape, cache.stride())
|
||||
kv = torch.randn(num_tokens + 3, 512, device="cuda", dtype=torch.bfloat16)
|
||||
positions = torch.arange(num_tokens + 3, device="cuda") + 7
|
||||
angles = torch.randn(num_tokens + 16, 32, device="cuda")
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_tokens]
|
||||
slots[::5] = -1
|
||||
scale = torch.tensor([0.7], device="cuda")
|
||||
# Only what _insert_context_kv actually reads: the record width comes off
|
||||
# the cache tensor, so no per-record width attribute is stubbed here.
|
||||
attn = SimpleNamespace(
|
||||
swa_cache_layer=SimpleNamespace(kv_cache=cache, block_size=block_size),
|
||||
kv_mxfp8=kv_mxfp8,
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
|
||||
_flashinfer_fp8_kv_scale=scale,
|
||||
)
|
||||
|
||||
def legacy_insert():
|
||||
q = torch.zeros(kv.shape[0], 8, 512, dtype=kv.dtype, device="cuda")
|
||||
if cache_dtype == torch.uint8:
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
reference.view(num_blocks, -1),
|
||||
slots,
|
||||
positions,
|
||||
cos_sin,
|
||||
8,
|
||||
1e-20,
|
||||
block_size,
|
||||
True,
|
||||
kv_mxfp8,
|
||||
)
|
||||
elif cache_dtype == torch.bfloat16:
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
q,
|
||||
kv,
|
||||
reference,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin,
|
||||
1e-20,
|
||||
block_size,
|
||||
)
|
||||
else:
|
||||
q_fp8 = torch.empty_like(q, dtype=cache_dtype)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
q,
|
||||
kv,
|
||||
q_fp8,
|
||||
reference,
|
||||
slots,
|
||||
positions,
|
||||
cos_sin,
|
||||
scale,
|
||||
scale,
|
||||
1e-20,
|
||||
block_size,
|
||||
)
|
||||
|
||||
def insert():
|
||||
_insert_context_kv(attn, kv, positions, slots)
|
||||
|
||||
insert()
|
||||
graph = None
|
||||
if use_graph:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
insert()
|
||||
for iteration in range(3):
|
||||
kv.normal_()
|
||||
positions.add_(1)
|
||||
if iteration == 2:
|
||||
slots.fill_(-1)
|
||||
backing.view(torch.uint8).fill_(165)
|
||||
reference_backing.copy_(backing)
|
||||
before = kv.clone()
|
||||
legacy_insert()
|
||||
insert() if graph is None else graph.replay()
|
||||
torch.testing.assert_close(
|
||||
backing.view(torch.uint8),
|
||||
reference_backing.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
torch.testing.assert_close(kv, before, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="graph capture coverage"
|
||||
)
|
||||
@@ -65,7 +181,7 @@ def test_v41_fused_save_compress_and_insert(
|
||||
a Torch reference with BF16 rounding before RoPE. Replays change both
|
||||
inputs and history so stale scratch/cache values cannot satisfy the test.
|
||||
"""
|
||||
from vllm.models.deepseek_v4_1.common.ops.fused_compress_quant_cache import (
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
fused_save_compress_norm,
|
||||
rope_quant_insert,
|
||||
)
|
||||
@@ -215,6 +331,136 @@ def test_v41_fused_save_compress_and_insert(
|
||||
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _mxfp8_record_reference(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reference V4.1 record for one token: 512 fp8 bytes + 16 UE8M0 scales.
|
||||
|
||||
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
|
||||
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
|
||||
"""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
|
||||
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
"""The 528-byte V4.1 record quantizes the RoPE dims instead of keeping them
|
||||
in bf16, so a page is [block_size x 512 fp8][block_size x 16 UE8M0].
|
||||
|
||||
Byte-exact against the reference quantizer, including the rows and page
|
||||
padding the kernel must leave alone.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
cache_block = 64
|
||||
cache_stride = math.ceil(cache_block * 528 / 512) * 512
|
||||
num_tokens = 12
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(64, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
|
||||
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
|
||||
cache = cache_backing.as_strided((2, cache_block, 528), (cache_stride, 528, 1))
|
||||
# Leave one token unmapped so the kernel's negative-slot skip is covered.
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
slots[3] = -1
|
||||
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
|
||||
|
||||
expected = torch.full_like(cache_backing, 165)
|
||||
for t in range(num_tokens):
|
||||
slot = slots[t].item()
|
||||
pos = positions[t].item()
|
||||
if slot < 0 or (pos + 1) % compress_ratio:
|
||||
continue
|
||||
page, row = divmod(slot, cache_block)
|
||||
values, scales = _mxfp8_record_reference(
|
||||
latent[t], cos_sin[pos // compress_ratio * compress_ratio]
|
||||
)
|
||||
expected[page, row * 512 : (row + 1) * 512] = values
|
||||
scale_offset = cache_block * 512 + row * 16
|
||||
expected[page, scale_offset : scale_offset + 16] = scales
|
||||
|
||||
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_v41_mxfp8_cache_round_trip():
|
||||
"""quantize_and_insert -> dequantize_and_gather recovers the V4.1 record.
|
||||
|
||||
The RoPE dims now go through fp8, so they carry quantization error too --
|
||||
the check is that every dim survives within one MXFP8 tile step, and that
|
||||
the bytes on the way match the reference quantizer.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
dequantize_and_gather_k_cache,
|
||||
quantize_and_insert_k_cache,
|
||||
)
|
||||
|
||||
torch.manual_seed(11)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
num_tokens = 70
|
||||
num_blocks = 4
|
||||
page_bytes = math.ceil(block_size * 528 / 512) * 512
|
||||
|
||||
k = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
cache = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
quantize_and_insert_k_cache(
|
||||
k, cache, slot_mapping, block_size=block_size, bytes_per_token=528
|
||||
)
|
||||
|
||||
for t in (0, 1, block_size, num_tokens - 1):
|
||||
values, scales = _ue8m0_reference(k[t].float(), 32, 448.0)
|
||||
page, row = divmod(t, block_size)
|
||||
torch.testing.assert_close(
|
||||
cache[page, row * 512 : (row + 1) * 512],
|
||||
values.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
scale_offset = block_size * 512 + row * 16
|
||||
torch.testing.assert_close(
|
||||
cache[page, scale_offset : scale_offset + 16],
|
||||
(scales.log2() + 127).to(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
dequantize_and_gather_k_cache(
|
||||
out,
|
||||
cache.view(num_blocks, block_size, 528),
|
||||
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
|
||||
gather_lens=None,
|
||||
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
|
||||
1, -1
|
||||
),
|
||||
block_size=block_size,
|
||||
offset=0,
|
||||
)
|
||||
# Half an e4m3 ULP at the top of the range (16 units) bounds the error.
|
||||
tile_amax = k.float().abs().view(num_tokens, 16, 32).amax(-1).clamp(min=1e-4)
|
||||
tolerance = 16.0 * (tile_amax / 448.0).log2().ceil().exp2()
|
||||
error = (out[0].float() - k.float()).abs()
|
||||
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
@@ -226,7 +472,7 @@ def test_v41_rope_insert_plain_row(compress_ratio: int, store_fp8: bool):
|
||||
Non-boundary positions, negative slots and untouched rows must keep their
|
||||
prior contents.
|
||||
"""
|
||||
from vllm.models.deepseek_v4_1.common.ops.fused_compress_quant_cache import (
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
@@ -301,7 +547,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
capacity`` and keep padding tokens at PAD."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.models.deepseek_v4_1.compressor import CompressorMetadataBuilder
|
||||
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec
|
||||
|
||||
@@ -351,11 +597,11 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph):
|
||||
"""Both reused-event joins must publish this forward's states and cache rows."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v4_1.attention import (
|
||||
from vllm.models.deepseek_v41.attention import (
|
||||
DeepseekV4Attention,
|
||||
DeepseekV4Indexer,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.compressor import DeepseekCompressor
|
||||
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
|
||||
|
||||
torch.manual_seed(43)
|
||||
raw = torch.randn(19, 1024, device="cuda")
|
||||
|
||||
@@ -7,15 +7,15 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4_1.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v4_1.common.engram import (
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram as CommonEngram,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.common.engram import (
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
NgramHashState,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.nvidia import engram as nvidia_engram_ops
|
||||
from vllm.models.deepseek_v4_1.nvidia.engram import Engram, ParallelEngramEmbedding
|
||||
from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops
|
||||
from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -503,7 +503,7 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
def test_v2_model_state_gathers_lookback_window():
|
||||
"""The window is gathered on device from the runner's token history."""
|
||||
from vllm.models.deepseek_v4_1.nvidia.model_state import DeepseekV41ModelState
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
|
||||
depth, max_num_reqs, max_model_len = 3, 4, 16
|
||||
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
|
||||
|
||||
@@ -573,6 +573,126 @@ def test_fused_norm_rope_ds_mla(num_tokens: int):
|
||||
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
|
||||
|
||||
|
||||
E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def quantize_to_e2m1(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Round to nearest e2m1, saturating to +-6 like cvt.rn.satfinite.e2m1x2.f32."""
|
||||
mags = torch.tensor(E2M1_MAGNITUDES, dtype=torch.float32, device=x.device)
|
||||
a = x.float().abs().clamp_max(6.0)
|
||||
mids = (mags[:-1] + mags[1:]) / 2
|
||||
code = torch.bucketize(a, mids, right=True)
|
||||
on_tie = (a.unsqueeze(-1) == mids).any(dim=-1)
|
||||
tie_code = torch.bucketize(a, mids, right=False)
|
||||
code = torch.where(on_tie, tie_code + (tie_code & 1), code)
|
||||
return (torch.signbit(x).to(torch.uint8) << 3) | code.to(torch.uint8)
|
||||
|
||||
|
||||
def nvfp4_sf_byte(s: torch.Tensor) -> torch.Tensor:
|
||||
"""Scale-factor byte permutation shared with FlashMLA: 8*(s&3) + (s>>2)."""
|
||||
return 8 * (s % 4) + (s // 4)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability(100),
|
||||
reason="nvfp4_ds_mla requires SM100 (Blackwell)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
|
||||
def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int):
|
||||
"""nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only).
|
||||
|
||||
Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble
|
||||
= even element) | 64 B unscaled e4m3 RoPE | 32 B byte-permuted e4m3 tile
|
||||
scales, one per 16 NoPE elements.
|
||||
"""
|
||||
torch.manual_seed(5)
|
||||
dev = "cuda"
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
|
||||
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
# Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3
|
||||
# subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale
|
||||
# BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a
|
||||
# future change to the scale rule cannot silently stop covering this.
|
||||
# amax must land in (6, 9) * 2^-9 after rms_norm; 7.5 sits mid-band, with
|
||||
# margin for the per-token RMS to vary.
|
||||
kv_c[:, :16] = (
|
||||
7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16)
|
||||
)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.ones(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
|
||||
|
||||
bs = max_pos
|
||||
mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8)
|
||||
slot = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
|
||||
|
||||
q_out = K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
mla_cos_sin,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
EPS,
|
||||
None,
|
||||
topk,
|
||||
slot_mapping=slot,
|
||||
indexer_k_cache=None,
|
||||
mla_kv_cache=mla_cache,
|
||||
mla_kv_cache_dtype="nvfp4_ds_mla",
|
||||
mla_k_scale=None,
|
||||
has_indexer=False,
|
||||
index_rope_interleave=False,
|
||||
)
|
||||
|
||||
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (nvfp4_ds_mla)")
|
||||
|
||||
kv_ref = rms_norm(kv_c, kvw) # [N, 512] fp32
|
||||
kpe_ref = rope(k_pe.float(), pos, mla_cos_sin, interleave=True) # [N, 64]
|
||||
tiles = kv_ref.view(num_tokens, 32, 16)
|
||||
amax = tiles.abs().amax(dim=-1)
|
||||
scale_target = torch.clamp_min(amax / 6.0, 2.0**-9)
|
||||
ref_scale = scale_target.to(FP8) # round-to-nearest
|
||||
|
||||
# The crafted tile must actually exercise the saturating path.
|
||||
assert (ref_scale[:, 0].float() < scale_target[:, 0]).all(), (
|
||||
"tile 0 should round its scale DOWN; adjust the crafted magnitude"
|
||||
)
|
||||
assert (amax[:, 0] / ref_scale[:, 0].float() > 6.0).all(), (
|
||||
"tile 0 should saturate e2m1; adjust the crafted magnitude"
|
||||
)
|
||||
|
||||
ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1))
|
||||
ref_codes = ref_codes.reshape(num_tokens, KV_LORA)
|
||||
|
||||
cache = mla_cache[0, :num_tokens] # [N, 352] uint8
|
||||
packed = cache[:, : KV_LORA // 2]
|
||||
got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1)
|
||||
got_codes = got_codes.reshape(num_tokens, KV_LORA) # low nibble = even elem
|
||||
torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0)
|
||||
|
||||
got_rope = cache[:, KV_LORA // 2 : KV_LORA // 2 + ROPE_DIM].view(FP8)
|
||||
assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3")
|
||||
|
||||
perm = nvfp4_sf_byte(torch.arange(32, device=dev))
|
||||
got_scale = cache[:, KV_LORA // 2 + ROPE_DIM :].view(FP8)[:, perm]
|
||||
torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0)
|
||||
|
||||
# No indexer on this call: top-k buffer must be untouched.
|
||||
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
|
||||
|
||||
|
||||
def test_fused_norm_rope_supports_large_token_count():
|
||||
"""Keep the token count off CUDA grid-y at its 65,536-block boundary."""
|
||||
num_tokens = 65536
|
||||
|
||||
@@ -424,6 +424,104 @@ def test_kv_path_matches_reference(num_tokens: int, block_size: int):
|
||||
)
|
||||
|
||||
|
||||
# ── Test 2c: DeepSeek V4.1 MXFP8 record (kv_mxfp8=True) ──────────────────────
|
||||
|
||||
V41_HEAD_BYTES = HEAD_DIM + HEAD_DIM // 32 # 512 fp8 + 16 UE8M0 scales = 528
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 64])
|
||||
@pytest.mark.parametrize("block_size", [32, 64])
|
||||
def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int):
|
||||
"""``kv_mxfp8=True`` writes DeepSeek's V4.1 record: all 512 dims as fp8 e4m3
|
||||
with one UE8M0 scale per 32 dims, no bf16 RoPE tail.
|
||||
|
||||
Both sides quantize the same bf16-rounded rotated row, so the bytes must
|
||||
match the Triton encoder exactly apart from the documented 1-ULP bf16
|
||||
rounding split on the rotation itself.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
dequantize_and_gather_k_cache as v41_dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
quantize_and_insert_k_cache as v41_quantize_and_insert_k_cache,
|
||||
)
|
||||
|
||||
torch.manual_seed(3)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
max_pos = 4096
|
||||
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
v41_quantize_and_insert_k_cache(
|
||||
apply_rope_gptj_last_k(kv, positions, cos_sin_cache),
|
||||
k_cache_ref,
|
||||
slot_mapping,
|
||||
block_size=block_size,
|
||||
use_fnuz=USE_FNUZ,
|
||||
bytes_per_token=V41_HEAD_BYTES,
|
||||
)
|
||||
|
||||
k_cache_fused = torch.zeros_like(k_cache_ref)
|
||||
q_dummy = torch.zeros(num_tokens, 1, HEAD_DIM, dtype=dtype, device=device)
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_dummy,
|
||||
kv,
|
||||
k_cache_fused,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
64,
|
||||
1e-6,
|
||||
block_size,
|
||||
True,
|
||||
True, # kv_mxfp8
|
||||
)
|
||||
|
||||
# A pre-quantization bf16 tie can land the two encoders on adjacent e4m3
|
||||
# codes, so compare after dequantizing and allow one step at the tile scale.
|
||||
def _dequant(cache_2d):
|
||||
out = torch.zeros(1, num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
v41_dequantize_and_gather_k_cache(
|
||||
out,
|
||||
cache_2d.view(num_blocks, block_size, V41_HEAD_BYTES),
|
||||
torch.tensor([num_tokens], dtype=torch.int32, device=device),
|
||||
None,
|
||||
torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(0),
|
||||
block_size,
|
||||
offset=0,
|
||||
use_fnuz=USE_FNUZ,
|
||||
)
|
||||
return out[0]
|
||||
|
||||
rec_ref = _dequant(k_cache_ref).float()
|
||||
rec_fused = _dequant(k_cache_fused).float()
|
||||
tile_scale = (
|
||||
(rec_ref.abs().view(num_tokens, 16, 32).amax(-1).clamp(min=1e-4) / FP8_MAX)
|
||||
.log2()
|
||||
.ceil()
|
||||
.exp2()
|
||||
)
|
||||
# One e4m3 ULP at the top of the range is 32 quantized units.
|
||||
step = 32.0 * tile_scale.repeat_interleave(32, dim=-1)
|
||||
assert ((rec_fused - rec_ref).abs() <= step).all()
|
||||
# The NoPE dims see no rotation at all, so they must be byte-identical.
|
||||
nope_bytes = slice(0, block_size * HEAD_DIM)
|
||||
ref_rows = k_cache_ref[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
|
||||
fused_rows = k_cache_fused[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
|
||||
torch.testing.assert_close(
|
||||
fused_rows[..., :NOPE_DIM], ref_rows[..., :NOPE_DIM], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -359,7 +359,7 @@ def test_indexer_k_inserts_only_valid_groups_into_padded_pages(
|
||||
num_tokens, compress_ratio, use_fp4, cache_dtype
|
||||
):
|
||||
"""Insert group ends and preserve skipped slots, graph rows, and page padding."""
|
||||
from vllm.models.deepseek_v4_1.common.ops.indexer_k_store import (
|
||||
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
|
||||
indexer_k_norm_rope_store,
|
||||
)
|
||||
|
||||
@@ -442,7 +442,7 @@ def test_indexer_k_inserts_only_valid_groups_into_padded_pages(
|
||||
@torch.inference_mode()
|
||||
def test_indexer_k_cuda_graph_replay_reads_current_projection(use_fp4):
|
||||
"""Replay must consume updated keys and slot mapping through the captured API."""
|
||||
from vllm.models.deepseek_v4_1.common.ops.indexer_k_store import (
|
||||
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
|
||||
indexer_k_norm_rope_store,
|
||||
)
|
||||
|
||||
@@ -501,7 +501,7 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
|
||||
``block_size > 1``, so a row-major store would feed the indexer permuted
|
||||
key bytes and randomize its top-k. Assert the write/read pair is exact.
|
||||
"""
|
||||
from vllm.models.deepseek_v4_1.common.ops.indexer_k_store import (
|
||||
from vllm.models.deepseek_v41.common.ops.indexer_k_store import (
|
||||
indexer_k_norm_rope_store,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
@@ -560,3 +560,55 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
|
||||
torch.testing.assert_close(
|
||||
k_scale[emitted], expected_scale[emitted], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_cutedsl", [False, True])
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
),
|
||||
reason="MXFP4 indexer cache requires an SM100-family GPU",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
|
||||
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
|
||||
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
|
||||
result rounded once, so the scoring kernel needs no cast."""
|
||||
if use_cutedsl and not has_cutedsl():
|
||||
pytest.skip("cutedsl (cutlass) not installed")
|
||||
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
num_tokens, n_head = 257, 32
|
||||
q = torch.randn(num_tokens, n_head, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.randint(
|
||||
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=torch.float32, device=device)
|
||||
weights = torch.randn(num_tokens, n_head, dtype=torch.bfloat16, device=device)
|
||||
softmax_scale, head_scale = HEAD_DIM**-0.5, 0.125 # pow2 head_scale, see above
|
||||
|
||||
_, weights_ref = _reference(
|
||||
positions, q, cos_sin_cache, weights, softmax_scale, head_scale, n_head, True
|
||||
)
|
||||
cutedsl_patch = (
|
||||
mock.patch(
|
||||
"vllm.models.deepseek_v4.common.ops.fused_indexer_q.has_cutedsl",
|
||||
return_value=False,
|
||||
)
|
||||
if not use_cutedsl
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
with cutedsl_patch:
|
||||
_, weights_fused = fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
q.clone(),
|
||||
cos_sin_cache,
|
||||
weights,
|
||||
softmax_scale,
|
||||
head_scale,
|
||||
use_fp4=True,
|
||||
weights_out_dtype=torch.bfloat16,
|
||||
)
|
||||
assert weights_fused.dtype == torch.bfloat16
|
||||
assert torch.equal(weights_fused, weights_ref.to(torch.bfloat16))
|
||||
|
||||
@@ -91,7 +91,7 @@ class MockLinearFp8Static(torch.nn.Module):
|
||||
|
||||
def __init__(self, input_scale: torch.Tensor):
|
||||
super().__init__()
|
||||
self.input_quant_key = kFp8StaticTensorSym
|
||||
self._input_quant_key = kFp8StaticTensorSym
|
||||
self.input_scale = input_scale
|
||||
|
||||
|
||||
@@ -100,15 +100,21 @@ class MockLinearFp8Dynamic128(torch.nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.input_quant_key = kFp8Dynamic128Sym
|
||||
self._input_quant_key = kFp8Dynamic128Sym
|
||||
|
||||
|
||||
class MockLinearNoQuant(torch.nn.Module):
|
||||
"""Mock linear layer with no input_quant_key (no fusion)."""
|
||||
"""Mock linear layer with no input quantization key."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class MockLinearRequiresUnquantized(MockLinearFp8Static):
|
||||
"""Mock consumer that needs the original activation for another branch."""
|
||||
|
||||
requires_unquantized_input = True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@@ -199,7 +205,7 @@ def test_maybe_fused_act_quant_fallback(
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Test maybe_fused_act_quant falls back when no input_quant_key."""
|
||||
"""Test maybe_fused_act_quant falls back without an input quantization key."""
|
||||
device = "cuda:0"
|
||||
torch.set_default_device(device)
|
||||
|
||||
@@ -216,3 +222,19 @@ def test_maybe_fused_act_quant_fallback(
|
||||
|
||||
ref_out = act_fn(x)
|
||||
torch.testing.assert_close(result, ref_out)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_maybe_fused_act_quant_preserves_required_unquantized_input(
|
||||
default_vllm_config,
|
||||
) -> None:
|
||||
device = "cuda:0"
|
||||
act_fn = SiluAndMul()
|
||||
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
|
||||
linear = MockLinearRequiresUnquantized(scale)
|
||||
x = torch.randn(17, 256, dtype=torch.bfloat16, device=device)
|
||||
|
||||
result = maybe_fused_act_quant(act_fn, x, linear)
|
||||
|
||||
assert isinstance(result, torch.Tensor)
|
||||
torch.testing.assert_close(result, act_fn(x))
|
||||
|
||||
@@ -17,6 +17,8 @@ from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
MhcFusedTileLangKernel,
|
||||
MhcPostTileLangKernel,
|
||||
MhcPreBigFuseTileLangKernel,
|
||||
mhc_fused_post_pre_split_config,
|
||||
require_fused_post_pre_config,
|
||||
)
|
||||
from vllm.model_executor.warmup import jit_warmup_tilelang_helper
|
||||
|
||||
@@ -113,17 +115,32 @@ def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"),
|
||||
[(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)],
|
||||
("num_tokens", "hidden_size", "expected_tile_n", "expected_n_splits"),
|
||||
[
|
||||
(1, 4096, 2, 8),
|
||||
(4, 8192, 2, 8),
|
||||
(8, 5120, 2, 8),
|
||||
(16, 5120, 6, 8),
|
||||
(32, 7168, 6, 8),
|
||||
],
|
||||
)
|
||||
def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
def test_mhc_fused_dispatch_matches_the_launch_config(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
expected_n_splits: int,
|
||||
expected_tile_n: int,
|
||||
expected_n_splits: int,
|
||||
) -> None:
|
||||
kernel = MhcFusedTileLangKernel()
|
||||
"""The compile key must be the config the launch will actually use.
|
||||
|
||||
Both read mhc_fused_post_pre_split_config, so this pins the tuned bands
|
||||
and guards against the two drifting apart again.
|
||||
"""
|
||||
kernel = MhcFusedTileLangKernel()
|
||||
tile_n, n_splits, n_thr = require_fused_post_pre_config(
|
||||
num_tokens, hidden_size, hc_mult=4
|
||||
)
|
||||
|
||||
assert (tile_n, n_splits) == (expected_tile_n, expected_n_splits)
|
||||
assert kernel.dispatch(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=hidden_size,
|
||||
@@ -131,11 +148,20 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
) == kernel.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=4,
|
||||
n_splits=expected_n_splits,
|
||||
tile_n=expected_tile_n,
|
||||
n_splits=n_splits,
|
||||
tile_n=tile_n,
|
||||
n_thr=n_thr,
|
||||
)
|
||||
|
||||
|
||||
def test_mhc_fused_declines_shapes_it_cannot_tile() -> None:
|
||||
"""Above the token cutoff, and for a hidden size the block cannot split."""
|
||||
assert mhc_fused_post_pre_split_config(33, 5120, 4) is None
|
||||
assert mhc_fused_post_pre_split_config(1, 5137, 4) is None
|
||||
with pytest.raises(ValueError, match="does not cover num_tokens"):
|
||||
require_fused_post_pre_config(33, 5120, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel", "compile_key"),
|
||||
[
|
||||
@@ -235,8 +261,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=4,
|
||||
tile_n=3,
|
||||
n_splits=8,
|
||||
tile_n=6,
|
||||
n_thr=128,
|
||||
),
|
||||
),
|
||||
(
|
||||
@@ -246,6 +273,7 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
hc_mult=4,
|
||||
n_splits=8,
|
||||
tile_n=2,
|
||||
n_thr=128,
|
||||
),
|
||||
),
|
||||
(
|
||||
@@ -253,8 +281,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=8192,
|
||||
hc_mult=4,
|
||||
n_splits=4,
|
||||
n_splits=8,
|
||||
tile_n=2,
|
||||
n_thr=128,
|
||||
),
|
||||
),
|
||||
(
|
||||
|
||||
@@ -11,12 +11,17 @@ import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.layers.mhc as mhc_layers
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
_torch_hc_prenorm_gemm,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_delayed_tilelang,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
mhc_fused_post_pre_split_config,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.torch import (
|
||||
mhc_post_torch,
|
||||
mhc_pre_delayed_torch,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch
|
||||
from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
@@ -32,9 +37,13 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4Model,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.nvidia.model import (
|
||||
from vllm.models.deepseek_v41.nvidia.model import (
|
||||
DeepseekV4DecoderLayer as DeepseekV41DecoderLayer,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.ops.mega_mhc import (
|
||||
is_mega_mhc_supported,
|
||||
mhc_shifted_post_pre_deep_gemm,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -104,7 +113,7 @@ def test_hc_collapse_custom_op_supports_compile():
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7])
|
||||
def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
"""Match reference DSparkBlock.forward_head's hc_pre before its RMSNorm."""
|
||||
from vllm.models.deepseek_v4_1.nvidia import dspark
|
||||
from vllm.models.deepseek_v41.nvidia import dspark
|
||||
|
||||
set_random_seed(0)
|
||||
hidden_size, hc_mult = 5120, 4
|
||||
@@ -115,9 +124,9 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
carried_mixes = []
|
||||
|
||||
def make_layer(mix):
|
||||
def forward(hidden, positions, ids, pre, post, res, residual):
|
||||
def forward(hidden, positions, ids, pre, post, res, residual, **kwargs):
|
||||
carried_mixes.append(pre)
|
||||
return hidden, None, None, None, mix
|
||||
return hidden, None, None, None, mix, None
|
||||
|
||||
return forward
|
||||
|
||||
@@ -143,6 +152,27 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32])
|
||||
@pytest.mark.parametrize("tp_size", [4, 8])
|
||||
def test_hc_collapse_before_gather_is_bitwise_equal(num_tokens, tp_size):
|
||||
"""Simulate SP on one GPU, including padding and ranks without valid tokens."""
|
||||
set_random_seed(0)
|
||||
padded_tokens = num_tokens + (-num_tokens) % tp_size
|
||||
streams = torch.randn(padded_tokens, 4, 5120, dtype=torch.bfloat16, device=DEVICE)
|
||||
pre_mix = torch.rand(padded_tokens, 4, device=DEVICE)
|
||||
stream_shards = streams.chunk(tp_size)
|
||||
mix_shards = pre_mix.chunk(tp_size)
|
||||
before = hc_collapse_triton(
|
||||
torch.cat(stream_shards)[:num_tokens], torch.cat(mix_shards)[:num_tokens]
|
||||
)
|
||||
after = torch.cat(
|
||||
[hc_collapse_triton(x, pre) for x, pre in zip(stream_shards, mix_shards)]
|
||||
)[:num_tokens]
|
||||
assert after.shape == (num_tokens, 5120)
|
||||
torch.testing.assert_close(after, before, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,sinkhorn_repeat,norm_eps",
|
||||
@@ -235,6 +265,171 @@ def test_deepseek_v41_mhc_pre_delayed(
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_ref(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
*mix_args,
|
||||
pre_mix: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Project the post result before it is rounded to BF16.
|
||||
|
||||
The fused kernel keeps the updated residual streams in registers, so its
|
||||
projection sees the FP32 post result while the streams it stores are the
|
||||
BF16 rounding of it.
|
||||
"""
|
||||
post = torch.einsum("tij,tih->tjh", comb_res_mix, residual.float())
|
||||
post = post + post_layer_mix * x.float().unsqueeze(-2)
|
||||
return mhc_pre_delayed_torch(
|
||||
post.bfloat16(), *mix_args, pre_mix=pre_mix, x=post.flatten(1)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [4096, 7168])
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carried):
|
||||
"""Fold the post block into the delayed pre without changing its outputs."""
|
||||
set_random_seed(0)
|
||||
hc_mult = 4
|
||||
mix_size = hc_mult * (hc_mult + 2)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
|
||||
residual = torch.randn(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
post_layer_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
|
||||
comb_res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
|
||||
fn = torch.randn(mix_size, hc_mult * hidden_size, device=DEVICE) * 0.02
|
||||
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
|
||||
base = torch.randn(mix_size, device=DEVICE)
|
||||
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE) if carried else None
|
||||
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE)
|
||||
weight.uniform_(0.5, 1.5)
|
||||
mix_args = (fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
|
||||
|
||||
residual_cur, post, comb, layer_input, next_pre, aux = (
|
||||
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
|
||||
)
|
||||
)
|
||||
assert aux.shape == (0, hidden_size)
|
||||
assert [
|
||||
residual_cur.shape,
|
||||
post.shape,
|
||||
comb.shape,
|
||||
layer_input.shape,
|
||||
next_pre.shape,
|
||||
] == [
|
||||
(num_tokens, hc_mult, hidden_size),
|
||||
(num_tokens, hc_mult, 1),
|
||||
(num_tokens, hc_mult, hc_mult),
|
||||
(num_tokens, hidden_size),
|
||||
(num_tokens, hc_mult),
|
||||
]
|
||||
if num_tokens == 0:
|
||||
return
|
||||
|
||||
# The post mapping and the collapse are what the unfused kernels produce.
|
||||
residual_ref = torch.ops.vllm.mhc_post_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
layer_input_ref = mhc_pre_delayed_tilelang(
|
||||
residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
|
||||
)[2]
|
||||
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0)
|
||||
|
||||
post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix=pre_mix
|
||||
)
|
||||
# Above the fused kernel's token count the projection falls back to a
|
||||
# split-k TF32 GEMM over the stored BF16 streams.
|
||||
tol = (
|
||||
1e-5
|
||||
if mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) is not None
|
||||
else 5e-3
|
||||
)
|
||||
for actual, expected in (
|
||||
(post, post_ref),
|
||||
(comb, comb_ref),
|
||||
(next_pre, next_pre_ref),
|
||||
):
|
||||
torch.testing.assert_close(actual, expected, atol=tol, rtol=tol)
|
||||
|
||||
# Folding the stream mean into the collapse must not change it, or anything
|
||||
# else the op returns.
|
||||
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
*mix_args,
|
||||
pre_mix,
|
||||
weight,
|
||||
1e-6,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(captured[5], residual_cur.mean(dim=1), atol=0, rtol=0)
|
||||
unchanged = (residual_cur, post, comb, layer_input, next_pre)
|
||||
for with_aux, without in zip(captured[:5], unchanged, strict=True):
|
||||
torch.testing.assert_close(with_aux, without, atol=0, rtol=0)
|
||||
|
||||
# The unnormalized epilogue has its own fused aux path.
|
||||
bare = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, None, 1e-6, True
|
||||
)
|
||||
torch.testing.assert_close(bare[5], bare[0].mean(dim=1), atol=0, rtol=0)
|
||||
|
||||
# Decode replays this op from a captured graph.
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
|
||||
)
|
||||
graph.replay()
|
||||
eager = (residual_cur, post, comb, layer_input, next_pre, aux)
|
||||
for replayed, expected in zip(captured, eager, strict=True):
|
||||
torch.testing.assert_close(replayed, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
@pytest.mark.parametrize("capture_aux", [False, True])
|
||||
def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_aux):
|
||||
set_random_seed(0)
|
||||
x = torch.randn(2, 5120, dtype=torch.bfloat16, device=DEVICE)
|
||||
residual = torch.randn(2, 4, 5120, dtype=torch.bfloat16, device=DEVICE)
|
||||
post_layer_mix = torch.rand(2, 4, 1, device=DEVICE)
|
||||
comb_res_mix = torch.rand(2, 4, 4, device=DEVICE)
|
||||
fn = torch.randn(24, 20480, device=DEVICE) * 0.02
|
||||
pre_mix = torch.rand(2, 4, device=DEVICE) if carried else None
|
||||
scale = torch.ones(3, device=DEVICE)
|
||||
base = torch.zeros(24, device=DEVICE)
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang.default,
|
||||
(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
1e-20,
|
||||
1e-6,
|
||||
1e-6,
|
||||
2.0,
|
||||
20,
|
||||
pre_mix,
|
||||
None,
|
||||
1e-6,
|
||||
capture_aux,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
|
||||
def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
@@ -295,15 +490,86 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
|
||||
return post, res, decoder.attn_norm(collapsed), pre
|
||||
|
||||
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
|
||||
return residual, *reference(residual, *args, **kw), aux
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v4_1.nvidia.model.mhc_pre_delayed_tilelang",
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
|
||||
reference,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre",
|
||||
fused_reference,
|
||||
)
|
||||
expected = decoder(x, positions, None, **kwargs)
|
||||
for result, ref in zip(actual, expected, strict=True):
|
||||
assert actual[-1] is None and expected[-1] is None
|
||||
for result, ref in zip(actual[:-1], expected[:-1], strict=True):
|
||||
torch.testing.assert_close(result, ref, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["fused", "engram"])
|
||||
def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_config):
|
||||
"""Read the aux hidden state back out of the seam that already computed it.
|
||||
|
||||
The draft model's input is the mean over hc streams of the previous
|
||||
layer's post. That post now runs inside this layer's fused pre, so the
|
||||
captured value must equal a standalone ``mhc_post_tilelang`` on the same
|
||||
inputs -- and Engram must not be folded into it, since the injection
|
||||
happens after the aux consumers read the stream.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
|
||||
nn.Module.__init__(decoder)
|
||||
decoder.hc_mult = 4
|
||||
decoder.hc_sinkhorn_iters = 20
|
||||
decoder.hc_eps = 1e-6
|
||||
decoder.rms_norm_eps = 1e-6
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
decoder.attn_norm = decoder.ffn_norm = RMSNorm(5120, 1e-6).to(
|
||||
device=DEVICE, dtype=torch.bfloat16
|
||||
)
|
||||
decoder.attn = lambda positions, x, _: x * 0.5
|
||||
decoder.ffn = lambda x, input_ids: x * 0.25
|
||||
with torch.device(DEVICE):
|
||||
decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02
|
||||
decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02
|
||||
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
|
||||
decoder.hc_attn_base = torch.randn(24)
|
||||
decoder.hc_ffn_base = torch.randn(24)
|
||||
x = torch.randn(3, 5120, dtype=torch.bfloat16)
|
||||
kwargs = dict(
|
||||
pre_mix=torch.rand(3, 4),
|
||||
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.rand(3, 4, 1),
|
||||
res_mix=torch.rand(3, 4, 4),
|
||||
)
|
||||
if entry == "engram":
|
||||
|
||||
class FakeEngram(nn.Module):
|
||||
layer_hash_index = 0
|
||||
|
||||
def forward(self, residual, hashes, mask):
|
||||
return residual + 0.125
|
||||
|
||||
decoder.engram = FakeEngram()
|
||||
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
|
||||
positions = torch.arange(3)
|
||||
|
||||
expected = torch.ops.vllm.mhc_post_tilelang(
|
||||
x, kwargs["residual"], kwargs["post_mix"], kwargs["res_mix"]
|
||||
).mean(dim=1)
|
||||
previous_aux = decoder(x, positions, None, **kwargs, capture_previous_aux=True)[-1]
|
||||
torch.testing.assert_close(previous_aux, expected, atol=0, rtol=0)
|
||||
assert decoder(x, positions, None, **kwargs)[-1] is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_mhc_pre_delayed_custom_op_supports_compile(carried):
|
||||
@@ -319,6 +585,104 @@ def test_mhc_pre_delayed_custom_op_supports_compile(carried):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="DeepGEMM Mega mHC requires SM100-family CUDA",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 17, 128, 1024])
|
||||
@pytest.mark.parametrize("hidden_size", [5120, 7168])
|
||||
def test_deep_gemm_mega_mhc_correctness(num_tokens, hidden_size):
|
||||
set_random_seed(0)
|
||||
hc_mult = 4
|
||||
if not is_mega_mhc_supported(hidden_size, 4):
|
||||
pytest.skip("DeepGEMM Mega mHC is not available")
|
||||
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
|
||||
residual = torch.randn(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
previous_mix = torch.rand(num_tokens, hc_mult, device=DEVICE)
|
||||
post_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
|
||||
res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
|
||||
fn = torch.randn(24, hc_mult * hidden_size, device=DEVICE) * 0.01
|
||||
scale = torch.randn(3, device=DEVICE) * 0.1
|
||||
base = torch.randn(24, device=DEVICE) * 0.1
|
||||
norm_weight = torch.empty(
|
||||
hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
).uniform_(0.9, 1.1)
|
||||
|
||||
expected_residual = (
|
||||
mhc_post_tilelang(x, residual, post_mix, res_mix) if num_tokens else residual
|
||||
)
|
||||
(
|
||||
expected_post_mix,
|
||||
expected_res_mix,
|
||||
expected_y_bf16,
|
||||
expected_previous_mix,
|
||||
) = mhc_pre_delayed_tilelang(
|
||||
expected_residual,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
2e-5,
|
||||
3e-4,
|
||||
2e-6,
|
||||
1.25,
|
||||
10,
|
||||
pre_mix=previous_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=7e-6,
|
||||
)
|
||||
args = (
|
||||
x,
|
||||
residual,
|
||||
previous_mix,
|
||||
post_mix,
|
||||
res_mix,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
2e-5,
|
||||
3e-4,
|
||||
1.25,
|
||||
2e-6,
|
||||
10,
|
||||
norm_weight,
|
||||
7e-6,
|
||||
)
|
||||
actual = mhc_shifted_post_pre_deep_gemm(*args)
|
||||
expected = (
|
||||
expected_residual,
|
||||
expected_post_mix,
|
||||
expected_res_mix,
|
||||
expected_y_bf16,
|
||||
expected_previous_mix,
|
||||
)
|
||||
for i, (result, ref) in enumerate(zip(actual, expected, strict=True)):
|
||||
# Match the existing mHC BF16 tolerance: post rounding differences
|
||||
# can be amplified by cancellation in the carried collapse.
|
||||
atol, rtol = (1.6e-2, 1e-2) if i in (0, 3) else (1e-6, 1e-3)
|
||||
torch.testing.assert_close(result, ref, atol=atol, rtol=rtol)
|
||||
|
||||
if num_tokens:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
# DeepGEMM initializes barriers per stream before capture.
|
||||
mhc_shifted_post_pre_deep_gemm(*args)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
captured = mhc_shifted_post_pre_deep_gemm(*args)
|
||||
for multiplier in (0.5, -1.0):
|
||||
x.mul_(multiplier)
|
||||
graph.replay()
|
||||
eager = mhc_shifted_post_pre_deep_gemm(*args)
|
||||
for result, ref in zip(captured, eager, strict=True):
|
||||
torch.testing.assert_close(result, ref, atol=0, rtol=0)
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
|
||||
def sinkhorn_normalize_ref(x: torch.Tensor, repeat: int, eps: float) -> torch.Tensor:
|
||||
x = x.softmax(-1) + eps
|
||||
x = x / (x.sum(-2, keepdim=True) + eps)
|
||||
@@ -743,8 +1107,12 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
|
||||
)
|
||||
|
||||
expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix)
|
||||
actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix)
|
||||
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
|
||||
*args, pre_mix=pre_mix
|
||||
)
|
||||
|
||||
# No post was requested, so the residual comes straight back.
|
||||
assert residual_out is residual
|
||||
for i in (0, 1, 3):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
# The collapse is the same FP32 multiply-and-sum in both paths.
|
||||
@@ -786,7 +1154,12 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
|
||||
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x)
|
||||
actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x)
|
||||
# forward_hip leads with the residual; no post was requested here, so it is
|
||||
# the caller's own tensor and the reference's four outputs follow it.
|
||||
residual_out, *actual = op.forward_hip(
|
||||
broadcast_residual, broadcast_fn, *args[2:], x=x
|
||||
)
|
||||
assert residual_out is broadcast_residual
|
||||
assert not took_aiter, "the broadcast seam must not reach AITER"
|
||||
for i in range(4):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
@@ -1028,3 +1401,68 @@ def test_deepseek_v4_mhc_broadcast_refit_refreshes_in_place(monkeypatch):
|
||||
assert layer.hc_attn_fn_broadcast is buffer
|
||||
expected = layer.hc_attn_fn.detach().view(-1, 2, 8).sum(dim=1)
|
||||
assert torch.equal(layer.hc_attn_fn_broadcast, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128])
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_mhc_fused_post_pre_delayed_rocm_aiter(
|
||||
num_tokens, carried, default_vllm_config
|
||||
):
|
||||
"""Folding the post into the pre projection must match doing them apart.
|
||||
|
||||
The residual is bfloat16, so the two accumulation orders are allowed to
|
||||
land a rounding step apart; the tolerance here is one bfloat16 ULP.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs(
|
||||
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
|
||||
)
|
||||
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16, device=DEVICE)
|
||||
post_layer_mix = (
|
||||
torch.rand(num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
)
|
||||
comb_res_mix = (
|
||||
torch.rand(num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=DEVICE)
|
||||
+ 0.2
|
||||
)
|
||||
pre_mix = (
|
||||
torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
if carried
|
||||
else None
|
||||
)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6
|
||||
pre_args = (fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, 1.0, 20)
|
||||
|
||||
next_residual = mhc_post_torch(x, residual, post_layer_mix, comb_res_mix)
|
||||
expected = (
|
||||
next_residual,
|
||||
*mhc_pre_delayed_torch(next_residual, *pre_args, pre_mix=pre_mix),
|
||||
)
|
||||
actual = MHCPreDelayedOp().forward_hip(
|
||||
residual,
|
||||
*pre_args,
|
||||
pre_mix=pre_mix,
|
||||
sublayer_out=x,
|
||||
post_layer_mix=post_layer_mix,
|
||||
comb_res_mix=comb_res_mix,
|
||||
)
|
||||
|
||||
for got, want in zip(actual, expected, strict=True):
|
||||
torch.testing.assert_close(got, want.reshape(got.shape), atol=2e-2, rtol=8e-3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
def test_mhc_fused_post_pre_delayed_falls_back_for_large_batches():
|
||||
"""AITER's heuristic hands large batches back to the unfused path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
assert not rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1)
|
||||
assert rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1 << 20)
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa:
|
||||
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
|
||||
gemma_fused_add_rmsnorm,
|
||||
gemma_rmsnorm,
|
||||
swiglu_oai_quantize_mxfp8,
|
||||
swiglu_oai_split,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
|
||||
@@ -167,6 +168,25 @@ def test_swiglu_oai_split(m, inter, limit, dtype):
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_swiglu_oai_quantize_mxfp8_uses_e4m3_range_for_scale():
|
||||
# Keep a small value in two MX blocks next to their maxima. The scale must
|
||||
# use E4M3's finite range (448), otherwise that value underflows to zero.
|
||||
# Keep the third block empty to cover the reference's finite tiny clamp.
|
||||
m, inter = 3, 96
|
||||
gate = torch.full((m, inter), 2.0, device=DEVICE, dtype=torch.float16)
|
||||
up = torch.full((m, inter), 2**-10, device=DEVICE, dtype=torch.float16)
|
||||
up[:, :64:32] = 1.0
|
||||
up[:, 64:] = 0.0
|
||||
gate_up = torch.cat((gate, up), dim=-1)
|
||||
|
||||
got_q, got_s = swiglu_oai_quantize_mxfp8(gate_up, alpha=0.0, beta=0.0, limit=None)
|
||||
ref_q, ref_s = _mxfp8_e4m3_quantize_torch(up, is_sf_swizzled_layout=False)
|
||||
|
||||
assert torch.equal(got_s, ref_s)
|
||||
assert torch.equal(got_q, ref_q)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fused MXFP8 activation quant (Triton vs torch reference)
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -179,8 +199,8 @@ def test_mxfp8_quant_triton_matches_torch(shape, dtype):
|
||||
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
|
||||
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
|
||||
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
|
||||
# E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
|
||||
# most a 1-step difference at exact powers of two.
|
||||
# Both paths use the E4M3-aware scale calculation; allow a 1-step
|
||||
# difference at exact powers of two due to floating-point rounding.
|
||||
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
|
||||
# Dequantized values agree to fp8 granularity.
|
||||
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
|
||||
@@ -424,6 +444,69 @@ def test_mxfp8_linear_emulation_bf16_at_load(
|
||||
assert _relerr(out.float(), out_ref.float()) < 2e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 linear: the K % 128 != 0 weight is dequantized once at load
|
||||
# --------------------------------------------------------------------------- #
|
||||
@requires_gfx950
|
||||
# (5120, 576) and (5120, 288): DSv4.1 shared-expert down_proj at TP4 and TP8.
|
||||
@pytest.mark.parametrize("shape", [(5120, 576), (5120, 288), (256, 2048)])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape):
|
||||
"""``dot_scaled`` tiles K by 128, so an unaligned weight must not reach it.
|
||||
|
||||
``process_weights_after_loading`` converts those to BF16 once; a weight that
|
||||
never went through it has to fall back per step rather than take the
|
||||
invalid path, which is what keying the dispatch on the weight dtype alone
|
||||
would do.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import rocm_native
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
|
||||
N, K = shape
|
||||
aligned = K % 128 == 0
|
||||
torch.manual_seed(0)
|
||||
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
x = torch.randn(7, K, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
out_ref = torch.nn.functional.linear(x, dequant_mxfp8_to_bf16(w_fp8, w_scale))
|
||||
# dot_scaled re-quantizes the activation, the dequant paths do not.
|
||||
tol = 5e-2 if aligned else 2e-3
|
||||
|
||||
def make_layer():
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
|
||||
return layer
|
||||
|
||||
kernel = rocm_native.RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig())
|
||||
layer = make_layer()
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert (layer.weight.element_size() >= 2) is not aligned
|
||||
assert _relerr(kernel.apply_weights(layer, x).float(), out_ref.float()) < tol
|
||||
|
||||
# An unaligned weight that is still FP8 must not reach dot_scaled.
|
||||
unprocessed = make_layer()
|
||||
unprocessed.weight_scale = torch.nn.Parameter(
|
||||
w_scale[:N, : K // 32].contiguous(), requires_grad=False
|
||||
)
|
||||
calls = []
|
||||
real = rocm_native._mxfp8_dot_scaled_linear
|
||||
|
||||
def _spy(*a, **k):
|
||||
calls.append(1)
|
||||
return real(*a, **k)
|
||||
|
||||
rocm_native._mxfp8_dot_scaled_linear = _spy
|
||||
try:
|
||||
out = kernel.apply_weights(unprocessed, x)
|
||||
finally:
|
||||
rocm_native._mxfp8_dot_scaled_linear = real
|
||||
assert bool(calls) is aligned
|
||||
assert _relerr(out.float(), out_ref.float()) < tol
|
||||
|
||||
|
||||
# ── EP expert_mask handling for the FlyDSL (AITER_MXFP8) MoE ────────────────
|
||||
# The map/mask choice lives in ``RoutedExperts.expert_map``: it hands AITER
|
||||
# experts (``consumes_expert_mask``) the precomputed 0/1 ``expert_mask`` and
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.vllm import OptimizationLevel
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
|
||||
reason="This test requires SM90 or newer",
|
||||
)
|
||||
|
||||
|
||||
class _StaticFp8Linear(torch.nn.Module):
|
||||
def __init__(self, input_scale: torch.Tensor) -> None:
|
||||
super().__init__()
|
||||
self._input_quant_key = kFp8StaticTensorSym
|
||||
self.input_scale = input_scale
|
||||
|
||||
|
||||
def _assert_fp8_bitwise_equal(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
assert actual.dtype == expected.dtype == current_platform.fp8_dtype()
|
||||
torch.testing.assert_close(
|
||||
actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
def _assert_quantized_activation(
|
||||
result: QuantizedActivation, x: torch.Tensor, scale: torch.Tensor
|
||||
) -> None:
|
||||
assert result.scale is scale
|
||||
assert result.orig_dtype == x.dtype
|
||||
assert result.orig_shape == x.shape
|
||||
assert result.quant_key == kFp8StaticTensorSym
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def o2_relu2_fp8_ops():
|
||||
config = VllmConfig(
|
||||
optimization_level=OptimizationLevel.O2,
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
yield (
|
||||
ReLUSquaredActivation(),
|
||||
QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR),
|
||||
)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize("shape", [(1, 1), (1, 16), (17, 5120), (128, 10240)])
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant(o2_relu2_fp8_ops, shape) -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.1495535671710968, device="cuda", dtype=torch.float32)
|
||||
|
||||
relu2, quant_fp8 = o2_relu2_fp8_ops
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize(
|
||||
"enabled_ops",
|
||||
[(), ("relu2",), ("quant_fp8",), ("relu2", "quant_fp8")],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_ignores_custom_op_dispatch(enabled_ops) -> None:
|
||||
custom_ops = ["none", *(f"+{op}" for op in enabled_ops)]
|
||||
config = VllmConfig(
|
||||
optimization_level=OptimizationLevel.O2,
|
||||
compilation_config=CompilationConfig(custom_ops=custom_ops),
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
torch.manual_seed(0)
|
||||
relu2 = ReLUSquaredActivation()
|
||||
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_without_compilation() -> None:
|
||||
config = VllmConfig(optimization_level=OptimizationLevel.O0)
|
||||
with set_current_vllm_config(config):
|
||||
torch.manual_seed(0)
|
||||
relu2 = ReLUSquaredActivation()
|
||||
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize(
|
||||
"scale_value",
|
||||
[
|
||||
0.00435965,
|
||||
0.007149832788854837,
|
||||
0.03125,
|
||||
0.0714285746216774,
|
||||
0.1495535671710968,
|
||||
0.1517857164144516,
|
||||
6.03571415,
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_all_bf16_values(o2_relu2_fp8_ops, scale_value) -> None:
|
||||
bits = torch.arange(1 << 16, dtype=torch.int32).to(torch.uint16)
|
||||
x = bits.view(torch.bfloat16).reshape(1, -1).to("cuda")
|
||||
scale = torch.tensor(scale_value, device="cuda", dtype=torch.float32)
|
||||
|
||||
relu2, quant_fp8 = o2_relu2_fp8_ops
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_empty(o2_relu2_fp8_ops) -> None:
|
||||
x = torch.empty(0, device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
|
||||
relu2, _ = o2_relu2_fp8_ops
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
assert result.data.shape == x.shape
|
||||
assert result.data.dtype == current_platform.fp8_dtype()
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize("unsupported", ["dtype", "layout", "scale"])
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_falls_back(o2_relu2_fp8_ops, unsupported: str) -> None:
|
||||
x = torch.randn((17, 32), device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
if unsupported == "dtype":
|
||||
x = x.to(torch.float16)
|
||||
elif unsupported == "layout":
|
||||
x = x.T
|
||||
else:
|
||||
scale = scale.to(torch.float16)
|
||||
|
||||
relu2, _ = o2_relu2_fp8_ops
|
||||
expected = relu2(x)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, torch.Tensor)
|
||||
assert not isinstance(result, QuantizedActivation)
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_torch_compile_dynamic(
|
||||
o2_relu2_fp8_ops,
|
||||
) -> None:
|
||||
torch.manual_seed(1)
|
||||
scale = torch.tensor(0.00435965, device="cuda", dtype=torch.float32)
|
||||
relu2, quant_fp8 = o2_relu2_fp8_ops
|
||||
linear = _StaticFp8Linear(scale)
|
||||
|
||||
def fused(x: torch.Tensor) -> torch.Tensor:
|
||||
result = maybe_fused_act_quant(relu2, x, linear)
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
return result.data
|
||||
|
||||
compile_count = 0
|
||||
|
||||
def counting_inductor_backend(gm, example_inputs):
|
||||
nonlocal compile_count
|
||||
compile_count += 1
|
||||
return torch._inductor.compile(gm, example_inputs)
|
||||
|
||||
torch._dynamo.reset()
|
||||
try:
|
||||
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
compiled_fused = torch.compile(
|
||||
fused, backend=counting_inductor_backend, fullgraph=True
|
||||
)
|
||||
|
||||
for candidate in (
|
||||
x,
|
||||
torch.randn((31, 5120), device="cuda", dtype=torch.bfloat16),
|
||||
torch.randn((63, 5120), device="cuda", dtype=torch.bfloat16),
|
||||
):
|
||||
expected, _ = quant_fp8(relu2(candidate), scale)
|
||||
_assert_fp8_bitwise_equal(compiled_fused(candidate), expected)
|
||||
|
||||
assert compile_count == 1
|
||||
finally:
|
||||
torch._dynamo.reset()
|
||||
@@ -22,11 +22,22 @@ def _has_device_capability(major: int) -> bool:
|
||||
return current_platform.is_cuda() and current_platform.has_device_capability(major)
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
# DeepSelect is compiled for sm_100a/sm_103a only.
|
||||
requires_sm100 = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="DeepSelect requires SM100a/SM103a",
|
||||
)
|
||||
requires_gfx950 = pytest.mark.skipif(
|
||||
not _on_gfx950(), reason="This test exercises the gfx950 launch configuration"
|
||||
)
|
||||
|
||||
|
||||
COOPERATIVE_TOPK_BACKEND = pytest.param(
|
||||
@@ -487,6 +498,150 @@ def test_top_k_per_row_decode_gfx950_long_c4a_1d_seq_lens() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _assert_exact_topk(
|
||||
logits: torch.Tensor, indices: torch.Tensor, row_ends: torch.Tensor
|
||||
) -> None:
|
||||
ends = row_ends.reshape(-1, 1).clamp_min(0)
|
||||
valid = (indices >= 0) & (indices < ends)
|
||||
assert torch.equal(valid.sum(1), ends.clamp_max(indices.shape[1]).flatten())
|
||||
assert torch.all(valid | (indices == -1))
|
||||
ordered = indices.sort(1).values
|
||||
assert not ((ordered[:, 1:] == ordered[:, :-1]) & (ordered[:, 1:] >= 0)).any()
|
||||
|
||||
width = int(ends.max())
|
||||
visible = logits[:, :width].clone()
|
||||
columns = torch.arange(width, device=logits.device)
|
||||
visible.masked_fill_(columns[None, :] >= ends, float("-inf"))
|
||||
expected = visible.topk(min(indices.shape[1], width), dim=1).values
|
||||
selected = logits.gather(1, indices.clamp_min(0).long())
|
||||
selected.masked_fill_(~valid, float("-inf"))
|
||||
selected = selected.sort(1, descending=True).values
|
||||
assert torch.equal(selected[:, : expected.shape[1]], expected)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_top_k_per_row_decode_gfx950_k512_1d_seq_lens() -> None:
|
||||
"""Honor causal offsets and exact compressed lengths, including short rows."""
|
||||
next_n, width, top_k = 6, 16_384, 512
|
||||
lengths = torch.tensor(
|
||||
[0, 1, 511, 512, 513, 1023, 1024, 16_000],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
offsets = torch.arange(next_n, dtype=torch.int32, device="cuda")
|
||||
row_ends = (lengths[:, None] - next_n + offsets + 1).clamp_min(0).flatten()
|
||||
rows = row_ends.numel()
|
||||
storage = torch.full((rows, width + 8), 1e20, dtype=torch.float32, device="cuda")
|
||||
logits = storage[:, 1 : width + 1]
|
||||
logits[:, :16_000] = torch.arange(16_000, dtype=torch.float32, device="cuda")
|
||||
indices = torch.full((rows, top_k), -777, dtype=torch.int32, device="cuda")
|
||||
_run_topk_backend(
|
||||
"top_k_per_row_decode", logits, lengths, indices, top_k, width, next_n
|
||||
)
|
||||
_assert_exact_topk(logits, indices, row_ends)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rows", "row_length"),
|
||||
[
|
||||
pytest.param(128, 65_536, id="single-before-64k"),
|
||||
pytest.param(128, 65_537, id="four-splits-after-64k"),
|
||||
pytest.param(129, 262_144, id="single-before-three-split-length"),
|
||||
pytest.param(129, 262_145, id="three-splits-after-256k"),
|
||||
pytest.param(160, 262_145, id="three-splits-upper-row-bound"),
|
||||
pytest.param(161, 262_144, id="single-before-two-split-length"),
|
||||
pytest.param(161, 262_145, id="two-splits-after-256k"),
|
||||
pytest.param(256, 262_145, id="two-splits-upper-row-bound"),
|
||||
pytest.param(257, 262_145, id="large-row-single-block"),
|
||||
pytest.param(384, 262_145, id="optimized-row-limit"),
|
||||
pytest.param(385, 262_145, id="native-fallback"),
|
||||
],
|
||||
)
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_top_k_per_row_decode_gfx950_k512_split_policy_boundaries(
|
||||
rows: int, row_length: int
|
||||
) -> None:
|
||||
width, top_k = 262_145, 512
|
||||
row = torch.arange(width, dtype=torch.float32, device="cuda")
|
||||
logits = row.expand(rows, -1)
|
||||
lengths = torch.full((rows,), row_length, dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((rows, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
_run_topk_backend("top_k_per_row_decode", logits, lengths, indices, top_k, width)
|
||||
|
||||
expected = torch.arange(
|
||||
row_length - top_k, row_length, dtype=torch.int32, device="cuda"
|
||||
).expand_as(indices)
|
||||
assert torch.equal(indices.sort(1).values, expected)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_top_k_per_row_decode_gfx950_k512_masked_graph_replay() -> None:
|
||||
"""Preserve visible top-k scores when masks and lengths change on replay."""
|
||||
rows, next_n, width, top_k = 24, 6, 262_145, 512
|
||||
logits = torch.full((rows, width), 1e20, dtype=torch.float32, device="cuda")
|
||||
lengths = torch.full(
|
||||
(rows // next_n, next_n), 70_000, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
indices = torch.full((rows, top_k), -777, dtype=torch.int32, device="cuda")
|
||||
logits[:, :70_000] = float("-inf")
|
||||
bin_sizes = [128, 129, 1024, 1025]
|
||||
for row in range(rows):
|
||||
pattern = row % 9
|
||||
if pattern < 4:
|
||||
count = [0, 1, 511, 513][pattern]
|
||||
logits[row, :count] = torch.arange(
|
||||
count, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
elif pattern == 4:
|
||||
logits[row, :511] = 0
|
||||
logits[row, 511] = torch.finfo(torch.float32).min
|
||||
else:
|
||||
count = bin_sizes[pattern - 5]
|
||||
higher = max(0, top_k - count // 2)
|
||||
logits[row, :higher] = 2
|
||||
logits[row, higher : higher + count] = 1
|
||||
logits[row, higher] = 1 + 2**-23
|
||||
|
||||
def run() -> None:
|
||||
_run_topk_backend(
|
||||
"top_k_per_row_decode", logits, lengths, indices, top_k, width, next_n
|
||||
)
|
||||
|
||||
run()
|
||||
_assert_exact_topk(logits, indices, lengths)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
indices.fill_(-777)
|
||||
graph.replay()
|
||||
_assert_exact_topk(logits, indices, lengths)
|
||||
|
||||
bounds = [
|
||||
0,
|
||||
511,
|
||||
512,
|
||||
513,
|
||||
16_384,
|
||||
16_385,
|
||||
131_072,
|
||||
131_073,
|
||||
262_144,
|
||||
262_145,
|
||||
]
|
||||
row_bounds = torch.tensor(bounds, dtype=torch.int32, device="cuda")
|
||||
repeats = (rows + len(bounds) - 1) // len(bounds)
|
||||
lengths.flatten().copy_(row_bounds.repeat(repeats)[:rows])
|
||||
logits[:] = torch.arange(width, dtype=torch.float32, device="cuda")
|
||||
indices.fill_(-777)
|
||||
graph.replay()
|
||||
_assert_exact_topk(logits, indices, lengths)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
@@ -620,6 +775,31 @@ def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
compress_ratio=4,
|
||||
num_rows=257,
|
||||
max_valid_seq_len=250_000,
|
||||
num_columns=524_288,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=385,
|
||||
max_valid_seq_len=250_000,
|
||||
num_columns=524_288,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=384,
|
||||
max_valid_seq_len=50_000,
|
||||
num_columns=1_048_577,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
@@ -638,6 +818,15 @@ def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
)
|
||||
ineligible_cases = [
|
||||
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=384,
|
||||
max_valid_seq_len=1_048_576,
|
||||
num_columns=1_048_576,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
@@ -743,6 +932,106 @@ def test_deepseek_workspace_topk(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _has_device_capability(80), reason="Requires SM80+")
|
||||
@pytest.mark.parametrize("rows", [65, 128])
|
||||
@pytest.mark.parametrize("top_k", [512, 1024, 2048])
|
||||
@pytest.mark.parametrize(
|
||||
"distribution", ["random", "10LSBits", "ascending", "constant", "sampled_peaks"]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_persistent_topk_sampled_graph(
|
||||
rows: int, top_k: int, distribution: str
|
||||
) -> None:
|
||||
"""Sampling and both fallbacks preserve exact values across graph replays."""
|
||||
if torch.cuda.get_device_properties(0).shared_memory_per_block_optin < 144 * 1024:
|
||||
pytest.skip("Sampled top-k requires at least 144 KiB of shared memory")
|
||||
set_random_seed(42)
|
||||
min_sampled_length = 98304 if top_k == 512 else 65536
|
||||
width = min_sampled_length + 3
|
||||
lengths = torch.full((rows,), width, dtype=torch.int32, device="cuda")
|
||||
logits = create_random_logits(
|
||||
torch.zeros_like(lengths),
|
||||
lengths,
|
||||
torch.float32,
|
||||
42,
|
||||
False,
|
||||
"10LSBits" if distribution == "10LSBits" else "random",
|
||||
)
|
||||
if distribution == "ascending":
|
||||
logits.copy_(torch.arange(width, device="cuda", dtype=torch.float32))
|
||||
elif distribution == "constant":
|
||||
logits.fill_(1.0)
|
||||
elif distribution == "sampled_peaks":
|
||||
# A biased sample leaves fewer than k survivors and must fall back.
|
||||
logits.zero_()
|
||||
sample = torch.arange(4096, device="cuda")
|
||||
positions = (sample // 32) * (width // 128) + sample % 32
|
||||
logits[:, positions] = 1 + sample.float() / 4096
|
||||
original = logits.clone()
|
||||
indices = torch.empty((rows, top_k), dtype=torch.int32, device="cuda")
|
||||
workspace = torch.empty(RADIX_TOPK_WORKSPACE_SIZE, dtype=torch.uint8, device="cuda")
|
||||
|
||||
def run() -> None:
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
lengths.view(-1, 4 if rows == 128 else 1),
|
||||
indices,
|
||||
workspace,
|
||||
top_k,
|
||||
width,
|
||||
)
|
||||
|
||||
run()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
positions = torch.arange(width, device="cuda")
|
||||
slots = torch.arange(top_k, device="cuda")
|
||||
for step in range(3):
|
||||
bounds = torch.tensor(
|
||||
[
|
||||
-1,
|
||||
0,
|
||||
1,
|
||||
top_k - 1,
|
||||
top_k,
|
||||
32769,
|
||||
min_sampled_length - 1,
|
||||
min_sampled_length,
|
||||
width,
|
||||
width + 17,
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
lengths.copy_(
|
||||
bounds[(torch.arange(rows, device="cuda") + step) % bounds.numel()]
|
||||
)
|
||||
logits.copy_(original if step != 1 else original.flip(1))
|
||||
logits.masked_fill_(positions[None] >= lengths[:, None], float("nan"))
|
||||
indices.fill_(-2)
|
||||
graph.replay()
|
||||
valid = slots[None] < lengths.clamp(max=top_k)[:, None]
|
||||
assert torch.all(indices[~valid] == -1)
|
||||
assert torch.all(((indices >= 0) & (indices < lengths[:, None])) == valid)
|
||||
ordered_indices = indices.sort(dim=1).values
|
||||
assert torch.all(
|
||||
(ordered_indices[:, 1:] != ordered_indices[:, :-1])
|
||||
| (ordered_indices[:, 1:] == -1)
|
||||
)
|
||||
selected = logits.gather(1, indices.clamp_min(0).long())
|
||||
selected.masked_fill_(~valid, -float("inf"))
|
||||
expected = logits.masked_fill(
|
||||
positions[None] >= lengths[:, None], -float("inf")
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
selected.sort(dim=1, descending=True).values,
|
||||
expected.topk(top_k, dim=1).values,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
def run_large_context_topk_test(
|
||||
batch_size: int,
|
||||
seq_lens: list[int],
|
||||
|
||||
@@ -31,6 +31,9 @@ from vllm.lora.layers import (
|
||||
)
|
||||
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
|
||||
from vllm.lora.punica_wrapper import get_punica_wrapper
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
@@ -56,6 +59,13 @@ TOLERANCES = {
|
||||
torch.bfloat16: (3e-2, 2e-2),
|
||||
}
|
||||
|
||||
|
||||
def test_lora_linear_requires_unquantized_input() -> None:
|
||||
layer = RowParallelLinearWithLoRA.__new__(RowParallelLinearWithLoRA)
|
||||
layer._input_quant_key = object()
|
||||
assert get_input_quant_key(layer) is None
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not (
|
||||
|
||||
@@ -115,7 +115,9 @@ def test_b12x_backend_registration_priority_and_selection(
|
||||
assert names.index(before) < names.index(kernel_cls.__name__) < names.index(after)
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
monkeypatch.setattr(linear_mod, "_get_linear_backend", lambda: "b12x")
|
||||
monkeypatch.setattr(
|
||||
linear_mod, "_get_linear_backend", lambda *, quantization: "b12x"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kernel_cls,
|
||||
"is_supported",
|
||||
@@ -784,7 +786,9 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear as linear_mod
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
monkeypatch.setattr(linear_mod, "_get_linear_backend", lambda: "b12x")
|
||||
monkeypatch.setattr(
|
||||
linear_mod, "_get_linear_backend", lambda *, quantization: "b12x"
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
linear_mod._POSSIBLE_NVFP4_KERNELS,
|
||||
PlatformEnum.CUDA,
|
||||
|
||||
@@ -450,3 +450,547 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
|
||||
keep = valid & (block >= 0) & ((cols - ks[:, None]) // block_size == block)
|
||||
reference = torch.where(keep, 3.0, -torch.inf)
|
||||
torch.testing.assert_close(logits, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
# --- DeepGEMM sparse-MQA indexer path (V4.1 two-level candidate filtering) ---
|
||||
|
||||
|
||||
def _make_candidates(lens: torch.Tensor, num_candidates: int, cbk: int):
|
||||
"""Production-faithful candidates: top min(avail, K) blocks per row —
|
||||
full coverage for lens <= K*cbk, a fixed-size subset beyond that."""
|
||||
avail = ((lens.cpu() + cbk - 1) // cbk).tolist()
|
||||
rows = []
|
||||
for a in avail:
|
||||
n = min(a, num_candidates)
|
||||
picks = torch.randperm(a)[:n].sort().values[:num_candidates].tolist()
|
||||
rows.append(picks + [-1] * (num_candidates - len(picks)))
|
||||
return torch.tensor(rows, dtype=torch.int32)
|
||||
|
||||
|
||||
def _candidate_token_mask(
|
||||
candidates: torch.Tensor, cbk: int, total: int, ks: torch.Tensor | None = None
|
||||
):
|
||||
"""[rows, total] bool mask of the candidate-covered token positions.
|
||||
|
||||
``ks`` converts request-local candidate blocks to absolute positions
|
||||
(prefill's packed workspace); pass None when already request-local.
|
||||
"""
|
||||
rows = candidates.shape[0]
|
||||
device = candidates.device
|
||||
# Sentinel block keeps -1 padding from scattering onto real columns.
|
||||
width = max(total, (candidates.shape[1] + 1) * cbk)
|
||||
if ks is not None:
|
||||
width += int(ks.max()) + cbk
|
||||
in_cand = torch.zeros(rows, width, dtype=torch.bool, device=device)
|
||||
block_id = torch.where(candidates >= 0, candidates.long(), candidates.shape[1])
|
||||
tok = block_id.unsqueeze(2) * cbk + torch.arange(cbk, device=device)
|
||||
if ks is not None:
|
||||
tok = tok + ks.unsqueeze(1).unsqueeze(2)
|
||||
keep = (candidates >= 0).unsqueeze(2).expand(-1, -1, cbk)
|
||||
in_cand.scatter_(1, tok.flatten(1), keep.flatten(1))
|
||||
return in_cand[:, :total]
|
||||
|
||||
|
||||
def _reference_topk(masked: torch.Tensor, topk: int) -> torch.Tensor:
|
||||
"""Plain ``torch.topk`` over -inf-masked logits; slots past a row's valid
|
||||
count are -1 (the indexer's "no token" sentinel)."""
|
||||
values, indices = masked.topk(topk, dim=1)
|
||||
return torch.where(values > float("-inf"), indices.int(), -1)
|
||||
|
||||
|
||||
def _assert_same_selection(
|
||||
logits: torch.Tensor,
|
||||
got: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
row_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Both top-k index sets must pick the same multiset of logit values per
|
||||
row (-1 padding must match too). Indices themselves may differ: the
|
||||
top-k kernels break ties among equal values in nondeterministic order."""
|
||||
for r in range(logits.shape[0]):
|
||||
base = int(row_starts[r]) if row_starts is not None else 0
|
||||
g, e = got[r][got[r] >= 0], expected[r][expected[r] >= 0]
|
||||
assert g.numel() == e.numel(), f"row {r}: {g.numel()} vs {e.numel()} selected"
|
||||
vg = logits[r][(base + g).long()].sort().values
|
||||
ve = logits[r][(base + e).long()].sort().values
|
||||
assert torch.equal(vg, ve), f"row {r}: selected values differ"
|
||||
|
||||
|
||||
def _sparse_scratch(rows: int, num_sparse: int, topk: int) -> dict:
|
||||
"""Caller-owned buffers the sparse path writes into (the metadata builder
|
||||
owns these in production)."""
|
||||
i32 = dict(dtype=torch.int32, device="cuda")
|
||||
return dict(
|
||||
sparse_indices=torch.zeros(rows, num_sparse, **i32),
|
||||
end=torch.zeros(rows, **i32),
|
||||
col_indices=torch.zeros(rows, topk, **i32),
|
||||
)
|
||||
|
||||
|
||||
def _expand_candidates_reference(
|
||||
candidates: torch.Tensor, ks: torch.Tensor, ke: torch.Tensor, cbk: int, sbk: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""PyTorch reference for `candidate_blocks_to_sparse_indices`: expand
|
||||
each candidate block into ``cbk // sbk`` sparse blocks, keep those inside
|
||||
``[0, ke - ks)``, sort ascending, anchor at ``ks // sbk``, pad by
|
||||
repeating the last valid block, and count the valid sparse columns."""
|
||||
ratio = cbk // sbk
|
||||
rows, num_candidates = candidates.shape
|
||||
num_sparse = num_candidates * ratio
|
||||
big = torch.iinfo(torch.int32).max
|
||||
offsets = torch.arange(ratio, device=candidates.device, dtype=torch.int32)
|
||||
blocks = (candidates.unsqueeze(-1) * ratio + offsets).flatten(1)
|
||||
max_block = ((ke - ks).clamp(min=0) + sbk - 1) // sbk
|
||||
valid = (blocks >= 0) & (blocks < max_block.unsqueeze(1))
|
||||
sorted_blocks, _ = torch.where(valid, blocks, big).sort(dim=1)
|
||||
n_valid = (sorted_blocks != big).sum(dim=1, dtype=torch.int32)
|
||||
base = (ks // sbk).unsqueeze(1)
|
||||
kernel_blocks = torch.where(sorted_blocks != big, sorted_blocks, 0) + base
|
||||
last_slot = (n_valid - 1).clamp(min=0).unsqueeze(1).long()
|
||||
last_block = kernel_blocks.gather(1, last_slot)
|
||||
pad = torch.arange(num_sparse, device=candidates.device) >= n_valid.unsqueeze(1)
|
||||
sparse_indices = torch.where(pad, last_block, kernel_blocks)
|
||||
last_start = last_block.squeeze(1) * sbk + ks % sbk
|
||||
last_fill = (ke - last_start).clamp(min=1, max=sbk)
|
||||
end = torch.where(n_valid > 0, (n_valid - 1) * sbk + last_fill, 0)
|
||||
return sparse_indices.int(), end.int()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
def test_candidate_blocks_to_sparse_indices_math():
|
||||
"""Candidate->sparse-block expansion: ks anchoring, range filtering,
|
||||
repeat-last padding, and the per-row valid column count (``end``)."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
|
||||
# Rows cover: exactly one block, an unaligned ks with a partial tail
|
||||
# block, an empty row, duplicates / -1 padding / out-of-range garbage
|
||||
# (warmup runs touch uninitialized buffers), and int32-overflow junk.
|
||||
ks = torch.tensor([64, 13, 100, 8, 0], dtype=torch.int32, device="cuda")
|
||||
ke = torch.tensor([72, 30, 100, 40, 1000], dtype=torch.int32, device="cuda")
|
||||
candidates = torch.tensor(
|
||||
[
|
||||
[0, -1, -1, -1],
|
||||
[2, 0, -1, -1],
|
||||
[3, -1, -1, -1],
|
||||
[1, 1, 7, 0],
|
||||
[2**30, -(2**30), 0, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
indices, end = candidate_blocks_to_sparse_indices(
|
||||
candidates, ks, ke, candidate_block_size=8, sparse_block_kv=8
|
||||
)
|
||||
assert indices.tolist() == [
|
||||
[8, 8, 8, 8],
|
||||
[1, 3, 3, 3],
|
||||
[12, 12, 12, 12],
|
||||
[1, 2, 2, 2],
|
||||
[0, 0, 0, 0],
|
||||
]
|
||||
# end: row 1's last block covers [29, 37) but ke = 30 -> one valid token;
|
||||
# row 3 keeps the duplicated block (production candidates are unique).
|
||||
assert end.tolist() == [8, 9, 0, 24, 8]
|
||||
|
||||
# Wider candidate blocks expand into multiple sparse blocks.
|
||||
idx2, end2 = candidate_blocks_to_sparse_indices(
|
||||
torch.tensor([[1, 0]], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([48], dtype=torch.int32, device="cuda"),
|
||||
candidate_block_size=16,
|
||||
sparse_block_kv=8,
|
||||
)
|
||||
assert idx2.tolist() == [[0, 1, 2, 3]]
|
||||
assert end2.tolist() == [32]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("cbk,sbk", [(8, 8), (16, 8), (16, 16)])
|
||||
def test_candidate_blocks_to_sparse_indices_matches_reference(cbk: int, sbk: int):
|
||||
"""The Triton expansion equals the PyTorch reference on production-shaped
|
||||
random inputs, including strided (column-sliced) candidate rows and
|
||||
strided output buffers."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_candidates = 41, 2048
|
||||
ratio = cbk // sbk
|
||||
lens = torch.randint(1, 40000, (rows,), dtype=torch.int32)
|
||||
lens[0], lens[1] = 1, 3
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke = ks.cuda(), (ks + lens).cuda()
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
# Row 2 gets out-of-range garbage past the padding, as a CUDA-graph
|
||||
# warm-up on uninitialized buffers would.
|
||||
candidates[2, -3:] = torch.tensor([2**30, -(2**29), 10**6], dtype=torch.int32)
|
||||
|
||||
ref_indices, ref_end = _expand_candidates_reference(candidates, ks, ke, cbk, sbk)
|
||||
|
||||
cand_buf = torch.full((rows, num_candidates + 64), -7, dtype=torch.int32).cuda()
|
||||
cand_buf[:, :num_candidates] = candidates
|
||||
si_buf = torch.zeros(rows, num_candidates * ratio + 32, dtype=torch.int32).cuda()
|
||||
end_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
|
||||
ks_buf = torch.zeros(rows * 2, dtype=torch.int32, device="cuda")
|
||||
ks_buf[::2] = ks
|
||||
got_indices, got_end = candidate_blocks_to_sparse_indices(
|
||||
cand_buf[:, :num_candidates],
|
||||
ks_buf[::2],
|
||||
ke,
|
||||
cbk,
|
||||
sbk,
|
||||
out=(si_buf[:, : num_candidates * ratio], end_buf[::2]),
|
||||
)
|
||||
assert torch.equal(got_indices, ref_indices)
|
||||
assert torch.equal(got_end, ref_end)
|
||||
assert torch.equal(
|
||||
cand_buf[:, num_candidates:], torch.full_like(cand_buf[:, num_candidates:], -7)
|
||||
)
|
||||
assert not si_buf[:, num_candidates * ratio :].any()
|
||||
assert not end_buf[1::2].any()
|
||||
|
||||
|
||||
def _skip_unless_sm100_sparse_kernels():
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
|
||||
pytest.skip("SM100 required")
|
||||
deep_gemm = pytest.importorskip("deep_gemm")
|
||||
if not hasattr(deep_gemm, "fp8_fp4_sparse_mqa_logits"):
|
||||
pytest.skip("DeepGEMM >= 2.8 required for sparse MQA logits")
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
has_deep_select,
|
||||
)
|
||||
|
||||
if not has_deep_select():
|
||||
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
|
||||
return deep_gemm
|
||||
|
||||
|
||||
def _quant_fp4(x: torch.Tensor):
|
||||
from deep_gemm.utils import per_token_cast_to_fp4
|
||||
|
||||
return per_token_cast_to_fp4(x, use_ue8m0=True, gran_k=32, use_packed_ue8m0=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
"""The sparse-MQA candidate path selects exactly the same logit values as
|
||||
``torch.topk`` over dense bf16 logits masked to the candidates (the two
|
||||
DeepGEMM kernels agree bitwise on candidate columns), incl. under CUDA
|
||||
graph capture, and a later layer reusing the step's expansion/schedule
|
||||
selects the same values."""
|
||||
deep_gemm = _skip_unless_sm100_sparse_kernels()
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
sparse_mqa_logits_prefill_chunk,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_heads, head_dim = 37, 32, 128
|
||||
cbk, topk, num_candidates = 8, 512, 2048 # V4.1-Flash production values
|
||||
lens = torch.randint(1, 30000, (rows,), dtype=torch.int32)
|
||||
lens[0], lens[1], lens[2] = 1, 5, 2048 # single token, sub-block, long
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
|
||||
total = int(lens.sum())
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
|
||||
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
|
||||
dense = deep_gemm.fp8_fp4_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
),
|
||||
(k_fp.view(torch.int8), k_sf.view(total)),
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
clean_logits=False,
|
||||
logits_dtype=torch.bfloat16,
|
||||
)
|
||||
cols = torch.arange(total, device="cuda")
|
||||
in_range = (cols - ks.unsqueeze(1) >= 0) & (
|
||||
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
|
||||
)
|
||||
masked = dense.masked_fill(
|
||||
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
|
||||
float("-inf"),
|
||||
).float()
|
||||
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
|
||||
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
|
||||
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
|
||||
def run_sparse(kernel_metadata=None):
|
||||
return sparse_mqa_logits_prefill_chunk(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
k_fp.view(torch.int8),
|
||||
k_sf.view(total),
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
candidates,
|
||||
cbk,
|
||||
cbk,
|
||||
topk,
|
||||
out,
|
||||
kernel_metadata=kernel_metadata,
|
||||
**scratch,
|
||||
)
|
||||
|
||||
if capture:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run_sparse() # warm up JIT + the DeepGEMM workspace pre-capture
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
run_sparse()
|
||||
out.fill_(-1)
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse()
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
out.fill_(-1)
|
||||
run_sparse(kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
"""MXFP4 paged cache in DeepGEMM's fused layout: per page, all tokens'
|
||||
fp4 data first, then their packed UE8M0 scales."""
|
||||
device = req_lens.device
|
||||
pages_per = (req_lens + page_kv - 1) // page_kv
|
||||
num_pages, max_pages = int(pages_per.sum()), int(pages_per.max())
|
||||
kv_cache = torch.zeros(
|
||||
num_pages, page_kv, 1, head_dim // 2 + 4, dtype=torch.uint8, device=device
|
||||
)
|
||||
block_table = torch.zeros(
|
||||
len(req_lens), max_pages, dtype=torch.int32, device=device
|
||||
)
|
||||
page_pool = torch.randperm(num_pages, device=device)
|
||||
offset = 0
|
||||
for b in range(len(req_lens)):
|
||||
n = int(pages_per[b])
|
||||
block_table[b, :n] = page_pool[offset : offset + n].to(torch.int32)
|
||||
offset += n
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(int(req_lens[b]), head_dim, device=device, dtype=torch.bfloat16)
|
||||
)
|
||||
rows68 = torch.zeros(n * page_kv, 68, dtype=torch.uint8, device=device)
|
||||
rows68[: int(req_lens[b]), : head_dim // 2] = k_fp.view(torch.uint8)
|
||||
rows68[: int(req_lens[b]), head_dim // 2 :] = k_sf.view(torch.uint8).view(-1, 4)
|
||||
by_page = rows68.view(n, page_kv, 68)
|
||||
split = torch.cat(
|
||||
[by_page[..., :64].reshape(n, -1), by_page[..., 64:].reshape(n, -1)], dim=1
|
||||
).view(n, page_kv, 1, head_dim // 2 + 4)
|
||||
kv_cache[block_table[b, :n].long()] = split
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
@pytest.mark.parametrize("next_n", [1, 2])
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool):
|
||||
"""Paged sparse-MQA decode selects exactly the values ``torch.topk`` picks
|
||||
from the dense bf16 masked logits, incl. MTP rows (next_n > 1 flattened to
|
||||
one query row each), CUDA graph capture/replay and schedule reuse by a
|
||||
later layer."""
|
||||
deep_gemm = _skip_unless_sm100_sparse_kernels()
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
sparse_mqa_logits_paged_decode,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
batch, num_heads, head_dim, page_kv = 9, 32, 128, 128
|
||||
cbk, topk, num_candidates = 8, 512, 2048
|
||||
rows = batch * next_n
|
||||
req_lens = torch.randint(1, 30000, (batch,), dtype=torch.int32, device="cuda")
|
||||
req_lens[0], req_lens[1] = 1, 30000 # tiny row + a filtered (subset) row
|
||||
# Row (b, j) sees L_b - next_n + j + 1 tokens (native MTP layout).
|
||||
row_lens = (
|
||||
(req_lens.unsqueeze(1) - next_n + torch.arange(1, next_n + 1, device="cuda"))
|
||||
.clamp(min=0)
|
||||
.to(torch.int32)
|
||||
)
|
||||
kv_cache, block_table = _build_fp4_paged_cache(req_lens, page_kv, head_dim)
|
||||
assert kv_cache.stride(0) % 512 == 0 # the paged sparse kernel requires it
|
||||
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
q = q_fp.view(torch.int8).view(batch, next_n, num_heads, head_dim // 2)
|
||||
q_scale = q_sf.view(batch, next_n, num_heads)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
|
||||
|
||||
def reference_topk():
|
||||
context_lens_2d = row_lens.view(batch, next_n)
|
||||
sched = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
|
||||
)
|
||||
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_table,
|
||||
sched,
|
||||
int(req_lens.max()),
|
||||
False,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
)
|
||||
lens_flat = row_lens.flatten()
|
||||
cols = torch.arange(logits.shape[1], device="cuda")
|
||||
masked = logits.masked_fill(
|
||||
~(
|
||||
(cols.unsqueeze(0) < lens_flat.unsqueeze(1))
|
||||
& _candidate_token_mask(candidates, cbk, logits.shape[1])
|
||||
),
|
||||
float("-inf"),
|
||||
).float()
|
||||
return _reference_topk(masked, topk), logits
|
||||
|
||||
expected, dense_logits = reference_topk()
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
|
||||
# The metadata builder flattens every query to one row: per-row context
|
||||
# lengths, an expanded block table and a row -> request map.
|
||||
context_lens = row_lens.flatten().contiguous()
|
||||
row_block_table = block_table.repeat_interleave(next_n, dim=0)
|
||||
row_indices = torch.arange(batch, dtype=torch.int32, device="cuda")
|
||||
row_indices = row_indices.repeat_interleave(next_n)
|
||||
row_ks = torch.zeros(rows, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
|
||||
def run_sparse(o: torch.Tensor, kernel_metadata=None):
|
||||
return sparse_mqa_logits_paged_decode(
|
||||
q.view(rows, 1, num_heads, head_dim // 2),
|
||||
q_scale.view(rows, 1, num_heads),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens,
|
||||
row_block_table,
|
||||
row_indices,
|
||||
candidates,
|
||||
cbk,
|
||||
cbk,
|
||||
topk,
|
||||
o,
|
||||
row_ks=row_ks,
|
||||
kernel_metadata=kernel_metadata,
|
||||
**scratch,
|
||||
)
|
||||
|
||||
if capture:
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
run_sparse(out)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
run_sparse(out)
|
||||
out.fill_(-1)
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse(out)
|
||||
_assert_same_selection(dense_logits, out, expected)
|
||||
out.fill_(-1)
|
||||
run_sparse(out, kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense_logits, out, expected)
|
||||
|
||||
|
||||
def test_sparse_mqa_logits_accuracy_vs_torch():
|
||||
"""Sparse-kernel numerics vs a PyTorch fp32 reference on the dequantized
|
||||
MXFP4 inputs."""
|
||||
_skip_unless_sm100_sparse_kernels()
|
||||
from deep_gemm.utils import cast_back_from_fp4
|
||||
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
candidate_blocks_to_sparse_indices,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_sparse_mqa_logits,
|
||||
get_sparse_mqa_logits_metadata,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rows, num_heads, head_dim = 17, 32, 128
|
||||
sbk, num_candidates = 8, 2048
|
||||
lens = torch.randint(1, 1500, (rows,), dtype=torch.int32)
|
||||
lens[0] = 1
|
||||
ks = torch.zeros(rows, dtype=torch.int32)
|
||||
ks[1:] = lens.cumsum(0)[:-1]
|
||||
ks, ke, lens = ks.cuda(), (ks + lens).cuda(), lens.cuda()
|
||||
total = int(lens.sum())
|
||||
q_fp, q_sf = _quant_fp4(
|
||||
torch.randn(rows * num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
k_fp, k_sf = _quant_fp4(
|
||||
torch.randn(total, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
weights = torch.randn(rows, num_heads, device="cuda", dtype=torch.bfloat16)
|
||||
candidates = _make_candidates(lens, num_candidates, sbk).cuda()
|
||||
|
||||
sparse_idx, end = candidate_blocks_to_sparse_indices(candidates, ks, ke, sbk, sbk)
|
||||
metadata = get_sparse_mqa_logits_metadata(
|
||||
ks, ke, total, sparse_idx, torch.int8, sbk
|
||||
)
|
||||
logits = fp8_fp4_sparse_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
q_sf.view(rows, num_heads),
|
||||
),
|
||||
(k_fp.view(torch.int8), k_sf.view(total)),
|
||||
weights,
|
||||
metadata,
|
||||
sparse_idx.shape[1],
|
||||
sbk,
|
||||
)
|
||||
|
||||
# PyTorch fp32 reference on the dequantized inputs.
|
||||
q_deq = cast_back_from_fp4(q_fp, q_sf, gran_k=32, use_packed_ue8m0=True)
|
||||
k_deq = cast_back_from_fp4(k_fp, k_sf, gran_k=32, use_packed_ue8m0=True)
|
||||
dots = torch.einsum(
|
||||
"rhd,td->rht",
|
||||
q_deq.view(rows, num_heads, head_dim).float(),
|
||||
k_deq.float(),
|
||||
)
|
||||
ref = (dots.relu() * weights.float().unsqueeze(-1)).sum(1) # [rows, total]
|
||||
|
||||
# Compare on valid sparse columns (absolute packed-workspace positions).
|
||||
width = sparse_idx.shape[1] * sbk
|
||||
cols = torch.arange(width, device="cuda")
|
||||
valid = cols.unsqueeze(0) < end.unsqueeze(1).long()
|
||||
pos = (
|
||||
sparse_idx.repeat_interleave(sbk, dim=1).long() * sbk
|
||||
+ (ks % sbk).unsqueeze(1)
|
||||
+ (cols % sbk).unsqueeze(0)
|
||||
).clamp(min=0, max=total - 1)
|
||||
abs_err = (logits.float() - ref.gather(1, pos)).abs()[valid]
|
||||
row_scale = ref.abs().amax(dim=1)
|
||||
norm_err = ((logits.float() - ref.gather(1, pos)).abs() / row_scale.unsqueeze(1))[
|
||||
valid
|
||||
]
|
||||
print(
|
||||
f"sparse vs torch fp32: max_abs={abs_err.max().item():.4f}; "
|
||||
f"rowmax-normalized p50={norm_err.median().item():.5f} "
|
||||
f"p99={norm_err.quantile(0.99).item():.5f} max={norm_err.max().item():.5f}"
|
||||
)
|
||||
# bf16-level numerics: far below the value gaps that change top-k selection.
|
||||
assert norm_err.quantile(0.99).item() < 1e-2
|
||||
assert norm_err.max().item() < 2e-2
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import importlib.machinery
|
||||
import inspect
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import Mock
|
||||
from weakref import WeakKeyDictionary, ref
|
||||
|
||||
@@ -817,6 +820,110 @@ def test_layerwise_reload_updates_loaded_non_persistent_buffers(monkeypatch):
|
||||
assert "0.scale" not in model.state_dict()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hpc_rope_norm(monkeypatch, default_vllm_config):
|
||||
"""Import HpcRopeNorm with the external ``hpc`` package stubbed out."""
|
||||
if "hpc" not in sys.modules:
|
||||
stub = types.ModuleType("hpc")
|
||||
stub.__spec__ = importlib.machinery.ModuleSpec("hpc", loader=None)
|
||||
stub.QuantType = types.SimpleNamespace( # type: ignore[attr-defined]
|
||||
QPERTOKEN_PERHEAD_KPERTENSOR_VPERTENSOR=types.SimpleNamespace(value=0)
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "hpc", stub)
|
||||
from vllm.model_executor.layers.hpc import rope_norm
|
||||
|
||||
monkeypatch.setattr(rope_norm, "_hpc_rope_norm_instances", {})
|
||||
return rope_norm
|
||||
|
||||
|
||||
def test_hpc_rope_norm_kernel_sees_refit_norm_weights(monkeypatch, hpc_rope_norm):
|
||||
"""The fused HPC kernel is handed the live QK-norm weights after a refit.
|
||||
|
||||
Drives the production ``_forward_impl`` with a recording ``hpc`` stub. The
|
||||
Q/K norm weights it receives must be the model's own float32 parameters,
|
||||
so a layerwise reload that rewrites them in place (same storage) is what
|
||||
the kernel sees. Previously the kernel read separate mirrors that no
|
||||
reload path refreshed.
|
||||
"""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
head_dim, num_heads, num_kv_heads, block_size = 128, 8, 1, 4
|
||||
layer = torch.nn.Module()
|
||||
layer.q_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
|
||||
layer.k_norm = RMSNorm(head_dim, 1e-6, dtype=torch.float32)
|
||||
layer.hpc_rope_norm = hpc_rope_norm.HpcRopeNorm(
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
cos_sin_cache=torch.ones(16, head_dim),
|
||||
use_qk_norm=True,
|
||||
fallback_qnorm=layer.q_norm,
|
||||
fallback_knorm=layer.k_norm,
|
||||
kv_cache_dtype="auto",
|
||||
layer_name="hpc_test_layer",
|
||||
)
|
||||
model = torch.nn.Sequential(layer)
|
||||
rnorm = layer.hpc_rope_norm
|
||||
|
||||
calls: list[dict] = []
|
||||
|
||||
def record(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
|
||||
monkeypatch.setattr(sys.modules["hpc"], "rope_norm_store_kv", record, raising=False)
|
||||
|
||||
def kernel_norm_weights():
|
||||
q_size, kv_size = num_heads * head_dim, num_kv_heads * head_dim
|
||||
qkv = torch.zeros(1, q_size + 2 * kv_size, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(
|
||||
2, num_kv_heads, block_size, 2 * head_dim, dtype=torch.bfloat16
|
||||
)
|
||||
attn_layer = types.SimpleNamespace(
|
||||
_k_scale=torch.ones(1), _v_scale=torch.ones(1)
|
||||
)
|
||||
attn_metadata = types.SimpleNamespace(
|
||||
num_actual_tokens=1,
|
||||
num_decodes=1,
|
||||
num_decode_tokens=1,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
max_query_len=1,
|
||||
decode_query_len=1,
|
||||
qo_indptr=None,
|
||||
qo_indptr_decode=None,
|
||||
slot_mapping=torch.tensor([4]),
|
||||
seq_lens=torch.tensor([1]),
|
||||
block_table_tensor=torch.tensor([[1]]),
|
||||
hpc_kv_written=False,
|
||||
)
|
||||
output = torch.zeros(1, q_size, dtype=torch.bfloat16)
|
||||
rnorm._forward_impl(qkv, kv_cache, attn_metadata, attn_layer, output)
|
||||
return calls[-1]["q_norm_weight"], calls[-1]["k_norm_weight"]
|
||||
|
||||
def loaded(value):
|
||||
return torch.full((head_dim,), value, dtype=torch.bfloat16)
|
||||
|
||||
default_weight_loader(layer.q_norm.weight, loaded(0.5))
|
||||
default_weight_loader(layer.k_norm.weight, loaded(0.25))
|
||||
q, k = kernel_norm_weights()
|
||||
assert torch.equal(q, loaded(0.5).float())
|
||||
assert torch.equal(k, loaded(0.25).float())
|
||||
q_ptr, k_ptr = q.data_ptr(), k.data_ptr()
|
||||
|
||||
record_metadata_for_reloading(model)
|
||||
initialize_layerwise_reload(model)
|
||||
layer.q_norm.weight.weight_loader(layer.q_norm.weight, loaded(2.0))
|
||||
layer.k_norm.weight.weight_loader(layer.k_norm.weight, loaded(3.0))
|
||||
finalize_layerwise_reload(model, model_config=None)
|
||||
|
||||
q, k = kernel_norm_weights()
|
||||
assert q.dtype == k.dtype == torch.float32
|
||||
assert torch.equal(q, loaded(2.0).float())
|
||||
assert torch.equal(k, loaded(3.0).float())
|
||||
assert (q.data_ptr(), k.data_ptr()) == (q_ptr, k_ptr)
|
||||
assert not hasattr(rnorm, "qnorm_weight")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -21,7 +22,6 @@ import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import get_socket_path
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DAEMON_TIMEOUT_S = 600
|
||||
@@ -79,9 +79,7 @@ class WeightCacheDaemon:
|
||||
# Poll for the socket files rather than a log line: model loading can
|
||||
# pull in JIT compilers that swap the process's stderr and swallow
|
||||
# everything logged afterwards, making log-based readiness flaky.
|
||||
expected = [
|
||||
get_socket_path(gpu_id, self.socket_dir) for gpu_id in range(self.tp_size)
|
||||
]
|
||||
pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock")
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._proc.poll() is not None:
|
||||
@@ -89,7 +87,7 @@ class WeightCacheDaemon:
|
||||
f"Weight cache daemon exited with {self._proc.returncode}:\n"
|
||||
f"{self._logs()}"
|
||||
)
|
||||
if all(os.path.exists(path) for path in expected):
|
||||
if len(glob.glob(pattern)) >= self.tp_size:
|
||||
return
|
||||
time.sleep(1.0)
|
||||
raise TimeoutError(
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def test_nemotron_h_lm_head_receives_quant_config():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
@@ -32,3 +34,28 @@ def test_nemotron_h_lm_head_receives_quant_config():
|
||||
MockLMHead.assert_called_once()
|
||||
call_kwargs = MockLMHead.call_args.kwargs
|
||||
assert call_kwargs["quant_config"] is mock_quant_config
|
||||
|
||||
|
||||
def test_relu2_fp8_fusion_uses_registry():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHMLP
|
||||
|
||||
projected = torch.empty((1, 1), dtype=torch.bfloat16)
|
||||
fused = Mock()
|
||||
act_fn = Mock()
|
||||
down_proj = Mock(side_effect=lambda x: (x, None))
|
||||
|
||||
mlp = NemotronHMLP.__new__(NemotronHMLP)
|
||||
torch.nn.Module.__init__(mlp)
|
||||
mlp.up_proj = Mock(return_value=(projected, None))
|
||||
mlp.down_proj = down_proj
|
||||
mlp.act_fn = act_fn
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
|
||||
return_value=fused,
|
||||
) as maybe_fused:
|
||||
result = mlp(Mock())
|
||||
|
||||
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
|
||||
act_fn.assert_not_called()
|
||||
assert result is fused
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
|
||||
|
||||
@@ -293,6 +294,25 @@ def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
|
||||
torch.testing.assert_close(aux_hidden_states[0], torch.tensor([[15.0]]))
|
||||
|
||||
|
||||
def _dspark_vocab_stub(input_vocab_size: int, draft_vocab_size: int):
|
||||
"""A Qwen3 DSpark model with only what `load_weights` reads."""
|
||||
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
|
||||
|
||||
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
|
||||
nn.Module.__init__(model)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
vocab_size=input_vocab_size,
|
||||
draft_vocab_size=draft_vocab_size,
|
||||
),
|
||||
)
|
||||
object.__setattr__(model, "target_vocab_size", 100)
|
||||
object.__setattr__(model, "model", Mock(confidence_head=None))
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
("input_vocab_size", "draft_vocab_size", "weights", "error"),
|
||||
@@ -311,24 +331,25 @@ def test_qwen3_omni_text_model_collects_post_deepstack_aux_hidden_states():
|
||||
def test_qwen3_dspark_rejects_incomplete_vocab_weights(
|
||||
input_vocab_size, draft_vocab_size, weights, error
|
||||
):
|
||||
from vllm.model_executor.models.qwen3_dspark import Qwen3DSparkForCausalLM
|
||||
|
||||
model = Qwen3DSparkForCausalLM.__new__(Qwen3DSparkForCausalLM)
|
||||
nn.Module.__init__(model)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
vocab_size=input_vocab_size,
|
||||
draft_vocab_size=draft_vocab_size,
|
||||
),
|
||||
)
|
||||
object.__setattr__(model, "target_vocab_size", 100)
|
||||
model = _dspark_vocab_stub(input_vocab_size, draft_vocab_size)
|
||||
|
||||
with pytest.raises(ValueError, match=error):
|
||||
model.load_weights(weights)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("draft_vocab_size", [100, 101])
|
||||
def test_qwen3_dspark_accepts_draft_vocab_at_least_target(draft_vocab_size):
|
||||
"""A draft vocab >= the target's maps identically, so needs no lm_head/d2t.
|
||||
|
||||
Guards the padded case (draft > target): the draft's unembedding is padded
|
||||
to its physical size, which is not a reduced vocabulary.
|
||||
"""
|
||||
model = _dspark_vocab_stub(100, draft_vocab_size)
|
||||
|
||||
model.load_weights([])
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark import utils as dspark_utils
|
||||
@@ -351,7 +372,7 @@ def test_dspark_shares_target_embedding_with_smaller_draft_vocabulary():
|
||||
attention_backend=None,
|
||||
kv_cache_dtype=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||
parallel_config=ParallelConfig(),
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
cache_config=SimpleNamespace(),
|
||||
load_config=SimpleNamespace(),
|
||||
|
||||
@@ -14,12 +14,14 @@ from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
|
||||
from ....utils import ROCM_ENGINE_KWARGS
|
||||
from ....utils import ROCM_ENGINE_KWARGS, create_new_process_for_each_test
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
|
||||
AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn"
|
||||
@@ -130,6 +132,45 @@ async def async_engine():
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
|
||||
def _build_inputs(
|
||||
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
|
||||
) -> tuple[list[dict], list[SamplingParams]]:
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
for audio_asset in audio_assets:
|
||||
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
tokens = tokenized.tokens
|
||||
audio_array = tokenized.audios[0].audio_array
|
||||
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
|
||||
inputs.append(
|
||||
{
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
)
|
||||
return inputs, sampling_params
|
||||
|
||||
|
||||
def _assert_expected_text(outputs) -> None:
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -138,51 +179,39 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypa
|
||||
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
assert_encoder_kv_cache_spec(vllm_model.llm)
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
def from_file(file_path: str):
|
||||
audio = Audio.from_file(file_path, strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
return (tokenized.tokens, tokenized.audios[0].audio_array)
|
||||
|
||||
tokenized_list = [
|
||||
from_file(audio_asset.get_local_path()) for audio_asset in audio_assets
|
||||
]
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
)
|
||||
@create_new_process_for_each_test(method="spawn")
|
||||
def test_voxtral_realtime_cudagraph(
|
||||
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
vllm_kwargs = {**ENGINE_CONFIG}
|
||||
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
|
||||
vllm_kwargs["enforce_eager"] = False
|
||||
vllm_kwargs["compilation_config"] = {
|
||||
"cudagraph_mode": cudagraph_mode,
|
||||
# max_num_seqs is 4, so this is the only decode size worth capturing.
|
||||
"cudagraph_capture_sizes": [4],
|
||||
}
|
||||
|
||||
for tokens, audio_array in tokenized_list:
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(
|
||||
SamplingParams(temperature=0.0, max_tokens=max_tokens)
|
||||
)
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
input_dict = {
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
inputs.append(input_dict)
|
||||
|
||||
outputs = vllm_model.llm.generate(
|
||||
inputs,
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -6,9 +6,13 @@ from collections.abc import Mapping
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ImagePixelInputs,
|
||||
_dummy_audio_num_samples,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
|
||||
@@ -204,3 +208,41 @@ def test_limit_mm_per_prompt(
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
|
||||
"""The unified audio feature extractor does not define ``fft_length``.
|
||||
|
||||
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
|
||||
unconditionally, so memory profiling raised ``AttributeError`` for every
|
||||
unified Gemma 4 deployment in which audio is the modality with the largest
|
||||
per-item token count -- which is the case as soon as ``audio_seq_length``
|
||||
is raised above the video budget.
|
||||
"""
|
||||
|
||||
class _UnifiedFeatureExtractor:
|
||||
sampling_rate = 16000
|
||||
|
||||
class _UnifiedProcessor:
|
||||
feature_extractor = _UnifiedFeatureExtractor()
|
||||
audio_seq_length = 3000
|
||||
audio_ms_per_token = 40
|
||||
|
||||
# 3000 tokens * 40 ms = 120 s at 16 kHz
|
||||
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
|
||||
|
||||
|
||||
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
|
||||
"""The mel/tower extractor keeps its existing behaviour."""
|
||||
|
||||
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
|
||||
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
|
||||
self.fft_length = 512
|
||||
self.sampling_rate = 16000
|
||||
|
||||
class _TowerProcessor:
|
||||
feature_extractor = _TowerFeatureExtractor()
|
||||
audio_seq_length = 750
|
||||
audio_ms_per_token = 40
|
||||
|
||||
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
|
||||
|
||||
@@ -14,7 +14,7 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
class _StubModel:
|
||||
"""Carries only the two attributes the token helpers read from
|
||||
"""Carries only the attributes the token-count method reads from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
|
||||
@@ -22,18 +22,15 @@ class _StubModel:
|
||||
pooled_grid_length: int
|
||||
|
||||
|
||||
get_num_mm_encoder_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
|
||||
)
|
||||
get_num_mm_connector_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
|
||||
get_mm_lora_token_counts = (
|
||||
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the two helpers must invert each other's
|
||||
the real HF config, and the LoRA token counts must preserve the
|
||||
frame-level scaling exactly."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
@@ -49,10 +46,13 @@ def test_num_mm_tokens_match_real_config(model_id):
|
||||
for num_frames in (1, 2, 8, 16, 32):
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@@ -72,10 +72,13 @@ def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_fram
|
||||
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@@ -84,5 +87,9 @@ def test_num_mm_tokens_zero():
|
||||
stub.patch_grid_length = 24
|
||||
stub.pooled_grid_length = 12
|
||||
|
||||
assert get_num_mm_encoder_tokens(stub, 0) == 0
|
||||
assert get_num_mm_connector_tokens(stub, 0) == 0
|
||||
assert get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=0,
|
||||
) == (0, 0)
|
||||
|
||||
@@ -1610,6 +1610,12 @@ def test_fused_conv_correctness(
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
outer_residual = torch.randn(
|
||||
inputs.shape,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
generator=rng,
|
||||
)
|
||||
null_state = conv_state[NULL_BLOCK_ID].clone()
|
||||
residual_kernel = residual.clone()
|
||||
residual_reference = residual.clone()
|
||||
@@ -1617,6 +1623,7 @@ def test_fused_conv_correctness(
|
||||
module._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual_kernel,
|
||||
outer_residual=outer_residual,
|
||||
metadata=metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=weights,
|
||||
@@ -1630,15 +1637,17 @@ def test_fused_conv_correctness(
|
||||
conv_state_len=module.conv_state_len,
|
||||
dilation=module.short_conv_dilation,
|
||||
)
|
||||
residual_reference = outer_residual + residual_reference
|
||||
|
||||
torch.testing.assert_close(
|
||||
residual_kernel.float(), residual_reference.float(), atol=3e-2, rtol=3e-2
|
||||
assert torch.equal(
|
||||
residual_kernel[:num_real_tokens], residual_reference[:num_real_tokens]
|
||||
)
|
||||
assert torch.equal(conv_state, state_reference)
|
||||
assert torch.equal(conv_state[NULL_BLOCK_ID], null_state)
|
||||
if case.graph_padding:
|
||||
assert torch.equal(
|
||||
residual_kernel[num_real_tokens:], residual[num_real_tokens:]
|
||||
residual_kernel[num_real_tokens:],
|
||||
(outer_residual + residual)[num_real_tokens:],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -944,6 +944,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
output_gate = torch.randn_like(q)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
@@ -1023,6 +1024,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
block_table,
|
||||
token_to_req,
|
||||
use_prefill_config=use_prefill_config,
|
||||
output_gate=output_gate,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
@@ -1033,6 +1035,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
token_to_req,
|
||||
scale,
|
||||
)
|
||||
expected = expected * torch.sigmoid(output_gate)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@@ -461,6 +461,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"NanbeigeForCausalLM": _HfExamplesInfo(
|
||||
"Nanbeige/Nanbeige4.2-3B",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"OlmoForCausalLM": _HfExamplesInfo("allenai/OLMo-1B-hf"),
|
||||
"Olmo2ForCausalLM": _HfExamplesInfo("allenai/OLMo-2-0425-1B"),
|
||||
"Olmo3ForCausalLM": _HfExamplesInfo("allenai/Olmo-3-7B-Instruct"),
|
||||
|
||||
@@ -18,8 +18,8 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.mtp import DeepSeekV4MTP
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.deepseek_v4_1.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.models.deepseek_v41.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4MoE as DeepseekV41MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
|
||||
|
||||
|
||||
@@ -216,7 +216,7 @@ def test_v41_dspark_loads_linear_scales(
|
||||
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
|
||||
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
|
||||
from vllm.models.deepseek_v4_1.nvidia import dspark
|
||||
from vllm.models.deepseek_v41.nvidia import dspark
|
||||
|
||||
mxfp8 = scale_dtype != torch.float32
|
||||
scale_name = "weight_scale" if mxfp8 else "weight_scale_inv"
|
||||
|
||||
@@ -574,7 +574,7 @@ class TestImplicitReasoningEnd:
|
||||
# the text lexer.
|
||||
assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID}
|
||||
assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1
|
||||
assert thinking_parser.find_reasoning_end_offset([7, 8]) is None
|
||||
assert thinking_parser.find_reasoning_end_offset([7, 8]) == 2
|
||||
|
||||
def test_streaming_reasoning_implicit_end(self, thinking_parser):
|
||||
chunks = [
|
||||
|
||||
@@ -1616,3 +1616,36 @@ class TestCommaInStringValueRegression:
|
||||
assert result.tools_called is True
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214"
|
||||
|
||||
|
||||
class TestGemma4IsReasoningEnd:
|
||||
@staticmethod
|
||||
def _parser(thinking: bool):
|
||||
return Gemma4Parser(
|
||||
_make_tokenizer(_PLAIN_ANSWER_TOKENS),
|
||||
chat_template_kwargs={"enable_thinking": thinking},
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ids", "thinking", "ended"),
|
||||
[
|
||||
([CHANNEL_START_ID, 3000, CHANNEL_END_ID], True, True),
|
||||
([CHANNEL_START_ID, 3000, TOOL_CALL_START_ID], True, True),
|
||||
([CHANNEL_START_ID, 3000], True, False),
|
||||
([NEW_TURN_ID, 9100], True, False),
|
||||
([CHANNEL_END_ID, NEW_TURN_ID, 9100], True, False),
|
||||
([NEW_TURN_ID, 9100], False, True),
|
||||
# The bundled template's thinking-off prompt: closed channel.
|
||||
([NEW_TURN_ID, 9100, CHANNEL_START_ID, 3000, CHANNEL_END_ID], False, True),
|
||||
# A marker still wins with thinking off: the grammar honours it.
|
||||
([CHANNEL_START_ID, 3000], False, False),
|
||||
# A tool-call opener after the channel closed: still ended.
|
||||
(
|
||||
[CHANNEL_START_ID, 3000, CHANNEL_END_ID, TOOL_CALL_START_ID, 9200],
|
||||
True,
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_is_reasoning_end(self, ids, thinking, ended):
|
||||
assert self._parser(thinking).is_reasoning_end(ids) is ended
|
||||
|
||||
@@ -185,8 +185,8 @@ class TestReasoningEndTokenIds:
|
||||
def test_find_reasoning_end_offset_returns_first_match(self):
|
||||
engine = _make_engine()
|
||||
assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1
|
||||
assert engine.find_reasoning_end_offset([5, 6]) is None
|
||||
assert engine.find_reasoning_end_offset([]) is None
|
||||
assert engine.find_reasoning_end_offset([5, 6]) == 2
|
||||
assert engine.find_reasoning_end_offset([]) == 0
|
||||
# Rejected-draft placeholders never match.
|
||||
assert engine.find_reasoning_end_offset([-1, 201]) == 1
|
||||
|
||||
|
||||
@@ -180,9 +180,8 @@ class TestIsReasoningEnd:
|
||||
"""Tool examples before the generation <think> must not end reasoning."""
|
||||
assert not parser.is_reasoning_end([_TOOL_CALL_ID, _TEXT_ID, _THINK_START_ID])
|
||||
|
||||
def test_paired_tool_call_not_end(self, parser):
|
||||
"""Paired <tool_call>...</tool_call> (from template) is NOT end."""
|
||||
assert not parser.is_reasoning_end(
|
||||
def test_completed_tool_call_after_open_think_is_end(self, parser):
|
||||
assert parser.is_reasoning_end(
|
||||
[_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
|
||||
)
|
||||
|
||||
@@ -196,6 +195,43 @@ class TestIsReasoningEnd:
|
||||
assert not parser.is_reasoning_end([])
|
||||
|
||||
|
||||
class TestIsReasoningEndDerivedFromGrammar:
|
||||
@pytest.mark.parametrize(
|
||||
("ids", "thinking", "ended"),
|
||||
[
|
||||
# Explicit close, and the implicit close on <tool_call>.
|
||||
([_THINK_START_ID, 1, _THINK_END_ID], True, True),
|
||||
([_THINK_START_ID, 1, _TOOL_CALL_ID], True, True),
|
||||
# Still inside the block.
|
||||
([_THINK_START_ID, 1], True, False),
|
||||
([_THINK_END_ID, _THINK_START_ID, 1], True, False),
|
||||
# Uncommitted: a bare turn, an empty sequence. Thinking on starts
|
||||
# in REASONING, so reasoning is expected.
|
||||
([_IM_END_ID, _IM_START_ID, 1], True, False),
|
||||
([], True, False),
|
||||
# Thinking off starts in CONTENT with no way into REASONING, so
|
||||
# a <think> the template left behind is inert, and a silent tail
|
||||
# is ended because no block is expected.
|
||||
([_THINK_START_ID, 1], False, True),
|
||||
([_THINK_END_ID, _THINK_START_ID, 1], False, True),
|
||||
([_IM_END_ID, _IM_START_ID, 1], False, True),
|
||||
([], False, True),
|
||||
],
|
||||
)
|
||||
def test_is_reasoning_end(self, mock_tokenizer, ids, thinking, ended):
|
||||
parser = Qwen3Parser(
|
||||
mock_tokenizer, chat_template_kwargs={"enable_thinking": thinking}
|
||||
)
|
||||
assert parser.is_reasoning_end(ids) is ended
|
||||
|
||||
@pytest.mark.parametrize("thinking", [True, False])
|
||||
def test_wait_for_reasoning_follows_thinking(self, mock_tokenizer, thinking):
|
||||
parser = Qwen3Parser(
|
||||
mock_tokenizer, chat_template_kwargs={"enable_thinking": thinking}
|
||||
)
|
||||
assert parser.parser_engine_config.wait_for_reasoning is thinking
|
||||
|
||||
|
||||
class TestIsReasoningEndTurnBoundaries:
|
||||
"""Reasoning markers from earlier turns must not mark the new turn's
|
||||
reasoning as finished (think-marker injection / interleaved-thinking
|
||||
|
||||
@@ -77,8 +77,8 @@ def test_deepseek_v41_mxfp8_scale_loading(
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1 import quant_config as quant_module
|
||||
from vllm.models.deepseek_v4_1.nvidia import model as model_module
|
||||
from vllm.models.deepseek_v41 import quant_config as quant_module
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
quant_config = quant_module.DeepseekV4FP8Config(
|
||||
@@ -214,8 +214,8 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
|
||||
"""The VL wrapper must map ``.scale`` keys to the parameter the linear
|
||||
quant method registers, as the text model does. A hardcoded
|
||||
``weight_scale_inv`` raised KeyError for native MXFP8 checkpoints."""
|
||||
from vllm.models.deepseek_v4_1.nvidia import model as model_module
|
||||
from vllm.models.deepseek_v4_1.nvidia import vl_model as vl_module
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
from vllm.models.deepseek_v41.nvidia import vl_model as vl_module
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
|
||||
@@ -318,8 +318,8 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
compute_fp8_einsum_recipe,
|
||||
deep_gemm_fp8_o_proj,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.nvidia.model import DeepseekV4Model
|
||||
from vllm.models.deepseek_v4_1.quant_config import DeepseekV4FP8Config
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4Model
|
||||
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if not is_deep_gemm_supported() or not current_platform.is_device_capability_family(
|
||||
|
||||
@@ -1668,3 +1668,148 @@ def test_suffix_match_at_module_boundary(prefix, ignored_layer, expected):
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
_GLM5_MXFP4_WEIGHT = {
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
}
|
||||
_GLM5_BLOCK_FP8_WEIGHT = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
_GLM5_BLOCK_FP8_INPUT = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
_GLM5_GATE_UP = "language_model.model.layers.0.mlp.gate_up_proj"
|
||||
|
||||
|
||||
def _glm5_mixed_precision_config() -> QuarkConfig:
|
||||
"""Global MXFP4 with the dense-MLP gate/up shards marked block-FP8."""
|
||||
fp8 = {"weight": _GLM5_BLOCK_FP8_WEIGHT, "input_tensors": _GLM5_BLOCK_FP8_INPUT}
|
||||
return QuarkConfig(
|
||||
{
|
||||
"global_quant_config": {
|
||||
"weight": _GLM5_MXFP4_WEIGHT,
|
||||
"input_tensors": None,
|
||||
},
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {"*mlp.gate_proj": fp8, "*mlp.up_proj": fp8},
|
||||
"exclude": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _GLM5RecordingParam:
|
||||
"""Stand-in for a BF16 target param that records what gets loaded."""
|
||||
|
||||
def __init__(self):
|
||||
self.loaded_weight: torch.Tensor | None = None
|
||||
self.loaded_shard: object = "unset"
|
||||
|
||||
def weight_loader(self, param, loaded_weight, shard_id=None):
|
||||
self.loaded_weight = loaded_weight
|
||||
self.loaded_shard = shard_id
|
||||
|
||||
|
||||
def _glm5_block_fp8(out_dim: int, in_dim: int, block: int = 128):
|
||||
"""Return a (fp8 weight, f32 per-block scale) pair."""
|
||||
weight = (torch.randn(out_dim, in_dim) * 0.1).to(torch.float8_e4m3fn)
|
||||
scale = torch.rand(out_dim // block, in_dim // block, dtype=torch.float32) + 0.5
|
||||
return weight, scale
|
||||
|
||||
|
||||
def test_glm5next_gate_up_proj_mapping_is_genuine_fusion():
|
||||
from vllm.models.glm5next.common.model import Glm5NextForConditionalGeneration
|
||||
|
||||
mapping = Glm5NextForConditionalGeneration.packed_modules_mapping
|
||||
assert mapping["gate_up_proj"] == ["gate_proj", "up_proj"]
|
||||
|
||||
|
||||
def test_glm5next_genuine_fusion_resolves_gate_up_to_block_fp8():
|
||||
# The fused module must expand to its real shard names so each resolves to
|
||||
# the per-layer block-FP8 entry (rather than the global MXFP4 scheme).
|
||||
config = _glm5_mixed_precision_config()
|
||||
config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
|
||||
_, _, scheme_cls = config.get_scheme_cls(LinearBase, _GLM5_GATE_UP)
|
||||
assert scheme_cls is QuarkW8A8Fp8PerBlock
|
||||
|
||||
|
||||
def test_glm5next_attn_loader_accepts_quark_weight_scale():
|
||||
# Regression for KeyError on '...kv_a_proj_with_mqa.weight_scale': the fused
|
||||
# q_a/kv_a projection is kept BF16 and dequantized on load; the Quark scale
|
||||
# name must be recognized and routed to the correct fused shard.
|
||||
from vllm.models.glm5next.common.model import (
|
||||
_dequant_fp8_block,
|
||||
_try_load_fp8_attn_proj,
|
||||
)
|
||||
|
||||
prefix = "layers.3.self_attn"
|
||||
target = _GLM5RecordingParam()
|
||||
params_dict = {f"{prefix}.fused_qkv_a_proj.weight": target}
|
||||
buf: dict = {}
|
||||
loaded: set = set()
|
||||
|
||||
weight, scale = _glm5_block_fp8(256, 256)
|
||||
# fp8 weight arrives first -> buffered, nothing loaded yet.
|
||||
assert (
|
||||
_try_load_fp8_attn_proj(
|
||||
f"{prefix}.kv_a_proj_with_mqa.weight", weight, buf, params_dict, loaded, 0
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert target.loaded_weight is None
|
||||
# Quark-named scale completes the pair -> dequantize + load.
|
||||
assert (
|
||||
_try_load_fp8_attn_proj(
|
||||
f"{prefix}.kv_a_proj_with_mqa.weight_scale",
|
||||
scale,
|
||||
buf,
|
||||
params_dict,
|
||||
loaded,
|
||||
0,
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert target.loaded_weight is not None
|
||||
assert target.loaded_weight.dtype == torch.bfloat16
|
||||
assert target.loaded_shard == 1 # kv_a is shard 1 of fused_qkv_a_proj
|
||||
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
|
||||
assert f"{prefix}.fused_qkv_a_proj.weight" in loaded
|
||||
|
||||
|
||||
def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv():
|
||||
# DeepSeek scale name keeps working unchanged.
|
||||
from vllm.models.glm5next.common.model import (
|
||||
_dequant_fp8_block,
|
||||
_try_load_fp8_attn_proj,
|
||||
)
|
||||
|
||||
prefix = "layers.7.self_attn"
|
||||
target = _GLM5RecordingParam()
|
||||
params_dict = {f"{prefix}.o_proj.weight": target}
|
||||
buf: dict = {}
|
||||
loaded: set = set()
|
||||
|
||||
weight, scale = _glm5_block_fp8(128, 256)
|
||||
_try_load_fp8_attn_proj(
|
||||
f"{prefix}.o_proj.weight", weight, buf, params_dict, loaded, 0
|
||||
)
|
||||
assert (
|
||||
_try_load_fp8_attn_proj(
|
||||
f"{prefix}.o_proj.weight_scale_inv", scale, buf, params_dict, loaded, 0
|
||||
)
|
||||
is True
|
||||
)
|
||||
assert target.loaded_shard is None # o_proj is a direct (non-fused) proj
|
||||
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
|
||||
|
||||
@@ -6,13 +6,17 @@ Run `pytest tests/samplers/test_beam_search.py`.
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
from transformers import AutoModelForSeq2SeqLM
|
||||
|
||||
from vllm import CompletionOutput, RequestOutput
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.entrypoints.llm import LLM
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
|
||||
|
||||
@@ -45,6 +49,87 @@ MM_BEAM_WIDTHS = [2]
|
||||
MODELS = ["TinyLlama/TinyLlama-1.1B-Chat-v1.0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
|
||||
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
|
||||
def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts(
|
||||
monkeypatch,
|
||||
abort_after: int,
|
||||
prompt_token: int,
|
||||
terminal_logprobs: SampleLogprobs | None,
|
||||
) -> None:
|
||||
"""Return tokens and scores from the last completed step for aborted prompts.
|
||||
|
||||
Other prompts in the batch continue generating.
|
||||
"""
|
||||
|
||||
def run_requests(prompts, **kwargs):
|
||||
results = []
|
||||
for prompt in prompts:
|
||||
tokens = prompt["prompt_token_ids"]
|
||||
if tokens[0] == prompt_token:
|
||||
assert len(tokens) <= abort_after + 1, "Continued after abort"
|
||||
aborted = (
|
||||
tokens[0] == prompt_token
|
||||
and len(tokens) == abort_after + 1
|
||||
and tokens[-1] != 12
|
||||
)
|
||||
results.append(
|
||||
RequestOutput(
|
||||
request_id="inner",
|
||||
prompt=None,
|
||||
prompt_token_ids=tokens,
|
||||
prompt_logprobs=None,
|
||||
finished=True,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[] if aborted else [11],
|
||||
cumulative_logprob=None,
|
||||
logprobs=terminal_logprobs
|
||||
if aborted
|
||||
else [{11: Logprob(-0.1), 12: Logprob(-0.2)}],
|
||||
finish_reason="abort" if aborted else "length",
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
llm = LLM.__new__(LLM)
|
||||
llm.llm_engine = Mock()
|
||||
tokenizer = SimpleNamespace(
|
||||
eos_token_id=0,
|
||||
decode=lambda tokens, skip_special_tokens=False: " ".join(
|
||||
str(token) for token in tokens if not skip_special_tokens or token != 0
|
||||
),
|
||||
)
|
||||
llm.renderer = Mock(get_tokenizer=Mock(return_value=tokenizer))
|
||||
monkeypatch.setattr(llm, "_preprocess_cmpl", lambda prompts: prompts)
|
||||
monkeypatch.setattr(llm, "_render_and_run_requests", run_requests)
|
||||
prompts = [
|
||||
{"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2]
|
||||
]
|
||||
params = BeamSearchParams(beam_width=2, max_tokens=3)
|
||||
outputs = llm.beam_search(prompts, params)
|
||||
|
||||
aborted, normal = outputs
|
||||
expected_tokens = (
|
||||
[[prompt_token, 11], [prompt_token, 12]] if abort_after else [[prompt_token]]
|
||||
)
|
||||
assert [beam.tokens for beam in aborted.sequences] == expected_tokens
|
||||
for beam in aborted.sequences:
|
||||
assert beam.finish_reason == "abort"
|
||||
assert beam.text == tokenizer.decode(
|
||||
beam.tokens, skip_special_tokens=params.skip_special_tokens
|
||||
)
|
||||
assert len(beam.logprobs) == abort_after
|
||||
assert aborted.sequences[0].cum_logprob == pytest.approx(-0.1 * abort_after)
|
||||
assert len(normal.sequences) == 2
|
||||
assert all(len(beam.tokens) == 4 for beam in normal.sequences)
|
||||
assert all(beam.finish_reason != "abort" for beam in normal.sequences)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", MAX_TOKENS)
|
||||
|
||||
@@ -5,14 +5,19 @@ import pytest
|
||||
|
||||
from vllm import CompletionOutput, RequestOutput
|
||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
eos_token_id = 0
|
||||
special_token_id = eos_token_id
|
||||
|
||||
def decode(self, token_ids: list[int]) -> str:
|
||||
def decode(self, token_ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
if skip_special_tokens:
|
||||
token_ids = [
|
||||
token_id for token_id in token_ids if token_id != self.special_token_id
|
||||
]
|
||||
return " ".join(str(token_id) for token_id in token_ids)
|
||||
|
||||
|
||||
@@ -72,3 +77,86 @@ async def test_beam_search_handles_extra_logprob_candidates() -> None:
|
||||
assert outputs[0].outputs[0].finish_reason == "stop"
|
||||
assert outputs[0].outputs[0].token_ids == []
|
||||
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("skip_special_tokens", "expected_text"),
|
||||
[
|
||||
pytest.param(True, "", id="skip"),
|
||||
pytest.param(False, "0", id="keep"),
|
||||
],
|
||||
)
|
||||
async def test_beam_search_respects_skip_special_tokens(
|
||||
skip_special_tokens: bool, expected_text: str
|
||||
) -> None:
|
||||
prompt = {
|
||||
"type": "token",
|
||||
"prompt": "prompt",
|
||||
"prompt_token_ids": [1],
|
||||
}
|
||||
params = BeamSearchParams(
|
||||
beam_width=1,
|
||||
max_tokens=1,
|
||||
ignore_eos=True,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
)
|
||||
|
||||
outputs = [
|
||||
output async for output in _Serving().beam_search(prompt, "request", params)
|
||||
]
|
||||
|
||||
assert outputs[0].outputs[0].text == expected_text
|
||||
assert outputs[0].outputs[0].token_ids == [_Tokenizer.special_token_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("abort_after", "prompt_token"), [(0, 1), (1, 1), (0, 0)])
|
||||
@pytest.mark.parametrize("terminal_logprobs", [None, [], [{11: Logprob(-0.1)}]])
|
||||
async def test_beam_search_abort_returns_partial_outputs(
|
||||
monkeypatch,
|
||||
abort_after: int,
|
||||
prompt_token: int,
|
||||
terminal_logprobs: SampleLogprobs | None,
|
||||
) -> None:
|
||||
"""Abort returns each beam's tokens and scores from the last completed step."""
|
||||
calls = 0
|
||||
|
||||
async def generate(prompt, *args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
depth = len(prompt["prompt_token_ids"]) - 1
|
||||
assert depth <= abort_after, "Beam search continued after abort"
|
||||
result = await anext(_EngineClient().generate(prompt, *args, **kwargs))
|
||||
output = result.outputs[0]
|
||||
output.token_ids = [11]
|
||||
output.logprobs = [{11: Logprob(-0.1), 12: Logprob(-0.2)}]
|
||||
output.finish_reason = "length"
|
||||
if depth == abort_after and prompt["prompt_token_ids"][-1] != 12:
|
||||
output.token_ids = []
|
||||
output.logprobs = terminal_logprobs
|
||||
output.finish_reason = "abort"
|
||||
yield result
|
||||
|
||||
serving = _Serving()
|
||||
monkeypatch.setattr(serving.engine_client, "generate", generate)
|
||||
prompt = {"type": "token", "prompt": "prompt", "prompt_token_ids": [prompt_token]}
|
||||
outputs = [
|
||||
output
|
||||
async for output in serving.beam_search(
|
||||
prompt, "request", BeamSearchParams(beam_width=2, max_tokens=4)
|
||||
)
|
||||
]
|
||||
|
||||
assert calls == 1 + 2 * abort_after
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].finished
|
||||
assert outputs[0].request_id == "request"
|
||||
expected_tokens = [[11], [12]] if abort_after else [[]]
|
||||
assert [output.token_ids for output in outputs[0].outputs] == expected_tokens
|
||||
for output in outputs[0].outputs:
|
||||
assert output.finish_reason == "abort"
|
||||
assert output.text == _Tokenizer().decode(list(output.token_ids))
|
||||
assert output.logprobs is not None
|
||||
assert len(output.logprobs) == abort_after
|
||||
assert outputs[0].outputs[0].cumulative_logprob == pytest.approx(-0.1 * abort_after)
|
||||
|
||||
@@ -337,6 +337,108 @@ def test_hisparse_rejects_non_cuda(monkeypatch):
|
||||
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kv_transfer_config",
|
||||
[
|
||||
KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 128},
|
||||
),
|
||||
KVTransferConfig(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [
|
||||
{
|
||||
"kv_connector": "OffloadingConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
|
||||
},
|
||||
{
|
||||
"kv_connector": "HiSparseConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"host_pool_gib": 128},
|
||||
},
|
||||
]
|
||||
},
|
||||
),
|
||||
KVTransferConfig(
|
||||
kv_connector="MultiConnector",
|
||||
kv_role="kv_consumer",
|
||||
kv_connector_extra_config={
|
||||
"connectors": [
|
||||
{"kv_connector": "NixlConnector", "kv_role": "kv_consumer"},
|
||||
{
|
||||
"kv_connector": "HiSparseConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"host_pool_gib": 128},
|
||||
},
|
||||
]
|
||||
},
|
||||
),
|
||||
],
|
||||
ids=["standalone", "multi-connector", "pd-decode"],
|
||||
)
|
||||
def test_hisparse_connector_implies_attention_config(monkeypatch, kv_transfer_config):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
|
||||
config = VllmConfig(
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
# Skip the HMA auto-detect block: it imports the connector class,
|
||||
# which pulls the platform's compiled attention extensions.
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
),
|
||||
)
|
||||
assert isinstance(config.attention_config.hisparse_config, HiSparseConfig)
|
||||
|
||||
|
||||
def test_hisparse_connector_preserves_explicit_attention_config(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
|
||||
config = VllmConfig(
|
||||
attention_config=AttentionConfig(
|
||||
hisparse_config=HiSparseConfig(device_buffer_size=512)
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 128},
|
||||
),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
),
|
||||
)
|
||||
assert config.attention_config.hisparse_config.device_buffer_size == 512
|
||||
|
||||
|
||||
def test_hisparse_connector_without_cuda_still_rejected(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
|
||||
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
|
||||
VllmConfig(
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="HiSparseConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"host_pool_gib": 128},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_no_hisparse_connector_keeps_attention_config_unset(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
config = VllmConfig(
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="OffloadingConnector", kv_role="kv_both"
|
||||
)
|
||||
)
|
||||
assert config.attention_config.hisparse_config is None
|
||||
|
||||
|
||||
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
|
||||
MRV1 paths and off breakable cudagraphs by default."""
|
||||
@@ -3322,6 +3424,22 @@ def test_ir_op_priority_default():
|
||||
assert priority_config.fused_add_rms_norm == ["native"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [CompilationMode.NONE, CompilationMode.VLLM_COMPILE])
|
||||
@pytest.mark.parametrize("backend", ["inductor", "eager"])
|
||||
def test_ir_op_platform_defaults_support_sparse_gelu(mode, backend):
|
||||
"""Worker initialization must not select an unregistered sparse GELU provider."""
|
||||
from vllm import ir
|
||||
|
||||
config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(mode=mode, backend=backend)
|
||||
)
|
||||
priority = current_platform.get_default_ir_op_priority(config)
|
||||
expected = ["triton", "native"] if current_platform.is_cuda() else ["native"]
|
||||
|
||||
with priority.set_priority():
|
||||
assert ir.ops.gelu_and_mul_sparse.get_priority() == expected
|
||||
|
||||
|
||||
def test_ir_op_priority_str():
|
||||
"""Test that passing a comma-delimited string works"""
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
@@ -43,41 +43,6 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
|
||||
assert "VLLM_API_KEY" not in envs.compile_factors()
|
||||
|
||||
|
||||
def test_scale_out_endpoints_flag_is_runtime_only(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is True
|
||||
assert "VLLM_ENABLE_SCALE_OUT_ENDPOINTS" not in envs.compile_factors()
|
||||
|
||||
|
||||
def test_scale_out_endpoints_flag_distinguishes_unset_from_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is False
|
||||
|
||||
|
||||
def test_scale_out_endpoints_flag_treats_empty_as_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "")
|
||||
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["-1", "2", "01", "+1", "invalid", " "])
|
||||
def test_scale_out_endpoints_flag_rejects_values_other_than_zero_or_one(
|
||||
monkeypatch: pytest.MonkeyPatch, value: str
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", value)
|
||||
|
||||
with pytest.raises(ValueError, match="must be 0 or 1"):
|
||||
_ = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS
|
||||
|
||||
|
||||
def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False)
|
||||
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False)
|
||||
|
||||
@@ -12,6 +12,7 @@ import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from typing import Protocol
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
@@ -27,10 +28,39 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [-1, 0, 2, 3, 4, 2**127])
|
||||
def test_routed_experts_offset_validated_before_engine_submission(offset):
|
||||
"""Reject invalid offsets even when routed-expert output is disabled."""
|
||||
processor = Mock(spec=InputProcessor)
|
||||
processor.tokenizer = None
|
||||
processor.generation_config_fields = {}
|
||||
processor.renderer = Mock()
|
||||
processor.renderer.get_eos_token_id.return_value = None
|
||||
processor.vllm_config = Mock()
|
||||
params = SamplingParams(routed_experts_prompt_start=offset)
|
||||
prompt = {"type": "token", "prompt_token_ids": [1, 2, 3]}
|
||||
|
||||
if not 0 <= offset <= 3:
|
||||
with pytest.raises(VLLMValidationError, match="routed_experts_prompt_start"):
|
||||
InputProcessor.process_inputs(
|
||||
processor, "invalid", prompt, params, ("generate",)
|
||||
)
|
||||
params.routed_experts_prompt_start = 0
|
||||
|
||||
request = InputProcessor.process_inputs(
|
||||
processor, "valid", prompt, params, ("generate",)
|
||||
)
|
||||
assert (
|
||||
request.sampling_params.routed_experts_prompt_start
|
||||
== params.routed_experts_prompt_start
|
||||
)
|
||||
|
||||
|
||||
# --- Stop strings: public requests cap the number of stop strings ---------
|
||||
|
||||
|
||||
|
||||
@@ -897,3 +897,111 @@ class TestStreamingExtraction:
|
||||
}
|
||||
|
||||
assert args_text.count("replace_all") == 1
|
||||
|
||||
|
||||
class TestBareAndMalformedOpeners:
|
||||
"""Tests for bare and malformed tool call openers.
|
||||
|
||||
See https://github.com/vllm-project/vllm/issues/53431.
|
||||
"""
|
||||
|
||||
def test_bare_opener_non_streaming(self, parser, mock_request):
|
||||
model_output = '<|tool_call>:get_weather{location:<|"|>London<|"|>}<tool_call|>'
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "London"}
|
||||
|
||||
def test_documented_form_still_works(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}<tool_call|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
def test_bare_opener_multiple_args(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>:get_weather{"
|
||||
'location:<|"|>Tokyo<|"|>,unit:<|"|>celsius<|"|>}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "Tokyo", "unit": "celsius"}
|
||||
|
||||
def test_malformed_no_brace_bounded(self, parser, mock_request):
|
||||
model_output = "<|tool_call>call:bad_func no brace<tool_call|>"
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
name = result.tool_calls[0].function.name
|
||||
assert "<tool_call|>" not in name
|
||||
|
||||
def test_malformed_then_well_formed(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>call:bad_func no brace<tool_call|>"
|
||||
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
assert result.tool_calls[0].function.name == "bad_func no brace"
|
||||
assert result.tool_calls[1].function.name == "get_weather"
|
||||
assert "London" in result.tool_calls[1].function.arguments
|
||||
|
||||
def test_bare_malformed_then_well_formed(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>:bad_func no brace<tool_call|>"
|
||||
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
assert result.tool_calls[0].function.name == "bad_func no brace"
|
||||
assert result.tool_calls[1].function.name == "get_weather"
|
||||
|
||||
def test_three_mixed_calls(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>:bad no brace<tool_call|>"
|
||||
'<|tool_call>call:search{input:<|"|>hello<|"|>}<tool_call|>'
|
||||
"<|tool_call>:set{flag:true,count:5}<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 3
|
||||
assert result.tool_calls[0].function.name == "bad no brace"
|
||||
assert result.tool_calls[1].function.name == "search"
|
||||
assert result.tool_calls[2].function.name == "set"
|
||||
|
||||
def test_colon_in_content_not_broken(self, parser, mock_request):
|
||||
model_output = (
|
||||
"Hello: world"
|
||||
'<|tool_call>:get_weather{location:<|"|>London<|"|>}<tool_call|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.content is not None
|
||||
assert "Hello" in result.content
|
||||
|
||||
def test_bare_opener_empty_args(self, parser, mock_request):
|
||||
model_output = "<|tool_call>:set_status{}<tool_call|>"
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "set_status"
|
||||
|
||||
@@ -5,7 +5,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar, StructuralTag
|
||||
from xgrammar import Grammar, StructuralTag, builtin_structural_tag
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -81,10 +81,14 @@ def test_supported_structural_tag_models_include_vllm_builtins():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
|
||||
def test_deepseek_v41_format_constraints_ignore_parameter_schema(
|
||||
def test_deepseek_v41_older_xgrammar_keeps_format_constraints(
|
||||
choice,
|
||||
sample_tools_strict,
|
||||
monkeypatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False
|
||||
)
|
||||
tool_choice = (
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name=choice)
|
||||
@@ -128,7 +132,11 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
|
||||
reasoning=False,
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
call = '<|DSML| invoke name="get_weather">\n</|DSML| invoke>\n'
|
||||
call = (
|
||||
'<|DSML| invoke name="get_weather">\n'
|
||||
'<|DSML| parameter name="city" string="true">Paris</|DSML| parameter>\n'
|
||||
"</|DSML| invoke>\n"
|
||||
)
|
||||
assert _is_grammar_accept_string(
|
||||
grammar, "\n\n<|DSML| calls>\n" + call + "</|DSML| calls>"
|
||||
)
|
||||
@@ -141,6 +149,72 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
|
||||
reason="Requires XGrammar's DeepSeek V4.1 builtin",
|
||||
)
|
||||
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
|
||||
@pytest.mark.parametrize("reasoning", [False, True])
|
||||
def test_deepseek_v41_constrains_parameters_after_reasoning(
|
||||
sample_tools_strict, choice, reasoning
|
||||
):
|
||||
"""Strict calls must enforce the schema without requiring another think close."""
|
||||
tool_choice = (
|
||||
ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name=choice)
|
||||
)
|
||||
if choice == "get_weather"
|
||||
else choice
|
||||
)
|
||||
sample_tools_strict[0].function.parameters["additionalProperties"] = False
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
begin = '\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
|
||||
end = "</|DSML| invoke>\n</|DSML| calls>"
|
||||
parameter = (
|
||||
'<|DSML| parameter name="city" string="true">Paris</|DSML| parameter>\n'
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, begin + parameter + end)
|
||||
for invalid in (
|
||||
"", # missing required city
|
||||
parameter * 2,
|
||||
parameter.replace('name="city"', 'name="unknown"'),
|
||||
parameter.replace('string="true">Paris', 'string="false">42'),
|
||||
parameter.replace("| parameter", "|parameter"),
|
||||
'{"city":"Paris"}', # the encoder emits DSML parameters, not a JSON body
|
||||
):
|
||||
assert not _is_grammar_accept_string(grammar, begin + invalid + end)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, "reason</think>" + begin + parameter + end
|
||||
)
|
||||
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
|
||||
reason="Requires XGrammar's DeepSeek V4.1 builtin",
|
||||
)
|
||||
def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools):
|
||||
sample_tools[0].function.strict = False
|
||||
tag = get_model_structural_tag(
|
||||
"deepseek_v41", sample_tools, "required", reasoning=False
|
||||
)
|
||||
grammar = Grammar.from_structural_tag(tag)
|
||||
call = (
|
||||
'<|DSML| invoke name="get_weather">\n'
|
||||
'<|DSML| parameter name="extra" string="false">'
|
||||
'{"nested":[true,null,1.5]}</|DSML| parameter>\n'
|
||||
"</|DSML| invoke>\n"
|
||||
)
|
||||
output = "\n\n<|DSML| calls>\n" + call * 2 + "</|DSML| calls>"
|
||||
assert _is_grammar_accept_string(grammar, output)
|
||||
assert not _is_grammar_accept_string(
|
||||
grammar, output.replace('{"nested":[true,null,1.5]}', "invalid")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
|
||||
def test_get_model_structural_tag_supports_all_xgrammar_builtins(
|
||||
model: str,
|
||||
|
||||
@@ -123,6 +123,24 @@ def test_suppressing_works_while_compiling(monkeypatch):
|
||||
with_gpu_sync_check(suppressed)()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_copy_checker_is_traceable_while_compiling(monkeypatch):
|
||||
"""The copy wrapper must bypass ContextVars while Dynamo is tracing."""
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
|
||||
def convert_like(x, like):
|
||||
return x.to(device=like.device, dtype=like.dtype)
|
||||
|
||||
compiled = torch.compile(convert_like, fullgraph=True)
|
||||
result = compiled(
|
||||
torch.ones(4, device="cuda", dtype=torch.float64),
|
||||
torch.ones(4, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
assert result.dtype == torch.float32
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_sync_debug_mode_restored_after_checked_call(monkeypatch):
|
||||
"""The mode is armed only for the duration of a checked call. Leaving it
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import (
|
||||
OMP_NUM_THREADS_SET_BY_VLLM,
|
||||
async_tensor_h2d,
|
||||
available_cpu_count,
|
||||
common_broadcastable_dtype,
|
||||
current_stream,
|
||||
@@ -192,3 +194,26 @@ def test_runtime_threads_override_vllm_set_omp_num_threads(
|
||||
torch.set_num_threads(3)
|
||||
set_torch_threads_for_runtime()
|
||||
assert torch.get_num_threads() == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device", ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]
|
||||
)
|
||||
def test_async_tensor_h2d_staging(device):
|
||||
"""Staging must preserve values while normalizing dtype and layout."""
|
||||
# Non-contiguous source with a dtype conversion: would previously sync.
|
||||
src = torch.arange(12, dtype=torch.int64).reshape(3, 4).T
|
||||
result = async_tensor_h2d(src, device=device, dtype=torch.int32)
|
||||
assert result.dtype == torch.int32
|
||||
assert result.is_contiguous()
|
||||
assert torch.equal(result.cpu(), src.to(torch.int32))
|
||||
|
||||
# numpy and list sources still work.
|
||||
assert torch.equal(
|
||||
async_tensor_h2d(np.arange(4, dtype=np.int64), device=device),
|
||||
torch.arange(4, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32),
|
||||
torch.tensor([1, 2, 3], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.v1.attention.ops.dcp import (
|
||||
mask_dcp_empty_shards_,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
requires_accelerator = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="needs a CUDA or ROCm device for the Triton pack kernel",
|
||||
)
|
||||
@@ -44,6 +44,7 @@ def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc)
|
||||
@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)])
|
||||
@pytest.mark.parametrize("tokens_per_req", [1, 2, 3])
|
||||
@pytest.mark.parametrize("num_pad_rows", [0, 2])
|
||||
@requires_accelerator
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens_list",
|
||||
[
|
||||
@@ -97,6 +98,7 @@ def test_fused_mask_matches_eager(
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("seed", range(8))
|
||||
@requires_accelerator
|
||||
def test_fused_mask_matches_eager_ragged(dtype, seed):
|
||||
"""Non-uniform per-request query lengths.
|
||||
|
||||
@@ -137,6 +139,7 @@ def test_fused_mask_matches_eager_ragged(dtype, seed):
|
||||
)
|
||||
|
||||
|
||||
@requires_accelerator
|
||||
def test_mask_disabled_is_unmasked():
|
||||
"""Passing no seq_lens/query_start_loc must leave the LSE untouched."""
|
||||
torch.manual_seed(0)
|
||||
@@ -149,3 +152,14 @@ def test_mask_disabled_is_unmasked():
|
||||
|
||||
packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None)
|
||||
assert torch.isfinite(packed.float()).all()
|
||||
|
||||
|
||||
def test_eager_mask_handles_rank_with_no_local_sequences():
|
||||
"""Padded graph rows are all empty when a DCP rank has no sequences."""
|
||||
lse = torch.randn(8, 4, dtype=torch.float32)
|
||||
seq_lens = torch.empty(0, dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0], dtype=torch.int32)
|
||||
|
||||
mask_dcp_empty_shards_(lse, seq_lens, query_start_loc)
|
||||
|
||||
assert torch.isneginf(lse).all()
|
||||
|
||||
@@ -151,32 +151,44 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
assert kwargs == {}
|
||||
|
||||
|
||||
def test_builder_attaches_its_state(monkeypatch):
|
||||
@pytest.mark.parametrize("use_mha", [False, True])
|
||||
@pytest.mark.parametrize("num_decodes", [0, 1])
|
||||
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
|
||||
"""MHA prefill rows must not make the MQA kernel read beyond its query."""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
builder.state = FakeState(TOPK)
|
||||
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
|
||||
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
|
||||
metadata.state = None
|
||||
metadata.num_prefills = 1
|
||||
metadata.num_decode_tokens = num_decodes
|
||||
monkeypatch.setattr(
|
||||
sm90_mod.FlashInferMLASparseMetadataBuilder,
|
||||
"build",
|
||||
lambda *_args, **_kwargs: metadata,
|
||||
)
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
|
||||
num_reqs=num_decodes + 1,
|
||||
query_start_loc_cpu=torch.tensor(
|
||||
[0, 1, 6] if num_decodes else [0, 5], dtype=torch.int32
|
||||
),
|
||||
seq_lens_cpu_upper_bound=torch.tensor(
|
||||
[1402, 5] if num_decodes else [5], dtype=torch.int32
|
||||
),
|
||||
positions=None,
|
||||
)
|
||||
|
||||
result = builder.build(0, cam)
|
||||
|
||||
assert result.state is builder.state
|
||||
assert builder.state.plan_calls[0][0] == 1
|
||||
assert builder.state.plan_calls[0][1].tolist() == [1]
|
||||
expected_lens = [1402] if num_decodes else []
|
||||
if not use_mha:
|
||||
expected_lens += [1, 2, 3, 4, 5]
|
||||
assert builder.state.plan_calls[0][0] == len(expected_lens)
|
||||
assert builder.state.plan_calls[0][1].tolist() == expected_lens
|
||||
|
||||
|
||||
def test_plan_uses_state_params(monkeypatch):
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for FlashInfer sparse MLA backend constraints."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
_required_workspace_bytes,
|
||||
compute_trtllm_sparse_mla_workspace_bytes,
|
||||
)
|
||||
|
||||
# Reporter config from vllm#50781: GLM-5.2 (64 q heads), TP=8, DCP=8,
|
||||
# max_num_batched_tokens=16384, FlashInfer 0.6.14.
|
||||
_REPORTER_HEADS_PER_RANK = 64 // 8
|
||||
_REPORTER_DCP = 8
|
||||
_REPORTER_MAX_TOKENS = 16384
|
||||
# From the reported crash: trtllm-gen requested exactly this many bytes for
|
||||
# trtllm_gen_softmax_workspace after an 8 MiB counter carve had left
|
||||
# 404,750,336 of the 413,138,944-byte default buffer.
|
||||
_REPORTER_OBSERVED_SOFTMAX_BYTES = 1_611_661_312
|
||||
_REPORTER_OBSERVED_PRECARVE_BYTES = 8_388_608
|
||||
|
||||
|
||||
def test_reporter_softmax_carve_is_reproduced_byte_exactly():
|
||||
# The crashing step scheduled 12288 tokens (a single 12K-token request);
|
||||
# the softmax slab formula must reproduce the kernel's request exactly:
|
||||
# 8 * (8 heads/rank * 8 dcp) * 12288 tokens * 256 + 1 MiB guard.
|
||||
softmax_bytes = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes=0,
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=12288,
|
||||
)
|
||||
assert softmax_bytes == _REPORTER_OBSERVED_SOFTMAX_BYTES
|
||||
|
||||
|
||||
def test_reporter_config_covers_observed_overflow():
|
||||
computed = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert computed >= (
|
||||
_REPORTER_OBSERVED_SOFTMAX_BYTES + _REPORTER_OBSERVED_PRECARVE_BYTES
|
||||
)
|
||||
|
||||
|
||||
def test_non_dcp_size_is_unchanged():
|
||||
computed = compute_trtllm_sparse_mla_workspace_bytes(
|
||||
base_workspace_bytes=_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
dcp_world_size=1,
|
||||
num_heads_per_rank=128,
|
||||
max_num_batched_tokens=65536,
|
||||
)
|
||||
assert computed == _DEFAULT_WORKSPACE_BUFFER_SIZE
|
||||
|
||||
|
||||
def test_combined_pcp_dcp_rejects_flashinfer_sparse():
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=4,
|
||||
decode_context_parallel_size=4,
|
||||
),
|
||||
model_config=None,
|
||||
)
|
||||
with patch("vllm.config.get_current_vllm_config", return_value=config):
|
||||
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=10),
|
||||
)
|
||||
|
||||
assert reason is not None
|
||||
assert "use FLASHMLA_SPARSE" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size,dcp_size", [(4, 1), (1, 4)])
|
||||
def test_single_context_parallel_mode_allows_flashinfer_sparse(
|
||||
pcp_size: int, dcp_size: int
|
||||
):
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=pcp_size,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
),
|
||||
model_config=None,
|
||||
)
|
||||
with patch("vllm.config.get_current_vllm_config", return_value=config):
|
||||
reason = FlashInferMLASparseTRTLLMBackend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=10),
|
||||
)
|
||||
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_default_constant_matches_envs_default(monkeypatch):
|
||||
from vllm import envs
|
||||
|
||||
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
|
||||
assert _DEFAULT_WORKSPACE_BUFFER_SIZE == envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE
|
||||
|
||||
|
||||
def test_env_unset_returns_computed(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", raising=False)
|
||||
required = _required_workspace_bytes(
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert required == compute_trtllm_sparse_mla_workspace_bytes(
|
||||
_DEFAULT_WORKSPACE_BUFFER_SIZE,
|
||||
_REPORTER_DCP,
|
||||
_REPORTER_HEADS_PER_RANK,
|
||||
_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
|
||||
|
||||
def test_env_override_below_computed_is_respected_with_warning(monkeypatch):
|
||||
override = 100 * 1024 * 1024
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
|
||||
) as mock_logger:
|
||||
required = _required_workspace_bytes(
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert required == override
|
||||
mock_logger.warning_once.assert_called_once()
|
||||
|
||||
|
||||
def test_env_override_above_computed_is_respected_without_warning(monkeypatch):
|
||||
override = 8 * 1024 * 1024 * 1024
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(override))
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.mla.flashinfer_mla_sparse.logger"
|
||||
) as mock_logger:
|
||||
required = _required_workspace_bytes(
|
||||
dcp_world_size=_REPORTER_DCP,
|
||||
num_heads_per_rank=_REPORTER_HEADS_PER_RANK,
|
||||
max_num_batched_tokens=_REPORTER_MAX_TOKENS,
|
||||
)
|
||||
assert required == override
|
||||
mock_logger.warning_once.assert_not_called()
|
||||
@@ -307,6 +307,23 @@ def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape():
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_reuses_device_rank_tensor(monkeypatch):
|
||||
seq_lens = torch.tensor([0, 1, 7, 8, 17], dtype=torch.int32)
|
||||
world = 4
|
||||
rank = 2
|
||||
rank_tensor = torch.tensor(rank, dtype=torch.int32)
|
||||
expected = get_dcp_local_seq_lens(seq_lens, world, rank, 2)
|
||||
|
||||
def fail_tensor_construction(*args, **kwargs):
|
||||
raise AssertionError("rank tensor must be reused")
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(torch, "tensor", fail_tensor_construction)
|
||||
actual = get_dcp_local_seq_lens(seq_lens, world, rank_tensor, 2)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_must_run_after_decode_expansion():
|
||||
world = 2
|
||||
rank = 1
|
||||
@@ -979,7 +996,7 @@ def test_pcp_plan_deinterleave_restores_global_order(
|
||||
|
||||
monkeypatch.setattr("vllm.v1.attention.backends.mla.indexer.PIN_MEMORY", False)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.indexer.async_copy_to_gpu",
|
||||
"vllm.v1.attention.backends.mla.indexer.async_tensor_h2d",
|
||||
lambda x, device: x.to(device),
|
||||
)
|
||||
scheduled = np.array(req_lens, dtype=np.int64)
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch
|
||||
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
|
||||
from vllm.models.deepseek_v4_1.sparse_mla import (
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
@@ -179,6 +179,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
),
|
||||
)
|
||||
@@ -193,6 +194,37 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
}
|
||||
|
||||
|
||||
def test_indexer_warmup_includes_pcp_normalized_dcp_key(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.indexer.get_dcp_group",
|
||||
lambda: SimpleNamespace(rank_in_group=2),
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(compress_ratios=[32], index_kpool=32)
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=4,
|
||||
prefill_context_parallel_size=4,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
),
|
||||
)
|
||||
|
||||
keys = BuildPrefillChunkMetadataKernel().get_warmup_keys(config)
|
||||
|
||||
# Triton's compile key normalizes generic i32 values to 2 and divisible
|
||||
# i32 values (including zero) to 16.
|
||||
assert {(key.dcp_rank, key.dcp_world) for key in keys} == {(2, 2), (16, 1)}
|
||||
assert {
|
||||
(
|
||||
key.input_variant.is_aligned("uncompressed_seq_lens"),
|
||||
key.input_variant.is_aligned("cu_compressed_seq_lens"),
|
||||
)
|
||||
for key in keys
|
||||
} == {(False, False), (False, True), (True, False), (True, True)}
|
||||
|
||||
|
||||
def test_compressed_slot_mapping_warmup_includes_index_kpool():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=256),
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention import mla_attention as mla_attention_module
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLAAttention,
|
||||
MLACommonBaseImpl,
|
||||
QueryLenSupport,
|
||||
_DecodeConcatQuantFP8,
|
||||
_use_masked_mha,
|
||||
@@ -166,6 +167,25 @@ def test_glm5_flashinfer_masked_mha_routing(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64, 0], ids=["rope", "nope"])
|
||||
def test_concat_k_nope_k_pe_matches_torch_cat(qk_rope_head_dim):
|
||||
"""The K concat used by the MLA prefill context loop must equal torch.cat of
|
||||
k_nope with the broadcast k_pe; with no RoPE part it returns k_nope itself
|
||||
instead of allocating and copying."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_heads, qk_nope_head_dim = 5, 4, 256
|
||||
k_nope = torch.randn(num_tokens, num_heads, qk_nope_head_dim, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, 1, qk_rope_head_dim, dtype=torch.bfloat16)
|
||||
impl = SimpleNamespace(_use_flashinfer_concat_mla_k=False)
|
||||
|
||||
k = MLACommonBaseImpl._concat_k_nope_k_pe(impl, k_nope, k_pe)
|
||||
|
||||
expected = torch.cat([k_nope, k_pe.expand(-1, num_heads, -1)], dim=-1)
|
||||
assert k.shape == (num_tokens, num_heads, qk_nope_head_dim + qk_rope_head_dim)
|
||||
torch.testing.assert_close(k, expected, rtol=0, atol=0)
|
||||
assert (k.data_ptr() == k_nope.data_ptr()) == (qk_rope_head_dim == 0)
|
||||
|
||||
|
||||
def test_masked_mha_routing_is_dimension_specific():
|
||||
assert _use_masked_mha(
|
||||
backend_name="FLASHMLA_SPARSE",
|
||||
|
||||
@@ -313,6 +313,40 @@ class TestBackendValidation:
|
||||
)
|
||||
assert invalid_reasons == []
|
||||
|
||||
def test_flash_attn_accepts_glm53_flash_nope_dimensions(self):
|
||||
"""(256, 0, 256) runs the same FA kernels as GLM-5's (192, 64, 256); a
|
||||
RoPE-carrying 320-wide query does not."""
|
||||
try:
|
||||
from vllm.v1.attention.backends.mla.prefill.flash_attn import (
|
||||
FlashAttnPrefillBackend,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("MLA prefill backend not available")
|
||||
return
|
||||
|
||||
capability = DeviceCapability(major=10, minor=0)
|
||||
|
||||
def validate(qk_rope_head_dim: int) -> list[str]:
|
||||
selector_config = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=256,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=256,
|
||||
),
|
||||
)
|
||||
with patch.object(
|
||||
FlashAttnPrefillBackend, "is_available", return_value=True
|
||||
):
|
||||
return FlashAttnPrefillBackend.validate_configuration(
|
||||
capability, selector_config
|
||||
)
|
||||
|
||||
assert validate(qk_rope_head_dim=0) == []
|
||||
invalid_reasons = validate(qk_rope_head_dim=64)
|
||||
assert len(invalid_reasons) == 1
|
||||
assert "supported MLA dimensions" in invalid_reasons[0]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
|
||||
@@ -176,3 +176,8 @@ def test_pad_unpad_round_trip_preserves_head_order(num_heads):
|
||||
|
||||
assert unpadded.shape == q.shape
|
||||
torch.testing.assert_close(unpadded, q)
|
||||
|
||||
|
||||
def test_a_non_causal_block_never_routes_to_gluon(gluon_available):
|
||||
"""A small-head block that normally uses Gluon must use ASM when non-causal."""
|
||||
assert not AiterMLAHelper.use_gluon_verify(12, 8, "auto", causal=False)
|
||||
|
||||
@@ -130,6 +130,7 @@ def _builder(
|
||||
_mla_q_dtype=torch.bfloat16,
|
||||
_mla_kv_dtype=torch.bfloat16,
|
||||
decode_attn_out_dtype=torch.bfloat16,
|
||||
_decode_causal=True,
|
||||
)
|
||||
# Bound to the stub rather than faked: the verify flatten's per-row view is
|
||||
# part of what _build_decode is being tested for.
|
||||
@@ -155,6 +156,25 @@ def test_backend_declares_uniform_batch_support():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("causal", [True, False])
|
||||
def test_build_hands_common_causal_flag_to_aiter_builder(monkeypatch, causal):
|
||||
seen = []
|
||||
|
||||
def fake_common_build(self, *args):
|
||||
seen.append(self._decode_causal)
|
||||
return SimpleNamespace(decode=None, prefill=None)
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla.MLACommonMetadataBuilder, "build", fake_common_build
|
||||
)
|
||||
builder = object.__new__(AiterMLAMetadataBuilder)
|
||||
builder._fp8_prefill_enabled = False
|
||||
|
||||
builder.build(0, SimpleNamespace(causal=causal))
|
||||
|
||||
assert seen == [causal]
|
||||
|
||||
|
||||
def test_dcp_verify_row_view_is_causal_per_row():
|
||||
"""Row t must cover this rank's share of positions [0, seq_len - qlen + t].
|
||||
|
||||
@@ -949,3 +969,81 @@ def test_persistent_metadata_gate_without_gluon_build(
|
||||
|
||||
assert metadata.has_persistent_metadata is expect_persistent
|
||||
assert get_mla_metadata_v1.called is expect_persistent
|
||||
|
||||
|
||||
def _build_non_causal(monkeypatch, *, num_heads, kv_cache_dtype, qlen, mtp_qlen):
|
||||
"""Drive _build_decode with causal=False and hand back the aiter mock."""
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"aiter",
|
||||
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
|
||||
)
|
||||
builder = _builder(
|
||||
mtp_decode_qlen=mtp_qlen,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
num_heads=num_heads,
|
||||
)
|
||||
builder._decode_causal = False
|
||||
metadata = AiterMLAMetadataBuilder._build_decode(
|
||||
builder,
|
||||
block_table_tensor=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
seq_lens_device=torch.tensor([7, 5], dtype=torch.int32),
|
||||
max_seq_len=7,
|
||||
query_start_loc_cpu=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
query_start_loc_device=torch.tensor([0, qlen, 2 * qlen], dtype=torch.int32),
|
||||
num_decode_tokens=2 * qlen,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
return metadata, get_mla_metadata_v1
|
||||
|
||||
|
||||
def test_non_causal_build_hands_the_mask_to_aiter(monkeypatch):
|
||||
"""The schedule carries the mask, so is_causal must follow the block."""
|
||||
metadata, get_mla_metadata_v1 = _build_non_causal(
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=8, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
# 6th positional arg of get_mla_metadata_v1 is is_causal.
|
||||
assert get_mla_metadata_v1.call_args.args[5] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [12, 16, 48])
|
||||
def test_a_two_token_fp8_block_is_refused_without_a_fold(monkeypatch, num_heads):
|
||||
"""16-head (and padded-to-16) qlen-2 fp8 has no non-causal kernel.
|
||||
|
||||
48/80/112 (and 33 padded to 48) fold to H16 while keeping Q2, so they
|
||||
must be refused the same way as native 16-head qlen 2.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="2-token"):
|
||||
_build_non_causal(
|
||||
monkeypatch,
|
||||
num_heads=num_heads,
|
||||
kv_cache_dtype="fp8",
|
||||
qlen=2,
|
||||
mtp_qlen=8,
|
||||
)
|
||||
|
||||
|
||||
def test_a_two_token_fp8_block_is_allowed_when_folded(monkeypatch):
|
||||
"""32 heads at qlen 2 folds onto the 16-head / 4-token kernel."""
|
||||
metadata, get_mla_metadata_v1 = _build_non_causal(
|
||||
monkeypatch,
|
||||
num_heads=32,
|
||||
kv_cache_dtype="fp8",
|
||||
qlen=2,
|
||||
mtp_qlen=8,
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
assert get_mla_metadata_v1.called
|
||||
|
||||
|
||||
def test_a_two_token_bf16_block_is_allowed(monkeypatch):
|
||||
"""The bf16 fold has no such hole, so the guard must not fire there."""
|
||||
metadata, _ = _build_non_causal(
|
||||
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
|
||||
)
|
||||
assert metadata.has_persistent_metadata
|
||||
|
||||
@@ -22,16 +22,24 @@ from tests.v1.attention.utils import (
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import HiSparseConfig, SpeculativeConfig, set_current_vllm_config
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
from vllm.config import (
|
||||
CUDAGraphMode,
|
||||
HiSparseConfig,
|
||||
SpeculativeConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import mla_attention
|
||||
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES,
|
||||
SparseMLACommonImpl,
|
||||
SparseMLAPrefillMetadata,
|
||||
_is_masked_mha_available,
|
||||
_masked_mha_workspace_fits,
|
||||
_topk_mask_shape,
|
||||
_use_dense_mha_prefill,
|
||||
@@ -896,6 +904,75 @@ def test_triton_convert_req_index_to_global_index_decode_only(
|
||||
torch.testing.assert_close(result, reference_result, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_index_group_convert_during_piecewise_capture():
|
||||
"""In piecewise cudagraph mode the indexer is captured while the convert
|
||||
runs in the following eager break. The convert's side stream must not
|
||||
wait on ``logical_topk_ready`` there: an event recorded inside a captured
|
||||
segment is graph-local, and an eager wait on it raises
|
||||
cudaErrorInvalidValue."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens, num_topk, num_requests, blocks_per_req, block_size = 8, 128, 4, 4, 16
|
||||
|
||||
logical_topk_indices = torch.randint(
|
||||
0,
|
||||
block_size * blocks_per_req,
|
||||
(num_tokens, num_topk),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
builder = SparseMLAIndexGroupBuilder(logical_topk_indices)
|
||||
group, layer_index = builder.register_layer(is_index_producing_layer=True)
|
||||
assert layer_index == 0 and group.has_indexer
|
||||
|
||||
req_id = torch.arange(num_tokens, dtype=torch.int32, device=device) % num_requests
|
||||
block_table = torch.arange(
|
||||
num_requests * blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(num_requests, blocks_per_req)
|
||||
attn_metadata = SimpleNamespace(
|
||||
req_id_per_token=req_id, block_table=block_table, block_size=block_size
|
||||
)
|
||||
expected = triton_convert_req_index_to_global_index(
|
||||
req_id,
|
||||
block_table,
|
||||
logical_topk_indices,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk,
|
||||
)
|
||||
|
||||
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
|
||||
marker = torch.zeros(1, device=device)
|
||||
capture_stream = torch.cuda.Stream(device=device)
|
||||
# torch.cuda.stream routes through vLLM's patched torch.cuda.set_stream,
|
||||
# so current_stream() tracks the capture stream as in production.
|
||||
with (
|
||||
torch.cuda.stream(capture_stream),
|
||||
set_forward_context(
|
||||
None, vllm_config, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
|
||||
),
|
||||
):
|
||||
capture = BreakableCUDAGraphCapture()
|
||||
with capture:
|
||||
marker.add_(1)
|
||||
# Record inside a captured segment, convert in the eager
|
||||
# break — mirrors mla.py -> unified_mla_attention_with_output.
|
||||
group.set_logical_topk_ready(layer_index)
|
||||
capture.add_eager(
|
||||
lambda: group.convert_logical_to_physical_topk(
|
||||
layer_index,
|
||||
logical_topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=False,
|
||||
)
|
||||
)
|
||||
marker.add_(1)
|
||||
capture.replay()
|
||||
capture_stream.synchronize()
|
||||
|
||||
result = group.physical_topk_indices[:num_tokens]
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
@@ -1165,6 +1242,33 @@ def test_masked_mha_workspace_guards_long_routing_policy(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_dims", "kv_cache_dtype", "fa_version", "expected"),
|
||||
[
|
||||
pytest.param((128, 512, 128, 64, 128), "auto", 4, True, id="deepseek_v32"),
|
||||
pytest.param((64, 512, 192, 64, 256), "auto", 4, True, id="glm5"),
|
||||
pytest.param((64, 512, 256, 0, 256), "auto", 4, True, id="glm53_flash_nope"),
|
||||
pytest.param((64, 512, 256, 0, 256), "fp8", 4, False, id="quantized_kv"),
|
||||
pytest.param((64, 512, 256, 0, 256), "auto", 3, False, id="no_fa4"),
|
||||
pytest.param((64, 512, 256, 64, 256), "auto", 4, False, id="rope_320"),
|
||||
pytest.param((128, 512, 256, 0, 256), "auto", 4, False, id="wrong_heads"),
|
||||
],
|
||||
)
|
||||
def test_is_masked_mha_available_model_dims(
|
||||
monkeypatch, model_dims, kv_cache_dtype, fa_version, expected
|
||||
):
|
||||
"""The allow-list gates masked MHA per exact model geometry: the DeepSeek-V3.2,
|
||||
GLM-5 and NoPE GLM-5.3-Flash layouts on an SM100-family GPU with FA4 and an
|
||||
unquantized KV cache, nothing else."""
|
||||
import vllm.model_executor.layers.attention.sparse_mla_attention as mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
mod.current_platform, "is_device_capability_family", lambda family: True
|
||||
)
|
||||
monkeypatch.setattr(mod, "get_flash_attn_version", lambda **kwargs: fa_version)
|
||||
assert _is_masked_mha_available(*model_dims, kv_cache_dtype) is expected
|
||||
|
||||
|
||||
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
|
||||
"""Request count and context chunk length are independent multipliers."""
|
||||
base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
|
||||
@@ -1198,6 +1302,7 @@ PREFILL_BATCH_SPECS = {
|
||||
[
|
||||
pytest.param(128, 128, 64, 128, id="deepseek_hd192_v128"),
|
||||
pytest.param(64, 192, 64, 256, id="glm5_hd256_v256"),
|
||||
pytest.param(64, 256, 0, 256, id="glm53_flash_nope_hd256_v256"),
|
||||
],
|
||||
)
|
||||
def test_sparse_backend_prefill_correctness(
|
||||
|
||||
@@ -299,7 +299,14 @@ def test_a_junction_past_the_prompt_falls_back_and_registers_nothing():
|
||||
running = request.num_computed_tokens
|
||||
assert not mamba.req_to_blocks["r"][running // block_size].is_null
|
||||
request.shared_prefix_boundary = running
|
||||
assert mamba._cache_partial_tail_block(request, running) is None
|
||||
assert (
|
||||
mamba._cache_partial_tail_block(
|
||||
request,
|
||||
running,
|
||||
retention_interval=manager.coordinator.retention_interval,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eagle_group", [None, 0])
|
||||
|
||||
@@ -615,6 +615,77 @@ def test_internal_checkpoint_uses_partial_hash_lifecycle():
|
||||
assert num_computed == 112
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"retention_interval,transient_published",
|
||||
[(None, True), (0, False)],
|
||||
)
|
||||
def test_internal_checkpoint_publication_respects_retention(
|
||||
retention_interval, transient_published
|
||||
):
|
||||
"""With retention_interval=0, a mid-prompt internal checkpoint is
|
||||
request-local and must not enter the prefix cache; the prompt-end
|
||||
checkpoint keeps the existing publication behavior. Dense retention
|
||||
(None) is unchanged.
|
||||
"""
|
||||
hash_block_size = 16
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=32,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
manager.coordinator.retention_interval = retention_interval
|
||||
|
||||
request = make_request("producer", list(range(240)), hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
|
||||
|
||||
# Mid-prompt chunk: exports a transient checkpoint at 112.
|
||||
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
|
||||
transient_hash = request.block_hashes[112 // hash_block_size - 1]
|
||||
transient_hit = manager.block_pool.get_cached_block(transient_hash, [1])
|
||||
assert (transient_hit is not None) == transient_published
|
||||
|
||||
request.num_computed_tokens = 128
|
||||
manager.new_step_starts()
|
||||
|
||||
# Final chunk: the prompt-end checkpoint at 224 stays published.
|
||||
assert manager.allocate_slots(request, 112) is not None
|
||||
end_hash = request.block_hashes[224 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
|
||||
|
||||
|
||||
def test_transient_checkpoint_evicts_retained_boundary_hash():
|
||||
"""With retention_interval=0, the checkpoint slot can coincide with a
|
||||
reachable boundary block that this step's full-block pass just hashed.
|
||||
The checkpoint state overwrites the slot, so the stale boundary hash
|
||||
must be evicted rather than left pointing at the wrong state.
|
||||
"""
|
||||
hash_block_size = 16
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=32,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
manager.coordinator.retention_interval = 0
|
||||
|
||||
request = make_request("producer", list(range(240)), hash_block_size, sha256)
|
||||
# Block 2 ends at the shared-prefix junction 96, so the full-block pass
|
||||
# hashes it as a retained boundary.
|
||||
request.shared_prefix_boundary = 96
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
|
||||
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
|
||||
|
||||
# The same slot is the transient checkpoint block (state@112): both the
|
||||
# boundary hash and the checkpoint itself must stay unpublished.
|
||||
boundary_hash = request.block_hashes[96 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(boundary_hash, [1]) is None
|
||||
checkpoint_hash = request.block_hashes[112 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
|
||||
|
||||
|
||||
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
|
||||
hash_block_size = mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
|
||||
@@ -334,7 +334,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
request = create_requests(num_requests=1, num_tokens=1)[0]
|
||||
request.structured_output_request = Mock()
|
||||
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
|
||||
request.structured_output_request.grammar.accept_tokens.return_value = False
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.num_computed_tokens = request.num_tokens
|
||||
request.num_output_placeholders = 1
|
||||
@@ -343,10 +342,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.connector = None
|
||||
scheduler.ec_connector = None
|
||||
scheduler.structured_output_manager = Mock()
|
||||
scheduler.structured_output_manager.should_advance.return_value = True
|
||||
scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
|
||||
lambda request, new_token_ids: new_token_ids
|
||||
)
|
||||
scheduler.structured_output_manager.accept_tokens.return_value = False
|
||||
scheduler.requests = {request.request_id: request}
|
||||
scheduler.running = [request]
|
||||
scheduler.waiting = Mock()
|
||||
|
||||
@@ -3208,6 +3208,45 @@ def test_mla_with_incompatible_swa_uses_one_full_allocation_group(caplog_vllm):
|
||||
assert "attention compute is unchanged" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_hidden_states_with_tp_scales_page_size():
|
||||
"""When TP shrinks KV pages below the hidden-state per-token cost,
|
||||
get_kv_cache_groups must scale up target block sizes so that the
|
||||
common page accommodates the unsharded hidden states."""
|
||||
# Simulate TP=4 sharding a model with 8 KV heads → 2 per rank.
|
||||
# KV page = block_size(16) * num_kv_heads(2) * head_size(64) * dtype(2)
|
||||
# = 16 * 2 * 64 * 2 = 4096 bytes.
|
||||
kv_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
# Hidden-state per-token cost = num_hidden_states(6) * hidden_size(512)
|
||||
# * dtype(2) = 6144 bytes, which exceeds the 4096-byte KV page.
|
||||
hs_spec = HiddenStateCacheSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=6,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
specs = {
|
||||
"target.0.attn": kv_spec,
|
||||
"target.1.attn": kv_spec,
|
||||
"cache_only_layers.48": hs_spec,
|
||||
}
|
||||
|
||||
groups = get_kv_cache_groups(_grouping_config(), specs)
|
||||
|
||||
# The hidden-state layer should be present and no assertion should fire.
|
||||
all_layers = {name for g in groups for name in g.layer_names}
|
||||
assert "cache_only_layers.48" in all_layers
|
||||
|
||||
# The target group block sizes must have been scaled up.
|
||||
for g in groups:
|
||||
if "cache_only_layers.48" not in g.layer_names:
|
||||
assert g.kv_cache_spec.block_size > kv_spec.block_size
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_kind_prefers_specific_attention_subclasses():
|
||||
assert get_kv_cache_spec_kind(new_mla_spec()) == KVCacheSpecKind.MLA_ATTENTION
|
||||
|
||||
@@ -4135,9 +4174,9 @@ def test_draft_group_not_annotated_without_spec_decode():
|
||||
|
||||
|
||||
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
|
||||
# No group carries the draft marker, so every consumer falls back to
|
||||
# flagging all groups -- including Mamba ones, which then can never report
|
||||
# a hit. That is silent today; it must at least be visible.
|
||||
# No group carries the draft marker, so consumers fall back to
|
||||
# conservative behavior that silently breaks reuse for Mamba groups.
|
||||
# That must at least be visible.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
|
||||
)
|
||||
@@ -4146,7 +4185,20 @@ def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
|
||||
assert "no KV cache group could be identified as the draft model's" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
assert "Mamba groups" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_unidentifiable_draft_without_mamba_does_not_warn(caplog_vllm):
|
||||
# Pure-attention models degrade gracefully under the consumers'
|
||||
# conservative fallback (a one-block hit drop at most), so the warning
|
||||
# stays silent to avoid noise on every unannotated EAGLE deployment.
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64),
|
||||
"target.attn.1": new_mla_spec(block_size=64),
|
||||
}
|
||||
groups = get_kv_cache_groups(_spec_decode_grouping_config(), specs)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
SamplingMaskLists,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
@@ -131,6 +132,44 @@ def test_add_requests():
|
||||
assert len(scheduler.waiting) == i + 1
|
||||
|
||||
|
||||
def test_routed_experts_prompt_start_at_prompt_end():
|
||||
"""A prompt-end offset should return empty routing data, not crash."""
|
||||
scheduler = create_scheduler()
|
||||
(request,) = create_requests(num_requests=1)
|
||||
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
|
||||
|
||||
manager = Mock()
|
||||
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
scheduler.enable_return_routed_experts = True
|
||||
scheduler.routed_experts_mgr = manager
|
||||
scheduler._re_block_ids = {request.request_id: []}
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[0]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
|
||||
slot_mapping=np.arange(num_tokens),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
manager.get.assert_called_once_with(
|
||||
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
|
||||
)
|
||||
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
|
||||
|
||||
|
||||
def test_finish_request():
|
||||
scheduler = create_scheduler()
|
||||
requests = create_requests(num_requests=10)
|
||||
@@ -3635,7 +3674,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
)
|
||||
request.structured_output_request = Mock()
|
||||
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
|
||||
request.structured_output_request.grammar.accept_tokens.return_value = False
|
||||
request.status = RequestStatus.RUNNING
|
||||
request.num_computed_tokens = request.num_tokens
|
||||
|
||||
@@ -3643,10 +3681,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
scheduler.connector = None
|
||||
scheduler.ec_connector = None
|
||||
scheduler.structured_output_manager = Mock()
|
||||
scheduler.structured_output_manager.should_advance.return_value = True
|
||||
scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
|
||||
lambda request, new_token_ids: new_token_ids
|
||||
)
|
||||
scheduler.structured_output_manager.accept_tokens.return_value = False
|
||||
scheduler.requests = {request.request_id: request}
|
||||
scheduler.running = [request]
|
||||
scheduler.waiting = Mock()
|
||||
@@ -3695,8 +3730,8 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
)
|
||||
engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
|
||||
|
||||
request.structured_output_request.grammar.accept_tokens.assert_called_once_with(
|
||||
request.request_id, [123]
|
||||
scheduler.structured_output_manager.accept_tokens.assert_called_once_with(
|
||||
request, [123]
|
||||
)
|
||||
assert request.resumable is False
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
@@ -6534,16 +6569,15 @@ def _decode_ready_request(scheduler):
|
||||
return request
|
||||
|
||||
|
||||
def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
|
||||
"""ngram_gpu pads unfilled draft slots with -1; update_draft_token_ids must
|
||||
def test_update_draft_token_ids_strips_ngram_padding():
|
||||
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
|
||||
strip them before grammar.validate_tokens (which otherwise raises)."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
grammar = _RecordingGrammar()
|
||||
request.structured_output_request = SimpleNamespace(grammar=grammar)
|
||||
monkeypatch.setattr(
|
||||
scheduler.structured_output_manager, "should_advance", lambda req: True
|
||||
request.structured_output_request = SimpleNamespace(
|
||||
grammar=grammar, reasoning_ended=True
|
||||
)
|
||||
|
||||
scheduler.update_draft_token_ids(
|
||||
@@ -6554,16 +6588,15 @@ def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
|
||||
assert request.spec_token_ids == [10, 11]
|
||||
|
||||
|
||||
def test_update_draft_token_ids_in_output_strips_padding(monkeypatch):
|
||||
def test_update_draft_token_ids_in_output_strips_padding():
|
||||
"""Same guard on the output path; the -1 pad-back for the rejected count
|
||||
is preserved (only the input to validate_tokens is stripped)."""
|
||||
is preserved (only the input to manager validation is stripped)."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=4)
|
||||
request = _decode_ready_request(scheduler)
|
||||
|
||||
grammar = _RecordingGrammar()
|
||||
request.structured_output_request = SimpleNamespace(grammar=grammar)
|
||||
monkeypatch.setattr(
|
||||
scheduler.structured_output_manager, "should_advance", lambda req: True
|
||||
request.structured_output_request = SimpleNamespace(
|
||||
grammar=grammar, reasoning_ended=True
|
||||
)
|
||||
|
||||
scheduler_output = SimpleNamespace(
|
||||
|
||||
@@ -43,6 +43,7 @@ def test_piecewise_capture_builds_fresh_metadata_for_both_passes():
|
||||
manager._capture_descs = {CUDAGraphMode.PIECEWISE: [desc]}
|
||||
manager._graphs_captured = False
|
||||
manager.use_breakable_cg = True
|
||||
manager.ubatch_runner = None
|
||||
|
||||
create_calls = []
|
||||
forward_calls = []
|
||||
|
||||
@@ -31,9 +31,8 @@ from vllm.v1.request import Request
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker import mamba_utils
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch
|
||||
from vllm.v1.worker.mamba_utils import get_mamba_groups
|
||||
|
||||
|
||||
@@ -315,7 +314,7 @@ def get_fake_process_mamba_fn(
|
||||
action: tuple[int, int],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
forward_context: dict[str, Any],
|
||||
input_batch: GPUInputBatch,
|
||||
input_batch: InputBatch,
|
||||
):
|
||||
assert copy_info is not None
|
||||
if action == (-1, -1):
|
||||
@@ -371,7 +370,7 @@ def get_fake_process_mamba_fn(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
cache_config: CacheConfig,
|
||||
mamba_state_idx: dict[str, int],
|
||||
input_batch: GPUInputBatch,
|
||||
input_batch: InputBatch,
|
||||
requests: dict[str, CachedRequestState],
|
||||
forward_context: dict[str, Any],
|
||||
mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...],
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU speculative-decoding end-to-end correctness."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from tests.v1.e2e.spec_decode.utils import compute_acceptance_len
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
pytestmark = pytest.mark.cpu_model
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
# Bound the KV cache so the run does not scale with host memory.
|
||||
os.environ.setdefault("VLLM_CPU_KVCACHE_SPACE", "1")
|
||||
|
||||
MODEL = "Qwen/Qwen3-0.6B"
|
||||
NUM_LOGPROBS = 5
|
||||
SP = SamplingParams(max_tokens=48, temperature=0, logprobs=NUM_LOGPROBS)
|
||||
|
||||
# Repetitive prompts so the ngram proposer finds matches and actually drafts;
|
||||
# without hits the run is indistinguishable from plain decoding.
|
||||
PROMPTS = [
|
||||
"The capital of France is Paris. The capital of Italy is Rome. "
|
||||
"The capital of Spain is Madrid. The capital of France is",
|
||||
"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - b\n\n"
|
||||
"def mul(a, b):\n return",
|
||||
"one two three four five one two three four five one two three four",
|
||||
]
|
||||
|
||||
|
||||
def _make_llm(**overrides) -> LLM:
|
||||
base = dict(
|
||||
model=MODEL,
|
||||
dtype="bfloat16",
|
||||
max_model_len=1024,
|
||||
disable_log_stats=False,
|
||||
)
|
||||
base.update(overrides)
|
||||
return LLM(**base)
|
||||
|
||||
|
||||
def _tuples(outputs) -> list[tuple[list[int], str, object]]:
|
||||
return [
|
||||
(list(o.outputs[0].token_ids), o.outputs[0].text, o.outputs[0].logprobs)
|
||||
for o in outputs
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def baseline_refs():
|
||||
llm = _make_llm()
|
||||
refs = _tuples(llm.generate(PROMPTS, SP))
|
||||
del llm
|
||||
return refs
|
||||
|
||||
|
||||
def _spec_metric(llm: LLM, name: str) -> float:
|
||||
for metric in llm.get_metrics():
|
||||
if metric.name == name:
|
||||
return float(getattr(metric, "value", 0.0))
|
||||
return 0.0
|
||||
|
||||
|
||||
def test_ngram_spec_decode_matches_baseline(baseline_refs):
|
||||
"""Greedy output must be unchanged by speculative decoding.
|
||||
|
||||
The drafted-token check keeps the test from passing vacuously when the
|
||||
proposer silently produces nothing.
|
||||
"""
|
||||
llm = _make_llm(
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
"prompt_lookup_max": 5,
|
||||
"prompt_lookup_min": 3,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
)
|
||||
got = _tuples(llm.generate(PROMPTS, SP))
|
||||
drafted = _spec_metric(llm, "vllm:spec_decode_num_draft_tokens")
|
||||
accepted = _spec_metric(llm, "vllm:spec_decode_num_accepted_tokens")
|
||||
del llm
|
||||
|
||||
assert drafted > 0, "ngram proposer drafted no tokens; the test would be vacuous"
|
||||
assert accepted > 0, f"no draft tokens accepted out of {drafted} drafted"
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_refs,
|
||||
outputs_1_lst=got,
|
||||
name_0="no_spec_decode",
|
||||
name_1="ngram_spec_decode",
|
||||
)
|
||||
|
||||
|
||||
SMOKE_PROMPTS = [
|
||||
"The capital of France is",
|
||||
"2 + 2 equals",
|
||||
"In one word, the color of the sky is",
|
||||
"Q: If a train travels 60 miles in 1.5 hours, what is its average speed?\nA:",
|
||||
]
|
||||
SMOKE_SP = SamplingParams(temperature=0.0, max_tokens=32, ignore_eos=True)
|
||||
|
||||
# Methods routed to the V2 runner, using the same target/draft pairs as the GPU
|
||||
# suite. MTP needs no separate draft checkpoint: its speculator layer is
|
||||
# trained jointly with, and shipped inside, the target model's own weights.
|
||||
SMOKE_CONFIGS = [
|
||||
pytest.param(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
{
|
||||
"method": "eagle3",
|
||||
"model": "nm-testing/Llama3_2_1B_speculator.eagle3",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
id="eagle3",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-4B",
|
||||
{
|
||||
"method": "dflash",
|
||||
"model": "z-lab/Qwen3-4B-DFlash-b16",
|
||||
# The checkpoint allows 16, but accepted tokens plateau around 64
|
||||
# regardless, so a larger draft only burns CPU on rejected tokens.
|
||||
"num_speculative_tokens": 4,
|
||||
},
|
||||
id="dflash",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
{
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
id="mtp",
|
||||
),
|
||||
]
|
||||
|
||||
# Measured 1.85 (eagle3), 1.99 (dflash), 2.29 (mtp) on CPU; this floor leaves
|
||||
# room for numeric drift while still failing if speculation stops paying off.
|
||||
MIN_ACCEPTANCE_LEN = 1.5
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TRITON, reason="the V2 runner needs triton-cpu, which CI builds"
|
||||
)
|
||||
@pytest.mark.parametrize("model,speculative_config", SMOKE_CONFIGS)
|
||||
def test_v2_speculator_smoke(model: str, speculative_config: dict):
|
||||
"""Gates that drafts are accepted often enough for speculation to pay off."""
|
||||
llm = _make_llm(model=model, speculative_config=speculative_config)
|
||||
try:
|
||||
assert llm.llm_engine.vllm_config.use_v2_model_runner, (
|
||||
f"{speculative_config['method']} fell back to the V1 runner; "
|
||||
"this test is meant to cover the V2 path"
|
||||
)
|
||||
llm.generate(SMOKE_PROMPTS, SMOKE_SP)
|
||||
acceptance = compute_acceptance_len(llm.get_metrics())
|
||||
assert acceptance >= MIN_ACCEPTANCE_LEN, (
|
||||
f"acceptance length {acceptance:.3f} below {MIN_ACCEPTANCE_LEN}; "
|
||||
"drafts are being rejected far more than expected"
|
||||
)
|
||||
finally:
|
||||
del llm
|
||||
@@ -116,7 +116,9 @@ def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch
|
||||
|
||||
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
reported = first["ec_transfer_params"]
|
||||
assert [handle] == [value for key, value in reported.items() if key != "ec_items"]
|
||||
assert {key: value for key, value in reported.items() if key != "ec_items"} == {
|
||||
"encoder-side-hash": handle
|
||||
}
|
||||
assert "ec_transfer_params" not in body
|
||||
|
||||
# Attempt 2's encode reports nothing: decode must be told nothing.
|
||||
@@ -154,10 +156,22 @@ def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("prefill", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"request_options",
|
||||
[
|
||||
{},
|
||||
{
|
||||
"mm_processor_kwargs": {"max_pixels": 262144},
|
||||
"media_io_kwargs": {"image": {"image_mode": "RGB"}},
|
||||
"priority": -3,
|
||||
"session_id": "session-123",
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
proxy, monkeypatch, stream, prefill
|
||||
proxy, monkeypatch, stream, prefill, request_options
|
||||
):
|
||||
"""Exercise real HTTP hops, rewrite and a decode retry without model servers."""
|
||||
"""Preserve media and scheduling context across HTTP hops and retries."""
|
||||
seen: dict[str, list[dict]] = {"encode": [], "prefill": [], "decode": []}
|
||||
prefix = b'data: {"content":"'
|
||||
# Split a multibyte character at the old 1024-byte forwarding boundary.
|
||||
@@ -174,10 +188,13 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
seen[stage].append(body)
|
||||
if stage == "encode":
|
||||
item = body["messages"][0]["content"][0]
|
||||
mm_hash = item["uuid"]
|
||||
if body.get("mm_processor_kwargs"):
|
||||
mm_hash += "-processed"
|
||||
return web.json_response(
|
||||
{
|
||||
"ec_transfer_params": {
|
||||
item["uuid"]: {
|
||||
mm_hash: {
|
||||
"metadata": {"image_grid_thw": [[1, 2, 2]]},
|
||||
"peer_port": len(seen[stage]),
|
||||
}
|
||||
@@ -230,6 +247,7 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
"max_tokens": 32,
|
||||
"seed": 42,
|
||||
"structured_outputs": {"choice": ["A", "B"]},
|
||||
**request_options,
|
||||
}
|
||||
await proxy.on_startup()
|
||||
try:
|
||||
@@ -247,6 +265,15 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
|
||||
assert len(seen["encode"]) == 4
|
||||
assert len(seen["decode"]) == 2
|
||||
for requests in seen.values():
|
||||
for forwarded in requests:
|
||||
for key in (
|
||||
"mm_processor_kwargs",
|
||||
"media_io_kwargs",
|
||||
"priority",
|
||||
"session_id",
|
||||
):
|
||||
assert forwarded.get(key) == body.get(key)
|
||||
final = seen["decode"][-1]
|
||||
for key in [
|
||||
"model",
|
||||
@@ -265,7 +292,11 @@ async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
"image_embeds": {"image_grid_thw": [1, 2, 2]},
|
||||
"uuid": proxy.content_uuid(item),
|
||||
}
|
||||
assert final["ec_transfer_params"][proxy.content_uuid(item)]["peer_port"] in (3, 4)
|
||||
mm_hash = proxy.content_uuid(item)
|
||||
if request_options:
|
||||
mm_hash += "-processed"
|
||||
assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash}
|
||||
assert final["ec_transfer_params"][mm_hash]["peer_port"] in (3, 4)
|
||||
if prefill:
|
||||
assert final["kv_transfer_params"] == {"remote_block_ids": [2]}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ These tests cover:
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import multiprocessing
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
@@ -34,6 +35,7 @@ from vllm.exceptions import (
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import human_readable_int
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
|
||||
@@ -66,6 +68,7 @@ def _make_async_llm(
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.get_num_unfinished_requests.return_value = num_unfinished
|
||||
llm.output_processor.get_num_queued_tokens.return_value = num_queued_tokens
|
||||
llm.admission_stats = None
|
||||
return llm
|
||||
|
||||
|
||||
@@ -84,6 +87,42 @@ def _make_scheduler_config(**kwargs) -> SchedulerConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_shared_stats(client_count: int = 2) -> list[SharedAdmissionStats]:
|
||||
counters = multiprocessing.RawArray(
|
||||
"q", SharedAdmissionStats.num_counters(client_count)
|
||||
)
|
||||
return [
|
||||
SharedAdmissionStats({"mp_admission_counters": counters}, client_count, index)
|
||||
for index in range(client_count)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_shared_admission_stats_aggregate_api_servers():
|
||||
server_0, server_1 = _make_shared_stats()
|
||||
|
||||
server_0.set_num_requests(2)
|
||||
assert server_0.get_num_requests() == 2
|
||||
server_1.set_num_requests(3)
|
||||
assert server_0.get_num_requests() == server_1.get_num_requests() == 5
|
||||
|
||||
server_0.set_num_requests(1)
|
||||
assert server_0.get_num_requests() == 4
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_shared_request_snapshot_enforces_global_limit():
|
||||
server_0, server_1 = _make_shared_stats()
|
||||
server_0.set_num_requests(2)
|
||||
llm = _make_async_llm(max_num_queued_reqs=2)
|
||||
llm.admission_stats = server_1
|
||||
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
|
||||
assert server_0.get_num_requests() == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exception classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _mock_cpu_backend_block_sizes(monkeypatch):
|
||||
"""Mock the CPU attention backend's kernel block-size declaration.
|
||||
|
||||
Connector unit tests exercise transfer logic, never the attention
|
||||
kernel, and hard-code block_size=16. On CPU-only machines
|
||||
`CPUAttentionBackend` declares `MultipleOf(32)` (#54042), so every
|
||||
worker construction would fail in `select_common_block_size` before
|
||||
reaching the code under test.
|
||||
"""
|
||||
if not current_platform.is_cpu():
|
||||
return
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend
|
||||
|
||||
monkeypatch.setattr(
|
||||
CPUAttentionBackend,
|
||||
"get_supported_kernel_block_sizes",
|
||||
staticmethod(lambda: [MultipleOf(16)]),
|
||||
)
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
@@ -74,7 +75,7 @@ from vllm.v1.kv_offload.base import (
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
@@ -195,15 +196,17 @@ def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
|
||||
|
||||
def _make_partial_tail_request(
|
||||
scheduler: OffloadingConnectorScheduler,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
) -> MagicMock:
|
||||
request = MagicMock()
|
||||
request.request_id = "req"
|
||||
request.kv_transfer_params = None
|
||||
request.kv_transfer_params = kv_transfer_params
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
request.all_token_ids = list(range(30))
|
||||
request.lora_request = None
|
||||
request.skip_reading_prefix_cache = False
|
||||
request.is_finished.return_value = False
|
||||
scheduler.on_new_request(request)
|
||||
return request
|
||||
@@ -427,6 +430,53 @@ def test_lookup_cap_stops_at_authoritative_prefix_boundary():
|
||||
assert scheduler._req_status["req"].partial_tail_boundary == 20
|
||||
|
||||
|
||||
def test_max_load_tokens_zero_skips_partial_tail_lookup():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": 0}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 16) == (0, False)
|
||||
scheduler.manager.lookup.assert_not_called()
|
||||
assert all(
|
||||
state.num_hit_chunks == 1 for state in scheduler._req_status["req"].group_states
|
||||
)
|
||||
|
||||
|
||||
def test_max_load_tokens_caps_tokens_beyond_gpu_prefix():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": 8}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 16) == (8, True)
|
||||
assert scheduler._req_status["req"].partial_tail_boundary == 24
|
||||
|
||||
|
||||
def test_max_load_tokens_rounds_down_without_partial_tail():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
scheduler.config = scheduler.config._replace(supports_partial_tail=False)
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": 20}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == (16, True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["8", 8.5, -1, True])
|
||||
def test_invalid_max_load_tokens_is_ignored(value):
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": value}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == (28, True)
|
||||
|
||||
|
||||
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
|
||||
"""A recurrent group may hold unhashed blocks inside the computed prefix.
|
||||
|
||||
@@ -1708,6 +1758,15 @@ class TestMaximalPrefixLookup:
|
||||
|
||||
|
||||
class TestSlidingWindowLookup:
|
||||
def test_pending_chunk_behind_gap_does_not_defer(self):
|
||||
"""A pending suffix too short for the window cannot improve the hit."""
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{1: LookupResult.HIT, 2: LookupResult.HIT, 4: LookupResult.HIT_PENDING}
|
||||
)
|
||||
assert (
|
||||
sched._sliding_window_lookup(to_keys([1, 2, 3, 4]), 2, _EMPTY_REQ_CTX) == 2
|
||||
)
|
||||
|
||||
def test_all_hit_exact_window(self):
|
||||
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
|
||||
assert sched._sliding_window_lookup(to_keys([1, 2]), 2, _EMPTY_REQ_CTX) == 2
|
||||
@@ -1826,6 +1885,73 @@ class TestSlidingWindowLookup:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_swa_load_does_not_wait_for_unusable_pending_suffix(
|
||||
request_runner, async_scheduling
|
||||
):
|
||||
"""A CPU store beyond a missing SWA chunk must not block a ready prefix."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=4, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa"],
|
||||
SlidingWindowSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
),
|
||||
),
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=32,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
sched = runner.connector_scheduler
|
||||
manager = CPUOffloadingManager(num_chunks=16)
|
||||
sched.manager = manager
|
||||
runner.new_request(token_ids=list(range(17)))
|
||||
state = sched._req_status["0"]
|
||||
state.update_offload_keys()
|
||||
full, swa = (g.offload_keys for g in state.group_states)
|
||||
ready = manager.prepare_store(full[:4] + swa[:2], state.req_context)
|
||||
assert ready is not None
|
||||
manager.complete_store(ready.keys_to_store, state.req_context)
|
||||
pending = manager.prepare_store(swa[3:4], state.req_context)
|
||||
assert pending is not None
|
||||
|
||||
# The last SWA chunk is still being written; the preceding one is missing.
|
||||
output = runner.scheduler.schedule()
|
||||
metadata = output.kv_connector_metadata
|
||||
assert isinstance(metadata, OffloadingConnectorMetadata)
|
||||
[(job_id, load)] = metadata.load_jobs.items()
|
||||
assert isinstance(load.dst_spec, GPULoadStoreSpec)
|
||||
assert load.dst_spec.group_sizes == [2, 2]
|
||||
assert manager.lookup(swa[3], state.req_context) is LookupResult.HIT_PENDING
|
||||
|
||||
runner.scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput.with_kv_conn_output_only(
|
||||
KVConnectorOutput(
|
||||
finished_recving={"0"},
|
||||
kv_connector_worker_meta=OffloadingWorkerMetadata(
|
||||
completed_jobs={job_id: 1}
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
resumed = runner.scheduler.schedule()
|
||||
assert resumed.num_scheduled_tokens == {"0": 9}
|
||||
assert manager.lookup(swa[3], state.req_context) is LookupResult.HIT_PENDING
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alignment_chunk_count", [4, 8, 64])
|
||||
@pytest.mark.parametrize("sliding_window_chunks", [1, 2, 3])
|
||||
@pytest.mark.parametrize("is_eagle_group", [False, True])
|
||||
@@ -2888,6 +3014,47 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
runner.manager.lookup.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_max_load_tokens_limits_external_load(request_runner, async_scheduling: bool):
|
||||
"""The load cap limits external reads without changing the store path."""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=(0, 1, 2),
|
||||
)
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.manager.lookup.reset_mock()
|
||||
runner.new_request(
|
||||
token_ids=[0] * tokens_per_chunk,
|
||||
kv_transfer_params={"max_load_tokens": 0},
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_loaded=(),
|
||||
expected_stored=(0, 1, 2),
|
||||
)
|
||||
|
||||
runner.manager.lookup.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Eagle/MTP test class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -39,10 +39,12 @@ from vllm.utils.network_utils import (
|
||||
make_zmq_path,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
)
|
||||
|
||||
@@ -338,7 +340,7 @@ def test_write_mode_saves_local_block_ids():
|
||||
req_meta = kv_connector_metadata.reqs_to_save[request_id]
|
||||
|
||||
for block_id, block in zip(
|
||||
req_meta.local_block_ids,
|
||||
req_meta.local_block_ids[0],
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
],
|
||||
@@ -393,7 +395,7 @@ def test_write_mode_with_chunked_prefill_saves_local_block_ids():
|
||||
req_meta = kv_connector_metadata.reqs_to_save[request_id]
|
||||
|
||||
for block_id, block in zip(
|
||||
req_meta.local_block_ids,
|
||||
req_meta.local_block_ids[0],
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
],
|
||||
@@ -432,7 +434,7 @@ def test_read_mode_loads_remote_block_ids():
|
||||
].req_to_blocks[request_id]
|
||||
|
||||
# Set remote block ids to be fetched.
|
||||
request.kv_transfer_params["remote_block_ids"] = block_list
|
||||
request.kv_transfer_params["remote_block_ids"] = [block_list]
|
||||
|
||||
# Remote Prefill, triggers MoRIIOConnectorMetadata.
|
||||
|
||||
@@ -457,7 +459,7 @@ def test_read_mode_loads_remote_block_ids():
|
||||
req_meta = kv_connector_metadata.reqs_to_recv[request_id]
|
||||
|
||||
for block_id, block in zip(
|
||||
req_meta.local_block_ids,
|
||||
req_meta.local_block_ids[0],
|
||||
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
|
||||
request_id
|
||||
],
|
||||
@@ -699,3 +701,225 @@ def test_resolve_host_ip_prefers_extra_config():
|
||||
fallback = get_ip()
|
||||
assert resolve_host_ip({}) == fallback
|
||||
assert resolve_host_ip({"host_ip": ""}) == fallback
|
||||
|
||||
|
||||
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
|
||||
"""One full-attention group + one sliding-window group (Gemma-like)."""
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
)
|
||||
sw_spec = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
sliding_window=32,
|
||||
)
|
||||
num_blocks = 2
|
||||
page = full_spec.page_size_bytes
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=2 * num_blocks * page,
|
||||
layers=["full0", "sw0"],
|
||||
layer_stride=num_blocks * page,
|
||||
block_stride=page,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
|
||||
KVCacheGroupSpec(layer_names=["sw0"], kv_cache_spec=sw_spec),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _read_scheduler(
|
||||
kv_cache_config: KVCacheConfig, disable_hma: bool = False
|
||||
) -> MoRIIOConnectorScheduler:
|
||||
vllm_config = create_vllm_config(role="kv_producer", read_mode=True)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
|
||||
with set_current_vllm_config(vllm_config):
|
||||
connector = MoRIIOConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
assert connector.connector_scheduler is not None
|
||||
return connector.connector_scheduler
|
||||
|
||||
|
||||
def test_hma_blocks_per_sw_two_groups():
|
||||
"""A Full + SlidingWindow config runs with HMA and computes block budgets
|
||||
correctly"""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
assert scheduler._is_hma_required is True
|
||||
# cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention.
|
||||
assert scheduler.blocks_per_sw == [0, 3]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"swa_enabled, disable_hma, expected_is_hma",
|
||||
[
|
||||
(True, False, True), # sliding-window group present, HMA enabled
|
||||
(True, True, False), # sliding-window group present but HMA disabled
|
||||
(False, False, False), # full-attention only, HMA not needed
|
||||
],
|
||||
)
|
||||
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
"""_is_hma_required tracks both the KV cache groups and the
|
||||
--disable-hybrid-kv-cache-manager flag. When HMA is off,
|
||||
get_exchange_clipped_blocks must be a no-op."""
|
||||
config = (
|
||||
_make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config()
|
||||
)
|
||||
scheduler = _read_scheduler(config, disable_hma=disable_hma)
|
||||
assert scheduler._is_hma_required is expected_is_hma
|
||||
if not expected_is_hma:
|
||||
blocks = [[1, 2, 3, 4, 5]]
|
||||
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
|
||||
|
||||
|
||||
def test_non_sliding_window_hybrid_is_rejected():
|
||||
"""A hybrid group that is not sliding-window (e.g. chunked-local
|
||||
attention) must fail closed rather than be silently mistransferred."""
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
)
|
||||
local_spec = ChunkedLocalAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
attention_chunk_size=32,
|
||||
)
|
||||
num_blocks = 2
|
||||
page = full_spec.page_size_bytes
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=2 * num_blocks * page,
|
||||
layers=["full0", "local0"],
|
||||
layer_stride=num_blocks * page,
|
||||
block_stride=page,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
|
||||
KVCacheGroupSpec(layer_names=["local0"], kv_cache_spec=local_spec),
|
||||
],
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="sliding-window hybrid"):
|
||||
_read_scheduler(config)
|
||||
|
||||
|
||||
def test_token_count_basis_uses_full_attention_group():
|
||||
"""Chunked-prefill token counting must use an unclipped full-attention
|
||||
group (blocks_per_sw == 0), not a clipped sliding-window group."""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
# Group 0 is the full-attention group
|
||||
assert scheduler._full_attn_group_idx == 0
|
||||
assert scheduler._full_attn_block_size == 16
|
||||
|
||||
|
||||
def test_get_exchange_clipped_blocks_clips_only_sw_group():
|
||||
"""get_exchange_clipped_blocks keeps the full attn group intact and clips the
|
||||
sliding-window group to its window tail."""
|
||||
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
|
||||
full = [10, 11, 12, 13, 14]
|
||||
sw = [20, 21, 22, 23, 24]
|
||||
clipped = scheduler.get_exchange_clipped_blocks([full, sw])
|
||||
assert clipped[0] == full
|
||||
assert clipped[1] == [22, 23, 24]
|
||||
|
||||
|
||||
def test_metadata_hma_block_ids_preserved_per_group():
|
||||
"""add_new_req stores per-group (BlockIds) block lists unchanged for both
|
||||
read and write, so the hybrid group structure is retained."""
|
||||
metadata = MoRIIOConnectorMetadata()
|
||||
|
||||
# Assume:
|
||||
# - Full-attention group (6 blocks) +
|
||||
# - sliding-window group already clipped to its window tail (3 blocks).
|
||||
fa_blocks = [0, 1, 2, 3, 4, 5]
|
||||
sw_blocks = [10, 11, 12]
|
||||
local_block_ids = [fa_blocks, sw_blocks]
|
||||
remote_block_ids = [[100, 101, 102, 103, 104, 105], [200, 201, 202]]
|
||||
|
||||
base_params = {
|
||||
"remote_engine_id": "remote-engine",
|
||||
"remote_host": "127.0.0.1",
|
||||
"remote_handshake_port": 6301,
|
||||
"remote_notify_port": 61005,
|
||||
"remote_block_ids": remote_block_ids,
|
||||
}
|
||||
|
||||
# Read mode: both local and remote ids stay per-group.
|
||||
metadata.add_new_req(
|
||||
request_id="recv-req",
|
||||
local_block_ids=local_block_ids,
|
||||
kv_transfer_params={**base_params, "transfer_id": "recv-req"},
|
||||
)
|
||||
recv_meta = metadata.reqs_to_recv["recv-req"]
|
||||
assert recv_meta.local_block_ids == [fa_blocks, sw_blocks]
|
||||
assert recv_meta.remote_block_ids == remote_block_ids
|
||||
|
||||
# Write mode: the decode peer allocates its own blocks, so #remote_block_ids may
|
||||
# be empty, Local group structure is kept.
|
||||
metadata.add_new_req(
|
||||
request_id="save-req",
|
||||
local_block_ids=local_block_ids,
|
||||
kv_transfer_params={
|
||||
**base_params,
|
||||
"transfer_id": "save-req",
|
||||
"remote_block_ids": [],
|
||||
},
|
||||
write_mode=True,
|
||||
)
|
||||
save_meta = metadata.reqs_to_save["save-req"]
|
||||
assert save_meta.local_block_ids == [fa_blocks, sw_blocks]
|
||||
assert save_meta.remote_block_ids == []
|
||||
|
||||
|
||||
def test_single_group_path_unchanged():
|
||||
"""A full attn config does not enable HMA and never clips blocks."""
|
||||
scheduler = _read_scheduler(_make_test_kv_cache_config())
|
||||
assert scheduler._is_hma_required is False
|
||||
assert scheduler.blocks_per_sw == [0]
|
||||
blocks = [[1, 2, 3, 4, 5]]
|
||||
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
|
||||
|
||||
|
||||
def test_hybrid_write_mode_rejected():
|
||||
"""Hybrid KV cache groups are unsupported in WRITE mode and fail closed."""
|
||||
vllm_config = create_vllm_config(role="kv_producer", read_mode=False)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
pytest.raises(NotImplementedError),
|
||||
):
|
||||
MoRIIOConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
_make_hybrid_kv_cache_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_worker_layer_to_group_routing(mock_parallel_groups):
|
||||
"""The worker maps every layer to its KV cache group correctly."""
|
||||
vllm_config = create_vllm_config(role="kv_consumer", read_mode=True)
|
||||
# Building the worker directly bypasses MoRIIOConnector._set_port_defaults,
|
||||
# so provide the ports manually.
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.update(
|
||||
{
|
||||
"http_port": 12346,
|
||||
"handshake_port": 12347,
|
||||
"notify_port": 12348,
|
||||
}
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
worker = FakeMoRIIOConnectorWorker(
|
||||
vllm_config,
|
||||
"engine0",
|
||||
hand_shake_latency=0,
|
||||
kv_cache_config=_make_hybrid_kv_cache_config(),
|
||||
)
|
||||
assert worker.layer_to_group == {"full0": 0, "sw0": 1}
|
||||
|
||||
@@ -20,12 +20,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
_request_finished = MoRIIOConnectorScheduler.request_finished
|
||||
_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks
|
||||
|
||||
|
||||
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
|
||||
return SimpleNamespace(
|
||||
s = SimpleNamespace(
|
||||
is_producer=True,
|
||||
_is_hma_required=False,
|
||||
blocks_per_sw=[0],
|
||||
_global_dp_rank=global_dp_rank,
|
||||
engine_id="engine-abc",
|
||||
host_ip="10.0.0.1",
|
||||
@@ -39,6 +42,8 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
)
|
||||
),
|
||||
)
|
||||
s.get_exchange_clipped_blocks = _clip_blocks.__get__(s)
|
||||
return s
|
||||
|
||||
|
||||
def _req(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import contextlib
|
||||
import inspect
|
||||
import os
|
||||
import queue
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
@@ -341,6 +342,10 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
|
||||
assert req_meta.remote.request_id == f"prefill-{42}"
|
||||
# do_remote_prefill is consumed by request_finished to prevent re-issuing.
|
||||
assert request.kv_transfer_params["do_remote_prefill"] is False
|
||||
# The scheduler is not waiting on this recv -- the request is already gone
|
||||
# from self.requests, so reporting it would trip `assert req_id in
|
||||
# self.requests` in _update_from_kv_xfer_finished.
|
||||
assert req_meta.awaiting_kvs is False
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -1948,17 +1953,43 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
|
||||
assert memory_types == ["DRAM", "VRAM"]
|
||||
|
||||
|
||||
def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
@pytest.fixture
|
||||
def recv_worker():
|
||||
"""Receive lifecycle state without distributed or device initialization."""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.transfer_topo.block_size_ratio.return_value = 1
|
||||
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_block_size=16, remote_physical_blocks_per_logical=1
|
||||
)
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker._recving_metadata = {"request": MagicMock(local_block_ids=([1, 2, 3],))}
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._failed_recv_reqs = queue.Queue()
|
||||
worker._recv_failures = set()
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._invalid_block_ids = queue.Queue()
|
||||
worker._pending_recv_notifs = {}
|
||||
worker._reqs_to_send = {}
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._is_hma_required = False
|
||||
worker._has_mamba = False
|
||||
worker.use_host_buffer = False
|
||||
worker.enable_permute_local_kv = False
|
||||
worker.enable_heterogeneous_attn_post_process = False
|
||||
worker.tp_rank = 0
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
worker.xfer_stats = NixlKVConnectorStats()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.get_xfer_telemetry.return_value = get_default_xfer_telemetry()
|
||||
return worker
|
||||
|
||||
|
||||
def test_mixed_memory_read_notifies_after_both_transfers_finish(recv_worker):
|
||||
worker = recv_worker
|
||||
worker._desc_is_dram_by_block_size = {16: np.array([True, True, False, False])}
|
||||
worker._desc_pos_by_block_size = {16: np.array([0, 1, 0, 1])}
|
||||
worker._dram_src_handles_by_block_size = {16: 10}
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker._recving_transfers = defaultdict(list)
|
||||
worker._pending_recv_notifs = {}
|
||||
worker._failed_inflight_recvs = set()
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
|
||||
worker.nixl_wrapper.check_xfer_state.return_value = "DONE"
|
||||
|
||||
@@ -1985,27 +2016,20 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
|
||||
np.testing.assert_array_equal(device_read.args[4], [7])
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
assert worker._pop_done_transfers(worker._recving_transfers) == {"request"}
|
||||
assert worker.get_finished() == (set(), {"request"})
|
||||
worker.nixl_wrapper.send_notif.assert_called_once_with(
|
||||
"prefill", notif_msg=b"request:1"
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_memory_read_failure_does_not_notify_producer():
|
||||
def test_mixed_memory_read_failure_does_not_notify_producer(recv_worker):
|
||||
"""A failed half of a split READ must suppress its success notification."""
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._recving_metadata = {"request": MagicMock()}
|
||||
worker = recv_worker
|
||||
worker._recving_transfers = {"request": [101, 102]}
|
||||
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
|
||||
worker._failed_inflight_recvs = set()
|
||||
worker._is_hma_required = True
|
||||
worker._failed_recv_reqs = MagicMock()
|
||||
worker._log_failure = MagicMock() # type: ignore[method-assign]
|
||||
worker.xfer_stats = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "DONE"]
|
||||
|
||||
worker._pop_done_transfers(worker._recving_transfers)
|
||||
assert worker.get_finished() == (set(), {"request"})
|
||||
|
||||
assert "request" not in worker._pending_recv_notifs
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
@@ -2695,6 +2719,74 @@ class FailingNixlWrapper(FakeNixlWrapper):
|
||||
return super().check_xfer_state(handle)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
@pytest.mark.parametrize("awaiting_kvs", [True, False])
|
||||
@pytest.mark.parametrize("notification_fails", [False, True])
|
||||
def test_empty_recv_is_reported_only_when_awaited(
|
||||
default_vllm_config, dist_init, awaiting_kvs, notification_fails
|
||||
):
|
||||
"""An empty recv is reported iff the scheduler parked the request on it.
|
||||
|
||||
_read_blocks returns early when the local block list is empty -- D already
|
||||
holds the KV and only the producer needs telling. Two kinds of caller reach
|
||||
that path, and they need opposite handling:
|
||||
|
||||
Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the
|
||||
request is named in neither _recving_transfers nor _failed_recv_reqs and
|
||||
never reaches finished_recving. The scheduler has no other way to release a
|
||||
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
|
||||
skipped_waiting holding its blocks for the life of the process.
|
||||
|
||||
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
|
||||
free P's blocks for a request aborted before it was scheduled, or a readback
|
||||
on a request that stays RUNNING. Reporting either one crashes the engine
|
||||
core -- _update_from_kv_xfer_finished asserts the id is still in
|
||||
self.requests and that the request is parked or finished.
|
||||
|
||||
A failed notification changes neither: the KV is local either way, and the
|
||||
producer frees its own blocks on a timeout.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0.0,
|
||||
kv_cache_config=connector._kv_cache_config,
|
||||
)
|
||||
connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails
|
||||
|
||||
request_id = f"test_empty_recv_awaited_{awaiting_kvs}"
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=(), # empty: the whole prompt is already cached locally
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": [[20, 21, 22]],
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
connector.bind_connector_metadata(metadata)
|
||||
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
connector.bind_connector_metadata(NixlConnectorMetadata())
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert (request_id in done_recving) is awaiting_kvs
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
@@ -2897,13 +2989,10 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
||||
# Wait for handshake to fail
|
||||
time.sleep(0.3)
|
||||
|
||||
# Check that blocks were marked invalid
|
||||
invalid_blocks = connector.get_block_ids_with_load_errors()
|
||||
assert invalid_blocks == {1, 2, 3}
|
||||
|
||||
# Check that request appears in get_finished
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert request_id in done_recving
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -2954,6 +3043,7 @@ def test_transfer_setup_failure_returns_finished(
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
results = connector.get_transfer_results(finished_req_ids=set())
|
||||
assert request_id in results.finished_recving
|
||||
assert results.failed_recving == {request_id}
|
||||
@@ -3176,74 +3266,49 @@ def test_failed_request_skips_kv_postprocessing(
|
||||
assert invalid_blocks == {1, 2, 3}
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
|
||||
default_vllm_config, dist_init
|
||||
):
|
||||
"""Failures across polls report once, after the last writer is terminal."""
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
@pytest.mark.parametrize("sibling_state", ["ERR", "DONE"])
|
||||
def test_recv_failure_waits_for_sibling_transfer(recv_worker, sibling_state):
|
||||
"""Failure must not release blocks while a sibling can still write to them."""
|
||||
worker = recv_worker
|
||||
worker._recving_transfers["request"] = [101, 102]
|
||||
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = ["ERR", "PROC", sibling_state]
|
||||
|
||||
request_id = "test_two_handles_failing_in_separate_polls"
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=([1, 2, 3],),
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": ([4, 5, 6],),
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
)
|
||||
connector.bind_connector_metadata(metadata)
|
||||
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
connector.bind_connector_metadata(NixlConnectorMetadata())
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == set()
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
assert "request" in worker._recving_metadata
|
||||
worker.nixl_wrapper.release_xfer_handle.assert_called_once_with(101)
|
||||
|
||||
# Give the request a second handle, as a second remote rank would.
|
||||
handles = worker._recving_transfers[request_id]
|
||||
assert len(handles) == 1
|
||||
first, second = handles[0], object()
|
||||
worker._recving_transfers[request_id] = [first, second]
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == {"request"}
|
||||
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
worker.nixl_wrapper.send_notif.assert_not_called()
|
||||
|
||||
# Poll 1: the first handle has failed, the second is still running.
|
||||
def first_failed(handle):
|
||||
return "ERR" if handle is first else "PROC"
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == set()
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
|
||||
with patch.object(
|
||||
worker.nixl_wrapper, "check_xfer_state", side_effect=first_failed
|
||||
):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
|
||||
assert request_id not in done_recving
|
||||
assert request_id in worker._recving_metadata
|
||||
assert worker._recving_transfers[request_id] == [second]
|
||||
assert connector.get_block_ids_with_load_errors() == set()
|
||||
def test_recv_failure_waits_for_unpollable_handle_to_be_released(recv_worker):
|
||||
"""A polling exception and failed release do not prove that DMA has stopped."""
|
||||
worker = recv_worker
|
||||
worker._recving_transfers["request"] = [101]
|
||||
worker.nixl_wrapper.check_xfer_state.side_effect = [RuntimeError("poll"), "DONE"]
|
||||
worker.nixl_wrapper.release_xfer_handle.side_effect = [
|
||||
RuntimeError("release"),
|
||||
None,
|
||||
]
|
||||
|
||||
# Only the last terminal handle permits the scheduler to reuse the blocks.
|
||||
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == set()
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
assert "request" in worker._recving_metadata
|
||||
|
||||
assert done_recving == {request_id}
|
||||
assert request_id not in worker._recving_transfers
|
||||
assert request_id not in worker._recving_metadata
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert done_recving == set()
|
||||
results = worker.get_transfer_results()
|
||||
assert results.finished_recving == results.failed_recving == {"request"}
|
||||
assert worker.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
assert worker.nixl_wrapper.release_xfer_handle.call_count == 2
|
||||
|
||||
|
||||
def _set_test_speculative_config(
|
||||
|
||||
@@ -149,8 +149,49 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
|
||||
|
||||
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
|
||||
_, _, local_num_computed_blocks, awaiting_kvs = scheduler._reqs_need_recv[
|
||||
request.request_id
|
||||
]
|
||||
assert local_num_computed_blocks == (2, 1)
|
||||
# There are external tokens to pull, so the scheduler parks the request and
|
||||
# is waiting to see it in finished_recving.
|
||||
assert awaiting_kvs is True
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_full_local_hit_is_not_awaited_by_the_scheduler():
|
||||
"""A recv with num_external_tokens == 0 must not be flagged as awaited.
|
||||
|
||||
The request keeps running -- there is nothing to pull, only a notification
|
||||
owed to the producer -- so the worker must not report it in
|
||||
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
|
||||
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
|
||||
request that is parked or already done."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
|
||||
scheduler = object.__new__(NixlPullConnectorScheduler)
|
||||
scheduler._reqs_in_batch = set()
|
||||
scheduler._reqs_need_save = {}
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.use_host_buffer = False
|
||||
scheduler.is_bidirectional_kv_xfer_enabled = False
|
||||
scheduler._is_hma_required = False
|
||||
scheduler.kv_cache_config = MagicMock(
|
||||
select_transfer_block_ids=lambda block_ids: block_ids
|
||||
)
|
||||
|
||||
blocks = KVCacheBlocks(blocks=([KVCacheBlock(block_id=0, _block_hash=object())],))
|
||||
request = create_request(do_remote_prefill=True)
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=0)
|
||||
|
||||
_, local_block_ids, _, awaiting_kvs = scheduler._reqs_need_recv[request.request_id]
|
||||
assert not local_block_ids # full local hit: nothing to pull
|
||||
assert awaiting_kvs is False
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@@ -877,6 +918,7 @@ def _make_mock_worker_for_desc_ids(
|
||||
worker = MagicMock(spec=NixlConnectorWorker)
|
||||
worker.num_regions = num_regions
|
||||
worker._uses_region_group_mapping = False
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._has_mamba = has_mamba
|
||||
worker._group_spec_types = group_spec_types
|
||||
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||
|
||||
@@ -12,6 +12,7 @@ mid-decode (silent corruption of an unrelated request).
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from threading import Event, Lock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
@@ -112,6 +113,7 @@ def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
worker.block_len_per_layer = [16, 16]
|
||||
worker.block_stride_per_layer = [16, 16]
|
||||
worker.region_num_blocks = [2, 3]
|
||||
worker._transfer_layer_region_indices = ()
|
||||
|
||||
descriptors = worker._build_fa_local([100, 1000], block_size_ratio=1)
|
||||
|
||||
@@ -119,7 +121,8 @@ def test_local_descriptors_follow_each_region_pool_capacity():
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_overlaid_transfer_groups_share_region_geometry():
|
||||
@pytest.mark.parametrize("push_pp", [False, True])
|
||||
def test_overlaid_transfer_groups_share_region_geometry(push_pp):
|
||||
"""Groups overlaid on one allocation share its transfer region."""
|
||||
import msgspec
|
||||
|
||||
@@ -127,6 +130,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlAgentMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
@@ -153,7 +159,14 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
}
|
||||
groups = [KVCacheGroupSpec([layer_name], spec) for layer_name in caches]
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker_cls = NixlPushConnectorWorker if push_pp else NixlConnectorWorker
|
||||
worker = object.__new__(worker_cls)
|
||||
if push_pp:
|
||||
worker._push_writer_stop = Event()
|
||||
worker._push_writer_wake = Event()
|
||||
worker._push_writer_thread = MagicMock()
|
||||
worker._sending_transfers_lock = Lock()
|
||||
worker._sending_transfers = defaultdict(list)
|
||||
worker.tp_rank = 0
|
||||
worker.world_size = 1
|
||||
worker.transfer_tp_rank = 0
|
||||
@@ -198,6 +211,9 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
worker._desc_is_dram_by_block_size = {}
|
||||
worker._desc_pos_by_block_size = {}
|
||||
worker._dram_src_handles_by_block_size = {}
|
||||
worker._transfer_layer_names = ()
|
||||
worker._transfer_layer_region_indices = ()
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._region_is_mla = []
|
||||
worker.block_len_per_layer = []
|
||||
worker.block_stride_per_layer = []
|
||||
@@ -205,7 +221,8 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
worker.use_host_buffer = False
|
||||
worker.host_xfer_buffers = {}
|
||||
worker.device_kv_caches = {}
|
||||
worker.pp_size = 1
|
||||
worker.pp_size = 2 if push_pp else 1
|
||||
worker._is_hma_required = True
|
||||
worker.dcp_size = 1
|
||||
worker.pcp_size = 1
|
||||
worker.kv_buffer_device = "cuda"
|
||||
@@ -240,7 +257,13 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
expected_addrs = [
|
||||
backing.data_ptr() + block * block_stride for block in range(num_blocks)
|
||||
]
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
|
||||
num_desc_regions = 2 if push_pp else 1
|
||||
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs * num_desc_regions
|
||||
assert worker.num_descs == num_blocks * num_desc_regions
|
||||
assert (
|
||||
worker.dst_region_num_blocks[worker.engine_id]
|
||||
== [num_blocks] * num_desc_regions
|
||||
)
|
||||
|
||||
metadata = msgspec.msgpack.decode(
|
||||
worker.xfer_handshake_metadata.agent_metadata_bytes,
|
||||
@@ -248,6 +271,7 @@ def test_overlaid_transfer_groups_share_region_geometry():
|
||||
)
|
||||
assert metadata.region_group_ids == [-1]
|
||||
assert metadata.region_num_blocks == [num_blocks]
|
||||
assert metadata.region_members == ([["layer.0", "layer.1"]] if push_pp else [])
|
||||
assert worker._block_ids_by_region(([0], [2]), worker.region_group_ids) == [[0, 2]]
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user