Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
@@ -17,6 +17,7 @@ import vllm.config as vllm_config_module
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils
from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
@@ -140,10 +141,15 @@ def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]:
return quantization_config
def _stub_quant_config() -> QuantizationConfig:
return cast(QuantizationConfig, object())
def get_fse_test_model_config(
model_type: str,
quantization_config: dict[str, Any],
) -> tuple[object, type[nn.Module]]:
config: object
if model_type == "minimax_m3":
config = MiniMaxM3TextConfig(
hidden_size=128,
@@ -301,7 +307,7 @@ def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled(
)
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
object(), "model.layers.0.mlp"
_stub_quant_config(), "model.layers.0.mlp"
)
@@ -316,7 +322,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
assert (
fused_moe_utils.resolve_layer_fused_shared_expert(
object(), "model.layers.0.mlp"
_stub_quant_config(), "model.layers.0.mlp"
)
is False
)
@@ -325,7 +331,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
def test_resolve_layer_fused_shared_expert_passes_module_prefixes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
quant_config = object()
quant_config = _stub_quant_config()
monkeypatch.setattr(
fused_moe_utils.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
@@ -374,7 +380,7 @@ def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization(
)
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
object(), "model.layers.0.mlp"
_stub_quant_config(), "model.layers.0.mlp"
)
assert "shared experts are excluded" in caplog.text
@@ -1214,7 +1220,7 @@ def test_quark_packed_layer_config_must_match_global_config() -> None:
def test_non_quark_shared_expert_fse_is_incompatible() -> None:
compatible, reason = is_shared_expert_quant_fse_compatible(
object(),
_stub_quant_config(),
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_experts",
)
@@ -205,6 +205,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
topk_indices,
False,
False,
False,
MLA_LAYER,
)
@@ -259,6 +260,7 @@ def test_skipped_k_cache_insert_accepts_no_k(
topk_indices,
True,
False,
False,
"",
)
@@ -267,9 +269,11 @@ def test_skipped_k_cache_insert_accepts_no_k(
@pytest.mark.parametrize("fp8_query", [False, True])
@pytest.mark.parametrize("prepared_mha", [False, True])
def test_deepseek_v32_dispatches_selected_mha(
monkeypatch: pytest.MonkeyPatch,
fp8_query: bool,
prepared_mha: bool,
) -> None:
attn_metadata = SimpleNamespace(num_actual_tokens=2)
kv_cache = torch.empty(1)
@@ -301,7 +305,11 @@ def test_deepseek_v32_dispatches_selected_mha(
q_nope = torch.randn(2, 1, 2)
q_pe = torch.randn(2, 1, 2)
mqa_q = torch.randn(2, 1, 2)
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
if fp8_query and not prepared_mha:
mqa_q = mqa_q.to(torch.float8_e4m3fn)
mha_q = torch.cat(
(q_nope, q_pe + 1 if fp8_query and not prepared_mha else mqa_q), dim=-1
)
kv_c = torch.empty(2, 2)
k_pe = torch.empty(2, 2)
output = torch.empty(2, 2)
@@ -6,6 +6,8 @@
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
@@ -263,6 +265,23 @@ class TestFusedMMInputNormKernel:
class TestFusedMMInputNormConstruction:
"""Weight/bias buffer semantics at init time."""
def test_input_norm_is_always_enabled(self):
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
compile_mm_encoder=True,
custom_ops=["none"],
)
)
with set_current_vllm_config(config):
FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE)
assert (
config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1
)
assert not config.compilation_config.disabled_custom_ops[
"fused_mm_input_norm"
]
def test_identity_config_buffers(self):
"""Numerically identity parameters still build plain weight/bias
buffers; the identity shortcut lives in ``from_model_config``."""
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import (
TokenEmbeddingPoolerHead,
)
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
_HIDDEN = 16
@@ -24,7 +25,7 @@ _BATCH = 3
def _make_params(
n: int,
*,
task: str = "embed",
task: PoolingTask = "embed",
dimensions: int | None = None,
use_activation: bool | None = None,
) -> list[PoolingParams]:
@@ -320,6 +321,7 @@ class TestTokenEmbeddingPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is not None
assert out.dtype == torch.float16
def test_projector(self):
@@ -328,6 +330,7 @@ class TestTokenEmbeddingPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 8)
assert torch.allclose(out, proj(x))
@@ -336,6 +339,7 @@ class TestTokenEmbeddingPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
assert torch.equal(out, x[..., :4])
@@ -360,6 +364,7 @@ class TestTokenEmbeddingPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
assert torch.equal(out, proj(x)[..., :4])
@@ -368,6 +373,7 @@ class TestTokenEmbeddingPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
@@ -416,6 +422,7 @@ class TestTokenClassifierPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert out is not None
assert out.dtype == torch.float16
def test_classifier(self):
@@ -424,6 +431,7 @@ class TestTokenClassifierPoolerHead:
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 3)
assert torch.allclose(out, clf(x))
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.pooler.tokwise.methods import (
get_tok_pooling_method,
)
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
_CPU = torch.device("cpu")
@@ -66,7 +67,7 @@ def _make_pooling_cursor(
def _make_metadata(
prompt_lens: list[int],
*,
tasks: list[str] | None = None,
tasks: list[PoolingTask] | None = None,
token_ids: list[list[int]] | None = None,
pooling_params: list[PoolingParams] | None = None,
num_scheduled_tokens: list[int] | None = None,
@@ -36,7 +36,7 @@ try:
import tensorizer
from tensorizer import EncryptionParams
except ImportError:
tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment]
tensorizer = PlaceholderModule("tensorizer")
EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
@@ -250,12 +250,12 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
dp_third_node = _parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
)
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
dp_tp = _parallel(
tensor_parallel_size=2,
@@ -263,29 +263,43 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
data_parallel_size_local=2,
data_parallel_rank=2,
)
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
assert plan_local_ranks(dp_tp) == [
(0, 2, 0, 0),
(1, 2, 0, 1),
(2, 3, 0, 0),
(3, 3, 0, 1),
]
# DP + nnodes: node-local TP replicas
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
dp_tp_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
)
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
tp_span_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
)
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
tp_span_node2 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
)
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
# PP adds one daemon placement per stage while preserving TP rank order.
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
assert plan_local_ranks(pp_tp) == [
(0, 0, 0, 0),
(1, 0, 0, 1),
(2, 0, 1, 0),
(3, 0, 1, 1),
]
def test_daemon_rejects_unmappable_parallelism():
@@ -302,8 +316,7 @@ def test_daemon_rejects_unmappable_parallelism():
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
)
with pytest.raises(ValueError, match="pipeline"):
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
with pytest.raises(ValueError, match="evenly divide"):
@@ -335,6 +348,24 @@ def test_weight_cache_key_distinguishes_dp_ranks():
vllm_version="v",
dp_size=16,
dp_rank=3,
pp_size=2,
pp_rank=1,
)
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
"""Copy mode clones the weights into the engine, so nothing is external
(vllm_config is never touched)."""
from vllm.config import LoadConfig
from vllm.model_executor.model_loader.weight_cache.ipc_loader import (
IpcModelLoader,
)
copy_mode = LoadConfig(
load_format="ipc_cache", model_loader_extra_config={"mode": "copy"}
)
loader = IpcModelLoader(copy_mode)
assert loader.get_external_weight_memory(None) == 0
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from torch.testing._internal.two_tensor import TwoTensor
from vllm.config import ParallelConfig
from vllm.model_executor import parameter
from vllm.model_executor.model_loader.weight_checksum import (
compute_tensor_digests,
zero_weights,
)
from vllm.v1.worker import gpu_worker
from vllm.v1.worker.gpu_worker import Worker
pytestmark = pytest.mark.cpu_test
class _Model(nn.Module):
def __init__(self):
super().__init__()
self.linear = nn.Linear(2, 2)
nn.init.ones_(self.linear.weight)
nn.init.ones_(self.linear.bias)
# A strided view, which cannot be reinterpreted as bytes in place.
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
self.scalar = nn.Parameter(torch.tensor(2.0))
# Loading never restores buffers, so reset must leave them alone.
self.register_buffer("k_scale", torch.tensor(2.0))
def test_digests_cover_every_parameter():
assert set(compute_tensor_digests(_Model())) == {
"linear.weight",
"linear.bias",
"strided",
"scalar",
}
def test_zero_weights_changes_every_parameter_and_no_buffer():
model = _Model()
before = compute_tensor_digests(model)
zero_weights(model)
after = compute_tensor_digests(model)
assert all(before[name] != after[name] for name in before)
assert model.k_scale.item() == 2.0
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
model = nn.Module()
model.transform = parameter.SharedWeightParameter(weight_loader=None)
model.transform.add_partition(0, object(), 2, 2)
model.transform.partitions[0].data.fill_(3.0)
before = compute_tensor_digests(model)
zero_weights(model)
assert set(before) == {"transform.0"}
assert compute_tensor_digests(model) != before
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
model = nn.Module()
model.weight = nn.Parameter(
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
)
before = compute_tensor_digests(model)
zero_weights(model)
assert set(before) == {"weight.a", "weight.b"}
assert all(
before[name] != digest for name, digest in compute_tensor_digests(model).items()
)
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
prefixes = []
for dp_rank in (0, 1):
worker = object.__new__(Worker)
worker.parallel_config = ParallelConfig(
data_parallel_size=2, data_parallel_rank=dp_rank
)
# Dense DP resets data_parallel_rank to 0 in each engine.
worker.parallel_config.reconfigure_for_independent_dp_rank()
prefixes.append(worker._weight_checksum_prefix())
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]
@@ -6,10 +6,61 @@ import pytest
import torch
from vllm.model_executor.layers import utils
from vllm.platforms import current_platform
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_default_torch_dtype
def test_dispatch_prepacks_arm_bf16_causal_conv(monkeypatch):
monkeypatch.setattr(
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
)
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
packed = torch.randn(32, 4, dtype=torch.bfloat16)
pack_inputs = []
def pack(weight):
pack_inputs.append(weight.clone())
return packed
monkeypatch.setattr(utils.ops, "causal_conv1d_weight_pack", pack)
original = torch.randn(32, 1, 4, dtype=torch.bfloat16)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(original.clone(), requires_grad=False)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert len(pack_inputs) == 1
# Check causal_conv1d_weight_pack received the expected weights
torch.testing.assert_close(pack_inputs[0], original.view(32, 4))
# Check we have correctly stashed the unpacked weights
torch.testing.assert_close(layer._cpu_unpacked_conv_weight, original.view(32, 4))
# Check we have correctly stored the packed weights
assert layer.weight.data_ptr() == packed.data_ptr()
def test_dispatch_does_not_pack_3d_expert_weight(monkeypatch):
monkeypatch.setattr(
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
)
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
pack_calls = []
monkeypatch.setattr(
utils.ops,
"causal_conv1d_weight_pack",
lambda weight: pack_calls.append(weight),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.randn(8, 32, 4, dtype=torch.bfloat16), requires_grad=False
)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert pack_calls == []
@pytest.fixture(scope="module")
def _mock_zentorch_linear_unary():
"""Register a mock zentorch_linear_unary op when zentorch is not installed.
@@ -1,16 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import sys
from collections import defaultdict
from contextlib import nullcontext
from inspect import signature
from types import SimpleNamespace
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
import pytest
from vllm.model_executor.warmup import kernel_warmup as warmup
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
flashinfer_autotune,
)
pytestmark = pytest.mark.cpu_test
@@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto
return SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
kernel_config=SimpleNamespace(linear_backend=linear_backend)
kernel_config=SimpleNamespace(linear_backend=linear_backend),
attention_config=SimpleNamespace(hisparse_config=None),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
@@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
**({"skip_attn": True} if skip_attn else {}),
),
]
class _AutotuneGroup:
def __init__(self, run, ranks):
self.run = run
self.ranks = tuple(ranks)
self.world_size = len(self.ranks)
self.rank_in_group = self.ranks.index(run.rank)
self.cpu_group = self
def record(self, operation):
self.run.collectives[self.ranks][self.run.rank].append(operation)
def broadcast_object(self, obj, src=0):
assert src == 0
self.record(("broadcast", src))
if self.rank_in_group == src:
self.run.broadcasts[self.ranks] = obj
return self.run.broadcasts[self.ranks]
def barrier(self):
self.record(("barrier",))
class _AutotuneTuner:
def __init__(self, run):
self.run = run
self.cache = {}
self.loaded = None
def load_configs(self, path):
self.loaded = json.loads(Path(path).read_text())
self.cache.update(self.loaded)
def save_configs(self, path):
self.run.saves.append((self.run.rank, Path(path), dict(self.cache)))
Path(path).write_text(json.dumps(self.cache))
def profile(self, operation):
if operation in self.cache:
return
group = self.run.tuning_group
self.run.profile_groups[self.run.rank].append(
None if group is None else group.ranks
)
for tactic in range(2):
if group is not None:
group.record(("all_reduce", operation, tactic))
self.cache[operation] = self.run.rank // self.run.tp
class _AutotuneRun:
def __init__(self, pp, tp):
self.pp, self.tp = pp, tp
self.rank = 0
self.tuning_group = None
self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = (
defaultdict(lambda: defaultdict(list))
)
self.broadcasts = {}
self.tuners = {}
self.saves = []
self.profile_groups = defaultdict(list)
def world(self):
return _AutotuneGroup(self, range(self.pp * self.tp))
def tensor_group(self):
start = self.rank // self.tp * self.tp
return _AutotuneGroup(self, range(start, start + self.tp))
def pipeline_group(self):
return SimpleNamespace(world_size=self.pp)
def set_group(self, group):
self.tuning_group = group
def dummy_runs(self, runner, *, skip_attn=False):
self.tuners[self.rank].profile("shared_gemm")
if self.pp > 1 and self.rank // self.tp == 0:
self.tuners[self.rank].profile("pp0_extra_gemm")
def execute(self):
for rank in range(self.pp * self.tp):
self.rank = rank
self.tuners[rank] = _AutotuneTuner(self)
flashinfer_autotune(_make_runner([]))
return self
def assert_collectives_match(self):
for ranks, traces in self.collectives.items():
assert set(traces) == set(ranks)
expected = traces[ranks[0]]
assert all(trace == expected for trace in traces.values()), dict(traces)
@pytest.fixture
def autotune_run(monkeypatch, tmp_path):
import vllm.utils.flashinfer as fi_utils
from vllm.distributed import parallel_state
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
"AutoTuner",
SimpleNamespace(get=lambda: run.tuners[run.rank]),
raising=False,
)
monkeypatch.setattr(
autotuner, "set_autotune_process_group", run.set_group, raising=False
)
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner)
monkeypatch.setattr(parallel_state, "get_world_group", run.world)
monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group)
monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group)
monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext())
monkeypatch.setattr(
warmup,
"resolve_flashinfer_autotune_file",
lambda runner: tmp_path / "autotune_configs.json",
)
monkeypatch.setattr(
warmup, "_flashinfer_autotune_skip_ops", lambda runner: None
)
monkeypatch.setattr(
warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs
)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None)
return run
return make_run
@pytest.mark.parametrize("tp", [1, 4])
def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp):
run = autotune_run(tp=tp).execute()
run.assert_collectives_match()
for rank, groups in run.profile_groups.items():
expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp))
assert groups and all(
group == (expected if tp > 1 else None) for group in groups
)
def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe(
autotune_run, tmp_path
):
legacy = tmp_path / "autotune_configs.json"
legacy.write_text('{"legacy_world_cache": 99}')
cold = autotune_run().execute()
assert [rank for rank, _, _ in cold.saves] == [0, 4]
paths = [path for _, path, _ in cold.saves]
assert len(set(paths)) == 2 and legacy not in paths
assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99}
assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0}
assert cold.saves[1][2] == {"shared_gemm": 1}
cold.assert_collectives_match()
warm = autotune_run().execute()
warm.assert_collectives_match()
assert not warm.profile_groups
for rank, tuner in warm.tuners.items():
assert tuner.loaded == cold.saves[rank // 4][2]
paths[1].unlink()
mixed = autotune_run().execute()
mixed.assert_collectives_match()
assert set(mixed.profile_groups) == {4, 5, 6, 7}
assert all(mixed.tuners[rank].loaded is not None for rank in range(4))
assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8))
def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run):
run = autotune_run(pp=1, tp=4).execute()
run.assert_collectives_match()
assert [(rank, path.name) for rank, path, _ in run.saves] == [
(0, "autotune_configs.json")
]
assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values())
@@ -0,0 +1,636 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
import vllm.model_executor.models.k2_horizon as k2_horizon
from tests.quantization.utils import is_quant_method_supported
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.models.k2_horizon import (
K2HorizonDecoderLayer,
K2HorizonModel,
K2HorizonRMSNorm,
_rope_weight_perm,
apply_partial_rope,
is_rope_weights_folding_supported,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed
DEVICE = current_platform.device_type
DTYPE = torch.bfloat16
SHAPES = [
(128, 64),
(128, 128),
(192, 128),
(96, 32),
]
def get_gptj_rope(
positions: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
max_position: int,
base: float,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
rotary_emb = get_rope(
rope_head_dim,
max_position=max_position,
is_neox_style=True,
rope_parameters={"rope_theta": base},
dtype=dtype,
)
return apply_partial_rope(
rotary_emb,
positions,
q,
k,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
fold_rope_weights=False,
)
def _apply_head_perm(
x: torch.Tensor, num: int, head_dim: int, idx: torch.Tensor
) -> torch.Tensor:
return (
x.reshape(*x.shape[:-1], num, head_dim)[..., idx].reshape(*x.shape).contiguous()
)
def get_neox_rope(
positions: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
max_position: int,
base: float,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
rotary_emb = get_rope(
head_dim,
max_position=max_position,
is_neox_style=True,
rope_parameters={"rope_theta": base, "rope_dim": rope_head_dim},
dtype=dtype,
)
return apply_partial_rope(
rotary_emb,
positions,
q,
k,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
fold_rope_weights=True,
)
def fold_qk_proj_weight(
weight: torch.Tensor, head_dim: int, idx: torch.Tensor
) -> torch.Tensor:
hidden = weight.shape[-1]
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
def fold_qk_channels(
vec: torch.Tensor, head_dim: int, idx: torch.Tensor
) -> torch.Tensor:
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
def test_rope_fold_matches_permute_then_rope(
head_dim, rope_head_dim, default_vllm_config
):
dtype = DTYPE
set_random_seed(0)
num_tokens = 17
num_heads = 8
num_kv_heads = 2
hidden_size = 512
max_position = 4096
base = 10000.0
torch.set_default_dtype(dtype)
positions = torch.randint(0, max_position, (num_tokens,), device=DEVICE)
scale = hidden_size**-0.5
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
w_q = (
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
* scale
)
w_k = (
torch.randn(num_kv_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
* scale
)
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
q = torch.nn.functional.linear(x, w_q)
k = torch.nn.functional.linear(x, w_k)
q_gptj, k_gptj = get_gptj_rope(
positions,
q,
k,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
max_position,
base,
dtype,
)
q_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_q, head_dim, idx))
k_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_k, head_dim, idx))
q_neox, k_neox = get_neox_rope(
positions,
q_folded,
k_folded,
num_heads,
num_kv_heads,
head_dim,
rope_head_dim,
max_position,
base,
dtype,
)
q_gptj_perm = _apply_head_perm(q_gptj, num_heads, head_dim, idx)
k_gptj_perm = _apply_head_perm(k_gptj, num_kv_heads, head_dim, idx)
atol, rtol = 1e-2, 1e-2
# Two complementary checks. First, an element-wise check that the folded
# NeoX path equals the GPT-J path up to the channel permutation P
# (neox_rope(Pq, Pk) == P @ gptj_rope(q, k)), hence permuting q_gptj/k_gptj.
# Second, the scores check below: P cancels in q @ k.T.
torch.testing.assert_close(q_neox, q_gptj_perm, atol=atol, rtol=rtol)
torch.testing.assert_close(k_neox, k_gptj_perm, atol=atol, rtol=rtol)
def scores(qt, kt):
qh = qt.reshape(num_tokens, num_heads, head_dim)
kh = kt.reshape(num_tokens, num_kv_heads, head_dim)
g = num_heads // num_kv_heads
qh = qh.reshape(num_tokens, num_kv_heads, g, head_dim).float()
kh = kh.float()
return torch.einsum("tkgd,skd->tksg", qh, kh)
torch.testing.assert_close(
scores(q_neox, k_neox), scores(q_gptj, k_gptj), atol=atol, rtol=rtol
)
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("num_heads", [8, 2])
def test_qk_bias_fold_matches_permute(
head_dim, rope_head_dim, num_heads, default_vllm_config
):
dtype = DTYPE
set_random_seed(0)
num_tokens = 17
hidden_size = 512
torch.set_default_dtype(dtype)
scale = hidden_size**-0.5
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE)
w = (
torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE)
* scale
)
b = torch.randn(num_heads * head_dim, dtype=dtype, device=DEVICE)
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
ref = _apply_head_perm(
torch.nn.functional.linear(x, w, b), num_heads, head_dim, idx
)
folded = torch.nn.functional.linear(
x,
fold_qk_proj_weight(w, head_dim, idx),
fold_qk_channels(b, head_dim, idx),
)
atol, rtol = 1e-2, 1e-2
torch.testing.assert_close(folded, ref, atol=atol, rtol=rtol)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a GPU (CUDA/ROCm)"
)
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("num_heads", [8, 2])
def test_qk_norm_scale_fold_matches_permute(
head_dim, rope_head_dim, num_heads, default_vllm_config
):
dtype = DTYPE
set_random_seed(0)
num_tokens = 13
torch.set_default_dtype(dtype)
hidden = num_heads * head_dim
idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE)
x = torch.randn(num_tokens, hidden, dtype=dtype, device=DEVICE)
norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
with torch.no_grad():
norm.weight.copy_(torch.randn(hidden, dtype=dtype, device=DEVICE))
y_ref = _apply_head_perm(norm(x.clone()), num_heads, head_dim, idx)
folded_norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE)
with torch.no_grad():
folded_norm.weight.copy_(fold_qk_channels(norm.weight, head_dim, idx))
# Permute x to emulate the folded q_proj/k_proj: at runtime the norm sees the
# already-permuted projection output (P @ x), not the raw activations.
y_folded = folded_norm(_apply_head_perm(x.clone(), num_heads, head_dim, idx))
atol, rtol = 1e-2, 1e-2
torch.testing.assert_close(y_folded, y_ref, atol=atol, rtol=rtol)
class _Fp8QKVOnlyConfig(Fp8Config):
def get_quant_method(self, layer, prefix):
if isinstance(layer, LinearBase) and not (
prefix.endswith("qkv_proj") or prefix.endswith("qk_proj")
):
return UnquantizedLinearMethod()
return super().get_quant_method(layer, prefix)
class _DummyAttention(nn.Module):
def __init__(self, *args, **kwargs) -> None:
super().__init__()
def forward(self, q, k, v):
return q
@pytest.fixture
def dist_init():
with set_current_vllm_config(VllmConfig()):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}",
local_rank=0,
backend="gloo",
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
def _make_hf_config(
*,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
attention_bias: bool,
query_key_norm: bool,
) -> SimpleNamespace:
return SimpleNamespace(
hidden_size=hidden_size,
max_position_embeddings=4096,
dual_chunk_attention_config=None,
mlp_only_layers=[],
num_experts=0, # dense layer: avoids the MoE path
decoder_sparse_step=1,
mova_num_experts=0,
num_attention_heads=num_heads,
num_key_value_heads=num_kv_heads,
rope_parameters={"rope_theta": 10000.0},
rope_head_dim=rope_head_dim,
query_key_norm=query_key_norm,
rms_norm_eps=1e-6,
attention_bias=attention_bias,
head_dim=head_dim,
attention_gate_func=None,
intermediate_size=2 * hidden_size,
hidden_act="silu",
layernorm_num_groups=1,
vocab_size=1000,
num_hidden_layers=1,
tie_word_embeddings=False,
)
def _build_single_layer_model(
monkeypatch, hf_config: SimpleNamespace, *, quant_config=None
) -> K2HorizonModel:
monkeypatch.setattr(k2_horizon, "Attention", _DummyAttention)
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
hf_text_config=hf_config, dtype=torch.get_default_dtype()
)
vllm_config.quant_config = quant_config
with set_current_vllm_config(vllm_config):
layer = K2HorizonDecoderLayer(vllm_config=vllm_config, prefix="model.layers.0")
model = K2HorizonModel.__new__(K2HorizonModel)
nn.Module.__init__(model)
model.config = hf_config
model.quant_config = quant_config
model.num_redundant_experts = 0
model.layers = nn.ModuleList([layer])
return model
def _fold_rows(weight: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
hidden = weight.shape[-1]
return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous()
def _fold_channels(vec: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor:
return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous()
def _non_qk_weights(prefix, *, hidden_size, intermediate_size, q_size, device=None):
return [
(
f"{prefix}.self_attn.o_proj.weight",
torch.randn(hidden_size, q_size, device=device),
),
(
f"{prefix}.mlp.gate_proj.weight",
torch.randn(intermediate_size, hidden_size, device=device),
),
(
f"{prefix}.mlp.up_proj.weight",
torch.randn(intermediate_size, hidden_size, device=device),
),
(
f"{prefix}.mlp.down_proj.weight",
torch.randn(hidden_size, intermediate_size, device=device),
),
(
f"{prefix}.input_layernorm.weight",
torch.randn(hidden_size, device=device),
),
(
f"{prefix}.post_attention_layernorm.weight",
torch.randn(hidden_size, device=device),
),
]
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("attention_bias", [False, True])
def test_load_weights_dense_folds_qk(
head_dim,
rope_head_dim,
attention_bias,
monkeypatch,
):
dtype = DTYPE
set_random_seed(0)
num_heads = 8
num_kv_heads = 2
hidden_size = 128
intermediate_size = 2 * hidden_size
q_size = num_heads * head_dim
kv_size = num_kv_heads * head_dim
prefix = "layers.0"
with set_default_torch_dtype(dtype):
hf_config = _make_hf_config(
hidden_size=hidden_size,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
rope_head_dim=rope_head_dim,
attention_bias=attention_bias,
query_key_norm=True,
)
model = _build_single_layer_model(monkeypatch, hf_config, quant_config=None)
assert model.layers[0].self_attn.fold_rope_weights is True
raw = {
"q_proj.weight": torch.randn(q_size, hidden_size),
"k_proj.weight": torch.randn(kv_size, hidden_size),
"v_proj.weight": torch.randn(kv_size, hidden_size),
"q_norm.weight": torch.randn(q_size),
"k_norm.weight": torch.randn(kv_size),
}
if attention_bias:
raw.update(
{
"q_proj.bias": torch.randn(q_size),
"k_proj.bias": torch.randn(kv_size),
"v_proj.bias": torch.randn(kv_size),
"o_proj.bias": torch.randn(hidden_size),
}
)
weights: list[tuple[str, torch.Tensor]] = [
(f"{prefix}.self_attn.{name}", tensor) for name, tensor in raw.items()
]
weights += _non_qk_weights(
prefix,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
q_size=q_size,
)
loaded = model.load_weights(iter(weights))
params = dict(model.named_parameters())
assert loaded == set(params)
idx = _rope_weight_perm(head_dim, rope_head_dim)
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
q_part = qkv[:q_size]
k_part = qkv[q_size : q_size + kv_size]
v_part = qkv[q_size + kv_size :]
q_norm = params[f"{prefix}.self_attn.q_norm.weight"]
k_norm = params[f"{prefix}.self_attn.k_norm.weight"]
assert torch.equal(q_part, _fold_rows(raw["q_proj.weight"], head_dim, idx))
assert torch.equal(k_part, _fold_rows(raw["k_proj.weight"], head_dim, idx))
assert torch.equal(v_part, raw["v_proj.weight"]) # v never folded
assert torch.equal(q_norm, _fold_channels(raw["q_norm.weight"], head_dim, idx))
assert torch.equal(k_norm, _fold_channels(raw["k_norm.weight"], head_dim, idx))
assert qkv.dtype == dtype
if attention_bias:
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
q_bias = qkv_bias[:q_size]
k_bias = qkv_bias[q_size : q_size + kv_size]
v_bias = qkv_bias[q_size + kv_size :]
assert torch.equal(q_bias, _fold_channels(raw["q_proj.bias"], head_dim, idx))
assert torch.equal(k_bias, _fold_channels(raw["k_proj.bias"], head_dim, idx))
assert torch.equal(v_bias, raw["v_proj.bias"]) # v bias never folded
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported.",
)
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES)
@pytest.mark.parametrize("attention_bias", [False, True])
def test_load_weights_fp8_qk_no_fold(
head_dim,
rope_head_dim,
attention_bias,
monkeypatch,
):
set_random_seed(0)
num_heads = 8
num_kv_heads = 2
hidden_size = 128
intermediate_size = 2 * hidden_size
q_size = num_heads * head_dim
kv_size = num_kv_heads * head_dim
prefix = "layers.0"
def rand_fp8(*shape):
return (torch.randn(*shape, device=DEVICE) * 32.0).to(torch.float8_e4m3fn)
def rand_scale():
return torch.rand(1, device=DEVICE, dtype=torch.float32) + 0.1
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
hf_config = _make_hf_config(
hidden_size=hidden_size,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
rope_head_dim=rope_head_dim,
attention_bias=attention_bias,
query_key_norm=True,
)
quant_config = _Fp8QKVOnlyConfig(
is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"
)
model = _build_single_layer_model(
monkeypatch, hf_config, quant_config=quant_config
)
qkv_proj = model.layers[0].self_attn.qkv_proj
assert isinstance(qkv_proj.quant_method, Fp8LinearMethod)
assert model.layers[0].self_attn.fold_rope_weights is False
raw_w = {
"q_proj.weight": rand_fp8(q_size, hidden_size),
"k_proj.weight": rand_fp8(kv_size, hidden_size),
"v_proj.weight": rand_fp8(kv_size, hidden_size),
}
raw_s = {
"q_proj.weight_scale": rand_scale(),
"k_proj.weight_scale": rand_scale(),
"v_proj.weight_scale": rand_scale(),
}
q_norm = torch.randn(q_size, device=DEVICE)
k_norm = torch.randn(kv_size, device=DEVICE)
weights: list[tuple[str, torch.Tensor]] = [
(f"{prefix}.self_attn.{name}", tensor)
for name, tensor in {**raw_w, **raw_s}.items()
]
weights += [
(f"{prefix}.self_attn.q_norm.weight", q_norm),
(f"{prefix}.self_attn.k_norm.weight", k_norm),
]
biases = {}
if attention_bias:
biases = {
"q_proj.bias": torch.randn(q_size, device=DEVICE),
"k_proj.bias": torch.randn(kv_size, device=DEVICE),
"v_proj.bias": torch.randn(kv_size, device=DEVICE),
"o_proj.bias": torch.randn(hidden_size, device=DEVICE),
}
weights += [
(f"{prefix}.self_attn.{name}", tensor)
for name, tensor in biases.items()
]
weights += _non_qk_weights(
prefix,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
q_size=q_size,
device=DEVICE,
)
loaded = model.load_weights(iter(weights))
params = dict(model.named_parameters())
assert loaded == set(params)
qkv = params[f"{prefix}.self_attn.qkv_proj.weight"]
weight_scale = params[f"{prefix}.self_attn.qkv_proj.weight_scale"]
assert qkv.dtype == torch.float8_e4m3fn
assert torch.equal(qkv[:q_size], raw_w["q_proj.weight"])
assert torch.equal(qkv[q_size : q_size + kv_size], raw_w["k_proj.weight"])
assert torch.equal(qkv[q_size + kv_size :], raw_w["v_proj.weight"])
assert torch.equal(
weight_scale,
torch.cat(
[
raw_s["q_proj.weight_scale"],
raw_s["k_proj.weight_scale"],
raw_s["v_proj.weight_scale"],
]
),
)
assert torch.equal(params[f"{prefix}.self_attn.q_norm.weight"], q_norm)
assert torch.equal(params[f"{prefix}.self_attn.k_norm.weight"], k_norm)
if attention_bias:
qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"]
assert torch.equal(qkv_bias[:q_size], biases["q_proj.bias"])
assert torch.equal(qkv_bias[q_size : q_size + kv_size], biases["k_proj.bias"])
assert torch.equal(qkv_bias[q_size + kv_size :], biases["v_proj.bias"])
def test_is_rope_weights_folding_supported(default_vllm_config):
dense = SimpleNamespace(quant_method=UnquantizedLinearMethod())
quantized = SimpleNamespace(quant_method=object())
assert is_rope_weights_folding_supported(dense, None) is True
assert is_rope_weights_folding_supported(quantized, None) is False
assert is_rope_weights_folding_supported(dense, {"chunk_size": 8}) is False
assert is_rope_weights_folding_supported(SimpleNamespace(), None) is False
+19 -4
View File
@@ -67,8 +67,13 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
with patch.object(
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
):
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
result = warmup._replayssm_autotune_kwargs(
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
)
assert result is not None
max_num_reqs, decode_kwargs = result
assert max_num_reqs == expected_num_reqs
expected_kwargs = {
**PREFILL_KWARGS,
"num_tokens": expected_num_reqs * query_len,
@@ -82,7 +87,7 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
force_attention=True,
profile_seq_lens=query_len + 1,
)
assert result == (expected_num_reqs, expected_kwargs)
assert decode_kwargs == expected_kwargs
@pytest.mark.parametrize(
@@ -91,8 +96,14 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
(dict(use_replayssm=False), True),
(dict(backend=MambaBackendEnum.TRITON), True),
({}, False),
(dict(num_blocks=1), True),
],
ids=[
"replayssm_disabled",
"non_flashinfer_backend",
"kernel_unavailable",
"zero_non_padding_slots",
],
ids=["replayssm_disabled", "non_flashinfer_backend", "kernel_unavailable"],
)
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
with patch.object(
@@ -100,7 +111,9 @@ def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
"flashinfer_replayssm_autotune_supported",
return_value=flashinfer_supported,
):
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
result = warmup._replayssm_autotune_kwargs(
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
)
assert result is None
@@ -116,10 +129,12 @@ def test_replayssm_autotune_slots_restore_state_and_trackers():
)
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
tracked = (
*mixer.kv_cache,
mixer._replayssm_ring_start,
mixer._replayssm_prev_num_accepted,
mixer._replayssm_prev_query_len,
)
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)