[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-914
View File
@@ -1,914 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests whether FP8 computation is enabled correctly.
Run `pytest tests/quantization/test_fp8.py --forked`.
"""
from types import SimpleNamespace
import pytest
import torch
from tests.quantization.utils import (
is_quant_method_supported,
load_model_without_vllm_runner,
)
from vllm import _custom_ops as ops
from vllm.config import set_current_vllm_config
from vllm.config.cache import CacheConfig
from vllm.config.kernel import KernelConfig
from vllm.config.model import ModelConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention.attention import (
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import (
Fp8Config,
Fp8LinearMethod,
Fp8MoEMethod,
)
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
prepare_fp8_moe_layer_for_fi,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_input_tensor_strategy_moe,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
MODELS = [
"neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
# The checkpoint below was removed from the HF.
# TODO: add a small replacement checkpoint.
pytest.param(
"nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
marks=pytest.mark.skip(reason="Checkpoint removed from HF."),
),
]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DSV4.1 requires CUDA")
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
def test_deepseek_v41_mxfp8_scale_loading(
dist_init, default_vllm_config, monkeypatch, tp_rank, scale_dtype
):
"""Expand linear scales before TP slicing, including padded shared experts."""
from vllm.model_executor.layers import linear as linear_module
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.models.deepseek_v41 import quant_config as quant_module
from vllm.models.deepseek_v41.nvidia import model as model_module
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
quant_config = quant_module.DeepseekV4FP8Config(
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
)
quant_config._resolved_expert_dtype = "fp4"
for module in (model_module, linear_module):
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(
linear_module, "get_tensor_model_parallel_rank", lambda: tp_rank
)
block = torch.nn.Module()
block.attn = torch.nn.Module()
block.attn.wq_b = ColumnParallelLinear(
128, 128, quant_config=quant_config, bias=False
)
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(
128, [128, 64], quant_config=quant_config, bias=False
)
block.ffn = torch.nn.Module()
block.ffn.shared_experts = torch.nn.Module()
block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear(
128, [128, 128], quant_config=quant_config, bias=False
)
block.ffn.shared_experts.down_proj = RowParallelLinear(
128, 128, quant_config=quant_config, bias=False
)
block.ffn.experts = torch.nn.Module()
block.ffn.experts.register_parameter(
"weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False)
)
block.engram = torch.nn.Module()
block.engram.register_parameter(
"weight_scale_inv",
torch.nn.Parameter(torch.empty(3, 4, dtype=scale_dtype), False),
)
def load_expert_scale(param, weight, *args, **kwargs):
param.data.copy_(weight)
return True
block.ffn.experts.weight_scale.weight_loader = load_expert_scale
model = model_module.DeepseekV4Model.__new__(model_module.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.layers = torch.nn.ModuleList([block])
model.config = SimpleNamespace(num_attention_heads=2)
model.quant_config = quant_config
model.use_sequence_parallel = False
monkeypatch.setattr(
model,
"get_expert_mapping",
lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")],
)
checkpoints = [
("attn.wq_b", "attn.wq_b", 128, 128, 0, None),
("attn.wq_a", "attn.fused_wqa_wkv", 128, 128, 0, slice(0, 64)),
("attn.wkv", "attn.fused_wqa_wkv", 64, 128, 0, slice(64, 96)),
(
"ffn.shared_experts.w1",
"ffn.shared_experts.gate_up_proj",
96,
128,
0,
slice(0, 64),
),
(
"ffn.shared_experts.w3",
"ffn.shared_experts.gate_up_proj",
96,
128,
0,
slice(64, 128),
),
(
"ffn.shared_experts.down_proj",
"ffn.shared_experts.down_proj",
128,
96,
1,
None,
),
]
weights = []
expected = []
for source, target, n, k, axis, shard in checkpoints:
weight = torch.randint(-4, 5, (n, k)).to(torch.float8_e4m3fn)
scale_bytes = torch.randint(124, 131, (n // 32, k // 32), dtype=torch.uint8)
weights.extend(
[
(f"layers.0.{source}.weight", weight),
(f"layers.0.{source}.scale", scale_bytes.view(scale_dtype)),
]
)
dequant = (
weight.float().reshape(n // 32, 32, k // 32, 32)
* torch.exp2(scale_bytes.float() - 127)[:, None, :, None]
).reshape(n, k)
if "shared_experts" in source:
padded = torch.zeros(128, 128)
padded[:n, :k] = dequant
dequant = padded
expected.append((target, shard, dequant.chunk(2, dim=axis)[tp_rank]))
expert_scale = torch.full((3, 4), 125, dtype=torch.uint8)
weights.append(
("layers.0.ffn.experts.0.w1.weight_scale", expert_scale.view(scale_dtype))
)
weights.append(("layers.0.engram.weight_scale_inv", expert_scale.view(scale_dtype)))
mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale")
loaded = model.load_weights(
(name.removeprefix("model."), weight) for name, weight in mapper.apply(weights)
)
assert "layers.0.ffn.experts.weight_scale" in loaded
assert torch.equal(block.ffn.experts.weight_scale, expert_scale)
assert torch.equal(block.engram.weight_scale_inv.view(torch.uint8), expert_scale)
for target, shard, reference in expected:
linear = block.get_submodule(target)
assert isinstance(linear.quant_method, ModelOptLinearMethod)
assert f"layers.0.{target}.weight_scale" in loaded
weight = linear.weight if shard is None else linear.weight[shard]
scale = linear.weight_scale if shard is None else linear.weight_scale[shard]
actual = (
weight.float().unflatten(-1, (-1, 32))
* torch.exp2(scale.float() - 127).unsqueeze(-1)
).flatten(-2)
torch.testing.assert_close(actual, reference, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DSV4.1 requires CUDA")
@pytest.mark.parametrize(
"weight_block_size,expert_dtype,scale_name",
[
([32, 32], "fp4", "weight_scale"),
([128, 128], "fp4", "weight_scale_inv"),
([32, 32], "fp8", "weight_scale_inv"),
],
)
def test_deepseek_v41_vl_mapper_routes_linear_scales(
weight_block_size, expert_dtype, scale_name
):
"""The VL wrapper must map ``.scale`` keys to the parameter the linear
quant method registers, as the text model does. A hardcoded
``weight_scale_inv`` raised KeyError for native MXFP8 checkpoints."""
from vllm.models.deepseek_v41.nvidia import model as model_module
from vllm.models.deepseek_v41.nvidia import vl_model as vl_module
vllm_config = SimpleNamespace(
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
)
resolved = model_module._linear_scale_param_name(
vllm_config,
expert_dtype,
)
assert resolved == scale_name
mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved)
weight = torch.empty(0)
mapped = [
name
for name, _ in mapper.apply(
[("layers.0.attn.wq_a.scale", weight), ("layers.0.attn.wkv.weight", weight)]
)
]
assert mapped == [
f"language_model.model.layers.0.attn.wq_a.{scale_name}",
"language_model.model.layers.0.attn.wkv.weight",
]
def test_deepseek_v41_vl_exposes_quant_mappings_on_class():
"""``configure_quant_config`` reads both mappings off the class, before
``__init__`` builds the instance mapper. Without them a Quark config's
per-layer keys never match this wrapper's ``language_model.``-rooted
prefixes and every attention shard falls back to the global spec."""
from vllm.models.deepseek_v41.amd.vl_model import DeepseekV41ForCausalLM
assert "fused_wqa_wkv" in DeepseekV41ForCausalLM.packed_modules_mapping
mapper = DeepseekV41ForCausalLM.hf_to_vllm_mapper.get_rename_mapper()
assert mapper.apply_list(["layers.0.attn.wq_a"]) == [
"language_model.model.layers.0.attn.wq_a"
]
def test_deepseek_v41_engram_scale_accepts_quark_name():
"""Quark exports name the engram scale ``embed.weight_scale``; the
``\\.scale$`` rules only match a literal ``.scale`` suffix, so without an
explicit rule the tensor is never routed and loading fails."""
from vllm.models.deepseek_v41.amd import model as model_module
mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale")
weight = torch.empty(0)
mapped = [
name
for name, _ in mapper.apply(
[
("layers.1.engram.embed.weight_scale", weight),
("layers.1.engram.embed.scale", weight),
]
)
]
assert mapped == [
"model.layers.1.engram.embed_tokens.weight_scale_inv",
"model.layers.1.engram.embed_tokens.weight_scale_inv",
]
def test_deepseek_v41_declines_quark_configs():
"""``from_config`` rewrites a Quark config into a single global FP8 scheme,
which would discard the per-layer specs of a mixed-precision export (MXFP4
experts + 2-D block MXFP8 attention in DeepSeek-V4.1-Flash). QuarkConfig
knows how to dispatch each scheme, so it must handle every Quark export."""
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
hf_config = SimpleNamespace(model_type="deepseek_v41")
mxfp4_global = {
"global_quant_config": {
"weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}
},
"quant_method": "quark",
}
assert (
DeepseekV4FP8Config.override_quantization_method(
mxfp4_global, None, hf_config=hf_config
)
is None
)
mixed = {
**mxfp4_global,
"layer_quant_config": {
"layers.0.attn.wkv": {
"weight": {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"block_size": [32, 32],
}
}
},
}
assert (
DeepseekV4FP8Config.override_quantization_method(
mixed, None, hf_config=hf_config
)
is None
)
_DSV41_NVFP4 = {
"moe_quant_algo": "NVFP4",
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"],
}
_DSV4_NVFP4_DSPARK = {
"moe_quant_algo": "NVFP4",
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head"],
}
@pytest.mark.parametrize(
("ckpt_quant_cfg", "prefix", "expected"),
[
pytest.param(_DSV41_NVFP4, "model.layers.1.ffn.experts", "nvfp4", id="target"),
pytest.param(
_DSV41_NVFP4,
"language_model.model.layers.0.ffn.experts",
"nvfp4",
id="target-vl-prefix",
),
pytest.param(
_DSV41_NVFP4, "model.layers.40.ffn.experts", "mxfp4", id="ignored-mtp0"
),
pytest.param(
_DSV41_NVFP4, "model.layers.42.ffn.experts", "mxfp4", id="ignored-mtp2"
),
pytest.param(
_DSV4_NVFP4_DSPARK,
"model.layers.41.ffn.experts",
"nvfp4",
id="quantized-mtp1",
),
pytest.param(
{"moe_quant_algo": "NVFP4"},
"model.layers.40.ffn.experts",
"nvfp4",
id="no-ignore-list",
),
pytest.param({}, "model.layers.1.ffn.experts", "mxfp4", id="not-nvfp4"),
],
)
def test_deepseek_v41_nvfp4_export_keeps_ignored_experts_mxfp4(
monkeypatch, ckpt_quant_cfg, prefix, expected
):
"""NVFP4 exports keep the routed experts they ``ignore`` in MXFP4, e.g. the
bundled DSpark draft (``mtp.*``, built as ``layers.{40 + k}``) of
DeepSeek-V4.1-Flash-NVFP4. The ignore list is read from the checkpoint
passed to ``from_config``: the current hf_config below carries none."""
import vllm.model_executor.layers.quantization.modelopt as modelopt
import vllm.models.deepseek_v41.quant_config as dsv41_qc
from vllm.model_executor.layers.fused_moe import RoutedExperts
hf_config = SimpleNamespace(
expert_dtype="fp4",
quantization_config={"moe_quant_algo": ckpt_quant_cfg.get("moe_quant_algo")},
)
model_config = SimpleNamespace(
hf_config=hf_config, hf_text_config=SimpleNamespace(num_hidden_layers=40)
)
monkeypatch.setattr(
dsv41_qc,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=model_config),
)
monkeypatch.setattr(dsv41_qc, "Mxfp4MoEMethod", lambda moe_config: "mxfp4")
monkeypatch.setattr(modelopt, "ModelOptNvFp4FusedMoE", lambda **kwargs: "nvfp4")
config = dsv41_qc.DeepseekV4FP8Config.from_config(
{"quant_method": "fp8", "activation_scheme": "dynamic", **ckpt_quant_cfg}
)
monkeypatch.setattr(config, "_get_nvfp4_config", lambda: None)
layer = RoutedExperts.__new__(RoutedExperts)
layer.moe_config = None
assert config.get_quant_method(layer, prefix) == expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
@pytest.mark.parametrize(
"weight_shape,is_bmm",
[
((129, 160), False),
((387, 160), True),
((3, 129, 160), False),
((3, 128, 512), False),
],
)
def test_deepgemm_mxfp8_preserves_weight_and_scale_values(
scale_dtype, weight_shape, is_bmm
):
"""DeepGEMM layout conversion preserves native weights and E8M0 scales."""
from vllm.model_executor.layers.quantization.utils import fp8_utils
from vllm.utils.deep_gemm import is_deep_gemm_supported
if not is_deep_gemm_supported() or not current_platform.is_device_capability_family(
100
):
pytest.skip("DeepGEMM MXFP8 requires Blackwell")
weight = torch.randn(weight_shape, device="cuda").to(torch.float8_e4m3fn)
scales = (
torch.randint(
1,
255,
(weight.numel() // 32 + 1,),
device="cuda",
dtype=torch.uint8,
)[1:]
.view(*weight_shape[:-1], weight_shape[-1] // 32)
.view(scale_dtype)
)
original_weight = weight.view(torch.uint8).clone()
original_scales = scales.view(torch.uint8).clone()
processed_weight, packed = fp8_utils.deepgemm_post_process_fp8_weight_block(
weight,
scales,
quant_block_shape=(1, 32),
use_e8m0=True,
is_bmm=is_bmm,
bmm_batch_size=3 if is_bmm else 0,
)
assert packed.dtype == torch.int32
unpacked = torch.stack(
[(packed >> (8 * i)) & 0xFF for i in range(4)], dim=-1
).flatten(-2)
unpacked = unpacked[..., : scales.shape[-1]].to(torch.uint8)
torch.testing.assert_close(
unpacked.reshape_as(original_scales), original_scales, rtol=0, atol=0
)
torch.testing.assert_close(
processed_weight.view(torch.uint8).reshape_as(original_weight),
original_weight,
rtol=0,
atol=0,
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
@pytest.mark.parametrize("prequantized", [False, True])
@pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
def test_mxfp8_bmm_loads_and_projects_grouped_weights(
dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens
):
"""BMM metadata set after construction selects grouped weight processing."""
from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import (
DeepGemmMxfp8BmmLinearKernel,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4Model
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
from vllm.utils.deep_gemm import is_deep_gemm_supported
if not is_deep_gemm_supported() or not current_platform.is_device_capability_family(
100
):
pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell")
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
deepseek_config = DeepseekV4FP8Config(
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
)
deepseek_config._resolved_expert_dtype = "fp4"
quant_config: QuantizationConfig = deepseek_config
if config_source == "mxfp8":
quant_config = get_quantization_config("mxfp8").from_config(
{"quant_method": "mxfp8"}
)
with torch.device("cuda"):
linear = ColumnParallelLinear(
512,
256,
bias=False,
quant_config=quant_config,
return_bias=False,
)
linear.is_bmm = True
linear.bmm_batch_size = 2
weight = torch.randn(256, 512, device="cuda").to(torch.float8_e4m3fn)
scales = torch.randint(124, 131, (8, 16), device="cuda", dtype=torch.uint8)
checkpoint_scales = (
scales.view(torch.float8_e8m0fnu)
if config_source == "deepseek"
else scales.repeat_interleave(32, dim=0)
)
model = DeepseekV4Model.__new__(DeepseekV4Model)
torch.nn.Module.__init__(model)
block = torch.nn.Module()
block.attn = torch.nn.Module()
block.attn.wo_a = linear
model.layers = torch.nn.ModuleList([block])
model.config = SimpleNamespace(num_attention_heads=2)
model.quant_config = quant_config
model.use_sequence_parallel = False
monkeypatch.setattr(model, "get_expert_mapping", lambda: [])
model.load_weights(
[
("layers.0.attn.wo_a.weight", weight),
("layers.0.attn.wo_a.weight_scale", checkpoint_scales),
]
)
reference_weight = (
weight.float().reshape(8, 32, 16, 32)
* torch.exp2(scales.float() - 127)[:, None, :, None]
).reshape(2, 128, 512)
assert isinstance(linear.quant_method, ModelOptLinearMethod)
linear.quant_method.process_weights_after_loading(linear)
linear.quant_method.process_weights_after_loading(linear)
assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel)
assert linear.weight.shape == (2, 128, 512)
torch.testing.assert_close(
linear.weight.view(torch.uint8).flatten(),
weight.view(torch.uint8).flatten(),
rtol=0,
atol=0,
)
unpacked_scales = torch.stack(
[(linear.weight_scale >> (8 * i)) & 0xFF for i in range(4)], dim=-1
).flatten(-2)
torch.testing.assert_close(
unpacked_scales.to(torch.uint8).reshape(256, 16),
scales.repeat_interleave(32, dim=0),
rtol=0,
atol=0,
)
x = torch.randn(2, num_tokens, 512, device="cuda", dtype=torch.bfloat16)
reference = torch.einsum("gmk,gnk->mgn", x.float(), reference_weight)
inputs = x.transpose(0, 1)
if prequantized:
cache = torch.cat(
(
torch.ones(num_tokens, 32, device="cuda"),
torch.zeros(num_tokens, 32, device="cuda"),
),
dim=1,
)
inputs = fused_inv_rope_fp8_quant(
x.permute(1, 0, 2),
torch.arange(num_tokens, device="cuda"),
cache,
n_groups=2,
heads_per_group=1,
nope_dim=448,
rope_dim=64,
quant_group_size=32,
tma_aligned_scales=current_platform.has_device_capability(100),
)
with torch.no_grad():
output = linear(inputs)
assert output.shape == reference.shape
assert (output.float() - reference).norm() / reference.norm() < 0.06
if prequantized:
recipe, tma_aligned_scales = compute_fp8_einsum_recipe(block_size=32)
projected = deep_gemm_fp8_o_proj(
x.permute(1, 0, 2),
torch.arange(num_tokens, device="cuda"),
cache,
linear,
torch.nn.Identity(),
n_groups=2,
heads_per_group=1,
nope_dim=448,
rope_dim=64,
o_lora_rank=128,
einsum_recipe=recipe,
tma_aligned_scales=tma_aligned_scales,
)
torch.testing.assert_close(projected, output.flatten(1), rtol=0, atol=0)
compiled = torch.compile(linear, backend="eager", fullgraph=True)
with torch.no_grad():
torch.testing.assert_close(compiled(inputs), output, rtol=0, atol=0)
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
monkeypatch.setattr(
flashinfer_utils,
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
lambda *args: None,
)
intermediate = 17
padded_intermediate = 32
hidden_size = 4
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
up = torch.full_like(gate, 2)
w13 = torch.cat((gate, up), dim=1)
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
layer = SimpleNamespace(
activation=SimpleNamespace(is_gated=True),
moe_config=SimpleNamespace(
is_act_and_mul=True,
intermediate_size_per_partition=intermediate,
),
)
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
layer,
w13,
w2,
w13_scale=torch.ones(1),
w13_input_scale=torch.ones(1),
w2_scale=torch.ones(1),
w2_input_scale=torch.ones(1),
is_trtllm=True,
)
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
expected[:, :intermediate] = up
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
assert torch.equal(padded_w31, expected)
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
torch.tensor([0.25, 0.5]),
torch.tensor([0.75, 0.6]),
enable_eplb=False,
)
assert a1_scale.ndim == a2_scale.ndim == 0
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
@pytest.mark.parametrize("model_id", MODELS)
@pytest.mark.parametrize(
"force_marlin", [True, False] if current_platform.is_cuda() else [False]
)
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
def test_model_load_and_run(
model_id: str,
force_marlin: bool,
use_rocm_aiter: bool,
monkeypatch,
dist_init,
workspace_init,
) -> None:
if use_rocm_aiter:
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
kernel_config = KernelConfig(
linear_backend="marlin" if force_marlin else "auto",
moe_backend="marlin" if force_marlin else "auto",
)
model, vllm_config = load_model_without_vllm_runner(
model_id,
model_config_kwargs={"hf_overrides": {"num_hidden_layers": 3}},
vllm_config_kwargs={"kernel_config": kernel_config},
)
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE_TYPE)
positions = torch.arange(input_ids.numel(), device=DEVICE_TYPE)
with (
set_current_vllm_config(vllm_config),
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
):
model(input_ids, positions, None)
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_scaled_fp8_quant(dtype) -> None:
def quantize_ref(tensor, inv_scale):
# The reference implementation that fully aligns to
# the kernel being tested.
finfo = torch.finfo(current_platform.fp8_dtype())
scale = inv_scale.reciprocal()
qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max)
qweight = qweight.to(current_platform.fp8_dtype())
return qweight
def per_tensor_dequantize(tensor, inv_scale, dtype):
fake_qweight = tensor.to(dtype)
dq_weight = fake_qweight * inv_scale
return dq_weight
# Note that we use a shape % 4 != 0 to cover edge cases,
# because scaled_fp8_quant is vectorized by 4.
x = (torch.randn(size=(11, 11), device=DEVICE_TYPE) * 13).to(dtype)
# Dynamic quantization
ref_y, inv_scale = ops.scaled_fp8_quant(x, None)
ref_y = per_tensor_dequantize(ref_y, inv_scale, dtype)
# Reference dynamic quantization
y = quantize_ref(x, inv_scale)
torch.testing.assert_close(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
# Static quantization
y, _ = ops.scaled_fp8_quant(x, inv_scale)
torch.testing.assert_close(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
# Padding
y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
assert y.shape[0] == 17
torch.testing.assert_close(
ref_y,
per_tensor_dequantize(torch.narrow(y, 0, 0, x.shape[0]), inv_scale, dtype),
)
# non-contiguous input with padding
m, n, padded_stride = 975, 512, 576
padded_tensor = (torch.randn(size=(m, padded_stride), device=DEVICE_TYPE) * 13).to(
dtype
)
x_nc = padded_tensor[:, :n] # shape (m, n) with stride (padded_stride, 1)
assert not x_nc.is_contiguous()
assert x_nc.stride(0) == padded_stride
# dynamic quantization
ref_y_nc, inv_scale_nc = ops.scaled_fp8_quant(x_nc, None)
ref_y_nc = per_tensor_dequantize(ref_y_nc, inv_scale_nc, dtype)
# reference dynamic quantization
y_nc = quantize_ref(x_nc, inv_scale_nc)
torch.testing.assert_close(
ref_y_nc, per_tensor_dequantize(y_nc, inv_scale_nc, dtype)
)
# static quantization
y_nc, _ = ops.scaled_fp8_quant(x_nc, inv_scale_nc)
torch.testing.assert_close(
ref_y_nc, per_tensor_dequantize(y_nc, inv_scale_nc, dtype)
)
# padding after non-contiguous input quantization
y_nc_pad, _ = ops.scaled_fp8_quant(x_nc, inv_scale_nc, num_token_padding=m + 10)
assert y_nc_pad.shape[0] == m + 10
torch.testing.assert_close(
ref_y_nc,
per_tensor_dequantize(
torch.narrow(y_nc_pad, 0, 0, x_nc.shape[0]), inv_scale_nc, dtype
),
)
@pytest.mark.skipif(
current_platform.is_fp8_fnuz(),
reason="FP8 e4m3fn weight reloading is not supported on e4m3fnuz platforms",
)
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
# FP8 weight reloading does not support online quantization
@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
# any postprocessing that is applied to the weights such as padding and repacking
# (excluding device sharding) must also be applied to the reloaded weights
#
# this is the case for marlin as well as per-tensor Fp8MoEMethod
@pytest.mark.parametrize("use_marlin", [False]) # skip True
def test_fp8_reloading(
default_vllm_config,
method_cls,
weight_block_size,
use_marlin,
dist_init,
monkeypatch,
):
# NOTE(rob): this test fails when using DeepGEMM because the
# shapes are invalid. Previously the test was passing because
# we set fp8_backend to None, which sidestepped the issue.
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
if method_cls is Fp8MoEMethod and weight_block_size is None:
pytest.skip(
"FP8 Tensor weight reloading does not support fusing w13_weight_scale. "
"If this is your use case, consider using a restore function like #26327"
)
# Set model config as model_config.dtype is required in Fp8LinearMethod.
default_vllm_config.model_config = ModelConfig()
default_vllm_config.kernel_config.moe_backend = "triton"
layer_size = 128 if weight_block_size is not None else 1
with torch.device(f"{DEVICE_TYPE}:0"):
config = Fp8Config(
weight_block_size=weight_block_size,
)
if method_cls is Fp8LinearMethod:
layer = torch.nn.Linear(layer_size, layer_size)
method = method_cls(config)
method.create_weights(
layer=layer,
input_size_per_partition=layer_size,
output_partition_sizes=[layer_size],
input_size=layer_size,
output_size=layer_size,
params_dtype=torch.bfloat16,
weight_loader=default_weight_loader,
)
method.use_marlin = use_marlin
else:
layer = FusedMoEFactory(
num_experts=1,
top_k=1,
hidden_size=layer_size,
intermediate_size=layer_size,
)
layer = layer.routed_experts
method = method_cls(config, layer)
method.create_weights(
layer=layer,
num_experts=1,
hidden_size=layer_size,
intermediate_size_per_partition=layer_size,
params_dtype=torch.bfloat16,
weight_loader=default_weight_loader,
)
# capture weights format during loading
original_metadata = [
(name, param.shape, getattr(param, "weight_loader", default_weight_loader))
for name, param in layer.named_parameters()
]
# test loading
for name, shape, _ in original_metadata:
param = getattr(layer, name)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, torch.zeros(shape)) # cannot use empty
method.process_weights_after_loading(layer)
# test reloading works after loading
for name, shape, _ in original_metadata:
param = getattr(layer, name)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, torch.zeros(shape)) # cannot use empty
method.process_weights_after_loading(layer)
def test_kv_cache_scale_sync_to_host_copies():
"""Test device-to-host sync of the k/v quantization scales, for both the
checkpoint-load and runtime-calc paths that produce them.
"""
layer = torch.nn.Module()
set_default_quant_scales(layer, register_buffer=True)
layer.kv_cache_dtype = "fp8"
method = BaseKVCacheMethod(quant_config=None)
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
method.process_weights_after_loading(layer)
assert layer._k_scale_float != 1.0
assert layer._v_scale_float != 1.0
# Host copy must mirror both the float and the device scale tensor.
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float)
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float)
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item())
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item())
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_kv_cache_dtype_skip_layers(monkeypatch, dist_init, workspace_init):
"""Test that kv_cache_dtype_skip_layers skips quantization for specified layers."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model, _ = load_model_without_vllm_runner(
"facebook/opt-125m",
vllm_config_kwargs={
"cache_config": CacheConfig(
cache_dtype="fp8", kv_cache_dtype_skip_layers=["0", "2"]
)
},
)
for i, layer in enumerate(model.model.decoder.layers):
expected = "auto" if str(i) in ["0", "2"] else "fp8"
assert layer.self_attn.attn.kv_cache_dtype == expected
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,637 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests ROCm AITER fused shared experts with online quantization."""
import json
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from tests.quantization.utils import load_model_without_vllm_runner
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.model import ModelConfig
from vllm.config.quantization import (
QuantizationConfigArgs,
resolve_quantization_config,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.experts import rocm_aiter_moe
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.quark_moe import (
QuarkOCP_MX_MoEMethod,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.model_executor.model_loader.reload.layerwise import (
finalize_layerwise_reload,
initialize_layerwise_reload,
record_metadata_for_reloading,
)
from vllm.model_executor.model_loader.utils import get_model_architecture
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER shared-expert tests require ROCm.",
)
_QUARK_MXFP4_CONFIG: dict[str, Any] = {
"global_quant_config": {
"input_tensors": {
"dtype": "fp4",
"is_dynamic": True,
"qscheme": "per_group",
"ch_axis": -1,
"group_size": 32,
"block_size": None,
"symmetric": None,
"round_method": "half_even",
"scale_type": "float",
"scale_format": "e8m0",
"scale_calculation_mode": "even",
"mx_element_dtype": None,
"observer_cls": "PerBlockMXObserver",
"is_scale_quant": False,
"enable_buffer_reuse": False,
"max_input_numel": 4194304,
},
"output_tensors": None,
"weight": {
"dtype": "fp4",
"is_dynamic": False,
"qscheme": "per_group",
"ch_axis": -1,
"group_size": 32,
"block_size": None,
"symmetric": None,
"round_method": "half_even",
"scale_type": "float",
"scale_format": "e8m0",
"scale_calculation_mode": "even",
"mx_element_dtype": None,
"observer_cls": "PerBlockMXObserver",
"is_scale_quant": False,
"enable_buffer_reuse": False,
"max_input_numel": 4194304,
},
"bias": None,
"target_device": None,
},
"algo_config": None,
"softmax_quant_spec": None,
"quant_method": "quark",
"layer_type_quant_config": {},
"layer_quant_config": {},
"exclude": ["re:^(?!.*\\.mlp\\.experts(?:\\.|$)).*$"],
"kv_cache_quant_config": {},
"kv_cache_post_rope": False,
"quant_mode": "eager_mode",
"version": "0.12+9d3d471cdf1",
"export": {
"kv_cache_group": [],
"min_kv_scale": 0.0,
"pack_method": "reorder",
"weight_format": "real_quantized",
"weight_merge_groups": None,
},
}
_QUARK_MXFP4_LAYER_CONFIG: dict[str, Any] = _QUARK_MXFP4_CONFIG["global_quant_config"]
_QUARK_MXFP4_CONFIG = {
**_QUARK_MXFP4_CONFIG,
"global_quant_config": {
"input_tensors": {},
"output_tensors": None,
"weight": {},
"bias": None,
"target_device": None,
},
"layer_quant_config": {"*mlp.experts*": _QUARK_MXFP4_LAYER_CONFIG},
}
class _StubAttention(torch.nn.Module):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__()
self.o_proj = SimpleNamespace(reduce_results=True)
@pytest.fixture(autouse=True)
def reset_aiter_shared_expert_topk_metadata(monkeypatch: pytest.MonkeyPatch):
"""Isolate AITER's process-global shared-expert routing buffer."""
monkeypatch.setattr(rocm_aiter_moe, "aiter_topK_meta_data", None)
rocm_aiter_moe.init_aiter_topK_meta_data.cache_clear()
yield
rocm_aiter_moe.init_aiter_topK_meta_data.cache_clear()
def _write_minimal_moe_config(model_path: Path, architecture: str) -> None:
"""Write a compact MoE config with one shared expert."""
config = {
"architectures": [architecture],
"hidden_size": 256,
"intermediate_size": 512,
"moe_intermediate_size": 256,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"head_dim": 64,
"num_hidden_layers": 1,
"vocab_size": 256,
"max_position_embeddings": 64,
"q_lora_rank": 64,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"n_routed_experts": 4,
"num_experts": 4,
"num_experts_per_tok": 1,
"n_group": 1,
"topk_group": 1,
"n_shared_experts": 1,
"first_k_dense_replace": 0,
"moe_layer_freq": 1,
"rms_norm_eps": 1e-6,
"hidden_act": "silu",
"tie_word_embeddings": False,
"rope_theta": 10000,
"quantization_config": _QUARK_MXFP4_CONFIG,
}
if architecture == "AXK1ForCausalLM":
config["model_type"] = "axk1"
elif architecture == "MiniMaxM3SparseForCausalLM":
text_config = {
**config,
"model_type": "minimax_m3_text",
"dense_intermediate_size": 512,
"hidden_act": "swigluoai",
"num_local_experts": 4,
"n_shared_experts": 1,
"moe_layer_freq": [1],
"sparse_attention_config": None,
"quantization_config": {
**_QUARK_MXFP4_CONFIG,
"layer_quant_config": {
"*block_sparse_moe.experts*": _QUARK_MXFP4_LAYER_CONFIG
},
"exclude": [r"re:^(?!.*\.block_sparse_moe\.experts(?:\.|$)).*$"],
},
}
config = {
"architectures": [architecture],
"model_type": "minimax_m3_vl",
"text_config": text_config,
}
elif architecture == "Glm4MoeForCausalLM":
config["model_type"] = "glm4_moe"
elif architecture == "Glm4MoeLiteForCausalLM":
config["model_type"] = "glm4_moe_lite"
elif architecture in ("Qwen3NextForCausalLM", "Qwen3_5MoeForCausalLM"):
config.update(
model_type=(
"qwen3_next"
if architecture == "Qwen3NextForCausalLM"
else "qwen3_5_moe_text"
),
layer_types=[
"linear_attention"
if architecture == "Qwen3NextForCausalLM"
else "full_attention"
],
head_dim=64,
linear_key_head_dim=64,
linear_value_head_dim=64,
linear_num_key_heads=4,
linear_num_value_heads=4,
shared_expert_intermediate_size=256,
)
elif architecture == "Qwen4ExpForCausalLM":
config.update(
model_type="qwen4_exp_text",
layer_types=["full_attention"],
head_dim=64,
linear_key_head_dim=64,
linear_value_head_dim=64,
linear_num_key_heads=4,
linear_num_value_heads=4,
shared_expert_intermediate_size=256,
eos_token_id=1,
hc_count=2,
hc_lowrank=4,
ple_layer_ids=[],
)
elif architecture == "Glm5NextForCausalLM":
config.update(
model_type="glm5_next",
layer_types=["linear_attention"],
mlp_layer_types=["sparse"],
qk_rope_head_dim=0,
index_topk=1,
index_kpool=1,
pad_token_id=None,
scoring_func="sigmoid",
topk_method="noaux_tc",
mhc=False,
)
else:
config["model_type"] = (
"deepseek_v3"
if "V3" in architecture
else "deepseek_v32"
if architecture == "DeepseekV32ForCausalLM"
else "deepseek_v2"
)
if architecture in ("DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"):
config.update(
index_topk=1,
index_kpool=1,
index_n_heads=1,
index_head_dim=32,
index_kv_lora_rank=32,
)
(model_path / "config.json").write_text(json.dumps(config))
@pytest.mark.parametrize(
"load_path", ["load_weights", "fused_gate_up", "weight_loader"]
)
def test_online_shared_expert_loads_bf16_weights_into_mxfp4_slot(
default_vllm_config,
dist_init,
load_path: str,
) -> None:
"""A BF16 shared expert is quantized while routed MXFP4 weights are loaded.
`weight_loader` covers DeepSeek-style model loaders, which bypass
`RoutedExperts.load_weights`.
"""
default_vllm_config.model_config = ModelConfig()
hidden_size = intermediate_size = 64
num_routed_experts = 2
device = current_platform.device_type
online_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"*shared_expert*": "mxfp4"})
)
quant_config = QuarkConfig(_QUARK_MXFP4_CONFIG)
quant_config.online_quantization_config = online_config
with torch.device(device):
runner = FusedMoEFactory(
num_experts=num_routed_experts,
top_k=1,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
n_shared_experts=1,
fuse_shared_experts=True,
shared_expert_prefix="model.layers.0.mlp.shared_expert",
prefix="model.layers.0.mlp.experts",
quant_config=quant_config,
)
layer = runner.routed_experts
assert isinstance(layer.quant_method, QuarkOCP_MX_MoEMethod)
routed_gate, _ = mxfp4_quantize(
torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
)
routed_up, _ = mxfp4_quantize(
torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
)
routed_down, _ = mxfp4_quantize(
torch.randn(hidden_size, intermediate_size, dtype=torch.bfloat16)
)
shared_gate = torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
shared_up = torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
shared_down = torch.randn(hidden_size, intermediate_size, dtype=torch.bfloat16)
weights = [
("0.gate_proj.weight", routed_gate),
("0.up_proj.weight", routed_up),
("0.down_proj.weight", routed_down),
]
if load_path == "load_weights":
weights += [
("2.gate_proj.weight", shared_gate),
("2.up_proj.weight", shared_up),
("2.down_proj.weight", shared_down),
]
elif load_path == "fused_gate_up":
weights += [
("2.gate_up_proj.weight", torch.cat([shared_gate, shared_up])),
("2.down_proj.weight", shared_down),
]
list(layer.load_weights(weights))
if load_path == "weight_loader":
for stem, shard_id, weight in (
("w13", "w1", shared_gate),
("w13", "w3", shared_up),
("w2", "w2", shared_down),
):
param = getattr(layer, f"{stem}_weight")
param.weight_loader(
param, weight, f"{layer.layer_name}.{stem}_weight", shard_id, 2
)
expected_gate, expected_gate_scale = mxfp4_quantize(shared_gate)
expected_up, expected_up_scale = mxfp4_quantize(shared_up)
expected_down, expected_down_scale = mxfp4_quantize(shared_down)
assert torch.equal(layer.w13_weight[2, :intermediate_size], expected_gate)
assert torch.equal(layer.w13_weight[2, intermediate_size:], expected_up)
assert torch.equal(layer.w2_weight[2], expected_down)
assert torch.equal(
layer.w13_weight_scale[2, :intermediate_size], expected_gate_scale
)
assert torch.equal(layer.w13_weight_scale[2, intermediate_size:], expected_up_scale)
assert torch.equal(layer.w2_weight_scale[2], expected_down_scale)
def test_online_shared_expert_reload_compatibility(
default_vllm_config,
dist_init,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Reload a BF16 shared weight into its MXFP4 slot via meta staging."""
default_vllm_config.model_config = ModelConfig()
online_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"*shared_expert*": "mxfp4"})
)
quant_config = QuarkConfig(_QUARK_MXFP4_CONFIG)
quant_config.online_quantization_config = online_config
with torch.device(current_platform.device_type):
layer = FusedMoEFactory(
num_experts=2,
top_k=1,
hidden_size=64,
intermediate_size=64,
n_shared_experts=1,
fuse_shared_experts=True,
shared_expert_prefix="model.layers.0.mlp.shared_expert",
prefix="model.layers.0.mlp.experts",
quant_config=quant_config,
).routed_experts
monkeypatch.setattr(
layer.quant_method, "process_weights_after_loading", lambda _: None
)
routed_weights = []
for expert_id in range(2):
for projection in ("gate_proj", "up_proj", "down_proj"):
weight, scale = mxfp4_quantize(
torch.randn(64, 64, dtype=torch.bfloat16)
)
prefix = f"{expert_id}.{projection}"
routed_weights.extend(
[(f"{prefix}.weight", weight), (f"{prefix}.weight_scale", scale)]
)
shared_gate = torch.randn(64, 64, dtype=torch.bfloat16)
shared_up = torch.randn(64, 64, dtype=torch.bfloat16)
shared_down = torch.randn(64, 64, dtype=torch.bfloat16)
def checkpoint_weights(gate: torch.Tensor):
return [
*routed_weights,
("2.gate_proj.weight", gate),
("2.up_proj.weight", shared_up),
("2.down_proj.weight", shared_down),
]
list(layer.load_weights(checkpoint_weights(shared_gate)))
old_weight = layer.w13_weight[2, :64].clone()
reloaded_gate = shared_gate + 1
expected_weight, expected_scale = mxfp4_quantize(reloaded_gate)
assert not torch.equal(old_weight, expected_weight)
record_metadata_for_reloading(layer)
initialize_layerwise_reload(layer)
assert layer.w13_weight.is_meta
list(layer.load_weights(checkpoint_weights(reloaded_gate)))
finalize_layerwise_reload(layer, default_vllm_config.model_config)
assert not layer.w13_weight.is_meta
assert torch.equal(layer.w13_weight[2, :64], expected_weight)
assert torch.equal(layer.w13_weight_scale[2, :64], expected_scale)
# TODO: Add DeepseekV4ForConditionalGeneration once it uses
# resolve_layer_fused_shared_expert.
@pytest.mark.parametrize(
"architecture",
[
"AXK1ForCausalLM",
"DeepseekForCausalLM",
"DeepseekV2ForCausalLM",
"DeepseekV32ForCausalLM",
"DeepseekV3ForCausalLM",
"Glm4MoeForCausalLM",
"Glm4MoeLiteForCausalLM",
"Glm5NextForCausalLM",
"GlmMoeDsaForCausalLM",
"MiniMaxM3SparseForCausalLM",
"Qwen3NextForCausalLM",
"Qwen3_5MoeForCausalLM",
"Qwen4ExpForCausalLM",
],
)
def test_online_quantization(
architecture: str,
tmp_path: Path,
monkeypatch,
dist_init,
workspace_init,
) -> None:
"""Online MXFP4 fuses each architecture's shared expert into its MoE."""
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "1")
rocm_aiter_ops.refresh_env_variables()
if architecture == "Glm5NextForCausalLM":
from vllm.models.glm5next.common import model as glm5_next_model
for attention in ("Glm5NextLinearAttention", "Glm5NextMLAAttention"):
monkeypatch.setattr(glm5_next_model, attention, _StubAttention)
monkeypatch.setattr(
glm5_next_model, "_fused_shared_experts_tuned", lambda _: True
)
elif architecture == "Glm4MoeLiteForCausalLM":
from vllm.model_executor.models import glm4_moe_lite
monkeypatch.setattr(glm4_moe_lite, "Glm4MoeLiteAttention", _StubAttention)
monkeypatch.setattr(glm4_moe_lite, "Glm4MoeLiteMLAAttention", _StubAttention)
elif architecture == "DeepseekV32ForCausalLM":
from vllm.models.deepseek_v32.amd import model as deepseek_v32_model
monkeypatch.setattr(
deepseek_v32_model, "DeepseekV32MLAAttention", _StubAttention
)
elif architecture == "Qwen3_5MoeForCausalLM":
from vllm.model_executor.models import qwen3_5
monkeypatch.setattr(qwen3_5, "Qwen3NextAttention", _StubAttention)
elif architecture == "Qwen4ExpForCausalLM":
from vllm.models.qwen4_exp.amd import model as qwen4_exp_model
monkeypatch.setattr(qwen4_exp_model, "Qwen3NextAttention", _StubAttention)
_write_minimal_moe_config(tmp_path, architecture)
expected_shared_expert_name = (
"shared_expert" if architecture.startswith("Qwen") else "shared_experts"
)
target_pattern = f"*{expected_shared_expert_name}*"
logged_messages: list[str] = []
logged_warnings: list[str] = []
def record_info(message: str, *args: object, **_kwargs: object) -> None:
if "Quantizing " in message and "of types" in message:
logged_messages.append(message % args)
def record_warning(message: str, *args: object, **_kwargs: object) -> None:
logged_warnings.append(message % args)
monkeypatch.setattr(
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.utils.logger.warning", record_warning
)
model, vllm_config = load_model_without_vllm_runner(
str(tmp_path),
dtype="bfloat16",
quantization="quark",
model_config_kwargs={
"quantization_config": resolve_quantization_config(
"quark", {"targets": {target_pattern: "mxfp4"}}
)
},
model_loader_cls=DummyModelLoader,
)
expected_model_cls, resolved_architecture = get_model_architecture(
vllm_config.model_config
)
assert resolved_architecture == architecture
assert isinstance(model, expected_model_cls)
fused_moes = [
module
for module in model.modules()
if getattr(module, "is_fused_shared_expert_enabled", False)
and hasattr(module, "experts")
]
assert len(fused_moes) == 1, f"{architecture} did not enable FSE"
moe = fused_moes[0]
shared_expert_name = (
"shared_expert" if hasattr(moe, "shared_expert") else "shared_experts"
)
assert shared_expert_name == expected_shared_expert_name
assert getattr(moe, shared_expert_name) is None
routed_experts = moe.experts.routed_experts
assert isinstance(routed_experts.quant_method, QuarkOCP_MX_MoEMethod)
shared_expert_prefix = routed_experts.moe_config.shared_expert_prefix
assert shared_expert_prefix is not None
assert shared_expert_prefix.endswith(shared_expert_name)
assert routed_experts._fused_shared_expert_quantizer is not None
expert_map_manager = routed_experts.expert_map_manager
assert expert_map_manager.num_fused_shared_experts == 1
assert expert_map_manager.map_global_to_local(
routed_experts.moe_config.num_experts
) == (routed_experts.moe_config.num_experts)
expected_weight_dtype = {
Mxfp4MoeBackend.AITER_MXFP4_BF16: torch.float4_e2m1fn_x2,
Mxfp4MoeBackend.AITER_MXFP4_MXFP4: torch.float4_e2m1fn_x2,
Mxfp4MoeBackend.EMULATION: torch.uint8,
}[routed_experts.quant_method.mxfp4_backend]
assert routed_experts.w13_weight.dtype == expected_weight_dtype
assert routed_experts.w13_weight_scale.dtype == torch.uint8
assert routed_experts.w2_weight.dtype == expected_weight_dtype
assert routed_experts.w2_weight_scale.dtype == torch.uint8
assert vllm_config.quant_config is not None
assert vllm_config.quant_config.online_quantization_config is not None
assert not any(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled" in warning
for warning in logged_warnings
)
if architecture == "MiniMaxM3SparseForCausalLM":
# TODO: Use resolve_layer_fused_shared_expert in MiniMax M3 and remove
# this branch once its virtual fused projections are registered.
assert logged_messages == ["Quantizing 0 layers of types: "]
else:
assert logged_messages == [
"Quantizing 2 layers of types: "
f"mlp.{shared_expert_name}.down_proj: 1 "
f"(from targets: {target_pattern}, mxfp4); "
f"mlp.{shared_expert_name}.gate_up_proj: 1 "
f"(from targets: {target_pattern}, mxfp4)"
]
def test_online_gate_up_proj_target_disables_fse(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
dist_init,
workspace_init,
) -> None:
"""A partial shared-expert target quantizes while disabling FSE."""
architecture = "DeepseekForCausalLM"
target_pattern = "*shared_experts.gate_up_proj*"
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "1")
rocm_aiter_ops.refresh_env_variables()
_write_minimal_moe_config(tmp_path, architecture)
logged_messages: list[str] = []
logged_warnings: list[str] = []
def record_info(message: str, *args: object, **_kwargs: object) -> None:
if "Quantizing " in message and "of types" in message:
logged_messages.append(message % args)
def record_warning(message: str, *args: object, **_kwargs: object) -> None:
logged_warnings.append(message % args)
monkeypatch.setattr(
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.utils.logger.warning", record_warning
)
model, _ = load_model_without_vllm_runner(
str(tmp_path),
dtype="bfloat16",
quantization="quark",
model_config_kwargs={
"quantization_config": resolve_quantization_config(
"quark", {"targets": {target_pattern: "mxfp4"}}
)
},
model_loader_cls=DummyModelLoader,
)
moe = next(
module
for module in model.modules()
if hasattr(module, "is_fused_shared_expert_enabled")
and hasattr(module, "shared_experts")
)
assert not moe.is_fused_shared_expert_enabled
assert moe.shared_experts is not None
assert logged_messages == [
"Quantizing 1 layers of types: "
"mlp.shared_experts.gate_up_proj: 1 "
f"(from targets: {target_pattern}, mxfp4)"
]
assert logged_warnings == [
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but cannot be "
"enabled - skipping for this layer: online quantization targets only "
"part of the shared expert at model.layers.0.mlp.shared_experts."
]
File diff suppressed because it is too large Load Diff
-782
View File
@@ -1,782 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for TurboQuant KV-cache quantization.
Run: .venv/bin/python -m pytest tests/quantization/test_turboquant.py -v
"""
import math
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.quantization.turboquant.centroids import (
get_centroids,
solve_lloyd_max,
)
from vllm.model_executor.layers.quantization.turboquant.config import (
TQ_PRESETS,
TurboQuantConfig,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
# ============================================================================
# Helpers
# ============================================================================
ALL_PRESETS = list(TQ_PRESETS.keys())
def _assert_strictly_sorted(seq, name="sequence"):
for i in range(len(seq) - 1):
assert seq[i] < seq[i + 1], f"{name} not sorted at index {i}"
def _is_power_of_2(n: int) -> bool:
return n > 0 and next_power_of_2(n) == n
# Expected concrete values for each preset at head_dim=128.
# fmt: off
PRESET_EXPECTED = {
"turboquant_k8v4": dict(
key_fp8=True, key_quant_bits=8,
key_mse_bits=0, value_quant_bits=4,
mse_bits=4, n_centroids=16, centroid_bits=4,
norm_correction=False,
key_packed_size=128, value_packed_size=68,
slot_size=196, slot_size_aligned=196,
),
"turboquant_4bit_nc": dict(
key_fp8=False, key_quant_bits=4,
key_mse_bits=4, value_quant_bits=4,
mse_bits=4, n_centroids=16, centroid_bits=4,
norm_correction=True,
key_packed_size=66, value_packed_size=68,
slot_size=134, slot_size_aligned=134,
),
"turboquant_k3v4_nc": dict(
key_fp8=False, key_quant_bits=3,
key_mse_bits=3, value_quant_bits=4,
mse_bits=3, n_centroids=8, centroid_bits=3,
norm_correction=True,
key_packed_size=50, value_packed_size=68,
slot_size=118, slot_size_aligned=118,
),
"turboquant_3bit_nc": dict(
key_fp8=False, key_quant_bits=3,
key_mse_bits=3, value_quant_bits=3,
mse_bits=3, n_centroids=8, centroid_bits=3,
norm_correction=True,
key_packed_size=50, value_packed_size=52,
slot_size=102, slot_size_aligned=102,
),
}
# fmt: on
# ============================================================================
# Config tests (CPU-only, no dependencies beyond config.py)
# ============================================================================
class TestTurboQuantConfig:
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_preset_parses(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert isinstance(cfg, TurboQuantConfig)
def test_invalid_preset_raises(self):
with pytest.raises(ValueError, match="Unknown TurboQuant"):
TurboQuantConfig.from_cache_dtype("turboquant_invalid", head_dim=128)
# ---- Per-preset concrete value checks (table-driven) ----
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_key_mode(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
exp = PRESET_EXPECTED[preset]
assert cfg.key_fp8 is exp["key_fp8"]
assert cfg.key_quant_bits == exp["key_quant_bits"]
assert cfg.key_mse_bits == exp["key_mse_bits"]
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_value_mode(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
exp = PRESET_EXPECTED[preset]
assert cfg.value_quant_bits == exp["value_quant_bits"]
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_bits_and_centroids(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
exp = PRESET_EXPECTED[preset]
assert cfg.mse_bits == exp["mse_bits"]
assert cfg.n_centroids == exp["n_centroids"]
assert cfg.centroid_bits == exp["centroid_bits"]
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_norm_correction(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert cfg.norm_correction is PRESET_EXPECTED[preset]["norm_correction"]
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_packed_sizes(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
exp = PRESET_EXPECTED[preset]
assert cfg.key_packed_size == exp["key_packed_size"]
assert cfg.value_packed_size == exp["value_packed_size"]
assert cfg.slot_size == exp["slot_size"]
assert cfg.slot_size_aligned == exp["slot_size_aligned"]
# ---- Cross-preset structural invariants ----
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_slot_equals_key_plus_value(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert cfg.slot_size == cfg.key_packed_size + cfg.value_packed_size
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_padded_slot_is_even(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert cfg.slot_size_aligned >= cfg.slot_size
assert cfg.slot_size_aligned % 2 == 0, (
f"slot_size_aligned={cfg.slot_size_aligned} is not even"
)
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_key_value_packed_sizes_positive(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert cfg.key_packed_size > 0
assert cfg.value_packed_size > 0
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_n_centroids_is_2_to_mse_bits(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert cfg.n_centroids == 2**cfg.mse_bits
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_centroid_bits_always_positive(self, preset):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
assert cfg.centroid_bits > 0
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_mse_key_or_fp8_exclusive(self, preset):
"""Each preset is either FP8 keys or MSE keys, never both."""
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
if cfg.key_fp8:
assert cfg.key_mse_bits == 0
assert cfg.key_quant_bits == 8
else:
assert cfg.key_mse_bits > 0
assert cfg.key_quant_bits in (3, 4)
@pytest.mark.parametrize("preset", ALL_PRESETS)
@pytest.mark.parametrize("head_dim", [64, 96, 128, 256])
def test_all_presets_all_head_dims(self, preset, head_dim):
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=head_dim)
assert cfg.head_dim == head_dim
assert cfg.slot_size == cfg.key_packed_size + cfg.value_packed_size
assert cfg.slot_size_aligned >= cfg.slot_size
assert cfg.slot_size_aligned % 2 == 0
# ---- Boundary skip layers ----
@staticmethod
def _dense_model_config(num_layers):
from types import SimpleNamespace
return SimpleNamespace(
is_hybrid=False,
hf_text_config=SimpleNamespace(num_hidden_layers=num_layers),
)
def test_boundary_skip_layers_basic(self):
mc = self._dense_model_config(32)
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
assert layers == ["0", "1", "30", "31"]
def test_boundary_skip_layers_zero(self):
mc = self._dense_model_config(32)
assert TurboQuantConfig.get_boundary_skip_layers(mc, 0) == []
def test_boundary_skip_layers_small_model(self):
mc = self._dense_model_config(4)
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
assert layers == ["0", "1", "2", "3"]
def test_boundary_skip_layers_cap_at_half(self):
mc = self._dense_model_config(8)
layers = TurboQuantConfig.get_boundary_skip_layers(mc, 10)
assert len(layers) == 8
class TestHybridAttentionIndices:
"""Regression tests for boundary protection on hybrid models.
Hybrid models (attention + Mamba / linear-attention) identify KV-carrying
layers via layer_types / layers_block_type / attn_type_list. The helper
must return the *global* layer indices of the full-attention layers so
that kv_cache_dtype_skip_layers matches what extract_layer_index(prefix)
reports on the Attention layers at runtime.
"""
@staticmethod
def _fake_model_config(text_cfg=None, hf_cfg=None):
from types import SimpleNamespace
return SimpleNamespace(
hf_text_config=text_cfg if text_cfg is not None else SimpleNamespace(),
hf_config=hf_cfg if hf_cfg is not None else SimpleNamespace(),
)
def test_layer_types_full_attention(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
cfg = type("C", (), {})()
cfg.layer_types = [
"linear_attention",
"linear_attention",
"full_attention",
"linear_attention",
"full_attention",
"full_attention",
]
mc = self._fake_model_config(text_cfg=cfg)
assert _get_full_attention_layer_indices(mc) == [2, 4, 5]
def test_layers_block_type_jamba(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
cfg = type("C", (), {})()
cfg.layers_block_type = ["mamba", "attention", "mamba", "attention"]
mc = self._fake_model_config(text_cfg=cfg)
assert _get_full_attention_layer_indices(mc) == [1, 3]
def test_attn_type_list_minimax(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
hf = type("C", (), {})()
hf.attn_type_list = [0, 1, 0, 1, 1]
mc = self._fake_model_config(hf_cfg=hf)
assert _get_full_attention_layer_indices(mc) == [1, 3, 4]
def test_no_hybrid_hints_returns_empty(self):
from vllm.model_executor.layers.quantization.turboquant.config import (
_get_full_attention_layer_indices,
)
mc = self._fake_model_config()
assert _get_full_attention_layer_indices(mc) == []
class TestTurboQuantKVCacheSpec:
@pytest.mark.parametrize("preset", ALL_PRESETS)
def test_kv_cache_spec_sets_kv_quant_mode(self, preset):
from vllm.model_executor.layers.attention.attention import Attention
from vllm.v1.attention.backends.turboquant_attn import (
TurboQuantAttentionBackend,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec
layer = SimpleNamespace(
attn_type="decoder",
kv_cache_dtype=preset,
kv_cache_torch_dtype=torch.uint8,
head_size=128,
head_size_v=128,
num_kv_heads=4,
sliding_window=None,
get_attn_backend=lambda: TurboQuantAttentionBackend,
)
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=32))
# The layer builds an unpacked spec; the worker's spec-collection
# loop applies TQ slot packing via the backend's customize_spec hook.
spec = Attention.get_kv_cache_spec(layer, vllm_config)
assert isinstance(spec, FullAttentionSpec)
assert spec.kv_quant_mode.is_turboquant
assert spec.state_content_bytes is None
spec = TurboQuantAttentionBackend.customize_spec(spec)
expected_slot = TurboQuantConfig.from_cache_dtype(preset, 128).slot_size_aligned
assert spec.state_content_bytes == expected_slot
class TestTurboQuantWorkspaceReservation:
@staticmethod
def _fake_vllm_config(
*,
max_num_seqs: int = 16,
max_num_batched_tokens: int = 4096,
enable_chunked_prefill: bool = True,
max_model_len: int = 8192,
dtype: torch.dtype = torch.float16,
max_num_kv_splits: int = 4,
):
return SimpleNamespace(
scheduler_config=SimpleNamespace(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=max_num_batched_tokens,
enable_chunked_prefill=enable_chunked_prefill,
),
model_config=SimpleNamespace(
max_model_len=max_model_len,
dtype=dtype,
get_num_attention_heads=lambda parallel_config: 8,
),
parallel_config=SimpleNamespace(
tensor_parallel_size=2,
decode_context_parallel_size=1,
),
attention_config=SimpleNamespace(
tq_max_kv_splits_for_cuda_graph=max_num_kv_splits
),
)
@staticmethod
def _fake_kv_cache_spec():
from vllm.v1.kv_cache_interface import FullAttentionSpec
return FullAttentionSpec(
block_size=32,
num_kv_heads=4,
head_size=128,
head_size_v=128,
dtype=torch.uint8,
state_content_bytes=102,
)
def test_metadata_builder_reserves_decode_and_continuation_prefill_workspace(
self, monkeypatch
):
from vllm.v1.attention.backends import turboquant_attn
calls = []
class FakeWorkspaceManager:
def get_simultaneous(self, *shapes_and_dtypes):
calls.append(shapes_and_dtypes)
monkeypatch.setattr(
turboquant_attn,
"current_workspace_manager",
lambda: FakeWorkspaceManager(),
)
monkeypatch.setattr(
turboquant_attn,
"is_workspace_manager_initialized",
lambda: True,
)
turboquant_attn.TurboQuantMetadataBuilder(
kv_cache_spec=self._fake_kv_cache_spec(),
layer_names=["layers.0.self_attn.attn"],
vllm_config=self._fake_vllm_config(),
device=torch.device("cuda"),
)
assert calls == [
(
((16, 8, 4, 129), torch.float32),
((16, 8, 128), torch.float16),
((16, 8), torch.float32),
),
(
((1, 4, 8192, 128), torch.float16),
((1, 4, 8192, 128), torch.float16),
),
]
def test_metadata_builder_skips_continuation_prefill_when_disabled(
self, monkeypatch
):
from vllm.v1.attention.backends import turboquant_attn
calls = []
class FakeWorkspaceManager:
def get_simultaneous(self, *shapes_and_dtypes):
calls.append(shapes_and_dtypes)
monkeypatch.setattr(
turboquant_attn,
"current_workspace_manager",
lambda: FakeWorkspaceManager(),
)
monkeypatch.setattr(
turboquant_attn,
"is_workspace_manager_initialized",
lambda: True,
)
turboquant_attn.TurboQuantMetadataBuilder(
kv_cache_spec=self._fake_kv_cache_spec(),
layer_names=["layers.0.self_attn.attn"],
vllm_config=self._fake_vllm_config(enable_chunked_prefill=False),
device=torch.device("cuda"),
)
assert calls == [
(
((16, 8, 4, 129), torch.float32),
((16, 8, 128), torch.float16),
((16, 8), torch.float32),
)
]
# ============================================================================
# Centroids tests (CPU-only)
# ============================================================================
class TestCentroids:
@pytest.mark.parametrize("bits,expected_n", [(2, 4), (3, 8), (4, 16)])
def test_centroids_shape(self, bits, expected_n):
c = get_centroids(128, bits)
assert c.shape == (expected_n,)
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_centroids_sorted(self, bits):
_assert_strictly_sorted(get_centroids(128, bits), "centroids")
def test_centroids_cached(self):
c1 = get_centroids(128, 3)
c2 = get_centroids(128, 3)
assert c1 is c2, "get_centroids should return cached object"
def test_centroids_different_dims_not_identical(self):
c64 = get_centroids(64, 3)
c128 = get_centroids(128, 3)
assert not torch.equal(c64, c128)
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_centroids_symmetric_around_zero(self, bits):
"""N(0, 1/d) is symmetric, so centroids should be ~symmetric."""
c = get_centroids(128, bits)
assert abs(c.mean().item()) < 0.01, "Centroids not centered near 0"
assert abs(c[0].item() + c[-1].item()) < 0.01
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_centroids_within_4sigma(self, bits):
"""All centroids should be within ~4 sigma of N(0, 1/d)."""
sigma = math.sqrt(1.0 / 128)
c = get_centroids(128, bits)
for i, val in enumerate(c):
assert abs(val.item()) < 4 * sigma, (
f"Centroid {i}={val:.6f} outside 4*sigma={4 * sigma:.6f}"
)
class TestLloydMax:
@pytest.mark.parametrize("bits,expected_n", [(2, 4), (3, 8), (4, 16)])
def test_solve_shapes(self, bits, expected_n):
centroids, boundaries = solve_lloyd_max(128, bits)
assert centroids.shape == (expected_n,)
assert boundaries.shape == (expected_n - 1,)
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_centroids_sorted(self, bits):
centroids, _ = solve_lloyd_max(128, bits)
_assert_strictly_sorted(centroids, "centroids")
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_boundaries_sorted(self, bits):
_, boundaries = solve_lloyd_max(128, bits)
_assert_strictly_sorted(boundaries, "boundaries")
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_boundaries_between_centroids(self, bits):
"""Each boundary must lie between its adjacent centroids."""
centroids, boundaries = solve_lloyd_max(128, bits)
for i in range(len(boundaries)):
assert centroids[i] < boundaries[i] < centroids[i + 1], (
f"Boundary {i}={boundaries[i]:.6f} not between "
f"c[{i}]={centroids[i]:.6f} and c[{i + 1}]={centroids[i + 1]:.6f}"
)
@pytest.mark.parametrize("bits", [2, 3, 4])
def test_boundaries_are_midpoints(self, bits):
"""Lloyd-Max boundaries are midpoints of adjacent centroids."""
centroids, boundaries = solve_lloyd_max(128, bits)
for i in range(len(boundaries)):
expected = (centroids[i] + centroids[i + 1]) / 2.0
assert abs(boundaries[i].item() - expected.item()) < 1e-6
def test_solve_deterministic(self):
c1, b1 = solve_lloyd_max(128, 3)
c2, b2 = solve_lloyd_max(128, 3)
assert torch.equal(c1, c2)
assert torch.equal(b1, b2)
def test_solve_dtype_float32(self):
centroids, boundaries = solve_lloyd_max(128, 3)
assert centroids.dtype == torch.float32
assert boundaries.dtype == torch.float32
@pytest.mark.parametrize("bits", [3, 4])
def test_centroids_match_scipy_reference(self, bits):
"""Verify _trapz(n=200) centroids match scipy.integrate.quad reference.
This ensures our scipy-free trapezoid integration doesn't silently
drift from the published Lloyd-Max quality.
"""
pytest.importorskip("scipy")
from scipy.integrate import quad
d = 128
sigma2 = 1.0 / d
sigma = math.sqrt(sigma2)
def pdf(x):
return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(
-x * x / (2 * sigma2)
)
n_levels = 2**bits
lo, hi = -3.5 * sigma, 3.5 * sigma
ref_centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
for _ in range(200):
boundaries = [
(ref_centroids[i] + ref_centroids[i + 1]) / 2.0
for i in range(n_levels - 1)
]
edges = [lo * 3] + boundaries + [hi * 3]
new_centroids = []
for i in range(n_levels):
a, b = edges[i], edges[i + 1]
num, _ = quad(lambda x: x * pdf(x), a, b)
den, _ = quad(pdf, a, b)
new_centroids.append(num / den if den > 1e-15 else ref_centroids[i])
if (
max(abs(new_centroids[i] - ref_centroids[i]) for i in range(n_levels))
< 1e-10
):
break
ref_centroids = new_centroids
# Compare our _trapz centroids against scipy reference
our_centroids, _ = solve_lloyd_max(d, bits)
ref_t = torch.tensor(ref_centroids, dtype=torch.float32)
max_err = (our_centroids - ref_t).abs().max().item()
# _trapz(n=200) has ~O(h^2) error vs adaptive quad; 1e-3 is tight
# enough to catch regression while allowing trapezoid approximation.
assert max_err < 1e-3, (
f"d={d}, bits={bits}: max centroid error vs scipy = {max_err:.2e}"
)
# ============================================================================
# Rotation matrix tests (GPU required)
# ============================================================================
GPGPU_AVAILABLE = torch.cuda.is_available() or torch.xpu.is_available()
DEVICE_TYPE = current_platform.device_type
def generate_rotation_matrix(d: int, seed: int, device: str = "cpu") -> torch.Tensor:
"""Haar-distributed random orthogonal matrix via QR (test/benchmark only)."""
gen = torch.Generator(device="cpu")
gen.manual_seed(seed)
G = torch.randn(d, d, generator=gen, device="cpu", dtype=torch.float32)
# torch.linalg.qr on CPU requires LAPACK, which some torch wheels
# (ROCm) ship without. Run QR on accelerator instead
qr_device = "cuda" if torch.cuda.is_available() else "cpu"
Q, R = torch.linalg.qr(G.to(qr_device))
diag_sign = torch.sign(torch.diag(R))
diag_sign[diag_sign == 0] = 1.0
Q = Q * diag_sign.unsqueeze(0)
return Q.to(device)
@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
class TestRotationMatrix:
"""Tests for the QR-based rotation (standalone benchmarks only)."""
@pytest.mark.parametrize("dim", [64, 96, 128, 256])
def test_rotation_matrix_shape_and_orthogonal(self, dim):
Pi = generate_rotation_matrix(dim, seed=42, device=DEVICE_TYPE)
assert Pi.shape == (dim, dim)
eye = Pi @ Pi.T
assert torch.allclose(eye, torch.eye(dim, device=DEVICE_TYPE), atol=1e-5), (
f"Pi not orthogonal for dim={dim}"
)
def test_rotation_matrix_deterministic(self):
Pi1 = generate_rotation_matrix(128, seed=42)
Pi2 = generate_rotation_matrix(128, seed=42)
assert torch.equal(Pi1, Pi2)
def test_rotation_matrix_different_seeds(self):
Pi1 = generate_rotation_matrix(128, seed=42)
Pi2 = generate_rotation_matrix(128, seed=99)
assert not torch.equal(Pi1, Pi2)
def test_rotation_matrix_det_is_pm1(self):
"""Orthogonal matrix determinant must be +1 or -1."""
Pi = generate_rotation_matrix(128, seed=42, device=DEVICE_TYPE)
det = torch.linalg.det(Pi)
assert abs(abs(det.item()) - 1.0) < 1e-4
# ============================================================================
# Hadamard rotation tests (serving path: _build_hadamard)
# ============================================================================
def _build_hadamard(d: int, device: str = "cpu") -> torch.Tensor:
"""Reproduce the serving-path Hadamard construction."""
H = torch.tensor([[1.0]])
while H.shape[0] < d:
H = torch.cat([torch.cat([H, H], 1), torch.cat([H, -H], 1)], 0)
return (H / math.sqrt(d)).to(torch.device(device))
@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
class TestHadamardRotation:
"""Tests for the Hadamard rotation used in serving."""
@pytest.mark.parametrize("dim", [64, 128, 256])
def test_hadamard_orthonormal(self, dim):
"""H must be orthonormal: H @ H^T = I."""
H = _build_hadamard(dim, DEVICE_TYPE)
eye = H @ H.T
assert torch.allclose(eye, torch.eye(dim, device=DEVICE_TYPE), atol=1e-5), (
f"Hadamard not orthonormal for dim={dim}"
)
@pytest.mark.parametrize("dim", [64, 128, 256])
def test_hadamard_symmetric(self, dim):
"""Sylvester Hadamard must be symmetric: H = H^T."""
H = _build_hadamard(dim, DEVICE_TYPE)
assert torch.allclose(H, H.T, atol=1e-6), (
f"Hadamard not symmetric for dim={dim}"
)
# ============================================================================
# Store → Decode round-trip test (GPU + Triton required)
# ============================================================================
@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
class TestStoreDecodeRoundTrip:
"""End-to-end: store KV into TQ cache, decode, compare vs fp16 ref."""
@pytest.mark.parametrize(
"preset",
["turboquant_k8v4", "turboquant_4bit_nc"],
)
def test_single_token_roundtrip(self, preset):
"""Store 1 token, decode with query=key, check attention output.
For a single token with query=key, attention output should equal
the value (softmax over single key = 1.0). Quantization error
means we check cosine similarity rather than exact equality.
"""
from vllm.model_executor.layers.quantization.turboquant.centroids import (
solve_lloyd_max,
)
from vllm.v1.attention.ops.triton_turboquant_decode import (
triton_turboquant_decode_attention,
)
from vllm.v1.attention.ops.triton_turboquant_store import (
triton_turboquant_store,
)
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
D = 128
Hk = 4 # num_kv_heads
Hq = 4 # num_q_heads (no GQA for simplicity)
B = 1 # single token
block_size = 16
num_blocks = 1
device = torch.device(DEVICE_TYPE)
# Pure Hadamard rotation (symmetric: H = H^T, so Pi = PiT = H)
H = _build_hadamard(D, DEVICE_TYPE)
PiT = H
Pi = H
# Generate centroids
centroids, _ = solve_lloyd_max(D, cfg.centroid_bits)
centroids = centroids.float().to(device)
c_sorted, _ = centroids.sort()
midpoints = ((c_sorted[:-1] + c_sorted[1:]) / 2).to(device)
# Random K, V
torch.manual_seed(123)
key = torch.randn(B, Hk, D, device=device, dtype=torch.float16)
value = torch.randn(B, Hk, D, device=device, dtype=torch.float16)
# Allocate KV cache
padded_slot = cfg.slot_size_aligned
kv_cache = torch.zeros(
num_blocks,
block_size,
Hk,
padded_slot,
device=device,
dtype=torch.uint8,
)
slot_mapping = torch.tensor([0], device=device, dtype=torch.int32)
# Store
triton_turboquant_store(
key,
value,
kv_cache,
slot_mapping,
PiT,
midpoints,
mse_bits=cfg.key_mse_bits,
key_packed_size=cfg.key_packed_size,
value_quant_bits=cfg.effective_value_quant_bits,
key_fp8=cfg.key_fp8,
)
# Decode: use key as query so attention = softmax([1]) * V = V
query = key.expand(B, Hq, D).contiguous().to(torch.float16)
block_table = torch.tensor([[0]], device=device, dtype=torch.int32)
seq_lens = torch.tensor([1], device=device, dtype=torch.int32)
output = triton_turboquant_decode_attention(
query=query,
kv_cache=kv_cache,
block_table=block_table,
seq_lens=seq_lens,
Pi=Pi,
centroids=centroids,
scale=1.0 / math.sqrt(D),
mse_bits=cfg.key_mse_bits,
key_packed_size=cfg.key_packed_size,
value_quant_bits=cfg.effective_value_quant_bits,
key_fp8=cfg.key_fp8,
norm_correction=cfg.norm_correction,
PiT=PiT,
max_num_kv_splits=4,
)
# With single KV, output should approximate the stored value.
# Check per-head cosine similarity > threshold.
out_fp32 = output.float()
val_fp32 = value.expand(B, Hq, D).float()
for h in range(Hq):
cos_sim = torch.nn.functional.cosine_similarity(
out_fp32[0, h].unsqueeze(0),
val_fp32[0, h].unsqueeze(0),
).item()
# FP8 keys should be very accurate; MSE keys have more error
threshold = 0.95 if cfg.key_fp8 else 0.85
assert cos_sim > threshold, (
f"Preset {preset} head {h}: cosine_sim={cos_sim:.4f} < {threshold}"
)