[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,914 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests whether FP8 computation is enabled correctly.
|
||||
|
||||
Run `pytest tests/quantization/test_fp8.py --forked`.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import (
|
||||
is_quant_method_supported,
|
||||
load_model_without_vllm_runner,
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8LinearMethod,
|
||||
Fp8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod
|
||||
from vllm.model_executor.layers.quantization.utils import flashinfer_utils
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
prepare_fp8_moe_layer_for_fi,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
MODELS = [
|
||||
"neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
|
||||
# The checkpoint below was removed from the HF.
|
||||
# TODO: add a small replacement checkpoint.
|
||||
pytest.param(
|
||||
"nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
|
||||
marks=pytest.mark.skip(reason="Checkpoint removed from HF."),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DSV4.1 requires CUDA")
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
|
||||
def test_deepseek_v41_mxfp8_scale_loading(
|
||||
dist_init, default_vllm_config, monkeypatch, tp_rank, scale_dtype
|
||||
):
|
||||
"""Expand linear scales before TP slicing, including padded shared experts."""
|
||||
from vllm.model_executor.layers import linear as linear_module
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.models.deepseek_v41 import quant_config as quant_module
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
quant_config = quant_module.DeepseekV4FP8Config(
|
||||
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
|
||||
)
|
||||
quant_config._resolved_expert_dtype = "fp4"
|
||||
for module in (model_module, linear_module):
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(
|
||||
linear_module, "get_tensor_model_parallel_rank", lambda: tp_rank
|
||||
)
|
||||
block = torch.nn.Module()
|
||||
block.attn = torch.nn.Module()
|
||||
block.attn.wq_b = ColumnParallelLinear(
|
||||
128, 128, quant_config=quant_config, bias=False
|
||||
)
|
||||
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
128, [128, 64], quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn = torch.nn.Module()
|
||||
block.ffn.shared_experts = torch.nn.Module()
|
||||
block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear(
|
||||
128, [128, 128], quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn.shared_experts.down_proj = RowParallelLinear(
|
||||
128, 128, quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn.experts = torch.nn.Module()
|
||||
block.ffn.experts.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False)
|
||||
)
|
||||
block.engram = torch.nn.Module()
|
||||
block.engram.register_parameter(
|
||||
"weight_scale_inv",
|
||||
torch.nn.Parameter(torch.empty(3, 4, dtype=scale_dtype), False),
|
||||
)
|
||||
|
||||
def load_expert_scale(param, weight, *args, **kwargs):
|
||||
param.data.copy_(weight)
|
||||
return True
|
||||
|
||||
block.ffn.experts.weight_scale.weight_loader = load_expert_scale
|
||||
model = model_module.DeepseekV4Model.__new__(model_module.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.layers = torch.nn.ModuleList([block])
|
||||
model.config = SimpleNamespace(num_attention_heads=2)
|
||||
model.quant_config = quant_config
|
||||
model.use_sequence_parallel = False
|
||||
monkeypatch.setattr(
|
||||
model,
|
||||
"get_expert_mapping",
|
||||
lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")],
|
||||
)
|
||||
|
||||
checkpoints = [
|
||||
("attn.wq_b", "attn.wq_b", 128, 128, 0, None),
|
||||
("attn.wq_a", "attn.fused_wqa_wkv", 128, 128, 0, slice(0, 64)),
|
||||
("attn.wkv", "attn.fused_wqa_wkv", 64, 128, 0, slice(64, 96)),
|
||||
(
|
||||
"ffn.shared_experts.w1",
|
||||
"ffn.shared_experts.gate_up_proj",
|
||||
96,
|
||||
128,
|
||||
0,
|
||||
slice(0, 64),
|
||||
),
|
||||
(
|
||||
"ffn.shared_experts.w3",
|
||||
"ffn.shared_experts.gate_up_proj",
|
||||
96,
|
||||
128,
|
||||
0,
|
||||
slice(64, 128),
|
||||
),
|
||||
(
|
||||
"ffn.shared_experts.down_proj",
|
||||
"ffn.shared_experts.down_proj",
|
||||
128,
|
||||
96,
|
||||
1,
|
||||
None,
|
||||
),
|
||||
]
|
||||
weights = []
|
||||
expected = []
|
||||
for source, target, n, k, axis, shard in checkpoints:
|
||||
weight = torch.randint(-4, 5, (n, k)).to(torch.float8_e4m3fn)
|
||||
scale_bytes = torch.randint(124, 131, (n // 32, k // 32), dtype=torch.uint8)
|
||||
weights.extend(
|
||||
[
|
||||
(f"layers.0.{source}.weight", weight),
|
||||
(f"layers.0.{source}.scale", scale_bytes.view(scale_dtype)),
|
||||
]
|
||||
)
|
||||
dequant = (
|
||||
weight.float().reshape(n // 32, 32, k // 32, 32)
|
||||
* torch.exp2(scale_bytes.float() - 127)[:, None, :, None]
|
||||
).reshape(n, k)
|
||||
if "shared_experts" in source:
|
||||
padded = torch.zeros(128, 128)
|
||||
padded[:n, :k] = dequant
|
||||
dequant = padded
|
||||
expected.append((target, shard, dequant.chunk(2, dim=axis)[tp_rank]))
|
||||
|
||||
expert_scale = torch.full((3, 4), 125, dtype=torch.uint8)
|
||||
weights.append(
|
||||
("layers.0.ffn.experts.0.w1.weight_scale", expert_scale.view(scale_dtype))
|
||||
)
|
||||
weights.append(("layers.0.engram.weight_scale_inv", expert_scale.view(scale_dtype)))
|
||||
mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale")
|
||||
loaded = model.load_weights(
|
||||
(name.removeprefix("model."), weight) for name, weight in mapper.apply(weights)
|
||||
)
|
||||
assert "layers.0.ffn.experts.weight_scale" in loaded
|
||||
assert torch.equal(block.ffn.experts.weight_scale, expert_scale)
|
||||
assert torch.equal(block.engram.weight_scale_inv.view(torch.uint8), expert_scale)
|
||||
for target, shard, reference in expected:
|
||||
linear = block.get_submodule(target)
|
||||
assert isinstance(linear.quant_method, ModelOptLinearMethod)
|
||||
assert f"layers.0.{target}.weight_scale" in loaded
|
||||
weight = linear.weight if shard is None else linear.weight[shard]
|
||||
scale = linear.weight_scale if shard is None else linear.weight_scale[shard]
|
||||
actual = (
|
||||
weight.float().unflatten(-1, (-1, 32))
|
||||
* torch.exp2(scale.float() - 127).unsqueeze(-1)
|
||||
).flatten(-2)
|
||||
torch.testing.assert_close(actual, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DSV4.1 requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"weight_block_size,expert_dtype,scale_name",
|
||||
[
|
||||
([32, 32], "fp4", "weight_scale"),
|
||||
([128, 128], "fp4", "weight_scale_inv"),
|
||||
([32, 32], "fp8", "weight_scale_inv"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v41_vl_mapper_routes_linear_scales(
|
||||
weight_block_size, expert_dtype, scale_name
|
||||
):
|
||||
"""The VL wrapper must map ``.scale`` keys to the parameter the linear
|
||||
quant method registers, as the text model does. A hardcoded
|
||||
``weight_scale_inv`` raised KeyError for native MXFP8 checkpoints."""
|
||||
from vllm.models.deepseek_v41.nvidia import model as model_module
|
||||
from vllm.models.deepseek_v41.nvidia import vl_model as vl_module
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
|
||||
)
|
||||
resolved = model_module._linear_scale_param_name(
|
||||
vllm_config,
|
||||
expert_dtype,
|
||||
)
|
||||
assert resolved == scale_name
|
||||
|
||||
mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved)
|
||||
weight = torch.empty(0)
|
||||
mapped = [
|
||||
name
|
||||
for name, _ in mapper.apply(
|
||||
[("layers.0.attn.wq_a.scale", weight), ("layers.0.attn.wkv.weight", weight)]
|
||||
)
|
||||
]
|
||||
assert mapped == [
|
||||
f"language_model.model.layers.0.attn.wq_a.{scale_name}",
|
||||
"language_model.model.layers.0.attn.wkv.weight",
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v41_vl_exposes_quant_mappings_on_class():
|
||||
"""``configure_quant_config`` reads both mappings off the class, before
|
||||
``__init__`` builds the instance mapper. Without them a Quark config's
|
||||
per-layer keys never match this wrapper's ``language_model.``-rooted
|
||||
prefixes and every attention shard falls back to the global spec."""
|
||||
from vllm.models.deepseek_v41.amd.vl_model import DeepseekV41ForCausalLM
|
||||
|
||||
assert "fused_wqa_wkv" in DeepseekV41ForCausalLM.packed_modules_mapping
|
||||
mapper = DeepseekV41ForCausalLM.hf_to_vllm_mapper.get_rename_mapper()
|
||||
assert mapper.apply_list(["layers.0.attn.wq_a"]) == [
|
||||
"language_model.model.layers.0.attn.wq_a"
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v41_engram_scale_accepts_quark_name():
|
||||
"""Quark exports name the engram scale ``embed.weight_scale``; the
|
||||
``\\.scale$`` rules only match a literal ``.scale`` suffix, so without an
|
||||
explicit rule the tensor is never routed and loading fails."""
|
||||
from vllm.models.deepseek_v41.amd import model as model_module
|
||||
|
||||
mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale")
|
||||
weight = torch.empty(0)
|
||||
mapped = [
|
||||
name
|
||||
for name, _ in mapper.apply(
|
||||
[
|
||||
("layers.1.engram.embed.weight_scale", weight),
|
||||
("layers.1.engram.embed.scale", weight),
|
||||
]
|
||||
)
|
||||
]
|
||||
assert mapped == [
|
||||
"model.layers.1.engram.embed_tokens.weight_scale_inv",
|
||||
"model.layers.1.engram.embed_tokens.weight_scale_inv",
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v41_declines_quark_configs():
|
||||
"""``from_config`` rewrites a Quark config into a single global FP8 scheme,
|
||||
which would discard the per-layer specs of a mixed-precision export (MXFP4
|
||||
experts + 2-D block MXFP8 attention in DeepSeek-V4.1-Flash). QuarkConfig
|
||||
knows how to dispatch each scheme, so it must handle every Quark export."""
|
||||
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
|
||||
|
||||
hf_config = SimpleNamespace(model_type="deepseek_v41")
|
||||
mxfp4_global = {
|
||||
"global_quant_config": {
|
||||
"weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}
|
||||
},
|
||||
"quant_method": "quark",
|
||||
}
|
||||
|
||||
assert (
|
||||
DeepseekV4FP8Config.override_quantization_method(
|
||||
mxfp4_global, None, hf_config=hf_config
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
mixed = {
|
||||
**mxfp4_global,
|
||||
"layer_quant_config": {
|
||||
"layers.0.attn.wkv": {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"block_size": [32, 32],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
assert (
|
||||
DeepseekV4FP8Config.override_quantization_method(
|
||||
mixed, None, hf_config=hf_config
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
_DSV41_NVFP4 = {
|
||||
"moe_quant_algo": "NVFP4",
|
||||
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"],
|
||||
}
|
||||
_DSV4_NVFP4_DSPARK = {
|
||||
"moe_quant_algo": "NVFP4",
|
||||
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ckpt_quant_cfg", "prefix", "expected"),
|
||||
[
|
||||
pytest.param(_DSV41_NVFP4, "model.layers.1.ffn.experts", "nvfp4", id="target"),
|
||||
pytest.param(
|
||||
_DSV41_NVFP4,
|
||||
"language_model.model.layers.0.ffn.experts",
|
||||
"nvfp4",
|
||||
id="target-vl-prefix",
|
||||
),
|
||||
pytest.param(
|
||||
_DSV41_NVFP4, "model.layers.40.ffn.experts", "mxfp4", id="ignored-mtp0"
|
||||
),
|
||||
pytest.param(
|
||||
_DSV41_NVFP4, "model.layers.42.ffn.experts", "mxfp4", id="ignored-mtp2"
|
||||
),
|
||||
pytest.param(
|
||||
_DSV4_NVFP4_DSPARK,
|
||||
"model.layers.41.ffn.experts",
|
||||
"nvfp4",
|
||||
id="quantized-mtp1",
|
||||
),
|
||||
pytest.param(
|
||||
{"moe_quant_algo": "NVFP4"},
|
||||
"model.layers.40.ffn.experts",
|
||||
"nvfp4",
|
||||
id="no-ignore-list",
|
||||
),
|
||||
pytest.param({}, "model.layers.1.ffn.experts", "mxfp4", id="not-nvfp4"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v41_nvfp4_export_keeps_ignored_experts_mxfp4(
|
||||
monkeypatch, ckpt_quant_cfg, prefix, expected
|
||||
):
|
||||
"""NVFP4 exports keep the routed experts they ``ignore`` in MXFP4, e.g. the
|
||||
bundled DSpark draft (``mtp.*``, built as ``layers.{40 + k}``) of
|
||||
DeepSeek-V4.1-Flash-NVFP4. The ignore list is read from the checkpoint
|
||||
passed to ``from_config``: the current hf_config below carries none."""
|
||||
import vllm.model_executor.layers.quantization.modelopt as modelopt
|
||||
import vllm.models.deepseek_v41.quant_config as dsv41_qc
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
|
||||
hf_config = SimpleNamespace(
|
||||
expert_dtype="fp4",
|
||||
quantization_config={"moe_quant_algo": ckpt_quant_cfg.get("moe_quant_algo")},
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config, hf_text_config=SimpleNamespace(num_hidden_layers=40)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dsv41_qc,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=model_config),
|
||||
)
|
||||
monkeypatch.setattr(dsv41_qc, "Mxfp4MoEMethod", lambda moe_config: "mxfp4")
|
||||
monkeypatch.setattr(modelopt, "ModelOptNvFp4FusedMoE", lambda **kwargs: "nvfp4")
|
||||
|
||||
config = dsv41_qc.DeepseekV4FP8Config.from_config(
|
||||
{"quant_method": "fp8", "activation_scheme": "dynamic", **ckpt_quant_cfg}
|
||||
)
|
||||
monkeypatch.setattr(config, "_get_nvfp4_config", lambda: None)
|
||||
layer = RoutedExperts.__new__(RoutedExperts)
|
||||
layer.moe_config = None
|
||||
assert config.get_quant_method(layer, prefix) == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
|
||||
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
|
||||
@pytest.mark.parametrize(
|
||||
"weight_shape,is_bmm",
|
||||
[
|
||||
((129, 160), False),
|
||||
((387, 160), True),
|
||||
((3, 129, 160), False),
|
||||
((3, 128, 512), False),
|
||||
],
|
||||
)
|
||||
def test_deepgemm_mxfp8_preserves_weight_and_scale_values(
|
||||
scale_dtype, weight_shape, is_bmm
|
||||
):
|
||||
"""DeepGEMM layout conversion preserves native weights and E8M0 scales."""
|
||||
from vllm.model_executor.layers.quantization.utils import fp8_utils
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if not is_deep_gemm_supported() or not current_platform.is_device_capability_family(
|
||||
100
|
||||
):
|
||||
pytest.skip("DeepGEMM MXFP8 requires Blackwell")
|
||||
|
||||
weight = torch.randn(weight_shape, device="cuda").to(torch.float8_e4m3fn)
|
||||
scales = (
|
||||
torch.randint(
|
||||
1,
|
||||
255,
|
||||
(weight.numel() // 32 + 1,),
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)[1:]
|
||||
.view(*weight_shape[:-1], weight_shape[-1] // 32)
|
||||
.view(scale_dtype)
|
||||
)
|
||||
original_weight = weight.view(torch.uint8).clone()
|
||||
original_scales = scales.view(torch.uint8).clone()
|
||||
processed_weight, packed = fp8_utils.deepgemm_post_process_fp8_weight_block(
|
||||
weight,
|
||||
scales,
|
||||
quant_block_shape=(1, 32),
|
||||
use_e8m0=True,
|
||||
is_bmm=is_bmm,
|
||||
bmm_batch_size=3 if is_bmm else 0,
|
||||
)
|
||||
assert packed.dtype == torch.int32
|
||||
unpacked = torch.stack(
|
||||
[(packed >> (8 * i)) & 0xFF for i in range(4)], dim=-1
|
||||
).flatten(-2)
|
||||
unpacked = unpacked[..., : scales.shape[-1]].to(torch.uint8)
|
||||
torch.testing.assert_close(
|
||||
unpacked.reshape_as(original_scales), original_scales, rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
processed_weight.view(torch.uint8).reshape_as(original_weight),
|
||||
original_weight,
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
|
||||
@pytest.mark.parametrize("prequantized", [False, True])
|
||||
@pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens
|
||||
):
|
||||
"""BMM metadata set after construction selects grouped weight processing."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import (
|
||||
DeepGemmMxfp8BmmLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.quantization import get_quantization_config
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
|
||||
compute_fp8_einsum_recipe,
|
||||
deep_gemm_fp8_o_proj,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.model import DeepseekV4Model
|
||||
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if not is_deep_gemm_supported() or not current_platform.is_device_capability_family(
|
||||
100
|
||||
):
|
||||
pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell")
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
deepseek_config = DeepseekV4FP8Config(
|
||||
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
|
||||
)
|
||||
deepseek_config._resolved_expert_dtype = "fp4"
|
||||
quant_config: QuantizationConfig = deepseek_config
|
||||
if config_source == "mxfp8":
|
||||
quant_config = get_quantization_config("mxfp8").from_config(
|
||||
{"quant_method": "mxfp8"}
|
||||
)
|
||||
with torch.device("cuda"):
|
||||
linear = ColumnParallelLinear(
|
||||
512,
|
||||
256,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
return_bias=False,
|
||||
)
|
||||
linear.is_bmm = True
|
||||
linear.bmm_batch_size = 2
|
||||
weight = torch.randn(256, 512, device="cuda").to(torch.float8_e4m3fn)
|
||||
scales = torch.randint(124, 131, (8, 16), device="cuda", dtype=torch.uint8)
|
||||
checkpoint_scales = (
|
||||
scales.view(torch.float8_e8m0fnu)
|
||||
if config_source == "deepseek"
|
||||
else scales.repeat_interleave(32, dim=0)
|
||||
)
|
||||
model = DeepseekV4Model.__new__(DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
block = torch.nn.Module()
|
||||
block.attn = torch.nn.Module()
|
||||
block.attn.wo_a = linear
|
||||
model.layers = torch.nn.ModuleList([block])
|
||||
model.config = SimpleNamespace(num_attention_heads=2)
|
||||
model.quant_config = quant_config
|
||||
model.use_sequence_parallel = False
|
||||
monkeypatch.setattr(model, "get_expert_mapping", lambda: [])
|
||||
model.load_weights(
|
||||
[
|
||||
("layers.0.attn.wo_a.weight", weight),
|
||||
("layers.0.attn.wo_a.weight_scale", checkpoint_scales),
|
||||
]
|
||||
)
|
||||
reference_weight = (
|
||||
weight.float().reshape(8, 32, 16, 32)
|
||||
* torch.exp2(scales.float() - 127)[:, None, :, None]
|
||||
).reshape(2, 128, 512)
|
||||
assert isinstance(linear.quant_method, ModelOptLinearMethod)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel)
|
||||
assert linear.weight.shape == (2, 128, 512)
|
||||
torch.testing.assert_close(
|
||||
linear.weight.view(torch.uint8).flatten(),
|
||||
weight.view(torch.uint8).flatten(),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
unpacked_scales = torch.stack(
|
||||
[(linear.weight_scale >> (8 * i)) & 0xFF for i in range(4)], dim=-1
|
||||
).flatten(-2)
|
||||
torch.testing.assert_close(
|
||||
unpacked_scales.to(torch.uint8).reshape(256, 16),
|
||||
scales.repeat_interleave(32, dim=0),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
x = torch.randn(2, num_tokens, 512, device="cuda", dtype=torch.bfloat16)
|
||||
reference = torch.einsum("gmk,gnk->mgn", x.float(), reference_weight)
|
||||
inputs = x.transpose(0, 1)
|
||||
if prequantized:
|
||||
cache = torch.cat(
|
||||
(
|
||||
torch.ones(num_tokens, 32, device="cuda"),
|
||||
torch.zeros(num_tokens, 32, device="cuda"),
|
||||
),
|
||||
dim=1,
|
||||
)
|
||||
inputs = fused_inv_rope_fp8_quant(
|
||||
x.permute(1, 0, 2),
|
||||
torch.arange(num_tokens, device="cuda"),
|
||||
cache,
|
||||
n_groups=2,
|
||||
heads_per_group=1,
|
||||
nope_dim=448,
|
||||
rope_dim=64,
|
||||
quant_group_size=32,
|
||||
tma_aligned_scales=current_platform.has_device_capability(100),
|
||||
)
|
||||
with torch.no_grad():
|
||||
output = linear(inputs)
|
||||
assert output.shape == reference.shape
|
||||
assert (output.float() - reference).norm() / reference.norm() < 0.06
|
||||
if prequantized:
|
||||
recipe, tma_aligned_scales = compute_fp8_einsum_recipe(block_size=32)
|
||||
projected = deep_gemm_fp8_o_proj(
|
||||
x.permute(1, 0, 2),
|
||||
torch.arange(num_tokens, device="cuda"),
|
||||
cache,
|
||||
linear,
|
||||
torch.nn.Identity(),
|
||||
n_groups=2,
|
||||
heads_per_group=1,
|
||||
nope_dim=448,
|
||||
rope_dim=64,
|
||||
o_lora_rank=128,
|
||||
einsum_recipe=recipe,
|
||||
tma_aligned_scales=tma_aligned_scales,
|
||||
)
|
||||
torch.testing.assert_close(projected, output.flatten(1), rtol=0, atol=0)
|
||||
compiled = torch.compile(linear, backend="eager", fullgraph=True)
|
||||
with torch.no_grad():
|
||||
torch.testing.assert_close(compiled(inputs), output, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_prepare_gated_trtllm_fp8_moe_weights_pads_each_projection(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
flashinfer_utils,
|
||||
"rotate_weights_for_fi_trtllm_fp8_per_tensor_moe",
|
||||
lambda *args: None,
|
||||
)
|
||||
intermediate = 17
|
||||
padded_intermediate = 32
|
||||
hidden_size = 4
|
||||
gate = torch.ones((1, intermediate, hidden_size), dtype=torch.float8_e4m3fn)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
w2 = torch.ones((1, hidden_size, intermediate), dtype=torch.float8_e4m3fn)
|
||||
layer = SimpleNamespace(
|
||||
activation=SimpleNamespace(is_gated=True),
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
),
|
||||
)
|
||||
|
||||
padded_w31, _, _, _ = prepare_fp8_moe_layer_for_fi(
|
||||
layer,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale=torch.ones(1),
|
||||
w13_input_scale=torch.ones(1),
|
||||
w2_scale=torch.ones(1),
|
||||
w2_input_scale=torch.ones(1),
|
||||
is_trtllm=True,
|
||||
)
|
||||
|
||||
expected = w13.new_zeros((1, 2 * padded_intermediate, hidden_size))
|
||||
expected[:, :intermediate] = up
|
||||
expected[:, padded_intermediate : padded_intermediate + intermediate] = gate
|
||||
assert layer.moe_config.intermediate_size_per_partition == padded_intermediate
|
||||
assert torch.equal(padded_w31, expected)
|
||||
|
||||
|
||||
def test_static_fp8_moe_input_scales_remain_scalar() -> None:
|
||||
a1_scale, a2_scale = process_fp8_input_tensor_strategy_moe(
|
||||
torch.tensor([0.25, 0.5]),
|
||||
torch.tensor([0.75, 0.6]),
|
||||
enable_eplb=False,
|
||||
)
|
||||
|
||||
assert a1_scale.ndim == a2_scale.ndim == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
@pytest.mark.parametrize(
|
||||
"force_marlin", [True, False] if current_platform.is_cuda() else [False]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
def test_model_load_and_run(
|
||||
model_id: str,
|
||||
force_marlin: bool,
|
||||
use_rocm_aiter: bool,
|
||||
monkeypatch,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
if use_rocm_aiter:
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
kernel_config = KernelConfig(
|
||||
linear_backend="marlin" if force_marlin else "auto",
|
||||
moe_backend="marlin" if force_marlin else "auto",
|
||||
)
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_id,
|
||||
model_config_kwargs={"hf_overrides": {"num_hidden_layers": 3}},
|
||||
vllm_config_kwargs={"kernel_config": kernel_config},
|
||||
)
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE_TYPE)
|
||||
positions = torch.arange(input_ids.numel(), device=DEVICE_TYPE)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
|
||||
):
|
||||
model(input_ids, positions, None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_scaled_fp8_quant(dtype) -> None:
|
||||
def quantize_ref(tensor, inv_scale):
|
||||
# The reference implementation that fully aligns to
|
||||
# the kernel being tested.
|
||||
finfo = torch.finfo(current_platform.fp8_dtype())
|
||||
scale = inv_scale.reciprocal()
|
||||
qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max)
|
||||
qweight = qweight.to(current_platform.fp8_dtype())
|
||||
return qweight
|
||||
|
||||
def per_tensor_dequantize(tensor, inv_scale, dtype):
|
||||
fake_qweight = tensor.to(dtype)
|
||||
dq_weight = fake_qweight * inv_scale
|
||||
return dq_weight
|
||||
|
||||
# Note that we use a shape % 4 != 0 to cover edge cases,
|
||||
# because scaled_fp8_quant is vectorized by 4.
|
||||
x = (torch.randn(size=(11, 11), device=DEVICE_TYPE) * 13).to(dtype)
|
||||
|
||||
# Dynamic quantization
|
||||
ref_y, inv_scale = ops.scaled_fp8_quant(x, None)
|
||||
ref_y = per_tensor_dequantize(ref_y, inv_scale, dtype)
|
||||
|
||||
# Reference dynamic quantization
|
||||
y = quantize_ref(x, inv_scale)
|
||||
torch.testing.assert_close(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
|
||||
|
||||
# Static quantization
|
||||
y, _ = ops.scaled_fp8_quant(x, inv_scale)
|
||||
torch.testing.assert_close(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
|
||||
|
||||
# Padding
|
||||
y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
|
||||
assert y.shape[0] == 17
|
||||
torch.testing.assert_close(
|
||||
ref_y,
|
||||
per_tensor_dequantize(torch.narrow(y, 0, 0, x.shape[0]), inv_scale, dtype),
|
||||
)
|
||||
|
||||
# non-contiguous input with padding
|
||||
m, n, padded_stride = 975, 512, 576
|
||||
padded_tensor = (torch.randn(size=(m, padded_stride), device=DEVICE_TYPE) * 13).to(
|
||||
dtype
|
||||
)
|
||||
x_nc = padded_tensor[:, :n] # shape (m, n) with stride (padded_stride, 1)
|
||||
|
||||
assert not x_nc.is_contiguous()
|
||||
assert x_nc.stride(0) == padded_stride
|
||||
|
||||
# dynamic quantization
|
||||
ref_y_nc, inv_scale_nc = ops.scaled_fp8_quant(x_nc, None)
|
||||
ref_y_nc = per_tensor_dequantize(ref_y_nc, inv_scale_nc, dtype)
|
||||
|
||||
# reference dynamic quantization
|
||||
y_nc = quantize_ref(x_nc, inv_scale_nc)
|
||||
torch.testing.assert_close(
|
||||
ref_y_nc, per_tensor_dequantize(y_nc, inv_scale_nc, dtype)
|
||||
)
|
||||
|
||||
# static quantization
|
||||
y_nc, _ = ops.scaled_fp8_quant(x_nc, inv_scale_nc)
|
||||
torch.testing.assert_close(
|
||||
ref_y_nc, per_tensor_dequantize(y_nc, inv_scale_nc, dtype)
|
||||
)
|
||||
|
||||
# padding after non-contiguous input quantization
|
||||
y_nc_pad, _ = ops.scaled_fp8_quant(x_nc, inv_scale_nc, num_token_padding=m + 10)
|
||||
assert y_nc_pad.shape[0] == m + 10
|
||||
torch.testing.assert_close(
|
||||
ref_y_nc,
|
||||
per_tensor_dequantize(
|
||||
torch.narrow(y_nc_pad, 0, 0, x_nc.shape[0]), inv_scale_nc, dtype
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_fp8_fnuz(),
|
||||
reason="FP8 e4m3fn weight reloading is not supported on e4m3fnuz platforms",
|
||||
)
|
||||
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
|
||||
# FP8 weight reloading does not support online quantization
|
||||
@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
|
||||
# any postprocessing that is applied to the weights such as padding and repacking
|
||||
# (excluding device sharding) must also be applied to the reloaded weights
|
||||
#
|
||||
# this is the case for marlin as well as per-tensor Fp8MoEMethod
|
||||
@pytest.mark.parametrize("use_marlin", [False]) # skip True
|
||||
def test_fp8_reloading(
|
||||
default_vllm_config,
|
||||
method_cls,
|
||||
weight_block_size,
|
||||
use_marlin,
|
||||
dist_init,
|
||||
monkeypatch,
|
||||
):
|
||||
# NOTE(rob): this test fails when using DeepGEMM because the
|
||||
# shapes are invalid. Previously the test was passing because
|
||||
# we set fp8_backend to None, which sidestepped the issue.
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
|
||||
|
||||
if method_cls is Fp8MoEMethod and weight_block_size is None:
|
||||
pytest.skip(
|
||||
"FP8 Tensor weight reloading does not support fusing w13_weight_scale. "
|
||||
"If this is your use case, consider using a restore function like #26327"
|
||||
)
|
||||
|
||||
# Set model config as model_config.dtype is required in Fp8LinearMethod.
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
default_vllm_config.kernel_config.moe_backend = "triton"
|
||||
layer_size = 128 if weight_block_size is not None else 1
|
||||
with torch.device(f"{DEVICE_TYPE}:0"):
|
||||
config = Fp8Config(
|
||||
weight_block_size=weight_block_size,
|
||||
)
|
||||
|
||||
if method_cls is Fp8LinearMethod:
|
||||
layer = torch.nn.Linear(layer_size, layer_size)
|
||||
method = method_cls(config)
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=layer_size,
|
||||
output_partition_sizes=[layer_size],
|
||||
input_size=layer_size,
|
||||
output_size=layer_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
method.use_marlin = use_marlin
|
||||
|
||||
else:
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=1,
|
||||
top_k=1,
|
||||
hidden_size=layer_size,
|
||||
intermediate_size=layer_size,
|
||||
)
|
||||
layer = layer.routed_experts
|
||||
method = method_cls(config, layer)
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=1,
|
||||
hidden_size=layer_size,
|
||||
intermediate_size_per_partition=layer_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
|
||||
# capture weights format during loading
|
||||
original_metadata = [
|
||||
(name, param.shape, getattr(param, "weight_loader", default_weight_loader))
|
||||
for name, param in layer.named_parameters()
|
||||
]
|
||||
|
||||
# test loading
|
||||
for name, shape, _ in original_metadata:
|
||||
param = getattr(layer, name)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, torch.zeros(shape)) # cannot use empty
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
# test reloading works after loading
|
||||
for name, shape, _ in original_metadata:
|
||||
param = getattr(layer, name)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, torch.zeros(shape)) # cannot use empty
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_kv_cache_scale_sync_to_host_copies():
|
||||
"""Test device-to-host sync of the k/v quantization scales, for both the
|
||||
checkpoint-load and runtime-calc paths that produce them.
|
||||
"""
|
||||
layer = torch.nn.Module()
|
||||
set_default_quant_scales(layer, register_buffer=True)
|
||||
layer.kv_cache_dtype = "fp8"
|
||||
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
|
||||
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer._k_scale_float != 1.0
|
||||
assert layer._v_scale_float != 1.0
|
||||
# Host copy must mirror both the float and the device scale tensor.
|
||||
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float)
|
||||
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float)
|
||||
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item())
|
||||
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_kv_cache_dtype_skip_layers(monkeypatch, dist_init, workspace_init):
|
||||
"""Test that kv_cache_dtype_skip_layers skips quantization for specified layers."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
model, _ = load_model_without_vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
vllm_config_kwargs={
|
||||
"cache_config": CacheConfig(
|
||||
cache_dtype="fp8", kv_cache_dtype_skip_layers=["0", "2"]
|
||||
)
|
||||
},
|
||||
)
|
||||
for i, layer in enumerate(model.model.decoder.layers):
|
||||
expected = "auto" if str(i) in ["0", "2"] else "fp8"
|
||||
assert layer.self_attn.attn.kv_cache_dtype == expected
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,637 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests ROCm AITER fused shared experts with online quantization."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.quantization import (
|
||||
QuantizationConfigArgs,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.experts import rocm_aiter_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import (
|
||||
QuarkOCP_MX_MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
mxfp4_quantize,
|
||||
)
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
finalize_layerwise_reload,
|
||||
initialize_layerwise_reload,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import get_model_architecture
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm AITER shared-expert tests require ROCm.",
|
||||
)
|
||||
|
||||
_QUARK_MXFP4_CONFIG: dict[str, Any] = {
|
||||
"global_quant_config": {
|
||||
"input_tensors": {
|
||||
"dtype": "fp4",
|
||||
"is_dynamic": True,
|
||||
"qscheme": "per_group",
|
||||
"ch_axis": -1,
|
||||
"group_size": 32,
|
||||
"block_size": None,
|
||||
"symmetric": None,
|
||||
"round_method": "half_even",
|
||||
"scale_type": "float",
|
||||
"scale_format": "e8m0",
|
||||
"scale_calculation_mode": "even",
|
||||
"mx_element_dtype": None,
|
||||
"observer_cls": "PerBlockMXObserver",
|
||||
"is_scale_quant": False,
|
||||
"enable_buffer_reuse": False,
|
||||
"max_input_numel": 4194304,
|
||||
},
|
||||
"output_tensors": None,
|
||||
"weight": {
|
||||
"dtype": "fp4",
|
||||
"is_dynamic": False,
|
||||
"qscheme": "per_group",
|
||||
"ch_axis": -1,
|
||||
"group_size": 32,
|
||||
"block_size": None,
|
||||
"symmetric": None,
|
||||
"round_method": "half_even",
|
||||
"scale_type": "float",
|
||||
"scale_format": "e8m0",
|
||||
"scale_calculation_mode": "even",
|
||||
"mx_element_dtype": None,
|
||||
"observer_cls": "PerBlockMXObserver",
|
||||
"is_scale_quant": False,
|
||||
"enable_buffer_reuse": False,
|
||||
"max_input_numel": 4194304,
|
||||
},
|
||||
"bias": None,
|
||||
"target_device": None,
|
||||
},
|
||||
"algo_config": None,
|
||||
"softmax_quant_spec": None,
|
||||
"quant_method": "quark",
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {},
|
||||
"exclude": ["re:^(?!.*\\.mlp\\.experts(?:\\.|$)).*$"],
|
||||
"kv_cache_quant_config": {},
|
||||
"kv_cache_post_rope": False,
|
||||
"quant_mode": "eager_mode",
|
||||
"version": "0.12+9d3d471cdf1",
|
||||
"export": {
|
||||
"kv_cache_group": [],
|
||||
"min_kv_scale": 0.0,
|
||||
"pack_method": "reorder",
|
||||
"weight_format": "real_quantized",
|
||||
"weight_merge_groups": None,
|
||||
},
|
||||
}
|
||||
|
||||
_QUARK_MXFP4_LAYER_CONFIG: dict[str, Any] = _QUARK_MXFP4_CONFIG["global_quant_config"]
|
||||
_QUARK_MXFP4_CONFIG = {
|
||||
**_QUARK_MXFP4_CONFIG,
|
||||
"global_quant_config": {
|
||||
"input_tensors": {},
|
||||
"output_tensors": None,
|
||||
"weight": {},
|
||||
"bias": None,
|
||||
"target_device": None,
|
||||
},
|
||||
"layer_quant_config": {"*mlp.experts*": _QUARK_MXFP4_LAYER_CONFIG},
|
||||
}
|
||||
|
||||
|
||||
class _StubAttention(torch.nn.Module):
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__()
|
||||
self.o_proj = SimpleNamespace(reduce_results=True)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_aiter_shared_expert_topk_metadata(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Isolate AITER's process-global shared-expert routing buffer."""
|
||||
monkeypatch.setattr(rocm_aiter_moe, "aiter_topK_meta_data", None)
|
||||
rocm_aiter_moe.init_aiter_topK_meta_data.cache_clear()
|
||||
yield
|
||||
rocm_aiter_moe.init_aiter_topK_meta_data.cache_clear()
|
||||
|
||||
|
||||
def _write_minimal_moe_config(model_path: Path, architecture: str) -> None:
|
||||
"""Write a compact MoE config with one shared expert."""
|
||||
config = {
|
||||
"architectures": [architecture],
|
||||
"hidden_size": 256,
|
||||
"intermediate_size": 512,
|
||||
"moe_intermediate_size": 256,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 4,
|
||||
"head_dim": 64,
|
||||
"num_hidden_layers": 1,
|
||||
"vocab_size": 256,
|
||||
"max_position_embeddings": 64,
|
||||
"q_lora_rank": 64,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"n_routed_experts": 4,
|
||||
"num_experts": 4,
|
||||
"num_experts_per_tok": 1,
|
||||
"n_group": 1,
|
||||
"topk_group": 1,
|
||||
"n_shared_experts": 1,
|
||||
"first_k_dense_replace": 0,
|
||||
"moe_layer_freq": 1,
|
||||
"rms_norm_eps": 1e-6,
|
||||
"hidden_act": "silu",
|
||||
"tie_word_embeddings": False,
|
||||
"rope_theta": 10000,
|
||||
"quantization_config": _QUARK_MXFP4_CONFIG,
|
||||
}
|
||||
if architecture == "AXK1ForCausalLM":
|
||||
config["model_type"] = "axk1"
|
||||
elif architecture == "MiniMaxM3SparseForCausalLM":
|
||||
text_config = {
|
||||
**config,
|
||||
"model_type": "minimax_m3_text",
|
||||
"dense_intermediate_size": 512,
|
||||
"hidden_act": "swigluoai",
|
||||
"num_local_experts": 4,
|
||||
"n_shared_experts": 1,
|
||||
"moe_layer_freq": [1],
|
||||
"sparse_attention_config": None,
|
||||
"quantization_config": {
|
||||
**_QUARK_MXFP4_CONFIG,
|
||||
"layer_quant_config": {
|
||||
"*block_sparse_moe.experts*": _QUARK_MXFP4_LAYER_CONFIG
|
||||
},
|
||||
"exclude": [r"re:^(?!.*\.block_sparse_moe\.experts(?:\.|$)).*$"],
|
||||
},
|
||||
}
|
||||
config = {
|
||||
"architectures": [architecture],
|
||||
"model_type": "minimax_m3_vl",
|
||||
"text_config": text_config,
|
||||
}
|
||||
elif architecture == "Glm4MoeForCausalLM":
|
||||
config["model_type"] = "glm4_moe"
|
||||
elif architecture == "Glm4MoeLiteForCausalLM":
|
||||
config["model_type"] = "glm4_moe_lite"
|
||||
elif architecture in ("Qwen3NextForCausalLM", "Qwen3_5MoeForCausalLM"):
|
||||
config.update(
|
||||
model_type=(
|
||||
"qwen3_next"
|
||||
if architecture == "Qwen3NextForCausalLM"
|
||||
else "qwen3_5_moe_text"
|
||||
),
|
||||
layer_types=[
|
||||
"linear_attention"
|
||||
if architecture == "Qwen3NextForCausalLM"
|
||||
else "full_attention"
|
||||
],
|
||||
head_dim=64,
|
||||
linear_key_head_dim=64,
|
||||
linear_value_head_dim=64,
|
||||
linear_num_key_heads=4,
|
||||
linear_num_value_heads=4,
|
||||
shared_expert_intermediate_size=256,
|
||||
)
|
||||
elif architecture == "Qwen4ExpForCausalLM":
|
||||
config.update(
|
||||
model_type="qwen4_exp_text",
|
||||
layer_types=["full_attention"],
|
||||
head_dim=64,
|
||||
linear_key_head_dim=64,
|
||||
linear_value_head_dim=64,
|
||||
linear_num_key_heads=4,
|
||||
linear_num_value_heads=4,
|
||||
shared_expert_intermediate_size=256,
|
||||
eos_token_id=1,
|
||||
hc_count=2,
|
||||
hc_lowrank=4,
|
||||
ple_layer_ids=[],
|
||||
)
|
||||
elif architecture == "Glm5NextForCausalLM":
|
||||
config.update(
|
||||
model_type="glm5_next",
|
||||
layer_types=["linear_attention"],
|
||||
mlp_layer_types=["sparse"],
|
||||
qk_rope_head_dim=0,
|
||||
index_topk=1,
|
||||
index_kpool=1,
|
||||
pad_token_id=None,
|
||||
scoring_func="sigmoid",
|
||||
topk_method="noaux_tc",
|
||||
mhc=False,
|
||||
)
|
||||
else:
|
||||
config["model_type"] = (
|
||||
"deepseek_v3"
|
||||
if "V3" in architecture
|
||||
else "deepseek_v32"
|
||||
if architecture == "DeepseekV32ForCausalLM"
|
||||
else "deepseek_v2"
|
||||
)
|
||||
if architecture in ("DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"):
|
||||
config.update(
|
||||
index_topk=1,
|
||||
index_kpool=1,
|
||||
index_n_heads=1,
|
||||
index_head_dim=32,
|
||||
index_kv_lora_rank=32,
|
||||
)
|
||||
(model_path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_path", ["load_weights", "fused_gate_up", "weight_loader"]
|
||||
)
|
||||
def test_online_shared_expert_loads_bf16_weights_into_mxfp4_slot(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
load_path: str,
|
||||
) -> None:
|
||||
"""A BF16 shared expert is quantized while routed MXFP4 weights are loaded.
|
||||
|
||||
`weight_loader` covers DeepSeek-style model loaders, which bypass
|
||||
`RoutedExperts.load_weights`.
|
||||
"""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
hidden_size = intermediate_size = 64
|
||||
num_routed_experts = 2
|
||||
device = current_platform.device_type
|
||||
online_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={"*shared_expert*": "mxfp4"})
|
||||
)
|
||||
quant_config = QuarkConfig(_QUARK_MXFP4_CONFIG)
|
||||
quant_config.online_quantization_config = online_config
|
||||
|
||||
with torch.device(device):
|
||||
runner = FusedMoEFactory(
|
||||
num_experts=num_routed_experts,
|
||||
top_k=1,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
n_shared_experts=1,
|
||||
fuse_shared_experts=True,
|
||||
shared_expert_prefix="model.layers.0.mlp.shared_expert",
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
layer = runner.routed_experts
|
||||
assert isinstance(layer.quant_method, QuarkOCP_MX_MoEMethod)
|
||||
|
||||
routed_gate, _ = mxfp4_quantize(
|
||||
torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
|
||||
)
|
||||
routed_up, _ = mxfp4_quantize(
|
||||
torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
|
||||
)
|
||||
routed_down, _ = mxfp4_quantize(
|
||||
torch.randn(hidden_size, intermediate_size, dtype=torch.bfloat16)
|
||||
)
|
||||
shared_gate = torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
|
||||
shared_up = torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
|
||||
shared_down = torch.randn(hidden_size, intermediate_size, dtype=torch.bfloat16)
|
||||
|
||||
weights = [
|
||||
("0.gate_proj.weight", routed_gate),
|
||||
("0.up_proj.weight", routed_up),
|
||||
("0.down_proj.weight", routed_down),
|
||||
]
|
||||
if load_path == "load_weights":
|
||||
weights += [
|
||||
("2.gate_proj.weight", shared_gate),
|
||||
("2.up_proj.weight", shared_up),
|
||||
("2.down_proj.weight", shared_down),
|
||||
]
|
||||
elif load_path == "fused_gate_up":
|
||||
weights += [
|
||||
("2.gate_up_proj.weight", torch.cat([shared_gate, shared_up])),
|
||||
("2.down_proj.weight", shared_down),
|
||||
]
|
||||
list(layer.load_weights(weights))
|
||||
|
||||
if load_path == "weight_loader":
|
||||
for stem, shard_id, weight in (
|
||||
("w13", "w1", shared_gate),
|
||||
("w13", "w3", shared_up),
|
||||
("w2", "w2", shared_down),
|
||||
):
|
||||
param = getattr(layer, f"{stem}_weight")
|
||||
param.weight_loader(
|
||||
param, weight, f"{layer.layer_name}.{stem}_weight", shard_id, 2
|
||||
)
|
||||
|
||||
expected_gate, expected_gate_scale = mxfp4_quantize(shared_gate)
|
||||
expected_up, expected_up_scale = mxfp4_quantize(shared_up)
|
||||
expected_down, expected_down_scale = mxfp4_quantize(shared_down)
|
||||
assert torch.equal(layer.w13_weight[2, :intermediate_size], expected_gate)
|
||||
assert torch.equal(layer.w13_weight[2, intermediate_size:], expected_up)
|
||||
assert torch.equal(layer.w2_weight[2], expected_down)
|
||||
assert torch.equal(
|
||||
layer.w13_weight_scale[2, :intermediate_size], expected_gate_scale
|
||||
)
|
||||
assert torch.equal(layer.w13_weight_scale[2, intermediate_size:], expected_up_scale)
|
||||
assert torch.equal(layer.w2_weight_scale[2], expected_down_scale)
|
||||
|
||||
|
||||
def test_online_shared_expert_reload_compatibility(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Reload a BF16 shared weight into its MXFP4 slot via meta staging."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
online_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={"*shared_expert*": "mxfp4"})
|
||||
)
|
||||
quant_config = QuarkConfig(_QUARK_MXFP4_CONFIG)
|
||||
quant_config.online_quantization_config = online_config
|
||||
|
||||
with torch.device(current_platform.device_type):
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=2,
|
||||
top_k=1,
|
||||
hidden_size=64,
|
||||
intermediate_size=64,
|
||||
n_shared_experts=1,
|
||||
fuse_shared_experts=True,
|
||||
shared_expert_prefix="model.layers.0.mlp.shared_expert",
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
quant_config=quant_config,
|
||||
).routed_experts
|
||||
monkeypatch.setattr(
|
||||
layer.quant_method, "process_weights_after_loading", lambda _: None
|
||||
)
|
||||
|
||||
routed_weights = []
|
||||
for expert_id in range(2):
|
||||
for projection in ("gate_proj", "up_proj", "down_proj"):
|
||||
weight, scale = mxfp4_quantize(
|
||||
torch.randn(64, 64, dtype=torch.bfloat16)
|
||||
)
|
||||
prefix = f"{expert_id}.{projection}"
|
||||
routed_weights.extend(
|
||||
[(f"{prefix}.weight", weight), (f"{prefix}.weight_scale", scale)]
|
||||
)
|
||||
|
||||
shared_gate = torch.randn(64, 64, dtype=torch.bfloat16)
|
||||
shared_up = torch.randn(64, 64, dtype=torch.bfloat16)
|
||||
shared_down = torch.randn(64, 64, dtype=torch.bfloat16)
|
||||
|
||||
def checkpoint_weights(gate: torch.Tensor):
|
||||
return [
|
||||
*routed_weights,
|
||||
("2.gate_proj.weight", gate),
|
||||
("2.up_proj.weight", shared_up),
|
||||
("2.down_proj.weight", shared_down),
|
||||
]
|
||||
|
||||
list(layer.load_weights(checkpoint_weights(shared_gate)))
|
||||
old_weight = layer.w13_weight[2, :64].clone()
|
||||
reloaded_gate = shared_gate + 1
|
||||
expected_weight, expected_scale = mxfp4_quantize(reloaded_gate)
|
||||
assert not torch.equal(old_weight, expected_weight)
|
||||
|
||||
record_metadata_for_reloading(layer)
|
||||
initialize_layerwise_reload(layer)
|
||||
|
||||
assert layer.w13_weight.is_meta
|
||||
list(layer.load_weights(checkpoint_weights(reloaded_gate)))
|
||||
finalize_layerwise_reload(layer, default_vllm_config.model_config)
|
||||
assert not layer.w13_weight.is_meta
|
||||
assert torch.equal(layer.w13_weight[2, :64], expected_weight)
|
||||
assert torch.equal(layer.w13_weight_scale[2, :64], expected_scale)
|
||||
|
||||
|
||||
# TODO: Add DeepseekV4ForConditionalGeneration once it uses
|
||||
# resolve_layer_fused_shared_expert.
|
||||
@pytest.mark.parametrize(
|
||||
"architecture",
|
||||
[
|
||||
"AXK1ForCausalLM",
|
||||
"DeepseekForCausalLM",
|
||||
"DeepseekV2ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV3ForCausalLM",
|
||||
"Glm4MoeForCausalLM",
|
||||
"Glm4MoeLiteForCausalLM",
|
||||
"Glm5NextForCausalLM",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"Qwen3NextForCausalLM",
|
||||
"Qwen3_5MoeForCausalLM",
|
||||
"Qwen4ExpForCausalLM",
|
||||
],
|
||||
)
|
||||
def test_online_quantization(
|
||||
architecture: str,
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
"""Online MXFP4 fuses each architecture's shared expert into its MoE."""
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
if architecture == "Glm5NextForCausalLM":
|
||||
from vllm.models.glm5next.common import model as glm5_next_model
|
||||
|
||||
for attention in ("Glm5NextLinearAttention", "Glm5NextMLAAttention"):
|
||||
monkeypatch.setattr(glm5_next_model, attention, _StubAttention)
|
||||
monkeypatch.setattr(
|
||||
glm5_next_model, "_fused_shared_experts_tuned", lambda _: True
|
||||
)
|
||||
elif architecture == "Glm4MoeLiteForCausalLM":
|
||||
from vllm.model_executor.models import glm4_moe_lite
|
||||
|
||||
monkeypatch.setattr(glm4_moe_lite, "Glm4MoeLiteAttention", _StubAttention)
|
||||
monkeypatch.setattr(glm4_moe_lite, "Glm4MoeLiteMLAAttention", _StubAttention)
|
||||
elif architecture == "DeepseekV32ForCausalLM":
|
||||
from vllm.models.deepseek_v32.amd import model as deepseek_v32_model
|
||||
|
||||
monkeypatch.setattr(
|
||||
deepseek_v32_model, "DeepseekV32MLAAttention", _StubAttention
|
||||
)
|
||||
elif architecture == "Qwen3_5MoeForCausalLM":
|
||||
from vllm.model_executor.models import qwen3_5
|
||||
|
||||
monkeypatch.setattr(qwen3_5, "Qwen3NextAttention", _StubAttention)
|
||||
elif architecture == "Qwen4ExpForCausalLM":
|
||||
from vllm.models.qwen4_exp.amd import model as qwen4_exp_model
|
||||
|
||||
monkeypatch.setattr(qwen4_exp_model, "Qwen3NextAttention", _StubAttention)
|
||||
_write_minimal_moe_config(tmp_path, architecture)
|
||||
expected_shared_expert_name = (
|
||||
"shared_expert" if architecture.startswith("Qwen") else "shared_experts"
|
||||
)
|
||||
target_pattern = f"*{expected_shared_expert_name}*"
|
||||
|
||||
logged_messages: list[str] = []
|
||||
logged_warnings: list[str] = []
|
||||
|
||||
def record_info(message: str, *args: object, **_kwargs: object) -> None:
|
||||
if "Quantizing " in message and "of types" in message:
|
||||
logged_messages.append(message % args)
|
||||
|
||||
def record_warning(message: str, *args: object, **_kwargs: object) -> None:
|
||||
logged_warnings.append(message % args)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.utils.logger.warning", record_warning
|
||||
)
|
||||
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
str(tmp_path),
|
||||
dtype="bfloat16",
|
||||
quantization="quark",
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
"quark", {"targets": {target_pattern: "mxfp4"}}
|
||||
)
|
||||
},
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
expected_model_cls, resolved_architecture = get_model_architecture(
|
||||
vllm_config.model_config
|
||||
)
|
||||
assert resolved_architecture == architecture
|
||||
assert isinstance(model, expected_model_cls)
|
||||
|
||||
fused_moes = [
|
||||
module
|
||||
for module in model.modules()
|
||||
if getattr(module, "is_fused_shared_expert_enabled", False)
|
||||
and hasattr(module, "experts")
|
||||
]
|
||||
assert len(fused_moes) == 1, f"{architecture} did not enable FSE"
|
||||
moe = fused_moes[0]
|
||||
shared_expert_name = (
|
||||
"shared_expert" if hasattr(moe, "shared_expert") else "shared_experts"
|
||||
)
|
||||
assert shared_expert_name == expected_shared_expert_name
|
||||
assert getattr(moe, shared_expert_name) is None
|
||||
|
||||
routed_experts = moe.experts.routed_experts
|
||||
assert isinstance(routed_experts.quant_method, QuarkOCP_MX_MoEMethod)
|
||||
shared_expert_prefix = routed_experts.moe_config.shared_expert_prefix
|
||||
assert shared_expert_prefix is not None
|
||||
assert shared_expert_prefix.endswith(shared_expert_name)
|
||||
assert routed_experts._fused_shared_expert_quantizer is not None
|
||||
expert_map_manager = routed_experts.expert_map_manager
|
||||
assert expert_map_manager.num_fused_shared_experts == 1
|
||||
assert expert_map_manager.map_global_to_local(
|
||||
routed_experts.moe_config.num_experts
|
||||
) == (routed_experts.moe_config.num_experts)
|
||||
expected_weight_dtype = {
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16: torch.float4_e2m1fn_x2,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_MXFP4: torch.float4_e2m1fn_x2,
|
||||
Mxfp4MoeBackend.EMULATION: torch.uint8,
|
||||
}[routed_experts.quant_method.mxfp4_backend]
|
||||
assert routed_experts.w13_weight.dtype == expected_weight_dtype
|
||||
assert routed_experts.w13_weight_scale.dtype == torch.uint8
|
||||
assert routed_experts.w2_weight.dtype == expected_weight_dtype
|
||||
assert routed_experts.w2_weight_scale.dtype == torch.uint8
|
||||
assert vllm_config.quant_config is not None
|
||||
assert vllm_config.quant_config.online_quantization_config is not None
|
||||
assert not any(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled" in warning
|
||||
for warning in logged_warnings
|
||||
)
|
||||
if architecture == "MiniMaxM3SparseForCausalLM":
|
||||
# TODO: Use resolve_layer_fused_shared_expert in MiniMax M3 and remove
|
||||
# this branch once its virtual fused projections are registered.
|
||||
assert logged_messages == ["Quantizing 0 layers of types: "]
|
||||
else:
|
||||
assert logged_messages == [
|
||||
"Quantizing 2 layers of types: "
|
||||
f"mlp.{shared_expert_name}.down_proj: 1 "
|
||||
f"(from targets: {target_pattern}, mxfp4); "
|
||||
f"mlp.{shared_expert_name}.gate_up_proj: 1 "
|
||||
f"(from targets: {target_pattern}, mxfp4)"
|
||||
]
|
||||
|
||||
|
||||
def test_online_gate_up_proj_target_disables_fse(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
"""A partial shared-expert target quantizes while disabling FSE."""
|
||||
architecture = "DeepseekForCausalLM"
|
||||
target_pattern = "*shared_experts.gate_up_proj*"
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
_write_minimal_moe_config(tmp_path, architecture)
|
||||
|
||||
logged_messages: list[str] = []
|
||||
logged_warnings: list[str] = []
|
||||
|
||||
def record_info(message: str, *args: object, **_kwargs: object) -> None:
|
||||
if "Quantizing " in message and "of types" in message:
|
||||
logged_messages.append(message % args)
|
||||
|
||||
def record_warning(message: str, *args: object, **_kwargs: object) -> None:
|
||||
logged_warnings.append(message % args)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.utils.logger.warning", record_warning
|
||||
)
|
||||
|
||||
model, _ = load_model_without_vllm_runner(
|
||||
str(tmp_path),
|
||||
dtype="bfloat16",
|
||||
quantization="quark",
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
"quark", {"targets": {target_pattern: "mxfp4"}}
|
||||
)
|
||||
},
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
|
||||
moe = next(
|
||||
module
|
||||
for module in model.modules()
|
||||
if hasattr(module, "is_fused_shared_expert_enabled")
|
||||
and hasattr(module, "shared_experts")
|
||||
)
|
||||
assert not moe.is_fused_shared_expert_enabled
|
||||
assert moe.shared_experts is not None
|
||||
assert logged_messages == [
|
||||
"Quantizing 1 layers of types: "
|
||||
"mlp.shared_experts.gate_up_proj: 1 "
|
||||
f"(from targets: {target_pattern}, mxfp4)"
|
||||
]
|
||||
assert logged_warnings == [
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but cannot be "
|
||||
"enabled - skipping for this layer: online quantization targets only "
|
||||
"part of the shared expert at model.layers.0.mlp.shared_experts."
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,782 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for TurboQuant KV-cache quantization.
|
||||
|
||||
Run: .venv/bin/python -m pytest tests/quantization/test_turboquant.py -v
|
||||
"""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.turboquant.centroids import (
|
||||
get_centroids,
|
||||
solve_lloyd_max,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TQ_PRESETS,
|
||||
TurboQuantConfig,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
# ============================================================================
|
||||
# Helpers
|
||||
# ============================================================================
|
||||
|
||||
ALL_PRESETS = list(TQ_PRESETS.keys())
|
||||
|
||||
|
||||
def _assert_strictly_sorted(seq, name="sequence"):
|
||||
for i in range(len(seq) - 1):
|
||||
assert seq[i] < seq[i + 1], f"{name} not sorted at index {i}"
|
||||
|
||||
|
||||
def _is_power_of_2(n: int) -> bool:
|
||||
return n > 0 and next_power_of_2(n) == n
|
||||
|
||||
|
||||
# Expected concrete values for each preset at head_dim=128.
|
||||
# fmt: off
|
||||
PRESET_EXPECTED = {
|
||||
"turboquant_k8v4": dict(
|
||||
key_fp8=True, key_quant_bits=8,
|
||||
key_mse_bits=0, value_quant_bits=4,
|
||||
mse_bits=4, n_centroids=16, centroid_bits=4,
|
||||
norm_correction=False,
|
||||
key_packed_size=128, value_packed_size=68,
|
||||
slot_size=196, slot_size_aligned=196,
|
||||
),
|
||||
"turboquant_4bit_nc": dict(
|
||||
key_fp8=False, key_quant_bits=4,
|
||||
key_mse_bits=4, value_quant_bits=4,
|
||||
mse_bits=4, n_centroids=16, centroid_bits=4,
|
||||
norm_correction=True,
|
||||
key_packed_size=66, value_packed_size=68,
|
||||
slot_size=134, slot_size_aligned=134,
|
||||
),
|
||||
"turboquant_k3v4_nc": dict(
|
||||
key_fp8=False, key_quant_bits=3,
|
||||
key_mse_bits=3, value_quant_bits=4,
|
||||
mse_bits=3, n_centroids=8, centroid_bits=3,
|
||||
norm_correction=True,
|
||||
key_packed_size=50, value_packed_size=68,
|
||||
slot_size=118, slot_size_aligned=118,
|
||||
),
|
||||
"turboquant_3bit_nc": dict(
|
||||
key_fp8=False, key_quant_bits=3,
|
||||
key_mse_bits=3, value_quant_bits=3,
|
||||
mse_bits=3, n_centroids=8, centroid_bits=3,
|
||||
norm_correction=True,
|
||||
key_packed_size=50, value_packed_size=52,
|
||||
slot_size=102, slot_size_aligned=102,
|
||||
),
|
||||
}
|
||||
# fmt: on
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Config tests (CPU-only, no dependencies beyond config.py)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTurboQuantConfig:
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_preset_parses(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert isinstance(cfg, TurboQuantConfig)
|
||||
|
||||
def test_invalid_preset_raises(self):
|
||||
with pytest.raises(ValueError, match="Unknown TurboQuant"):
|
||||
TurboQuantConfig.from_cache_dtype("turboquant_invalid", head_dim=128)
|
||||
|
||||
# ---- Per-preset concrete value checks (table-driven) ----
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_key_mode(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
exp = PRESET_EXPECTED[preset]
|
||||
assert cfg.key_fp8 is exp["key_fp8"]
|
||||
assert cfg.key_quant_bits == exp["key_quant_bits"]
|
||||
assert cfg.key_mse_bits == exp["key_mse_bits"]
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_value_mode(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
exp = PRESET_EXPECTED[preset]
|
||||
assert cfg.value_quant_bits == exp["value_quant_bits"]
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_bits_and_centroids(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
exp = PRESET_EXPECTED[preset]
|
||||
assert cfg.mse_bits == exp["mse_bits"]
|
||||
assert cfg.n_centroids == exp["n_centroids"]
|
||||
assert cfg.centroid_bits == exp["centroid_bits"]
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_norm_correction(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert cfg.norm_correction is PRESET_EXPECTED[preset]["norm_correction"]
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_packed_sizes(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
exp = PRESET_EXPECTED[preset]
|
||||
assert cfg.key_packed_size == exp["key_packed_size"]
|
||||
assert cfg.value_packed_size == exp["value_packed_size"]
|
||||
assert cfg.slot_size == exp["slot_size"]
|
||||
assert cfg.slot_size_aligned == exp["slot_size_aligned"]
|
||||
|
||||
# ---- Cross-preset structural invariants ----
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_slot_equals_key_plus_value(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert cfg.slot_size == cfg.key_packed_size + cfg.value_packed_size
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_padded_slot_is_even(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert cfg.slot_size_aligned >= cfg.slot_size
|
||||
assert cfg.slot_size_aligned % 2 == 0, (
|
||||
f"slot_size_aligned={cfg.slot_size_aligned} is not even"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_key_value_packed_sizes_positive(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert cfg.key_packed_size > 0
|
||||
assert cfg.value_packed_size > 0
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_n_centroids_is_2_to_mse_bits(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert cfg.n_centroids == 2**cfg.mse_bits
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_centroid_bits_always_positive(self, preset):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
assert cfg.centroid_bits > 0
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_mse_key_or_fp8_exclusive(self, preset):
|
||||
"""Each preset is either FP8 keys or MSE keys, never both."""
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
if cfg.key_fp8:
|
||||
assert cfg.key_mse_bits == 0
|
||||
assert cfg.key_quant_bits == 8
|
||||
else:
|
||||
assert cfg.key_mse_bits > 0
|
||||
assert cfg.key_quant_bits in (3, 4)
|
||||
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
@pytest.mark.parametrize("head_dim", [64, 96, 128, 256])
|
||||
def test_all_presets_all_head_dims(self, preset, head_dim):
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=head_dim)
|
||||
assert cfg.head_dim == head_dim
|
||||
assert cfg.slot_size == cfg.key_packed_size + cfg.value_packed_size
|
||||
assert cfg.slot_size_aligned >= cfg.slot_size
|
||||
assert cfg.slot_size_aligned % 2 == 0
|
||||
|
||||
# ---- Boundary skip layers ----
|
||||
|
||||
@staticmethod
|
||||
def _dense_model_config(num_layers):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(
|
||||
is_hybrid=False,
|
||||
hf_text_config=SimpleNamespace(num_hidden_layers=num_layers),
|
||||
)
|
||||
|
||||
def test_boundary_skip_layers_basic(self):
|
||||
mc = self._dense_model_config(32)
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
|
||||
assert layers == ["0", "1", "30", "31"]
|
||||
|
||||
def test_boundary_skip_layers_zero(self):
|
||||
mc = self._dense_model_config(32)
|
||||
assert TurboQuantConfig.get_boundary_skip_layers(mc, 0) == []
|
||||
|
||||
def test_boundary_skip_layers_small_model(self):
|
||||
mc = self._dense_model_config(4)
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(mc)
|
||||
assert layers == ["0", "1", "2", "3"]
|
||||
|
||||
def test_boundary_skip_layers_cap_at_half(self):
|
||||
mc = self._dense_model_config(8)
|
||||
layers = TurboQuantConfig.get_boundary_skip_layers(mc, 10)
|
||||
assert len(layers) == 8
|
||||
|
||||
|
||||
class TestHybridAttentionIndices:
|
||||
"""Regression tests for boundary protection on hybrid models.
|
||||
|
||||
Hybrid models (attention + Mamba / linear-attention) identify KV-carrying
|
||||
layers via layer_types / layers_block_type / attn_type_list. The helper
|
||||
must return the *global* layer indices of the full-attention layers so
|
||||
that kv_cache_dtype_skip_layers matches what extract_layer_index(prefix)
|
||||
reports on the Attention layers at runtime.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fake_model_config(text_cfg=None, hf_cfg=None):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(
|
||||
hf_text_config=text_cfg if text_cfg is not None else SimpleNamespace(),
|
||||
hf_config=hf_cfg if hf_cfg is not None else SimpleNamespace(),
|
||||
)
|
||||
|
||||
def test_layer_types_full_attention(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
cfg = type("C", (), {})()
|
||||
cfg.layer_types = [
|
||||
"linear_attention",
|
||||
"linear_attention",
|
||||
"full_attention",
|
||||
"linear_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
]
|
||||
mc = self._fake_model_config(text_cfg=cfg)
|
||||
assert _get_full_attention_layer_indices(mc) == [2, 4, 5]
|
||||
|
||||
def test_layers_block_type_jamba(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
cfg = type("C", (), {})()
|
||||
cfg.layers_block_type = ["mamba", "attention", "mamba", "attention"]
|
||||
mc = self._fake_model_config(text_cfg=cfg)
|
||||
assert _get_full_attention_layer_indices(mc) == [1, 3]
|
||||
|
||||
def test_attn_type_list_minimax(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
hf = type("C", (), {})()
|
||||
hf.attn_type_list = [0, 1, 0, 1, 1]
|
||||
mc = self._fake_model_config(hf_cfg=hf)
|
||||
assert _get_full_attention_layer_indices(mc) == [1, 3, 4]
|
||||
|
||||
def test_no_hybrid_hints_returns_empty(self):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
_get_full_attention_layer_indices,
|
||||
)
|
||||
|
||||
mc = self._fake_model_config()
|
||||
assert _get_full_attention_layer_indices(mc) == []
|
||||
|
||||
|
||||
class TestTurboQuantKVCacheSpec:
|
||||
@pytest.mark.parametrize("preset", ALL_PRESETS)
|
||||
def test_kv_cache_spec_sets_kv_quant_mode(self, preset):
|
||||
from vllm.model_executor.layers.attention.attention import Attention
|
||||
from vllm.v1.attention.backends.turboquant_attn import (
|
||||
TurboQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
layer = SimpleNamespace(
|
||||
attn_type="decoder",
|
||||
kv_cache_dtype=preset,
|
||||
kv_cache_torch_dtype=torch.uint8,
|
||||
head_size=128,
|
||||
head_size_v=128,
|
||||
num_kv_heads=4,
|
||||
sliding_window=None,
|
||||
get_attn_backend=lambda: TurboQuantAttentionBackend,
|
||||
)
|
||||
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=32))
|
||||
|
||||
# The layer builds an unpacked spec; the worker's spec-collection
|
||||
# loop applies TQ slot packing via the backend's customize_spec hook.
|
||||
spec = Attention.get_kv_cache_spec(layer, vllm_config)
|
||||
assert isinstance(spec, FullAttentionSpec)
|
||||
assert spec.kv_quant_mode.is_turboquant
|
||||
assert spec.state_content_bytes is None
|
||||
|
||||
spec = TurboQuantAttentionBackend.customize_spec(spec)
|
||||
expected_slot = TurboQuantConfig.from_cache_dtype(preset, 128).slot_size_aligned
|
||||
assert spec.state_content_bytes == expected_slot
|
||||
|
||||
|
||||
class TestTurboQuantWorkspaceReservation:
|
||||
@staticmethod
|
||||
def _fake_vllm_config(
|
||||
*,
|
||||
max_num_seqs: int = 16,
|
||||
max_num_batched_tokens: int = 4096,
|
||||
enable_chunked_prefill: bool = True,
|
||||
max_model_len: int = 8192,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
max_num_kv_splits: int = 4,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=max_model_len,
|
||||
dtype=dtype,
|
||||
get_num_attention_heads=lambda parallel_config: 8,
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=1,
|
||||
),
|
||||
attention_config=SimpleNamespace(
|
||||
tq_max_kv_splits_for_cuda_graph=max_num_kv_splits
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fake_kv_cache_spec():
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
return FullAttentionSpec(
|
||||
block_size=32,
|
||||
num_kv_heads=4,
|
||||
head_size=128,
|
||||
head_size_v=128,
|
||||
dtype=torch.uint8,
|
||||
state_content_bytes=102,
|
||||
)
|
||||
|
||||
def test_metadata_builder_reserves_decode_and_continuation_prefill_workspace(
|
||||
self, monkeypatch
|
||||
):
|
||||
from vllm.v1.attention.backends import turboquant_attn
|
||||
|
||||
calls = []
|
||||
|
||||
class FakeWorkspaceManager:
|
||||
def get_simultaneous(self, *shapes_and_dtypes):
|
||||
calls.append(shapes_and_dtypes)
|
||||
|
||||
monkeypatch.setattr(
|
||||
turboquant_attn,
|
||||
"current_workspace_manager",
|
||||
lambda: FakeWorkspaceManager(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
turboquant_attn,
|
||||
"is_workspace_manager_initialized",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
turboquant_attn.TurboQuantMetadataBuilder(
|
||||
kv_cache_spec=self._fake_kv_cache_spec(),
|
||||
layer_names=["layers.0.self_attn.attn"],
|
||||
vllm_config=self._fake_vllm_config(),
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
((16, 8, 4, 129), torch.float32),
|
||||
((16, 8, 128), torch.float16),
|
||||
((16, 8), torch.float32),
|
||||
),
|
||||
(
|
||||
((1, 4, 8192, 128), torch.float16),
|
||||
((1, 4, 8192, 128), torch.float16),
|
||||
),
|
||||
]
|
||||
|
||||
def test_metadata_builder_skips_continuation_prefill_when_disabled(
|
||||
self, monkeypatch
|
||||
):
|
||||
from vllm.v1.attention.backends import turboquant_attn
|
||||
|
||||
calls = []
|
||||
|
||||
class FakeWorkspaceManager:
|
||||
def get_simultaneous(self, *shapes_and_dtypes):
|
||||
calls.append(shapes_and_dtypes)
|
||||
|
||||
monkeypatch.setattr(
|
||||
turboquant_attn,
|
||||
"current_workspace_manager",
|
||||
lambda: FakeWorkspaceManager(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
turboquant_attn,
|
||||
"is_workspace_manager_initialized",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
turboquant_attn.TurboQuantMetadataBuilder(
|
||||
kv_cache_spec=self._fake_kv_cache_spec(),
|
||||
layer_names=["layers.0.self_attn.attn"],
|
||||
vllm_config=self._fake_vllm_config(enable_chunked_prefill=False),
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
((16, 8, 4, 129), torch.float32),
|
||||
((16, 8, 128), torch.float16),
|
||||
((16, 8), torch.float32),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Centroids tests (CPU-only)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestCentroids:
|
||||
@pytest.mark.parametrize("bits,expected_n", [(2, 4), (3, 8), (4, 16)])
|
||||
def test_centroids_shape(self, bits, expected_n):
|
||||
c = get_centroids(128, bits)
|
||||
assert c.shape == (expected_n,)
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_centroids_sorted(self, bits):
|
||||
_assert_strictly_sorted(get_centroids(128, bits), "centroids")
|
||||
|
||||
def test_centroids_cached(self):
|
||||
c1 = get_centroids(128, 3)
|
||||
c2 = get_centroids(128, 3)
|
||||
assert c1 is c2, "get_centroids should return cached object"
|
||||
|
||||
def test_centroids_different_dims_not_identical(self):
|
||||
c64 = get_centroids(64, 3)
|
||||
c128 = get_centroids(128, 3)
|
||||
assert not torch.equal(c64, c128)
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_centroids_symmetric_around_zero(self, bits):
|
||||
"""N(0, 1/d) is symmetric, so centroids should be ~symmetric."""
|
||||
c = get_centroids(128, bits)
|
||||
assert abs(c.mean().item()) < 0.01, "Centroids not centered near 0"
|
||||
assert abs(c[0].item() + c[-1].item()) < 0.01
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_centroids_within_4sigma(self, bits):
|
||||
"""All centroids should be within ~4 sigma of N(0, 1/d)."""
|
||||
sigma = math.sqrt(1.0 / 128)
|
||||
c = get_centroids(128, bits)
|
||||
for i, val in enumerate(c):
|
||||
assert abs(val.item()) < 4 * sigma, (
|
||||
f"Centroid {i}={val:.6f} outside 4*sigma={4 * sigma:.6f}"
|
||||
)
|
||||
|
||||
|
||||
class TestLloydMax:
|
||||
@pytest.mark.parametrize("bits,expected_n", [(2, 4), (3, 8), (4, 16)])
|
||||
def test_solve_shapes(self, bits, expected_n):
|
||||
centroids, boundaries = solve_lloyd_max(128, bits)
|
||||
assert centroids.shape == (expected_n,)
|
||||
assert boundaries.shape == (expected_n - 1,)
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_centroids_sorted(self, bits):
|
||||
centroids, _ = solve_lloyd_max(128, bits)
|
||||
_assert_strictly_sorted(centroids, "centroids")
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_boundaries_sorted(self, bits):
|
||||
_, boundaries = solve_lloyd_max(128, bits)
|
||||
_assert_strictly_sorted(boundaries, "boundaries")
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_boundaries_between_centroids(self, bits):
|
||||
"""Each boundary must lie between its adjacent centroids."""
|
||||
centroids, boundaries = solve_lloyd_max(128, bits)
|
||||
for i in range(len(boundaries)):
|
||||
assert centroids[i] < boundaries[i] < centroids[i + 1], (
|
||||
f"Boundary {i}={boundaries[i]:.6f} not between "
|
||||
f"c[{i}]={centroids[i]:.6f} and c[{i + 1}]={centroids[i + 1]:.6f}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("bits", [2, 3, 4])
|
||||
def test_boundaries_are_midpoints(self, bits):
|
||||
"""Lloyd-Max boundaries are midpoints of adjacent centroids."""
|
||||
centroids, boundaries = solve_lloyd_max(128, bits)
|
||||
for i in range(len(boundaries)):
|
||||
expected = (centroids[i] + centroids[i + 1]) / 2.0
|
||||
assert abs(boundaries[i].item() - expected.item()) < 1e-6
|
||||
|
||||
def test_solve_deterministic(self):
|
||||
c1, b1 = solve_lloyd_max(128, 3)
|
||||
c2, b2 = solve_lloyd_max(128, 3)
|
||||
assert torch.equal(c1, c2)
|
||||
assert torch.equal(b1, b2)
|
||||
|
||||
def test_solve_dtype_float32(self):
|
||||
centroids, boundaries = solve_lloyd_max(128, 3)
|
||||
assert centroids.dtype == torch.float32
|
||||
assert boundaries.dtype == torch.float32
|
||||
|
||||
@pytest.mark.parametrize("bits", [3, 4])
|
||||
def test_centroids_match_scipy_reference(self, bits):
|
||||
"""Verify _trapz(n=200) centroids match scipy.integrate.quad reference.
|
||||
|
||||
This ensures our scipy-free trapezoid integration doesn't silently
|
||||
drift from the published Lloyd-Max quality.
|
||||
"""
|
||||
pytest.importorskip("scipy")
|
||||
from scipy.integrate import quad
|
||||
|
||||
d = 128
|
||||
sigma2 = 1.0 / d
|
||||
sigma = math.sqrt(sigma2)
|
||||
|
||||
def pdf(x):
|
||||
return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(
|
||||
-x * x / (2 * sigma2)
|
||||
)
|
||||
|
||||
n_levels = 2**bits
|
||||
lo, hi = -3.5 * sigma, 3.5 * sigma
|
||||
ref_centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
|
||||
for _ in range(200):
|
||||
boundaries = [
|
||||
(ref_centroids[i] + ref_centroids[i + 1]) / 2.0
|
||||
for i in range(n_levels - 1)
|
||||
]
|
||||
edges = [lo * 3] + boundaries + [hi * 3]
|
||||
new_centroids = []
|
||||
for i in range(n_levels):
|
||||
a, b = edges[i], edges[i + 1]
|
||||
num, _ = quad(lambda x: x * pdf(x), a, b)
|
||||
den, _ = quad(pdf, a, b)
|
||||
new_centroids.append(num / den if den > 1e-15 else ref_centroids[i])
|
||||
if (
|
||||
max(abs(new_centroids[i] - ref_centroids[i]) for i in range(n_levels))
|
||||
< 1e-10
|
||||
):
|
||||
break
|
||||
ref_centroids = new_centroids
|
||||
|
||||
# Compare our _trapz centroids against scipy reference
|
||||
our_centroids, _ = solve_lloyd_max(d, bits)
|
||||
ref_t = torch.tensor(ref_centroids, dtype=torch.float32)
|
||||
max_err = (our_centroids - ref_t).abs().max().item()
|
||||
# _trapz(n=200) has ~O(h^2) error vs adaptive quad; 1e-3 is tight
|
||||
# enough to catch regression while allowing trapezoid approximation.
|
||||
assert max_err < 1e-3, (
|
||||
f"d={d}, bits={bits}: max centroid error vs scipy = {max_err:.2e}"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Rotation matrix tests (GPU required)
|
||||
# ============================================================================
|
||||
|
||||
GPGPU_AVAILABLE = torch.cuda.is_available() or torch.xpu.is_available()
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def generate_rotation_matrix(d: int, seed: int, device: str = "cpu") -> torch.Tensor:
|
||||
"""Haar-distributed random orthogonal matrix via QR (test/benchmark only)."""
|
||||
gen = torch.Generator(device="cpu")
|
||||
gen.manual_seed(seed)
|
||||
G = torch.randn(d, d, generator=gen, device="cpu", dtype=torch.float32)
|
||||
# torch.linalg.qr on CPU requires LAPACK, which some torch wheels
|
||||
# (ROCm) ship without. Run QR on accelerator instead
|
||||
qr_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
Q, R = torch.linalg.qr(G.to(qr_device))
|
||||
diag_sign = torch.sign(torch.diag(R))
|
||||
diag_sign[diag_sign == 0] = 1.0
|
||||
Q = Q * diag_sign.unsqueeze(0)
|
||||
return Q.to(device)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
|
||||
class TestRotationMatrix:
|
||||
"""Tests for the QR-based rotation (standalone benchmarks only)."""
|
||||
|
||||
@pytest.mark.parametrize("dim", [64, 96, 128, 256])
|
||||
def test_rotation_matrix_shape_and_orthogonal(self, dim):
|
||||
Pi = generate_rotation_matrix(dim, seed=42, device=DEVICE_TYPE)
|
||||
assert Pi.shape == (dim, dim)
|
||||
eye = Pi @ Pi.T
|
||||
assert torch.allclose(eye, torch.eye(dim, device=DEVICE_TYPE), atol=1e-5), (
|
||||
f"Pi not orthogonal for dim={dim}"
|
||||
)
|
||||
|
||||
def test_rotation_matrix_deterministic(self):
|
||||
Pi1 = generate_rotation_matrix(128, seed=42)
|
||||
Pi2 = generate_rotation_matrix(128, seed=42)
|
||||
assert torch.equal(Pi1, Pi2)
|
||||
|
||||
def test_rotation_matrix_different_seeds(self):
|
||||
Pi1 = generate_rotation_matrix(128, seed=42)
|
||||
Pi2 = generate_rotation_matrix(128, seed=99)
|
||||
assert not torch.equal(Pi1, Pi2)
|
||||
|
||||
def test_rotation_matrix_det_is_pm1(self):
|
||||
"""Orthogonal matrix determinant must be +1 or -1."""
|
||||
Pi = generate_rotation_matrix(128, seed=42, device=DEVICE_TYPE)
|
||||
det = torch.linalg.det(Pi)
|
||||
assert abs(abs(det.item()) - 1.0) < 1e-4
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Hadamard rotation tests (serving path: _build_hadamard)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _build_hadamard(d: int, device: str = "cpu") -> torch.Tensor:
|
||||
"""Reproduce the serving-path Hadamard construction."""
|
||||
H = torch.tensor([[1.0]])
|
||||
while H.shape[0] < d:
|
||||
H = torch.cat([torch.cat([H, H], 1), torch.cat([H, -H], 1)], 0)
|
||||
return (H / math.sqrt(d)).to(torch.device(device))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
|
||||
class TestHadamardRotation:
|
||||
"""Tests for the Hadamard rotation used in serving."""
|
||||
|
||||
@pytest.mark.parametrize("dim", [64, 128, 256])
|
||||
def test_hadamard_orthonormal(self, dim):
|
||||
"""H must be orthonormal: H @ H^T = I."""
|
||||
H = _build_hadamard(dim, DEVICE_TYPE)
|
||||
eye = H @ H.T
|
||||
assert torch.allclose(eye, torch.eye(dim, device=DEVICE_TYPE), atol=1e-5), (
|
||||
f"Hadamard not orthonormal for dim={dim}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("dim", [64, 128, 256])
|
||||
def test_hadamard_symmetric(self, dim):
|
||||
"""Sylvester Hadamard must be symmetric: H = H^T."""
|
||||
H = _build_hadamard(dim, DEVICE_TYPE)
|
||||
assert torch.allclose(H, H.T, atol=1e-6), (
|
||||
f"Hadamard not symmetric for dim={dim}"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Store → Decode round-trip test (GPU + Triton required)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
|
||||
class TestStoreDecodeRoundTrip:
|
||||
"""End-to-end: store KV into TQ cache, decode, compare vs fp16 ref."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"preset",
|
||||
["turboquant_k8v4", "turboquant_4bit_nc"],
|
||||
)
|
||||
def test_single_token_roundtrip(self, preset):
|
||||
"""Store 1 token, decode with query=key, check attention output.
|
||||
|
||||
For a single token with query=key, attention output should equal
|
||||
the value (softmax over single key = 1.0). Quantization error
|
||||
means we check cosine similarity rather than exact equality.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.turboquant.centroids import (
|
||||
solve_lloyd_max,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_turboquant_decode import (
|
||||
triton_turboquant_decode_attention,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_turboquant_store import (
|
||||
triton_turboquant_store,
|
||||
)
|
||||
|
||||
cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
|
||||
D = 128
|
||||
Hk = 4 # num_kv_heads
|
||||
Hq = 4 # num_q_heads (no GQA for simplicity)
|
||||
B = 1 # single token
|
||||
block_size = 16
|
||||
num_blocks = 1
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
|
||||
# Pure Hadamard rotation (symmetric: H = H^T, so Pi = PiT = H)
|
||||
H = _build_hadamard(D, DEVICE_TYPE)
|
||||
PiT = H
|
||||
Pi = H
|
||||
|
||||
# Generate centroids
|
||||
centroids, _ = solve_lloyd_max(D, cfg.centroid_bits)
|
||||
centroids = centroids.float().to(device)
|
||||
c_sorted, _ = centroids.sort()
|
||||
midpoints = ((c_sorted[:-1] + c_sorted[1:]) / 2).to(device)
|
||||
|
||||
# Random K, V
|
||||
torch.manual_seed(123)
|
||||
key = torch.randn(B, Hk, D, device=device, dtype=torch.float16)
|
||||
value = torch.randn(B, Hk, D, device=device, dtype=torch.float16)
|
||||
|
||||
# Allocate KV cache
|
||||
padded_slot = cfg.slot_size_aligned
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
block_size,
|
||||
Hk,
|
||||
padded_slot,
|
||||
device=device,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
slot_mapping = torch.tensor([0], device=device, dtype=torch.int32)
|
||||
|
||||
# Store
|
||||
triton_turboquant_store(
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
PiT,
|
||||
midpoints,
|
||||
mse_bits=cfg.key_mse_bits,
|
||||
key_packed_size=cfg.key_packed_size,
|
||||
value_quant_bits=cfg.effective_value_quant_bits,
|
||||
key_fp8=cfg.key_fp8,
|
||||
)
|
||||
|
||||
# Decode: use key as query so attention = softmax([1]) * V = V
|
||||
query = key.expand(B, Hq, D).contiguous().to(torch.float16)
|
||||
block_table = torch.tensor([[0]], device=device, dtype=torch.int32)
|
||||
seq_lens = torch.tensor([1], device=device, dtype=torch.int32)
|
||||
|
||||
output = triton_turboquant_decode_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
Pi=Pi,
|
||||
centroids=centroids,
|
||||
scale=1.0 / math.sqrt(D),
|
||||
mse_bits=cfg.key_mse_bits,
|
||||
key_packed_size=cfg.key_packed_size,
|
||||
value_quant_bits=cfg.effective_value_quant_bits,
|
||||
key_fp8=cfg.key_fp8,
|
||||
norm_correction=cfg.norm_correction,
|
||||
PiT=PiT,
|
||||
max_num_kv_splits=4,
|
||||
)
|
||||
|
||||
# With single KV, output should approximate the stored value.
|
||||
# Check per-head cosine similarity > threshold.
|
||||
out_fp32 = output.float()
|
||||
val_fp32 = value.expand(B, Hq, D).float()
|
||||
for h in range(Hq):
|
||||
cos_sim = torch.nn.functional.cosine_similarity(
|
||||
out_fp32[0, h].unsqueeze(0),
|
||||
val_fp32[0, h].unsqueeze(0),
|
||||
).item()
|
||||
# FP8 keys should be very accurate; MSE keys have more error
|
||||
threshold = 0.95 if cfg.key_fp8 else 0.85
|
||||
assert cos_sim > threshold, (
|
||||
f"Preset {preset} head {h}: cosine_sim={cos_sim:.4f} < {threshold}"
|
||||
)
|
||||
Reference in New Issue
Block a user