[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,239 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for AutoAWQConfig behavior after unification.
|
||||
|
||||
These tests verify the bug fixes for:
|
||||
1. CPU platform override conflict (auto_awq should not override on CPU)
|
||||
2. MoE fallback compatibility (full_config["quant_method"] should be "awq")
|
||||
3. Config attribute consistency
|
||||
4. End-to-end quantization method loading (auto_awq loads and runs correctly)
|
||||
|
||||
Note: Tests that require importing the full auto_awq module (which has GPU-dependent
|
||||
imports) should use subprocess or be run in a GPU environment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import (
|
||||
is_quant_method_supported,
|
||||
load_model_without_vllm_runner,
|
||||
)
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _get_auto_awq_config_source() -> str:
|
||||
"""Read the AutoAWQConfig class source code for isolated testing."""
|
||||
import inspect
|
||||
|
||||
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
|
||||
|
||||
return inspect.getsource(auto_awq_module.AutoAWQConfig)
|
||||
|
||||
|
||||
class TestAutoAWQConfigFromConfig:
|
||||
"""Tests for AutoAWQConfig.from_config behavior.
|
||||
|
||||
These tests require GPU environment to import the full module.
|
||||
They are skipped on non-GPU platforms.
|
||||
"""
|
||||
|
||||
def test_full_config_quant_method_is_awq_for_moe_fallback(self):
|
||||
"""full_config should have quant_method='awq' for MoE fallback compatibility.
|
||||
|
||||
MoeWNA16Config only accepts 'gptq' or 'awq' as linear_quant_method.
|
||||
If full_config has 'auto_awq', the MoE fallback will fail.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
config = {
|
||||
"w_bit": 4,
|
||||
"q_group_size": 128,
|
||||
"zero_point": True,
|
||||
"lm_head": False,
|
||||
}
|
||||
awq_config = AutoAWQConfig.from_config(config)
|
||||
|
||||
# Verify quant_method is 'awq' for MoE fallback
|
||||
assert awq_config.full_config["quant_method"] == "awq", (
|
||||
f"Expected quant_method='awq', got {awq_config.full_config['quant_method']}"
|
||||
)
|
||||
|
||||
def test_full_config_preserves_other_fields(self):
|
||||
"""full_config should preserve all original config fields."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
config = {
|
||||
"w_bit": 4,
|
||||
"q_group_size": 128,
|
||||
"zero_point": True,
|
||||
"lm_head": False,
|
||||
"custom_field": "custom_value",
|
||||
}
|
||||
awq_config = AutoAWQConfig.from_config(config)
|
||||
|
||||
assert awq_config.full_config["w_bit"] == 4
|
||||
assert awq_config.full_config["q_group_size"] == 128
|
||||
assert awq_config.full_config["zero_point"] is True
|
||||
assert awq_config.full_config["lm_head"] is False
|
||||
assert awq_config.full_config["custom_field"] == "custom_value"
|
||||
|
||||
def test_full_config_is_copy_not_original(self):
|
||||
"""full_config should be a copy, not the original dict."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
config = {
|
||||
"w_bit": 4,
|
||||
"q_group_size": 128,
|
||||
"zero_point": True,
|
||||
"lm_head": False,
|
||||
}
|
||||
original_quant_method = config.get("quant_method")
|
||||
|
||||
AutoAWQConfig.from_config(config)
|
||||
|
||||
# Original config should not be modified
|
||||
assert config.get("quant_method") == original_quant_method
|
||||
|
||||
|
||||
class TestAutoAWQConfigAttributes:
|
||||
"""Tests for AutoAWQConfig attribute consistency.
|
||||
|
||||
These tests require GPU environment to import the full module.
|
||||
They are skipped on non-GPU platforms.
|
||||
"""
|
||||
|
||||
def test_config_attributes_match_input(self):
|
||||
"""Config attributes should match input values."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
awq_config = AutoAWQConfig(
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
zero_point=True,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=["lm_head"],
|
||||
)
|
||||
|
||||
assert awq_config.weight_bits == 4
|
||||
assert awq_config.group_size == 128
|
||||
assert awq_config.zero_point is True
|
||||
assert awq_config.lm_head_quantized is False
|
||||
assert awq_config.modules_to_not_convert == ["lm_head"]
|
||||
|
||||
def test_pack_factor_for_4bit(self):
|
||||
"""Pack factor should be 8 for 4-bit quantization."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
awq_config = AutoAWQConfig(
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
zero_point=True,
|
||||
lm_head_quantized=False,
|
||||
)
|
||||
|
||||
assert awq_config.pack_factor == 8 # 32 // 4
|
||||
|
||||
|
||||
class TestAutoAWQConfigOverrideLogic:
|
||||
"""Tests for override logic by parsing source code (no GPU import required)."""
|
||||
|
||||
def _get_auto_awq_source(self) -> str:
|
||||
"""Read the auto_awq.py source file."""
|
||||
import inspect
|
||||
import pathlib
|
||||
|
||||
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
|
||||
|
||||
source_path = inspect.getfile(auto_awq_module)
|
||||
return pathlib.Path(source_path).read_text()
|
||||
|
||||
def test_cpu_check_in_override_method(self):
|
||||
"""override_quantization_method should check current_platform.is_cpu()."""
|
||||
source = self._get_auto_awq_source()
|
||||
|
||||
# Verify the CPU check exists in override method
|
||||
assert "current_platform.is_cpu()" in source, (
|
||||
"override_quantization_method should check is_cpu()"
|
||||
)
|
||||
assert "return None" in source, (
|
||||
"override_quantization_method should return None on CPU"
|
||||
)
|
||||
|
||||
def test_quant_method_normalization_in_from_config(self):
|
||||
"""from_config should normalize quant_method to 'awq' for MoE fallback."""
|
||||
source = self._get_auto_awq_source()
|
||||
|
||||
# Verify the normalization exists
|
||||
assert (
|
||||
'"quant_method"] = "awq"' in source or "'quant_method'] = 'awq'" in source
|
||||
), "from_config should set quant_method='awq' in full_config"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# End-to-end integration tests (require GPU environment)
|
||||
# =============================================================================
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
# Small AWQ model for testing - using Qwen2 1.5B which has official AWQ checkpoint
|
||||
AWQ_MODELS = [
|
||||
"Qwen/Qwen2-1.5B-Instruct-AWQ",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("auto_awq"),
|
||||
reason="auto_awq is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", AWQ_MODELS)
|
||||
def test_auto_awq_quantization_method(
|
||||
model_id: str, monkeypatch, dist_init, workspace_init
|
||||
):
|
||||
"""Test that quantization='auto_awq' loads and runs correctly."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_id,
|
||||
dtype=torch.float16,
|
||||
quantization="auto_awq",
|
||||
model_config_kwargs={"max_model_len": 2048},
|
||||
)
|
||||
target_device = torch.device(current_platform.device_type)
|
||||
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQLinearMethod,
|
||||
AutoAWQMarlinLinearMethod,
|
||||
)
|
||||
|
||||
qkv_proj = model.model.layers[0].self_attn.qkv_proj
|
||||
assert isinstance(
|
||||
qkv_proj.quant_method,
|
||||
(AutoAWQLinearMethod, AutoAWQMarlinLinearMethod),
|
||||
), (
|
||||
"Expected AutoAWQLinearMethod or AutoAWQMarlinLinearMethod, "
|
||||
f"got {type(qkv_proj.quant_method)}"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
input_ids = torch.tensor([1, 2, 3, 4], device=target_device)
|
||||
positions = torch.arange(input_ids.numel(), device=target_device)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
|
||||
):
|
||||
hidden_states = model(input_ids, positions, None)
|
||||
logits = model.compute_logits(hidden_states)
|
||||
assert torch.isfinite(logits).all()
|
||||
|
||||
|
||||
def test_auto_awq_config_get_name():
|
||||
"""Test that AutoAWQConfig.get_name() returns 'auto_awq'."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
assert AutoAWQConfig.get_name() == "auto_awq"
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests that the auto_gptq quantization method works correctly.
|
||||
|
||||
Run `pytest tests/quantization/test_auto_gptq.py -v -s`.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import (
|
||||
is_quant_method_supported,
|
||||
load_model_without_vllm_runner,
|
||||
)
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQConfig,
|
||||
AutoGPTQLinearMethod,
|
||||
AutoGPTQMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
is_layer_gptq_quantized,
|
||||
override_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
MODELS = [
|
||||
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("auto_gptq"),
|
||||
reason="auto_gptq is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_auto_gptq_quantization_method(
|
||||
model_id: str, monkeypatch, dist_init, workspace_init
|
||||
):
|
||||
"""Test that quantization='auto_gptq' loads and runs correctly."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_id,
|
||||
dtype=torch.float16,
|
||||
quantization="auto_gptq",
|
||||
model_config_kwargs={"max_model_len": 2048},
|
||||
)
|
||||
|
||||
qkv_proj = model.model.layers[0].self_attn.qkv_proj
|
||||
assert isinstance(qkv_proj.quant_method, AutoGPTQLinearMethod)
|
||||
|
||||
target_device = torch.device(current_platform.device_type)
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
input_ids = torch.tensor([1, 2, 3, 4], device=target_device)
|
||||
positions = torch.arange(input_ids.numel(), device=target_device)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
|
||||
):
|
||||
hidden_states = model(input_ids, positions, None)
|
||||
logits = model.compute_logits(hidden_states)
|
||||
assert torch.isfinite(logits).all()
|
||||
|
||||
|
||||
def test_auto_gptq_config_get_name():
|
||||
"""Test that AutoGPTQConfig.get_name() returns 'auto_gptq'."""
|
||||
assert AutoGPTQConfig.get_name() == "auto_gptq"
|
||||
|
||||
|
||||
def test_auto_gptq_quantizes_every_layer_when_the_module_list_is_unknown():
|
||||
"""An empty module list means the safetensors metadata could not be read,
|
||||
not that the checkpoint holds no quantized layer."""
|
||||
assert is_layer_gptq_quantized(
|
||||
prefix="model.layers.0.mlp.down_proj",
|
||||
quantized_layers=[],
|
||||
)
|
||||
assert not is_layer_gptq_quantized(
|
||||
prefix="model.layers.0.mlp.down_proj",
|
||||
quantized_layers=["self_attn.q_proj"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dynamic",
|
||||
[
|
||||
{},
|
||||
{r"+:model\.layers\.0\..*": {"desc_act": True}},
|
||||
{
|
||||
r"+:model\.layers\.0\..*": {
|
||||
"desc_act": True,
|
||||
"group_size": 32,
|
||||
}
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_auto_gptq_rejects_group_activation_order(dynamic):
|
||||
desc_act = not dynamic
|
||||
with pytest.raises(ValueError, match="group activation ordering"):
|
||||
AutoGPTQConfig(4, 128, desc_act, True, False, dynamic, {})
|
||||
|
||||
|
||||
def test_auto_gptq_normalizes_channelwise_activation_order():
|
||||
config = AutoGPTQConfig(4, -1, True, True, False, {}, {})
|
||||
assert not config.desc_act
|
||||
|
||||
config = AutoGPTQConfig(
|
||||
4,
|
||||
128,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
{r"+:model\.layers\.0\..*": {"desc_act": True, "group_size": -1}},
|
||||
{},
|
||||
)
|
||||
override_config(config, "model.layers.0.self_attn.q_proj")
|
||||
assert config.group_size == -1
|
||||
assert not config.desc_act
|
||||
|
||||
with pytest.raises(ValueError, match="group activation ordering"):
|
||||
AutoGPTQConfig(
|
||||
4,
|
||||
-1,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
{r"+:model\.layers\.0\..*": {"group_size": 128}},
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
|
||||
method = object.__new__(AutoGPTQMoEMethod)
|
||||
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
|
||||
method.input_dtype = None
|
||||
method.experts_cls = None
|
||||
method.moe = SimpleNamespace(w13_num_shards=2)
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=2,
|
||||
hidden_size=8,
|
||||
intermediate_size_per_partition=4,
|
||||
params_dtype=torch.float16,
|
||||
weight_loader=lambda *args, **kwargs: None,
|
||||
)
|
||||
|
||||
assert layer.w13_bias.shape == (2, 8)
|
||||
assert layer.w2_bias.shape == (2, 8)
|
||||
assert torch.count_nonzero(layer.w13_bias) == 0
|
||||
assert torch.count_nonzero(layer.w2_bias) == 0
|
||||
|
||||
|
||||
def test_routed_experts_loads_per_expert_biases():
|
||||
class Loader:
|
||||
quant_config = None
|
||||
quant_method = object()
|
||||
_fused_shared_expert_quantizer = None
|
||||
moe_config = SimpleNamespace(
|
||||
is_act_and_mul=True,
|
||||
tp_rank=0,
|
||||
moe_parallel_config=SimpleNamespace(tp_size=1),
|
||||
)
|
||||
_get_hidden_dim = staticmethod(RoutedExperts._get_hidden_dim)
|
||||
_narrow_expert_data_for_padding = staticmethod(
|
||||
RoutedExperts._narrow_expert_data_for_padding
|
||||
)
|
||||
_load_w13 = RoutedExperts._load_w13
|
||||
_loaded_expert_biases = set()
|
||||
|
||||
@staticmethod
|
||||
def _map_global_expert_id_to_local_expert_id(expert_id):
|
||||
return expert_id
|
||||
|
||||
loader = Loader()
|
||||
w13_bias = torch.nn.Parameter(torch.zeros(1, 8), requires_grad=False)
|
||||
w2_bias = torch.nn.Parameter(torch.zeros(1, 4), requires_grad=False)
|
||||
|
||||
for shard_id, loaded in (
|
||||
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
|
||||
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
|
||||
):
|
||||
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
|
||||
loader,
|
||||
w13_bias,
|
||||
loaded,
|
||||
weight_name="model.layers.0.mlp.experts.w13_bias",
|
||||
shard_id=shard_id,
|
||||
expert_id=0,
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
|
||||
loader,
|
||||
w2_bias,
|
||||
torch.tensor([9.0, 10.0, 11.0, 12.0]),
|
||||
weight_name="model.layers.0.mlp.experts.w2_bias",
|
||||
shard_id="w2",
|
||||
expert_id=0,
|
||||
return_success=True,
|
||||
)
|
||||
assert torch.equal(w13_bias, torch.arange(1, 9, dtype=torch.float32).reshape(1, 8))
|
||||
assert torch.equal(w2_bias, torch.arange(9, 13, dtype=torch.float32).reshape(1, 4))
|
||||
assert loader._loaded_expert_biases == {"w13_bias", "w2_bias"}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,312 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip(
|
||||
"This test only runs on Blackwell GPUs (SM10x).", allow_module_level=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def set_test_environment():
|
||||
"""Sets environment variables required for this test module."""
|
||||
# Make sure TRTLLM attention is available
|
||||
os.environ["VLLM_HAS_FLASHINFER_CUBIN"] = "1"
|
||||
# Set compilation threads to 16 to speed up startup
|
||||
os.environ["FLASHINFER_NVCC_THREADS"] = "16"
|
||||
|
||||
|
||||
# Override the backbone layers to 4 for faster startup
|
||||
HF_OVERRIDE_TEXT = {
|
||||
"num_layers": 4,
|
||||
"num_hidden_layers": 4,
|
||||
}
|
||||
HF_OVERRIDE_MM = {
|
||||
"text_config": {"num_layers": 4, "num_hidden_layers": 4},
|
||||
}
|
||||
|
||||
|
||||
def can_initialize(
|
||||
model: str,
|
||||
hf_overrides: dict[str, Any] | None = None,
|
||||
extra_args: list[str] | None = None,
|
||||
):
|
||||
# Server arguments
|
||||
extra_args = extra_args if extra_args is not None else []
|
||||
server_args = [
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-batched-tokens",
|
||||
"256",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--trust-remote-code",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"image": 0}),
|
||||
*extra_args,
|
||||
]
|
||||
|
||||
# Launch server and make a simple request
|
||||
with RemoteOpenAIServer(
|
||||
model,
|
||||
server_args,
|
||||
max_wait_seconds=1500, # Due to FlashInfer compile
|
||||
override_hf_configs=hf_overrides,
|
||||
) as server:
|
||||
client = server.get_client()
|
||||
# Make a simple request to verify the server works
|
||||
completion = client.completions.create(
|
||||
model=model,
|
||||
prompt=["Hello, World!"],
|
||||
temperature=0,
|
||||
max_tokens=2,
|
||||
)
|
||||
print(completion)
|
||||
assert completion.choices[0].text is not None
|
||||
|
||||
|
||||
## Llama4 ##
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"RuntimeError: run_moe() Expected a value of type "
|
||||
"'Optional[List[Tensor]]' for argument '_9' but instead found type "
|
||||
"'list'."
|
||||
)
|
||||
)
|
||||
def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
|
||||
hf_overrides=HF_OVERRIDE_MM,
|
||||
extra_args=["--moe-backend=flashinfer_cutlass"],
|
||||
)
|
||||
|
||||
|
||||
def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
|
||||
hf_overrides=HF_OVERRIDE_MM,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
|
||||
hf_overrides=HF_OVERRIDE_MM,
|
||||
extra_args=["--moe-backend=flashinfer_cutlass"],
|
||||
)
|
||||
|
||||
|
||||
def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
|
||||
hf_overrides=HF_OVERRIDE_MM,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
## DeepSeekV3 ##
|
||||
|
||||
|
||||
def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"deepseek-ai/DeepSeek-V3.1",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=deep_gemm"],
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_fp8_block_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"deepseek-ai/DeepSeek-V3.1",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=triton"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"Known issue: lack of kernel support. "
|
||||
"Expected failure: assert self.block_quant is None"
|
||||
)
|
||||
)
|
||||
def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"deepseek-ai/DeepSeek-V3.1",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_cutlass"],
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"deepseek-ai/DeepSeek-V3.1",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_nvfp4_moe_flashinfer_vllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=cutlass"],
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_cutlass"],
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
## GPT-OSS ##
|
||||
|
||||
|
||||
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=[
|
||||
"--moe-backend",
|
||||
"flashinfer_cutlass",
|
||||
"--quantization-config.moe.activation",
|
||||
"mxfp8",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=[
|
||||
"--quantization-config.moe.activation",
|
||||
"mxfp8",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--enforce-eager"],
|
||||
)
|
||||
|
||||
|
||||
## Qwen3 Next ##
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"FLASHINFER TRTLLM MoE has a bug with all negative router logits "
|
||||
"for models with RENORMALIZE. This will be re-enabled once the "
|
||||
"issue is fixed in flashinfer."
|
||||
)
|
||||
)
|
||||
def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
## NemoTron ##
|
||||
|
||||
|
||||
def test_nemotron_fp8_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_cutlass"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"FP8 MoE backend FLASHINFER_TRTLLM does not support the "
|
||||
"deployment configuration since kernel does not support "
|
||||
"no act_and_mul MLP layer."
|
||||
)
|
||||
)
|
||||
def test_nemotron_fp8_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"FP8 MoE backend TRITON does not support the "
|
||||
"deployment configuration since kernel does not support "
|
||||
"no act_and_mul MLP layer."
|
||||
)
|
||||
)
|
||||
def test_nemotron_fp8_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=triton"],
|
||||
)
|
||||
|
||||
|
||||
def test_nemotron_fp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_cutlass"],
|
||||
)
|
||||
|
||||
|
||||
def test_nemotron_fp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
def test_nemotron_nvfp4_per_token_moe_flashinfer_trtllm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
can_initialize(
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=[
|
||||
"--moe-backend=flashinfer_trtllm",
|
||||
"--quantization-config.moe.activation=nvfp4_per_token",
|
||||
],
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests whether Marlin models can be loaded from the autogptq config.
|
||||
|
||||
Run `pytest tests/quantization/test_configs.py --forked`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# Model Id // Quantization Arg // Expected Type
|
||||
MODEL_ARG_EXPTYPES = [
|
||||
# AUTOGPTQ
|
||||
# compat: autogptq <=0.7.1 is_marlin_format: bool
|
||||
# Model Serialized in Exllama Format.
|
||||
("TheBloke/Llama-2-7B-Chat-GPTQ", None, "auto_gptq"),
|
||||
(
|
||||
"TheBloke/Llama-2-7B-Chat-GPTQ",
|
||||
"marlin",
|
||||
"auto_gptq" if current_platform.is_cuda_alike() else "ERROR",
|
||||
),
|
||||
("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "auto_gptq"),
|
||||
("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
|
||||
# compat: autogptq >=0.8.0 use checkpoint_format: str
|
||||
# Model Serialized in Exllama Format.
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "auto_gptq"),
|
||||
(
|
||||
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
|
||||
"marlin",
|
||||
"auto_gptq" if current_platform.is_cuda_alike() else "ERROR",
|
||||
),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
|
||||
# AUTOAWQ
|
||||
# AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models
|
||||
# when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq"
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
None,
|
||||
"auto_awq",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"),
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
"marlin",
|
||||
"auto_awq" if current_platform.is_cuda_alike() else "ERROR",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)
|
||||
def test_auto_gptq(model_arg_exptype: tuple[str, None, str]) -> None:
|
||||
model_path, quantization_arg, expected_type = model_arg_exptype
|
||||
|
||||
try:
|
||||
model_config = ModelConfig(model_path, quantization=quantization_arg)
|
||||
found_quantization_type = model_config.quantization
|
||||
except ValueError:
|
||||
found_quantization_type = "ERROR"
|
||||
|
||||
assert found_quantization_type == expected_type, (
|
||||
f"Expected quant_type == {expected_type} for {model_path}, "
|
||||
f"but found {found_quantization_type} "
|
||||
f"for no --quantization {quantization_arg} case"
|
||||
)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
MODELS = [
|
||||
"RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8", # INT8 W8A8 MoE
|
||||
]
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
def test_cpu_w8a8(vllm_runner, model, dtype):
|
||||
with vllm_runner(model, dtype=dtype) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
|
||||
assert output
|
||||
print(output)
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
from vllm._custom_ops import cpu_gemm_wna16 # noqa: E402
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( # noqa: E402
|
||||
_get_isa_hint,
|
||||
)
|
||||
|
||||
# Isolated runs of cpu_gemm_wna16 can hang unless AMX tiles are initialized.
|
||||
# Guard so non-AMX hosts still take the vec/rvv path.
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
torch.cpu._init_amx()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_cpu_gemm_wna16_3d_input():
|
||||
"""cpu_gemm_wna16 flattens 3D [B, T, K] to 2D and reshapes the output back."""
|
||||
if not hasattr(torch.ops._C, "cpu_gemm_wna16"):
|
||||
pytest.skip("cpu_gemm_wna16 op is not available")
|
||||
|
||||
B, T, K, N = 2, 5, 64, 64
|
||||
pack_factor = 8
|
||||
group_size = 32
|
||||
dtype = torch.bfloat16
|
||||
q_weight = torch.zeros(N // 16, K * 16 // pack_factor, dtype=torch.int32)
|
||||
scales = torch.ones(K // group_size, N, dtype=dtype)
|
||||
|
||||
x_3d = torch.randn(B, T, K, dtype=dtype)
|
||||
x_2d = x_3d.reshape(-1, K)
|
||||
isa_hint = _get_isa_hint(dtype)
|
||||
kwargs = dict(
|
||||
q_weight=q_weight,
|
||||
scales=scales,
|
||||
zeros=None,
|
||||
bias=None,
|
||||
pack_factor=pack_factor,
|
||||
isa_hint=isa_hint,
|
||||
)
|
||||
|
||||
out_3d = cpu_gemm_wna16(input=x_3d, **kwargs)
|
||||
out_2d = cpu_gemm_wna16(input=x_2d, **kwargs)
|
||||
|
||||
assert out_3d.shape == (B, T, N)
|
||||
assert out_2d.shape == (B * T, N)
|
||||
torch.testing.assert_close(out_3d.reshape(-1, N), out_2d)
|
||||
|
||||
|
||||
MODELS = [
|
||||
"TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ",
|
||||
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
|
||||
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
|
||||
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
|
||||
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
|
||||
"Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE
|
||||
"openai/gpt-oss-20b", # MXFP4 W4A16
|
||||
"QuixiAI/Qwen3-30B-A3B-AWQ", # AWQ W4A16 MoE
|
||||
"Qwen/Qwen3-30B-A3B-GPTQ-Int4", # GPTQ W4A16 MoE
|
||||
"RedHatAI/Qwen3-30B-A3B-quantized.w4a16", # compressed-tensors W4A16 MoE
|
||||
]
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.get_cpu_architecture() != CpuArchEnum.POWERPC,
|
||||
reason="PowerPC-only test",
|
||||
)
|
||||
def test_cpu_wna16_power_bf16_uses_vsx():
|
||||
"""Use the POWER10 BF16 MMA microkernel instead of generic vectors."""
|
||||
assert _get_isa_hint(torch.bfloat16) == "vsx"
|
||||
assert _get_isa_hint(torch.float16) == "vec"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
def test_cpu_quant(vllm_runner, model, dtype):
|
||||
with vllm_runner(model, dtype=dtype) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
|
||||
assert output
|
||||
print(output)
|
||||
@@ -1,180 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Cutlass W4A16 (Machete) kernel on Hopper.
|
||||
|
||||
Verifies that W4A16 quantized models loaded through vllm select the
|
||||
MacheteLinearKernel on sm_90 GPUs, that weights are correctly repacked,
|
||||
and that inference produces valid output.
|
||||
|
||||
Run `pytest tests/quantization/test_cutlass_w4a16.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.has_device_capability(90) or current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"Machete W4A16 requires Hopper (sm_90).",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision import (
|
||||
MacheteLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsLinearMethod,
|
||||
CompressedTensorsWNA16,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def enable_pickle(monkeypatch):
|
||||
"""`LLM.apply_model` requires pickling a function."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"act_type,weight_type,group_size,zero_points",
|
||||
[
|
||||
(torch.float16, scalar_types.uint4b8, 128, False),
|
||||
(torch.bfloat16, scalar_types.uint4b8, 128, False),
|
||||
(torch.float16, scalar_types.uint4, 128, True),
|
||||
(torch.float16, scalar_types.uint4b8, -1, False),
|
||||
],
|
||||
ids=[
|
||||
"fp16-gptq-g128",
|
||||
"bf16-gptq-g128",
|
||||
"fp16-awq-g128",
|
||||
"fp16-channelwise",
|
||||
],
|
||||
)
|
||||
def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points):
|
||||
"""Verify choose_mp_linear_kernel picks MacheteLinearKernel."""
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(4096, 4096),
|
||||
partition_weight_shape=(4096, 4096),
|
||||
act_type=act_type,
|
||||
weight_type=weight_type,
|
||||
group_size=group_size,
|
||||
zero_points=zero_points,
|
||||
)
|
||||
kernel = choose_mp_linear_kernel(config)
|
||||
assert kernel is MacheteLinearKernel, (
|
||||
f"Expected MacheteLinearKernel, got {kernel.__name__}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_shape,part_shape,weight_type,group_size,expected_reason",
|
||||
[
|
||||
(
|
||||
(4096, 4096),
|
||||
(4096, 4096),
|
||||
scalar_types.float6_e3m2f,
|
||||
128,
|
||||
"Quant type",
|
||||
),
|
||||
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, "Group size"),
|
||||
],
|
||||
ids=["unsupported-quant-type", "unsupported-group-size"],
|
||||
)
|
||||
def test_machete_rejects_invalid_config(
|
||||
full_shape, part_shape, weight_type, group_size, expected_reason
|
||||
):
|
||||
"""Verify Machete rejects unsupported configurations."""
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=full_shape,
|
||||
partition_weight_shape=part_shape,
|
||||
act_type=torch.float16,
|
||||
weight_type=weight_type,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
)
|
||||
can_impl, reason = MacheteLinearKernel.can_implement(config)
|
||||
assert not can_impl
|
||||
assert reason is not None
|
||||
assert expected_reason in reason
|
||||
|
||||
|
||||
def test_kernel_selection_with_disabled_machete(monkeypatch):
|
||||
"""Verify kernel selection falls back when Machete is disabled."""
|
||||
monkeypatch.setattr("vllm.envs.VLLM_DISABLED_KERNELS", ["MacheteLinearKernel"])
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(4096, 4096),
|
||||
partition_weight_shape=(4096, 4096),
|
||||
act_type=torch.float16,
|
||||
weight_type=scalar_types.uint4b8,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
)
|
||||
kernel = choose_mp_linear_kernel(config)
|
||||
assert kernel is not MacheteLinearKernel, "MacheteLinearKernel should be disabled"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"nm-testing/tinyllama-oneshot-w4a16-channel-v2",
|
||||
],
|
||||
)
|
||||
def test_w4a16_machete_e2e(vllm_runner, model_name):
|
||||
"""Load a W4A16 model, verify Machete kernel is used, and generate."""
|
||||
with vllm_runner(model_name, enforce_eager=True, gpu_memory_utilization=0.5) as llm:
|
||||
|
||||
def check_model(model):
|
||||
layer = model.model.layers[0]
|
||||
qkv_proj = layer.self_attn.qkv_proj
|
||||
|
||||
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
|
||||
assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
|
||||
assert isinstance(qkv_proj.scheme.kernel, MacheteLinearKernel), (
|
||||
f"Expected MacheteLinearKernel on Hopper, "
|
||||
f"got {type(qkv_proj.scheme.kernel).__name__}"
|
||||
)
|
||||
|
||||
assert hasattr(qkv_proj, "weight_packed")
|
||||
assert hasattr(qkv_proj, "weight_scale")
|
||||
assert qkv_proj.weight_packed.dtype == torch.int32
|
||||
|
||||
llm.apply_model(check_model)
|
||||
|
||||
output = llm.generate_greedy("Hello my name is", max_tokens=10)
|
||||
assert output
|
||||
assert len(output[0][1]) > 0
|
||||
|
||||
|
||||
def test_w4a16_machete_bfloat16_deterministic(vllm_runner):
|
||||
"""Verify Machete works with bf16 activations and is deterministic."""
|
||||
model_name = "nm-testing/tinyllama-oneshot-w4a16-channel-v2"
|
||||
prompt = "The capital of France is"
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
enforce_eager=True,
|
||||
dtype="bfloat16",
|
||||
gpu_memory_utilization=0.5,
|
||||
) as llm:
|
||||
|
||||
def check_kernel_type(model):
|
||||
layer = model.model.layers[0]
|
||||
scheme = layer.self_attn.qkv_proj.scheme
|
||||
assert isinstance(scheme.kernel, MacheteLinearKernel), (
|
||||
f"Expected MacheteLinearKernel with bf16, "
|
||||
f"got {type(scheme.kernel).__name__}"
|
||||
)
|
||||
|
||||
llm.apply_model(check_kernel_type)
|
||||
|
||||
out1 = llm.generate_greedy(prompt, max_tokens=10)
|
||||
out2 = llm.generate_greedy(prompt, max_tokens=10)
|
||||
assert out1[0][1] == out2[0][1], (
|
||||
f"Non-deterministic: '{out1[0][1]}' vs '{out2[0][1]}'"
|
||||
)
|
||||
@@ -1,71 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FP8 per-channel online quantization.
|
||||
|
||||
Per-output-channel weight scale + dynamic per-token activation scale.
|
||||
bf16/fp16 checkpoints are quantized at load time with one fp32 scale per
|
||||
output channel for weights and one fp32 scale per token for activations
|
||||
(computed dynamically inside the kernel). Run via
|
||||
`pytest tests/quantization/test_fp8_per_channel.py --forked`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config.quantization import (
|
||||
_ONLINE_SHORTHANDS,
|
||||
QUANT_KEY_NAMES,
|
||||
QuantizationConfigArgs,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
_ONLINE_LINEAR_METHODS,
|
||||
_ONLINE_MOE_METHODS,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
Fp8PtpcOnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticChannelSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_fp8_per_channel_shorthand_registered() -> None:
|
||||
"""The `fp8_per_channel` CLI shorthand must resolve to a config that
|
||||
dispatches the per-channel methods. Guards against regressions in
|
||||
`_ONLINE_SHORTHANDS` / `_ONLINE_LINEAR_METHODS` / `_ONLINE_MOE_METHODS`
|
||||
drifting out of sync.
|
||||
"""
|
||||
args = _ONLINE_SHORTHANDS["fp8_per_channel"]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
assert args.linear is not None
|
||||
assert args.moe is not None
|
||||
assert args.linear.weight is kFp8StaticChannelSym
|
||||
assert args.moe.weight is kFp8StaticChannelSym
|
||||
|
||||
assert _ONLINE_LINEAR_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineLinearMethod
|
||||
assert _ONLINE_MOE_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineMoEMethod
|
||||
|
||||
assert QUANT_KEY_NAMES["fp8_per_channel_static"] is kFp8StaticChannelSym
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_scaled_fp8_quant_per_channel_shape() -> None:
|
||||
"""Verify the kernel call per-channel quant depends on: passing a 2D
|
||||
weight to `ops.scaled_fp8_quant` with `use_per_token_if_dynamic=True`
|
||||
yields one scale per output row -- a [N, 1] fp32 tensor.
|
||||
"""
|
||||
x = (torch.randn(size=(96, 256), device=current_platform.device_type) * 13).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
y, s = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
|
||||
assert y.shape == (96, 256)
|
||||
assert y.dtype == current_platform.fp8_dtype()
|
||||
assert s.shape == (96, 1)
|
||||
assert s.dtype == torch.float32
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MXFP4 MoE oracle backend selection on mi355x (GFX950).
|
||||
|
||||
These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
_requires_qwen38_tep8_emulation,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
ROCM_AVAILABLE = current_platform.is_rocm()
|
||||
ROCM_GFX950 = False
|
||||
ROCM_AITER_SUPPORTED = False
|
||||
|
||||
if ROCM_AVAILABLE:
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
ROCM_GFX950 = on_gfx950()
|
||||
ROCM_AITER_SUPPORTED = is_aiter_found_and_supported()
|
||||
|
||||
|
||||
def set_rocm_aiter(monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None:
|
||||
value = "1" if enabled else "0"
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", value)
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", value)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", enabled)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "_FMOE_ENABLED", enabled)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def enable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
|
||||
set_rocm_aiter(monkeypatch, True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
|
||||
set_rocm_aiter(monkeypatch, False)
|
||||
|
||||
|
||||
def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend=moe_backend,
|
||||
)
|
||||
|
||||
|
||||
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
num_experts=512,
|
||||
experts_per_token=9,
|
||||
hidden_dim=2560,
|
||||
intermediate_size=640,
|
||||
num_local_experts=64,
|
||||
num_logical_experts=512,
|
||||
moe_parallel_config=FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=1,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=False,
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
)
|
||||
|
||||
|
||||
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
|
||||
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config.moe_parallel_config.ep_size = 4
|
||||
config.num_local_experts = 128
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
|
||||
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested_backend,expected_backend",
|
||||
[
|
||||
("auto", Mxfp4MoeBackend.EMULATION),
|
||||
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
|
||||
],
|
||||
)
|
||||
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
|
||||
requested_backend,
|
||||
expected_backend,
|
||||
monkeypatch,
|
||||
):
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
|
||||
class SupportedExperts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args, **kwargs):
|
||||
return True, None
|
||||
|
||||
config = _make_qwen38_tep8_moe_config()
|
||||
config.moe_backend = requested_backend
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
|
||||
)
|
||||
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
|
||||
assert backend == expected_backend
|
||||
assert experts_cls is SupportedExperts
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mxfp4_oracle_config():
|
||||
"""Stub the config the oracle reads (``model_config.quantization_config``)
|
||||
so backend dispatch resolves without a real model / user override."""
|
||||
from unittest.mock import patch
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.mxfp4.get_current_vllm_config"
|
||||
) as mock_get_config:
|
||||
mock_get_config.return_value.model_config.quantization_config = None
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
|
||||
@pytest.mark.skipif(not ROCM_AITER_SUPPORTED, reason="Requires supported AITER")
|
||||
def test_w4a4_dispatches_to_aiter(mxfp4_oracle_config, enable_rocm_aiter):
|
||||
"""With AITER enabled + GFX950, W4A4 selects AITER_MXFP4_MXFP4."""
|
||||
config = _make_w4a4_moe_config()
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
assert backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
|
||||
def test_w4a4_falls_back_without_aiter(
|
||||
mxfp4_oracle_config,
|
||||
disable_rocm_aiter,
|
||||
):
|
||||
config = _make_w4a4_moe_config()
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
assert backend == Mxfp4MoeBackend.EMULATION
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
|
||||
def test_w4a4_dispatches_to_emulation_with_moe_backend(mxfp4_oracle_config):
|
||||
"""With --moe-backend emulation, W4A4 selects EMULATION."""
|
||||
config = _make_w4a4_moe_config(moe_backend="emulation")
|
||||
backend, experts_cls = select_mxfp4_moe_backend(
|
||||
config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
assert backend == Mxfp4MoeBackend.EMULATION
|
||||
assert experts_cls is not None
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (has_humming() and current_platform.is_cuda()),
|
||||
reason="requires humming and CUDA",
|
||||
)
|
||||
|
||||
|
||||
def test_humming_input_quant_compiles_fullgraph():
|
||||
"""Humming's forward helpers look up ``DeviceInfo`` on every call; vLLM
|
||||
traces them with ``fullgraph=True`` (vllm-project/humming#147)."""
|
||||
import vllm.utils.humming as hm
|
||||
|
||||
weight_schema = hm.HummingWeightSchema(b_dtype="int4", weight_scale_group_size=128)
|
||||
input_schema = hm.HummingInputSchema(a_dtype="int8")
|
||||
from humming.transform import prepare_layer_config
|
||||
|
||||
config = prepare_layer_config(
|
||||
shape_n=256,
|
||||
shape_k=256,
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
torch_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
x = torch.randn(16, 256, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def quant(t: torch.Tensor):
|
||||
return hm.may_process_input(config, t)
|
||||
|
||||
expected = quant(x)
|
||||
actual = torch.compile(quant, fullgraph=True)(x)
|
||||
for a, e in zip(actual, expected):
|
||||
if e is None:
|
||||
assert a is None
|
||||
else:
|
||||
torch.testing.assert_close(a, e)
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation.
|
||||
|
||||
The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a
|
||||
grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8
|
||||
`test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling
|
||||
it in vLLM comes down to two things, both exercised here:
|
||||
|
||||
1. A grouped FP8 activation must classify to a float32-scaled key
|
||||
(kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale
|
||||
activations (group size 32) keep their e8m0 (uint8) scale.
|
||||
2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE
|
||||
(weight, activation) pair.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
# make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the
|
||||
# config-plumbing asserts below never read them, so a placeholder satisfies the
|
||||
# required arg without pulling in the humming package.
|
||||
_PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None}
|
||||
|
||||
|
||||
def test_mxfp4_weight_with_block_fp8_activation_is_supported():
|
||||
"""MXFP4 weight + block-FP8 (group-128) activation is an allowed pair."""
|
||||
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym)
|
||||
|
||||
|
||||
def test_mxfp4_weight_with_per_token_fp8_activation_still_supported():
|
||||
"""The pre-existing per-token FP8 pairing must keep working."""
|
||||
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_key",
|
||||
[
|
||||
# per-token FP8 -> float32 per-token scale
|
||||
(0, kFp8DynamicTokenSym),
|
||||
# block FP8 -> float32 group-128 scale (the newly-fixed case)
|
||||
(128, kFp8Dynamic128Sym),
|
||||
# MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved)
|
||||
(32, kMxfp8Dynamic),
|
||||
],
|
||||
)
|
||||
def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
input_schema_to_quant_key,
|
||||
)
|
||||
from vllm.utils.humming import HummingInputSchema
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
schema = HummingInputSchema(
|
||||
a_dtype=humming_dtypes.float8e4m3,
|
||||
input_scale_group_size=group_size,
|
||||
)
|
||||
assert input_schema_to_quant_key(schema, param_dtype=torch.bfloat16) == expected_key
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
|
||||
def test_humming_bf16_input_schema_is_unquantized():
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
input_schema_to_quant_key,
|
||||
)
|
||||
from vllm.utils.humming import HummingInputSchema
|
||||
|
||||
# No a_dtype -> unquantized (bf16/fp16) inputs -> None.
|
||||
assert (
|
||||
input_schema_to_quant_key(HummingInputSchema(), param_dtype=torch.bfloat16)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_block_fp8_activation_quant_config_is_block_quantized():
|
||||
"""A block-FP8 activation group shape yields a block-quantized MoE config
|
||||
(FP8 dtype, [1, 128] block, not per-act-token). This is what makes the
|
||||
DeepEP prepare/finalize step quantize activations to block FP8 *before* the
|
||||
all-to-all dispatch instead of deferring to Humming -- and what makes
|
||||
HummingExpertsBase.expects_unquantized_inputs return False."""
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
make_humming_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
qc = make_humming_moe_quant_config(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
weight_dtype="float4e2m1",
|
||||
activation_group_shape=GroupShape(row=1, col=128),
|
||||
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
|
||||
)
|
||||
assert qc.is_block_quantized
|
||||
assert qc.block_shape == [1, 128]
|
||||
assert not qc.per_act_token_quant
|
||||
assert qc.quant_dtype == current_platform.fp8_dtype()
|
||||
|
||||
|
||||
def test_default_activation_quant_config_defers_to_humming():
|
||||
"""Without a block activation shape the config stays per-token (the deferred
|
||||
path): Humming quantizes internally, preserving pre-existing behavior for
|
||||
every non-block-FP8 scheme."""
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
make_humming_moe_quant_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
qc = make_humming_moe_quant_config(
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
weight_dtype="float4e2m1",
|
||||
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
|
||||
)
|
||||
assert not qc.is_block_quantized
|
||||
assert qc.per_act_token_quant
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for INT8 (W8A8) fused-MoE oracle backend selection.
|
||||
|
||||
These exercise ``select_int8_moe_backend`` only (no MoE kernels are launched),
|
||||
so they run on any platform where the Triton INT8 MoE kernel is available —
|
||||
CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
Int8MoeBackend,
|
||||
select_int8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTensorSym,
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
kInt8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# The Triton int8_w8a8 fused-MoE kernel is available on CUDA (Turing+), on
|
||||
# ROCm CDNA GPUs and on XPU. Gate on that rather than on a specific arch.
|
||||
INT8_MOE_SUPPORTED = (
|
||||
(current_platform.is_cuda() and current_platform.has_device_capability((7, 5)))
|
||||
or current_platform.is_rocm()
|
||||
or current_platform.is_xpu()
|
||||
)
|
||||
|
||||
requires_int8_moe = pytest.mark.skipif(
|
||||
not INT8_MOE_SUPPORTED,
|
||||
reason="Requires Triton INT8 MoE support (CUDA SM>=7.5, ROCm, or XPU)",
|
||||
)
|
||||
|
||||
# So FusedMoEConfig.device names the device the test actually runs on.
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend=moe_backend,
|
||||
)
|
||||
|
||||
|
||||
@requires_int8_moe
|
||||
@pytest.mark.parametrize(
|
||||
"weight_key,activation_key",
|
||||
[
|
||||
pytest.param(
|
||||
kInt8StaticChannelSym,
|
||||
kInt8DynamicTokenSym,
|
||||
id="per_channel_weight-per_token_act",
|
||||
),
|
||||
pytest.param(
|
||||
kInt8StaticTensorSym,
|
||||
kInt8DynamicTensorSym,
|
||||
id="per_tensor_weight-per_tensor_act",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_int8_dynamic_schemes_dispatch_to_triton(weight_key, activation_key):
|
||||
"""Both dynamic-activation INT8 MoE schemes select the Triton backend."""
|
||||
config = _make_int8_moe_config()
|
||||
backend, experts_cls = select_int8_moe_backend(
|
||||
config, weight_key=weight_key, activation_key=activation_key
|
||||
)
|
||||
assert backend == Int8MoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU-only behaviour")
|
||||
def test_scaled_int8_quant_is_available_on_xpu():
|
||||
"""The per-tensor scheme's activation quantization works on XPU.
|
||||
|
||||
``_int8_quantize`` sends per-tensor activations through
|
||||
``ops.scaled_int8_quant``, which has an XPU branch. This is what lets the
|
||||
per-tensor scheme be enabled here alongside CUDA and ROCm; pin it so a
|
||||
future change cannot silently remove that branch.
|
||||
"""
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
x = torch.randn(4, 16, device=DEVICE, dtype=torch.bfloat16)
|
||||
scale = torch.full((1,), 0.05, device=DEVICE, dtype=torch.float32)
|
||||
|
||||
q, returned_scale, azp = ops.scaled_int8_quant(x, scale=scale)
|
||||
|
||||
assert q.dtype == torch.int8
|
||||
assert q.shape == x.shape
|
||||
assert azp is None
|
||||
torch.testing.assert_close(returned_scale, scale)
|
||||
|
||||
|
||||
@requires_int8_moe
|
||||
def test_int8_explicit_moe_backend_triton():
|
||||
"""An explicit --moe-backend triton selects the Triton INT8 backend."""
|
||||
config = _make_int8_moe_config(moe_backend="triton")
|
||||
backend, experts_cls = select_int8_moe_backend(
|
||||
config,
|
||||
weight_key=kInt8StaticChannelSym,
|
||||
activation_key=kInt8DynamicTokenSym,
|
||||
)
|
||||
assert backend == Int8MoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@requires_int8_moe
|
||||
def test_int8_unsupported_moe_backend_raises():
|
||||
"""An unsupported --moe-backend for INT8 MoE raises a clear error."""
|
||||
config = _make_int8_moe_config(moe_backend="cutlass")
|
||||
with pytest.raises(ValueError, match="not supported for Int8 MoE"):
|
||||
select_int8_moe_backend(
|
||||
config,
|
||||
weight_key=kInt8StaticChannelSym,
|
||||
activation_key=kInt8DynamicTokenSym,
|
||||
)
|
||||
@@ -1,122 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests whether gptq models with quantized lm_head can be loaded.
|
||||
|
||||
Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.quantization import preset_name_to_scheme
|
||||
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
MODELS_QUANT = [
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", False),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id, lm_head_quantized", MODELS_QUANT)
|
||||
def test_lm_head(
|
||||
model_id: str,
|
||||
lm_head_quantized: bool,
|
||||
monkeypatch,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
) -> None:
|
||||
# `LLM.apply_model` requires pickling a function.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
model, _ = load_model_without_vllm_runner(
|
||||
model_id,
|
||||
dtype=torch.float16,
|
||||
model_config_kwargs={
|
||||
"max_model_len": 2048,
|
||||
"hf_overrides": {"num_hidden_layers": 3},
|
||||
},
|
||||
)
|
||||
lm_head_layer = model.lm_head
|
||||
if lm_head_quantized:
|
||||
assert isinstance(lm_head_layer.quant_method, AutoGPTQLinearMethod)
|
||||
else:
|
||||
assert isinstance(lm_head_layer.quant_method, UnquantizedEmbeddingMethod)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not has_humming(), reason="requires Humming/CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"preset,quant_format",
|
||||
[
|
||||
("FP8_DYNAMIC", "float-quantized"),
|
||||
("NVFP4A16", "nvfp4-pack-quantized"),
|
||||
("W4A16", "pack-quantized"),
|
||||
("W4A16_NVFP4", None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("bias", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_quantized_lm_head_matches_linear(
|
||||
dist_init, default_vllm_config, preset, quant_format, bias
|
||||
):
|
||||
"""An explicitly quantized head must produce the same logits as a linear layer."""
|
||||
default_vllm_config.model_config = SimpleNamespace(
|
||||
dtype=torch.bfloat16, head_dtype=None
|
||||
)
|
||||
default_vllm_config.kernel_config.linear_backend = "humming"
|
||||
quant_config: QuantizationConfig
|
||||
if quant_format is None:
|
||||
quant_config = ModelOptNvFp4Config(
|
||||
quant_method=preset,
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
else:
|
||||
scheme = preset_name_to_scheme(preset, targets=["Linear", "lm_head"])
|
||||
quant_config = CompressedTensorsConfig.from_config(
|
||||
{"config_groups": {"group_0": scheme.model_dump()}, "format": quant_format}
|
||||
)
|
||||
with torch.device("cuda"):
|
||||
kwargs = dict(
|
||||
bias=bias,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
disable_tp=True,
|
||||
)
|
||||
head = ParallelLMHead(500, 256, prefix="lm_head", **kwargs)
|
||||
linear = ColumnParallelLinear(256, 512, prefix="proj", **kwargs)
|
||||
for name, param in linear.named_parameters():
|
||||
if "scale" in name:
|
||||
param.fill_(1.0)
|
||||
elif name == "weight_shape":
|
||||
param.copy_(torch.tensor([512, 256]))
|
||||
elif param.is_floating_point():
|
||||
param.copy_(torch.randn(param.shape, dtype=torch.float32))
|
||||
else:
|
||||
param.random_(0, 127)
|
||||
head.load_state_dict(linear.state_dict())
|
||||
for layer in (head, linear):
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
x = torch.randn(8, 256, dtype=torch.bfloat16)
|
||||
expected, _ = linear(x)
|
||||
actual = LogitsProcessor(500)(head, x, head.bias)
|
||||
assert torch.isfinite(actual).all()
|
||||
torch.testing.assert_close(actual, expected[:, :500], rtol=0, atol=0)
|
||||
@@ -1,624 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
QuantizationType,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
WNA16MoEBackend,
|
||||
_backend_incompatibility_reason,
|
||||
_convert_moe_wna16_humming_tensors,
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
map_wna16_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import moe_wna16
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_map_wna16_backend_supports_triton():
|
||||
assert map_wna16_backend("triton") == WNA16MoEBackend.TRITON
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config",
|
||||
[
|
||||
{"desc_act": True, "group_size": 128},
|
||||
{
|
||||
"desc_act": False,
|
||||
"group_size": 128,
|
||||
"dynamic": {r"+:model\.layers\.0\..*": {"desc_act": True}},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_moe_wna16_rejects_gptq_group_activation_order(config):
|
||||
config.update({"quant_method": "gptq", "bits": 4, "sym": True})
|
||||
with pytest.raises(ValueError, match="group activation ordering"):
|
||||
MoeWNA16Config.from_config(config)
|
||||
|
||||
|
||||
def test_moe_wna16_accepts_channelwise_gptq_activation_order():
|
||||
config = {
|
||||
"quant_method": "gptq",
|
||||
"bits": 4,
|
||||
"group_size": -1,
|
||||
"desc_act": True,
|
||||
"sym": True,
|
||||
}
|
||||
assert MoeWNA16Config.is_moe_wna16_compatible(config)
|
||||
MoeWNA16Config.from_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "quant_config", "may_have_zp", "may_have_bias", "expected"),
|
||||
[
|
||||
(
|
||||
WNA16MoEBackend.TRITON,
|
||||
AutoAWQConfig(4, 128, True, False),
|
||||
True,
|
||||
False,
|
||||
"AutoAWQ weight layout",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.TRITON,
|
||||
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
|
||||
False,
|
||||
True,
|
||||
"bias",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.MARLIN,
|
||||
MoeWNA16Config(
|
||||
linear_quant_method="gptq",
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
has_zp=False,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=None,
|
||||
full_config={},
|
||||
),
|
||||
False,
|
||||
False,
|
||||
"MoeWNA16 checkpoint layout",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.RDNA3,
|
||||
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
|
||||
False,
|
||||
False,
|
||||
"compressed-tensors",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.RDNA3,
|
||||
QuantizationArgs(
|
||||
num_bits=4,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=False,
|
||||
dynamic=False,
|
||||
group_size=128,
|
||||
),
|
||||
True,
|
||||
False,
|
||||
"asymmetric",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.RDNA3,
|
||||
QuantizationArgs(
|
||||
num_bits=4,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.CHANNEL,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
),
|
||||
False,
|
||||
False,
|
||||
"group-wise scales",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_wna16_oracle_rejects_incompatible_quant_structures(
|
||||
backend, quant_config, may_have_zp, may_have_bias, expected
|
||||
):
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
reason = _backend_incompatibility_reason(
|
||||
backend=backend,
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=may_have_zp,
|
||||
may_have_bias=may_have_bias,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
|
||||
assert reason is not None
|
||||
assert expected in reason
|
||||
|
||||
|
||||
def test_compressed_tensors_nfirst_weights_pass_through_for_triton():
|
||||
"""CT provides N-first [E, N, K_packed] int32; Triton expects N-first uint8.
|
||||
|
||||
No transpose needed — just a view from int32 to uint8.
|
||||
"""
|
||||
quant_config = QuantizationArgs(
|
||||
num_bits=4,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
group_size=32,
|
||||
)
|
||||
# N-first layout: [E, N, K_packed]
|
||||
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2)
|
||||
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2)
|
||||
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4)
|
||||
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3)
|
||||
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=WNA16MoEBackend.TRITON,
|
||||
layer=torch.nn.Module(),
|
||||
quant_config=quant_config,
|
||||
input_dtype=None,
|
||||
w13=w13,
|
||||
w2=w2,
|
||||
w13_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
)
|
||||
|
||||
assert converted is not None
|
||||
assert torch.equal(converted[0], w13.contiguous().view(torch.uint8))
|
||||
assert torch.equal(converted[1], w2.contiguous().view(torch.uint8))
|
||||
assert torch.equal(converted[2], w13_scale)
|
||||
assert torch.equal(converted[3], w2_scale)
|
||||
|
||||
|
||||
def test_gptq_nfirst_weights_pass_through_for_triton():
|
||||
"""GPTQ frontend normalizes to N-first before calling oracle.
|
||||
|
||||
Verifies the oracle handles N-first GPTQ inputs the same way as CT.
|
||||
"""
|
||||
quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
|
||||
# N-first layout (after GPTQ frontend transpose): [E, N, K_packed]
|
||||
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2)
|
||||
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2)
|
||||
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4)
|
||||
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3)
|
||||
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=WNA16MoEBackend.TRITON,
|
||||
layer=torch.nn.Module(),
|
||||
quant_config=quant_config,
|
||||
input_dtype=None,
|
||||
w13=w13,
|
||||
w2=w2,
|
||||
w13_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
)
|
||||
|
||||
assert converted is not None
|
||||
assert torch.equal(converted[0], w13.contiguous().view(torch.uint8))
|
||||
assert torch.equal(converted[1], w2.contiguous().view(torch.uint8))
|
||||
assert torch.equal(converted[2], w13_scale)
|
||||
assert torch.equal(converted[3], w2_scale)
|
||||
|
||||
|
||||
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
method = object.__new__(MoeWNA16Method)
|
||||
method.experts_cls = object
|
||||
method.wna16_backend = WNA16MoEBackend.HUMMING
|
||||
method.moe = object()
|
||||
quant_config = object()
|
||||
monkeypatch.setattr(
|
||||
method, "get_fused_moe_quant_config", lambda layer: quant_config
|
||||
)
|
||||
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
|
||||
captured = {}
|
||||
kernel = object()
|
||||
|
||||
def fake_make_wna16_moe_kernel(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return kernel
|
||||
|
||||
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
|
||||
|
||||
method._setup_kernel(layer)
|
||||
|
||||
assert method.moe_kernel is kernel
|
||||
assert captured["backend"] == WNA16MoEBackend.HUMMING
|
||||
|
||||
|
||||
def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
|
||||
qweight = torch.arange(32, dtype=torch.uint8).reshape(1, 4, 8)
|
||||
scales = torch.arange(16, dtype=torch.float16).reshape(1, 4, 4)
|
||||
qzeros = torch.arange(16, dtype=torch.uint8).reshape(1, 8, 2)
|
||||
|
||||
converted = _convert_moe_wna16_humming_tensors(
|
||||
{"qweight": qweight, "scales": scales, "qzeros": qzeros},
|
||||
has_zero_point=True,
|
||||
)
|
||||
|
||||
assert torch.equal(converted["weight"], qweight.view(torch.int32))
|
||||
assert converted["weight"].shape == (1, 4, 2)
|
||||
assert torch.equal(converted["weight_scale"], scales)
|
||||
expected_qzeros = (
|
||||
qzeros.transpose(-1, -2)
|
||||
.contiguous()
|
||||
.view(torch.int32)
|
||||
.transpose(-1, -2)
|
||||
.contiguous()
|
||||
)
|
||||
assert torch.equal(converted["zero_point"], expected_qzeros)
|
||||
assert converted["zero_point"].shape == (1, 2, 2)
|
||||
|
||||
|
||||
def test_moe_wna16_uses_humming_quant_config(monkeypatch):
|
||||
from vllm.model_executor.layers.quantization.utils import humming as humming_utils
|
||||
|
||||
method = object.__new__(MoeWNA16Method)
|
||||
method.wna16_backend = WNA16MoEBackend.HUMMING
|
||||
layer = object()
|
||||
quant_config = object()
|
||||
monkeypatch.setattr(
|
||||
humming_utils,
|
||||
"get_humming_moe_quant_config",
|
||||
lambda actual_layer, *args, **kwargs: (
|
||||
quant_config if actual_layer is layer else None
|
||||
),
|
||||
)
|
||||
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
|
||||
)
|
||||
@pytest.mark.parametrize("num_bits", [3, 5, 6, 7])
|
||||
def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
|
||||
num_bits,
|
||||
):
|
||||
pytest.importorskip("humming")
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
quant_args = QuantizationArgs(
|
||||
num_bits=num_bits,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
group_size=128,
|
||||
)
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
)
|
||||
moe_config.moe_backend = "humming"
|
||||
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=512,
|
||||
params_dtype=torch.float16,
|
||||
)
|
||||
|
||||
packed_hidden = (256 * num_bits + 31) // 32
|
||||
packed_intermediate = (512 * num_bits + 31) // 32
|
||||
assert method.wna16_backend == WNA16MoEBackend.HUMMING
|
||||
assert layer.w13_weight_packed.shape == (2, 1024, packed_hidden)
|
||||
assert layer.w2_weight_packed.shape == (2, 256, packed_intermediate)
|
||||
assert layer.w13_weight_scale.shape == (2, 1024, 2)
|
||||
assert layer.w2_weight_scale.shape == (2, 256, 4)
|
||||
assert layer.w13_weight_packed.dtype is torch.int32
|
||||
assert layer.w2_weight_scale.dtype is torch.float16
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
|
||||
)
|
||||
def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
|
||||
pytest.importorskip("humming")
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
quant_args = QuantizationArgs(
|
||||
num_bits=3,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
group_size=128,
|
||||
)
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
)
|
||||
moe_config.moe_backend = "humming"
|
||||
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
|
||||
layer = torch.nn.Module()
|
||||
layer.moe_config = moe_config
|
||||
layer.params_dtype = torch.bfloat16
|
||||
layer.layer_name = "test.humming_moe"
|
||||
layer._expert_routing_tables = lambda: (None, None, None)
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
layer.cuda()
|
||||
for parameter in layer.parameters():
|
||||
parameter.data.zero_()
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert method.wna16_backend == WNA16MoEBackend.HUMMING
|
||||
assert method.moe_kernel is not None
|
||||
assert set(layer.weight_schemas) == {"w13", "w2"}
|
||||
assert set(layer.humming_configs) == {"w13", "w2"}
|
||||
assert not hasattr(layer, "w13_weight_packed")
|
||||
assert not hasattr(layer, "w2_weight_packed")
|
||||
assert layer.w13_weight.dtype is torch.int32
|
||||
assert layer.w2_weight.dtype is torch.int32
|
||||
|
||||
|
||||
def test_moe_wna16_forwards_packed_modules_mapping_to_linear_delegate(monkeypatch):
|
||||
"""The linear delegate must receive packed_modules_mapping.
|
||||
|
||||
It is rebuilt from the raw HF quantization dict, which lists shard names and
|
||||
never fused ones, so without the mapping a fused layer resolves to
|
||||
`UnquantizedLinearMethod` and the checkpoint's qweight has nowhere to load.
|
||||
"""
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
|
||||
config = MoeWNA16Config(
|
||||
linear_quant_method="gptq",
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
has_zp=False,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=None,
|
||||
full_config={
|
||||
"bits": 4,
|
||||
"group_size": 128,
|
||||
"desc_act": False,
|
||||
"sym": True,
|
||||
"quant_method": "gptq",
|
||||
# As emitted by AutoGPTQ: shard names, never the fused name.
|
||||
"modules_in_block_to_quantize": [["mlp.gate_proj", "mlp.up_proj"]],
|
||||
},
|
||||
)
|
||||
config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
|
||||
|
||||
seen: dict[str, dict[str, list[str]]] = {}
|
||||
monkeypatch.setattr(
|
||||
AutoGPTQConfig,
|
||||
"get_quant_method",
|
||||
lambda self, layer, prefix: seen.setdefault(
|
||||
"mapping", self.packed_modules_mapping
|
||||
),
|
||||
)
|
||||
layer = ColumnParallelLinear.__new__(ColumnParallelLinear)
|
||||
config.get_quant_method(layer, "model.layers.0.mlp.gate_up_proj")
|
||||
|
||||
assert seen["mapping"] == {"gate_up_proj": ["gate_proj", "up_proj"]}
|
||||
|
||||
|
||||
def test_xpu_platform_supports_moe_wna16():
|
||||
"""Regression guard for the XPU quantization allowlist."""
|
||||
try:
|
||||
from vllm.platforms.xpu import XPUPlatform
|
||||
except ImportError:
|
||||
pytest.skip("vllm_xpu_kernels not importable outside an XPU stack")
|
||||
|
||||
assert "moe_wna16" in XPUPlatform.supported_quantization
|
||||
|
||||
|
||||
def _channelwise_int4_args() -> QuantizationArgs:
|
||||
"""A per-channel int4 checkpoint, which leaves ``group_size`` unset."""
|
||||
args = QuantizationArgs(
|
||||
num_bits=4,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.CHANNEL,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
)
|
||||
assert args.group_size is None, "premise: CHANNEL leaves group_size unset"
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="check_moe_marlin_supports_config rejects every config on ROCm",
|
||||
)
|
||||
@pytest.mark.parametrize("backend", [WNA16MoEBackend.MARLIN, WNA16MoEBackend.TRITON])
|
||||
def test_wna16_oracle_accepts_unset_group_size(backend):
|
||||
"""Both backends must *accept* a per-channel config, not just survive it.
|
||||
|
||||
-1 is a supported Marlin group size and the shapes below pass the Marlin
|
||||
tiling checks, while Triton never reads group_size for QuantizationArgs.
|
||||
A reason string from either backend would mean the unset group_size cost
|
||||
the layer its preferred kernel instead of raising TypeError.
|
||||
"""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
|
||||
# hidden_dim % 128 and intermediate % 64 must hold, or the Marlin shape
|
||||
# check rejects the config before group_size is ever read.
|
||||
reason = _backend_incompatibility_reason(
|
||||
backend=backend,
|
||||
moe_config=make_dummy_moe_config(
|
||||
num_experts=2, hidden_dim=256, intermediate_size=512
|
||||
),
|
||||
quant_config=_channelwise_int4_args(),
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
|
||||
assert reason is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Marlin is only a candidate WNA16 MoE backend on CUDA; elsewhere "
|
||||
"__init__ takes the non-Marlin branch, which rejects channelwise",
|
||||
)
|
||||
def test_compressed_tensors_wna16_moe_marlin_prep_with_unset_group_size():
|
||||
"""Load a per-channel checkpoint through the Marlin path end to end.
|
||||
|
||||
``__init__`` and Marlin weight prep read ``group_size`` independently, so
|
||||
both have to normalise the unset value. The post-repack shapes prove prep
|
||||
ran with the Marlin K/N rather than merely returning something.
|
||||
"""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
num_experts, hidden_size, intermediate_size = 2, 256, 512
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
)
|
||||
moe_config.moe_backend = "marlin"
|
||||
|
||||
method = CompressedTensorsWNA16MoEMethod(_channelwise_int4_args(), None, moe_config)
|
||||
assert method.wna16_backend == WNA16MoEBackend.MARLIN
|
||||
assert method.group_size == -1
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.intermediate_size_per_partition = intermediate_size
|
||||
layer._expert_routing_tables = lambda: (None, None, None)
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
intermediate_size_full=intermediate_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
layer.cuda()
|
||||
for parameter in layer.parameters():
|
||||
parameter.data.zero_()
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
# gptq_marlin_moe_repack packs to (size_k // 16, size_n * 2) for int4; w13
|
||||
# is repacked with size_k=hidden_size, size_n=2*intermediate_size, and w2
|
||||
# the other way round. Channelwise keeps one scale group per channel.
|
||||
assert layer.w13_weight_packed.shape == (
|
||||
num_experts,
|
||||
hidden_size // 16,
|
||||
4 * intermediate_size,
|
||||
)
|
||||
assert layer.w2_weight_packed.shape == (
|
||||
num_experts,
|
||||
intermediate_size // 16,
|
||||
2 * hidden_size,
|
||||
)
|
||||
assert layer.w13_weight_scale.shape == (num_experts, 1, 2 * intermediate_size)
|
||||
assert layer.w2_weight_scale.shape == (num_experts, 1, hidden_size)
|
||||
|
||||
|
||||
def test_moe_wna16_w13_qzeros_shard_split_int8_asym_gptq(monkeypatch):
|
||||
"""8-bit asymmetric GPTQ MoE must split w13 zero-points by pack factor.
|
||||
|
||||
``w13_qzeros`` is packed along the intermediate dim by ``bit8_pack_factor``,
|
||||
so the gate/up (w1/w3) boundary in the weight loader is
|
||||
``shard_size // bit8_pack_factor``. Hardcoding ``shard_size // 2`` only
|
||||
holds for 4-bit and raises a shape mismatch when loading an 8-bit
|
||||
(unpacked) checkpoint.
|
||||
"""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
|
||||
monkeypatch.setattr(
|
||||
moe_wna16, "get_tp_group", lambda: SimpleNamespace(device="cpu")
|
||||
)
|
||||
monkeypatch.setattr(moe_wna16, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
|
||||
num_experts, hidden, intermediate, group_size = 2, 256, 512, 128
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
hidden_dim=hidden,
|
||||
intermediate_size=intermediate,
|
||||
)
|
||||
quant_config = MoeWNA16Config(
|
||||
linear_quant_method="gptq",
|
||||
weight_bits=8,
|
||||
group_size=group_size,
|
||||
has_zp=True,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=[],
|
||||
full_config={},
|
||||
)
|
||||
method = object.__new__(MoeWNA16Method)
|
||||
method.quant_config = quant_config
|
||||
method.moe = moe_config
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.intermediate_size_per_partition = intermediate
|
||||
layer.moe_config = moe_config
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
params_dtype=torch.float16,
|
||||
weight_loader=lambda *a, **k: True,
|
||||
)
|
||||
|
||||
# 8-bit is unpacked along the intermediate dim: dim1 == 2 * intermediate.
|
||||
assert layer.w13_qzeros.shape == (
|
||||
num_experts,
|
||||
2 * intermediate,
|
||||
hidden // group_size,
|
||||
)
|
||||
loader = layer.w13_qzeros.weight_loader
|
||||
|
||||
# Raw GPTQ 8-bit qzeros per shard: int32 (num_groups, intermediate // 4);
|
||||
# the loader views them as uint8 -> (num_groups, intermediate), then
|
||||
# transposes and offsets by 1.
|
||||
num_groups = hidden // group_size
|
||||
raw_w1 = torch.arange(num_groups * (intermediate // 4), dtype=torch.int32).reshape(
|
||||
num_groups, intermediate // 4
|
||||
)
|
||||
raw_w3 = raw_w1 + 1000
|
||||
|
||||
# Must not raise (old code raised a shape mismatch here for 8-bit).
|
||||
loader(layer.w13_qzeros, raw_w1, "w13_qzeros", "w1", 0, False)
|
||||
loader(layer.w13_qzeros, raw_w3, "w13_qzeros", "w3", 0, False)
|
||||
|
||||
expected_w1 = raw_w1.view(torch.uint8).T + 1
|
||||
expected_w3 = raw_w3.view(torch.uint8).T + 1
|
||||
assert torch.equal(layer.w13_qzeros[0, :intermediate].cpu(), expected_w1)
|
||||
assert torch.equal(layer.w13_qzeros[0, intermediate:].cpu(), expected_w3)
|
||||
@@ -1,629 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the online MXFP4 weight quantization backends.
|
||||
|
||||
Each backend (Triton, aiter, XPU) quantizes a synthetic bf16/fp16
|
||||
tensor to MXFP4, the result is dequantized back to bf16/fp16, and compared
|
||||
against the pure-torch reference in `reference_mxfp4.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
AiterMxfp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.online.mxfp4 import (
|
||||
Mxfp4OnlineLinearMethod,
|
||||
Mxfp4OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import (
|
||||
QuarkOCP_MX_MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_ocp_mx import (
|
||||
QuarkOCP_MX,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
quark_quantize_weight_to_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
downcast_to_mxfp,
|
||||
mxfp4_quantize,
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import PlatformEnum, current_platform
|
||||
|
||||
from .reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
|
||||
|
||||
|
||||
def fix_negative_zeros(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Normalize FP4 e2m1 negative-zero codewords (0b1000) to positive zero
|
||||
(0b0000) in a packed uint8 tensor (two e2m1 values per byte, low nibble
|
||||
first).
|
||||
|
||||
-0.0 and +0.0 dequantize to the same float value, but different MXFP4
|
||||
quantization backends disagree on which one they emit for values that
|
||||
round to zero magnitude. This normalizes both packed representations so
|
||||
tensors from different backends can be compared with `torch.equal`.
|
||||
"""
|
||||
assert tensor.dtype == torch.uint8, (
|
||||
f"Expected a torch.uint8 tensor, got {tensor.dtype}"
|
||||
)
|
||||
|
||||
low_nibble = tensor & 0x0F
|
||||
high_nibble = tensor & 0xF0
|
||||
|
||||
low_nibble = torch.where(
|
||||
low_nibble == 0x08, torch.zeros_like(low_nibble), low_nibble
|
||||
)
|
||||
high_nibble = torch.where(
|
||||
high_nibble == 0x80, torch.zeros_like(high_nibble), high_nibble
|
||||
)
|
||||
|
||||
return low_nibble | high_nibble
|
||||
|
||||
|
||||
def assert_quantized_weights_equal(
|
||||
checkpoint_layer: torch.nn.Module,
|
||||
online_layer: torch.nn.Module,
|
||||
keys: tuple[str, ...],
|
||||
packed_weight_keys: tuple[str, ...],
|
||||
) -> None:
|
||||
"""Assert the checkpoint and online paths produced byte-identical weights.
|
||||
|
||||
`packed_weight_keys` lists the entries holding packed FP4 codewords, which
|
||||
need negative-zero normalization before comparison.
|
||||
"""
|
||||
for key in keys:
|
||||
checkpoint_tensor = getattr(checkpoint_layer, key).view(torch.uint8)
|
||||
online_tensor = getattr(online_layer, key).view(torch.uint8)
|
||||
|
||||
# NOTE: AMD Quark checkpoints use exclusively **positive** zeros,
|
||||
# while other mxfp4_quantize implementations from mxfp4_utils.py may not.
|
||||
if key in packed_weight_keys:
|
||||
checkpoint_tensor = fix_negative_zeros(checkpoint_tensor)
|
||||
online_tensor = fix_negative_zeros(online_tensor)
|
||||
|
||||
assert checkpoint_tensor.shape == online_tensor.shape
|
||||
assert checkpoint_tensor.dtype == online_tensor.dtype
|
||||
num_mismatched = (checkpoint_tensor != online_tensor).sum().item()
|
||||
total = checkpoint_tensor.numel()
|
||||
|
||||
assert torch.equal(checkpoint_tensor, online_tensor), (
|
||||
f"{key}: {num_mismatched}/{total} "
|
||||
f"({100 * num_mismatched / total:.4f}%) mismatched bytes"
|
||||
)
|
||||
|
||||
|
||||
def _skip_reason_if_unavailable(backend: str, dtype: torch.dtype) -> str | None:
|
||||
"""Return a skip reason if `backend` cannot run on the current host."""
|
||||
if backend == "triton":
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
return "Triton MXFP4 kernel requires a CUDA or ROCm GPU."
|
||||
return None
|
||||
if backend == "aiter":
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
return "aiter is not available/supported on this platform."
|
||||
if dtype != torch.bfloat16:
|
||||
return "aiter's dynamic_mxfp4_quant only supports bfloat16 input."
|
||||
return None
|
||||
if backend == "xpu":
|
||||
if not current_platform.is_xpu():
|
||||
return "not on XPU platform."
|
||||
return None
|
||||
if backend == "quark":
|
||||
try:
|
||||
import quark.torch.kernel.mx # noqa: F401
|
||||
except ImportError:
|
||||
return "amd-quark is not installed."
|
||||
return None
|
||||
raise ValueError(f"Unknown backend {backend}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("backend", ["triton", "aiter", "xpu", "quark"])
|
||||
def test_mxfp4_quantization_correctness(backend: str, dtype: torch.dtype):
|
||||
"""Tests that the different implementations of mxfp4_quantize
|
||||
in mxfp4_utils.py all match.
|
||||
"""
|
||||
skip_reason = _skip_reason_if_unavailable(backend, dtype)
|
||||
if skip_reason is not None:
|
||||
pytest.skip(skip_reason)
|
||||
|
||||
torch.manual_seed(3)
|
||||
|
||||
num_rows = 64
|
||||
hidden_size = 32 * 32 # multiple 32-element MXFP4 blocks
|
||||
device = current_platform.device_type
|
||||
|
||||
x = (torch.rand(num_rows, hidden_size, dtype=dtype, device=device) - 0.5) * 2
|
||||
# Vary the magnitude block-to-block so several scale exponents are
|
||||
# exercised, rather than a single one for the whole tensor.
|
||||
scalings = [2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]
|
||||
for i in range(hidden_size // 32):
|
||||
x[:, i * 32 : (i + 1) * 32] *= scalings[i % len(scalings)]
|
||||
|
||||
if backend == "triton":
|
||||
x_fp4, x_scale, _ = downcast_to_mxfp(x, axis=-1)
|
||||
elif backend == "aiter":
|
||||
x_fp4, x_scale = quark_quantize_weight_to_mxfp4(x)
|
||||
elif backend == "xpu":
|
||||
# TODO: enable this test on XPU
|
||||
pytest.skip("xpu mxfp4 quantization match to reference is untested")
|
||||
# from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
# xpu_mxfp4_quantize,
|
||||
# )
|
||||
|
||||
# x_fp4, x_scale = xpu_mxfp4_quantize(x)
|
||||
elif backend == "quark":
|
||||
result = quant_dequant_mxfp4(x, scale_calculation_mode="even")
|
||||
else:
|
||||
raise ValueError(f"Unknown backend {backend}")
|
||||
|
||||
if backend != "quark":
|
||||
result = dq_mxfp4_torch(x_fp4, x_scale, x.dtype)
|
||||
reference = qdq_mxfp4_torch(x, scale_calculation_mode="even")
|
||||
|
||||
assert torch.equal(result, reference)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only tested on ROCm/CUDA."
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int):
|
||||
"""TP-aligned MXFP4 shards match slices of unsharded quantization."""
|
||||
torch.manual_seed(3)
|
||||
|
||||
device = current_platform.device_type
|
||||
hidden_size = 1024
|
||||
intermediate_size = 512
|
||||
|
||||
dense_weight = torch.randn(
|
||||
hidden_size, intermediate_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
w13_weight = torch.randn(
|
||||
2,
|
||||
2 * intermediate_size,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
w2_weight = torch.randn(
|
||||
2,
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
dense_quantized, dense_scales = mxfp4_quantize(dense_weight)
|
||||
w13_quantized, w13_scales = mxfp4_quantize(w13_weight)
|
||||
w2_quantized, w2_scales = mxfp4_quantize(w2_weight)
|
||||
|
||||
for tp_rank in range(tp_size):
|
||||
output_per_partition = hidden_size // tp_size
|
||||
input_per_partition = intermediate_size // tp_size
|
||||
output_start = tp_rank * output_per_partition
|
||||
input_start = tp_rank * input_per_partition
|
||||
|
||||
column_quantized, column_scales = mxfp4_quantize(
|
||||
dense_weight.narrow(0, output_start, output_per_partition)
|
||||
)
|
||||
assert torch.equal(
|
||||
column_quantized,
|
||||
dense_quantized.narrow(0, output_start, output_per_partition),
|
||||
)
|
||||
assert torch.equal(
|
||||
column_scales,
|
||||
dense_scales.narrow(0, output_start, output_per_partition),
|
||||
)
|
||||
|
||||
row_quantized, row_scales = mxfp4_quantize(
|
||||
dense_weight.narrow(1, input_start, input_per_partition)
|
||||
)
|
||||
assert torch.equal(
|
||||
row_quantized,
|
||||
dense_quantized.narrow(1, input_start // 2, input_per_partition // 2),
|
||||
)
|
||||
assert torch.equal(
|
||||
row_scales,
|
||||
dense_scales.narrow(1, input_start // 32, input_per_partition // 32),
|
||||
)
|
||||
|
||||
w13_quantized_shard, w13_scales_shard = mxfp4_quantize(
|
||||
w13_weight.narrow(1, 2 * input_start, 2 * input_per_partition)
|
||||
)
|
||||
assert torch.equal(
|
||||
w13_quantized_shard,
|
||||
w13_quantized.narrow(1, 2 * input_start, 2 * input_per_partition),
|
||||
)
|
||||
assert torch.equal(
|
||||
w13_scales_shard,
|
||||
w13_scales.narrow(1, 2 * input_start, 2 * input_per_partition),
|
||||
)
|
||||
|
||||
w2_quantized_shard, w2_scales_shard = mxfp4_quantize(
|
||||
w2_weight.narrow(2, input_start, input_per_partition)
|
||||
)
|
||||
assert torch.equal(
|
||||
w2_quantized_shard,
|
||||
w2_quantized.narrow(2, input_start // 2, input_per_partition // 2),
|
||||
)
|
||||
assert torch.equal(
|
||||
w2_scales_shard,
|
||||
w2_scales.narrow(2, input_start // 32, input_per_partition // 32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only tested on ROCm/CUDA."
|
||||
)
|
||||
@pytest.mark.parametrize("moe_backend", ["aiter", "emulation"])
|
||||
@pytest.mark.parametrize(
|
||||
"unpadded_hidden_size,unpadded_intermediate_size",
|
||||
[
|
||||
pytest.param(256, 256, id="no_padding"),
|
||||
# Not multiples of the MXFP4 block size (32), so that every backend --
|
||||
# including emulation, whose round-up granularity is 32 -- has to pad.
|
||||
pytest.param(240, 208, id="padding"),
|
||||
],
|
||||
)
|
||||
def test_online_mxfp4_moe_matches_quark(
|
||||
moe_backend: MoEBackend,
|
||||
unpadded_hidden_size: int,
|
||||
unpadded_intermediate_size: int,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Ensures `Mxfp4OnlineMoEMethod` (online quantization)
|
||||
and `QuarkOCP_MX_MoEMethod` (AMD Quark checkpoints) produce the same weights,
|
||||
with same MOE backend used.
|
||||
|
||||
The `padding` case additionally covers `maybe_roundup_sizes`: the online
|
||||
path allocates its bf16 weights with `torch.empty_strided` and the loader
|
||||
only writes the unpadded slice, so the padding is filled with NaN here to
|
||||
reproduce that uninitialized state. It must be zeroed before quantization
|
||||
for the two paths to agree, since Quark allocates its padding with
|
||||
`torch.zeros`.
|
||||
"""
|
||||
if moe_backend == "aiter":
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
if current_platform.is_cuda():
|
||||
pytest.skip(
|
||||
"mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4 requires "
|
||||
"rocm_aiter_ops in weight conversion, not compatible on cuda"
|
||||
)
|
||||
elif moe_backend == "emulation":
|
||||
# `OCP_MXQuantizationEmulationTritonExperts.is_supported_config` gates
|
||||
# on `has_quark()`, but its weight processing does not require it.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.experts."
|
||||
"ocp_mx_emulation_moe.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
|
||||
num_experts = 4
|
||||
device = current_platform.device_type
|
||||
|
||||
def make_layer(prefix: str) -> RoutedExperts:
|
||||
runner = FusedMoEFactory(
|
||||
num_experts=num_experts,
|
||||
top_k=2,
|
||||
hidden_size=unpadded_hidden_size,
|
||||
intermediate_size=unpadded_intermediate_size,
|
||||
prefix=prefix,
|
||||
)
|
||||
layer = runner.routed_experts
|
||||
layer.moe_config.moe_backend = moe_backend
|
||||
return layer
|
||||
|
||||
# `create_weights` implementations use plain `torch.zeros`/`torch.randn`
|
||||
# with no explicit device, so without this context they would default
|
||||
# to CPU and diverge from the (GPU) tensors produced by
|
||||
# `mxfp4_quantize`/`replace_parameter`.
|
||||
with torch.device(device):
|
||||
checkpoint_layer = make_layer("checkpoint_layer")
|
||||
online_layer = make_layer("online_layer")
|
||||
|
||||
checkpoint_method = QuarkOCP_MX_MoEMethod(
|
||||
moe=checkpoint_layer.moe_config,
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
online_method = Mxfp4OnlineMoEMethod(moe=online_layer.moe_config)
|
||||
|
||||
# `RoutedExperts.__init__` applies this round-up in production; these
|
||||
# layers are built without a quant config, so it is applied explicitly.
|
||||
# Both methods must agree on the padded sizes.
|
||||
roundup_arguments = dict(
|
||||
hidden_size=unpadded_hidden_size,
|
||||
intermediate_size_per_partition=unpadded_intermediate_size,
|
||||
act_dtype=torch.bfloat16,
|
||||
moe_parallel_config=online_layer.moe_config.moe_parallel_config,
|
||||
)
|
||||
hidden_size, intermediate_size = online_method.maybe_roundup_sizes(
|
||||
**roundup_arguments
|
||||
)
|
||||
assert (
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
) == checkpoint_method.maybe_roundup_sizes(**roundup_arguments)
|
||||
|
||||
if unpadded_hidden_size == 240:
|
||||
# Padded case.
|
||||
assert hidden_size > unpadded_hidden_size
|
||||
assert intermediate_size > unpadded_intermediate_size
|
||||
|
||||
for layer in (checkpoint_layer, online_layer):
|
||||
assert layer.moe_config.hidden_dim_unpadded == unpadded_hidden_size
|
||||
assert (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
== unpadded_intermediate_size
|
||||
)
|
||||
layer.moe_config.hidden_dim = hidden_size
|
||||
layer.moe_config.intermediate_size_per_partition = intermediate_size
|
||||
|
||||
gate_up_weight = torch.randn(
|
||||
num_experts,
|
||||
2 * unpadded_intermediate_size,
|
||||
unpadded_hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
down_weight = torch.randn(
|
||||
num_experts,
|
||||
unpadded_hidden_size,
|
||||
unpadded_intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
scalings = [2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]
|
||||
for i, start in enumerate(range(0, unpadded_hidden_size, 32)):
|
||||
gate_up_weight[..., start : start + 32] *= scalings[i % len(scalings)]
|
||||
for i, start in enumerate(range(0, unpadded_intermediate_size, 32)):
|
||||
down_weight[..., start : start + 32] *= scalings[i % len(scalings)]
|
||||
|
||||
def scatter_into_padded(
|
||||
gate_up_weight: torch.Tensor,
|
||||
down_weight: torch.Tensor,
|
||||
padding_value: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Place the source weights into buffers of the padded size, the
|
||||
way the weight loader writes only the unpadded slice of a larger
|
||||
allocation."""
|
||||
w13 = torch.full(
|
||||
(num_experts, 2 * intermediate_size, hidden_size),
|
||||
padding_value,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
w2 = torch.full(
|
||||
(num_experts, hidden_size, intermediate_size),
|
||||
padding_value,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
# w13 stacks gate and up, each padded to `intermediate_size`.
|
||||
w13[:, :unpadded_intermediate_size, :unpadded_hidden_size] = gate_up_weight[
|
||||
:, :unpadded_intermediate_size, :
|
||||
]
|
||||
w13[
|
||||
:,
|
||||
intermediate_size : intermediate_size + unpadded_intermediate_size,
|
||||
:unpadded_hidden_size,
|
||||
] = gate_up_weight[:, unpadded_intermediate_size:, :]
|
||||
w2[:, :unpadded_hidden_size, :unpadded_intermediate_size] = down_weight
|
||||
return w13, w2
|
||||
|
||||
# Checkpoint path: pre-quantize the source weights to MXFP4 with the
|
||||
# ~same RTN recipe a real Quark checkpoint, then feed the
|
||||
# already-packed tensors into `QuarkOCP_MX_MoEMethod`. Quark's
|
||||
# `create_weights` allocates with `torch.zeros`, so its padding is zero.
|
||||
checkpoint_w13, checkpoint_w2 = scatter_into_padded(
|
||||
gate_up_weight, down_weight, padding_value=0.0
|
||||
)
|
||||
checkpoint_w13, checkpoint_w13_scale = mxfp4_quantize(checkpoint_w13)
|
||||
checkpoint_w2, checkpoint_w2_scale = mxfp4_quantize(checkpoint_w2)
|
||||
|
||||
checkpoint_method.create_weights(
|
||||
layer=checkpoint_layer,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
checkpoint_layer.w13_weight.data.copy_(checkpoint_w13)
|
||||
checkpoint_layer.w2_weight.data.copy_(checkpoint_w2)
|
||||
checkpoint_layer.w13_weight_scale.data.copy_(checkpoint_w13_scale)
|
||||
checkpoint_layer.w2_weight_scale.data.copy_(checkpoint_w2_scale)
|
||||
checkpoint_method.process_weights_after_loading(checkpoint_layer)
|
||||
|
||||
# Online path: feed the *same* raw bf16 source weights and let
|
||||
# `Mxfp4OnlineMoEMethod` quantize them during
|
||||
# `process_weights_after_loading`. The padding starts out
|
||||
# uninitialized, standing in for the materialized meta tensor.
|
||||
online_w13, online_w2 = scatter_into_padded(
|
||||
gate_up_weight, down_weight, padding_value=float("nan")
|
||||
)
|
||||
|
||||
online_method.create_weights(
|
||||
layer=online_layer,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
replace_parameter(online_layer, "w13_weight", online_w13)
|
||||
replace_parameter(online_layer, "w2_weight", online_w2)
|
||||
online_method.process_weights_after_loading(online_layer)
|
||||
|
||||
assert checkpoint_method.mxfp4_backend == online_method.mxfp4_backend
|
||||
|
||||
assert_quantized_weights_equal(
|
||||
checkpoint_layer,
|
||||
online_layer,
|
||||
keys=("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale"),
|
||||
packed_weight_keys=("w13_weight", "w2_weight"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only tested on ROCm/CUDA."
|
||||
)
|
||||
@pytest.mark.parametrize("linear_backend", ["emulation", "aiter", "marlin"])
|
||||
def test_online_mxfp4_dense_matches_quark(
|
||||
linear_backend: str, default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""Ensures `Mxfp4OnlineLinearMethod` (online quantization)
|
||||
and `QuarkOCP_MX` (AMD Quark checkpoints) produce the same weights,
|
||||
with same linear backend used.
|
||||
"""
|
||||
if linear_backend == "marlin" and not current_platform.is_cuda():
|
||||
# `MarlinMxFp4LinearKernel.process_weights_after_loading` is CUDA-only
|
||||
pytest.skip(
|
||||
"MarlinMxFp4LinearKernel.process_weights_after_loading is CUDA-only."
|
||||
)
|
||||
|
||||
if linear_backend == "aiter":
|
||||
# `AiterMxfp4LinearKernel` is registered only for ROCm in
|
||||
# `_POSSIBLE_MXFP4_KERNELS`, and `is_supported` gates on
|
||||
# `current_platform.supports_mx()`. Force it onto the CUDA
|
||||
# kernel list and bypass the platform gate so this backend can
|
||||
# be exercised on CUDA hosts too.
|
||||
|
||||
monkeypatch.setattr(
|
||||
AiterMxfp4LinearKernel,
|
||||
"is_supported",
|
||||
classmethod(lambda cls, compute_capability=None: (True, None)),
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
_POSSIBLE_MXFP4_KERNELS,
|
||||
PlatformEnum.CUDA,
|
||||
[
|
||||
AiterMxfp4LinearKernel,
|
||||
*_POSSIBLE_MXFP4_KERNELS.get(PlatformEnum.CUDA, []),
|
||||
],
|
||||
)
|
||||
elif linear_backend == "emulation":
|
||||
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
|
||||
# EmulationMxfp4LinearKernel.process_weights_after_loading does not require it.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
default_vllm_config.kernel_config.linear_backend = linear_backend
|
||||
|
||||
input_size = 256
|
||||
output_size = 128
|
||||
device = current_platform.device_type
|
||||
|
||||
# `create_weights` implementations use plain `torch.empty` with no explicit
|
||||
# device, so without this context they would default to CPU and diverge
|
||||
# from the (GPU) tensors produced by `mxfp4_quantize`/`replace_parameter`.
|
||||
with torch.device(device):
|
||||
checkpoint_layer = torch.nn.Module()
|
||||
online_layer = torch.nn.Module()
|
||||
for layer in (checkpoint_layer, online_layer):
|
||||
# `LinearBase` subclasses normally set these attributes on the
|
||||
# layer before `create_weights` is called; the marlin kernel's
|
||||
# `process_weights_after_loading` reads them directly off the
|
||||
# layer, so a bare `torch.nn.Module` needs them set explicitly.
|
||||
layer.input_size = input_size
|
||||
layer.output_size = output_size
|
||||
layer.input_size_per_partition = input_size
|
||||
layer.output_size_per_partition = output_size
|
||||
layer.params_dtype = torch.bfloat16
|
||||
|
||||
weight = torch.randn(output_size, input_size, dtype=torch.bfloat16)
|
||||
|
||||
scalings = [2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]
|
||||
for i in range(input_size // 32):
|
||||
weight[:, i * 32 : (i + 1) * 32] *= scalings[i % len(scalings)]
|
||||
|
||||
# Checkpoint path: pre-quantize the source weight to MXFP4 with the
|
||||
# ~same RTN recipe as a real Quark checkpoint, then feed the
|
||||
# already-packed tensors into `QuarkOCP_MX`.
|
||||
checkpoint_weight, checkpoint_weight_scale = mxfp4_quantize(weight)
|
||||
|
||||
checkpoint_scheme = QuarkOCP_MX(
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
checkpoint_scheme.create_weights(
|
||||
layer=checkpoint_layer,
|
||||
output_partition_sizes=[output_size],
|
||||
input_size_per_partition=input_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
checkpoint_layer.weight.data.copy_(checkpoint_weight)
|
||||
checkpoint_layer.weight_scale.data.copy_(checkpoint_weight_scale)
|
||||
checkpoint_scheme.process_weights_after_loading(checkpoint_layer)
|
||||
|
||||
# Online path: feed the *same* raw bf16 source weight and let
|
||||
# `Mxfp4OnlineLinearMethod` quantize it during
|
||||
# `process_weights_after_loading`.
|
||||
online_method = Mxfp4OnlineLinearMethod()
|
||||
online_method.create_weights(
|
||||
layer=online_layer,
|
||||
input_size_per_partition=input_size,
|
||||
output_partition_sizes=[output_size],
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
replace_parameter(online_layer, "weight", weight.clone())
|
||||
online_method.process_weights_after_loading(online_layer)
|
||||
|
||||
assert type(checkpoint_scheme.ocp_mx_linear) is type(online_method.kernel)
|
||||
|
||||
assert_quantized_weights_equal(
|
||||
checkpoint_layer,
|
||||
online_layer,
|
||||
keys=("weight", "weight_scale"),
|
||||
packed_weight_keys=("weight",),
|
||||
)
|
||||
|
||||
|
||||
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
|
||||
scheme = object.__new__(QuarkOCP_MX)
|
||||
layer = torch.nn.Module()
|
||||
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"input size per partition of 80, which must be divisible by "
|
||||
"the OCP MX group size 32"
|
||||
),
|
||||
):
|
||||
scheme.create_weights(
|
||||
layer=layer,
|
||||
output_partition_sizes=[2560],
|
||||
input_size_per_partition=80,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *_: None,
|
||||
)
|
||||
@@ -1,725 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for per-token-head KV cache quantization (INT4, INT8 and FP8).
|
||||
|
||||
Covers:
|
||||
- Per-token-head Triton reshape-and-cache kernel
|
||||
- Round-trip quantize/dequantize accuracy
|
||||
- process_weights_after_loading early-return path
|
||||
- End-to-end integration with Triton unified attention kernel
|
||||
|
||||
Run: pytest tests/quantization/test_per_token_kv_cache.py -v -s
|
||||
"""
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.int4_per_token_head import single_rht
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode, is_quantized_kv_cache
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
# Skip entire module if no CUDA/ROCm GPU available
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="Per-token-head KV cache tests require GPU.",
|
||||
),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test parameters
|
||||
# ---------------------------------------------------------------------------
|
||||
NUM_TOKENS = [1, 7, 42]
|
||||
NUM_KV_HEADS = [1, 4, 8]
|
||||
HEAD_SIZES = [64, 128]
|
||||
BLOCK_SIZES = [16]
|
||||
SEEDS = [0]
|
||||
|
||||
# Platform-dependent FP8 dtype and range
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP8_MIN, FP8_MAX = get_fp8_min_max()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-dtype quantization config
|
||||
# ---------------------------------------------------------------------------
|
||||
@dataclass(frozen=True)
|
||||
class QuantConfig:
|
||||
"""Quantization parameters for a given cache dtype."""
|
||||
|
||||
cache_dtype: torch.dtype # torch.int8 or FP8_DTYPE
|
||||
kv_cache_dtype_str: str # "int8_per_token_head" or "fp8_per_token_head"
|
||||
quant_max: float
|
||||
quant_min: float
|
||||
kv_quant_mode: KVQuantMode
|
||||
# INT8 rounds explicitly; FP8 relies on dtype cast rounding.
|
||||
rounds_before_store: bool
|
||||
|
||||
|
||||
INT8_CONFIG = QuantConfig(
|
||||
cache_dtype=torch.int8,
|
||||
kv_cache_dtype_str="int8_per_token_head",
|
||||
quant_max=127.0,
|
||||
quant_min=-128.0,
|
||||
kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
|
||||
rounds_before_store=True,
|
||||
)
|
||||
FP8_CONFIG = QuantConfig(
|
||||
cache_dtype=FP8_DTYPE,
|
||||
kv_cache_dtype_str="fp8_per_token_head",
|
||||
quant_max=FP8_MAX,
|
||||
quant_min=FP8_MIN,
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
|
||||
rounds_before_store=False,
|
||||
)
|
||||
INT4_CONFIG = QuantConfig(
|
||||
cache_dtype=torch.uint8,
|
||||
kv_cache_dtype_str="int4_per_token_head",
|
||||
quant_max=7.0,
|
||||
quant_min=-8.0,
|
||||
kv_quant_mode=KVQuantMode.INT4_PER_TOKEN_HEAD,
|
||||
# Unused for int4 (handled by its own rint path); kept for the dataclass.
|
||||
rounds_before_store=False,
|
||||
)
|
||||
QUANT_CONFIGS = [INT4_CONFIG, INT8_CONFIG, FP8_CONFIG]
|
||||
|
||||
|
||||
@pytest.fixture(params=QUANT_CONFIGS, ids=["int4", "int8", "fp8"])
|
||||
def qcfg(request) -> QuantConfig:
|
||||
return request.param
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _quantize_per_token_head_ref(
|
||||
data: torch.Tensor, # [num_tokens, num_heads, head_size]
|
||||
cfg: QuantConfig,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reference per-token-head quantization (one scale per token per head).
|
||||
|
||||
Returns (quantized, scales) where scales is [num_tokens, num_heads].
|
||||
"""
|
||||
absmax = data.float().abs().amax(dim=2) # [num_tokens, num_heads]
|
||||
scales = (absmax / cfg.quant_max).clamp(min=1e-6)
|
||||
scaled = data.float() * (1.0 / scales[:, :, None])
|
||||
if cfg.rounds_before_store:
|
||||
q = scaled.round().clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
|
||||
else:
|
||||
q = scaled.clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
|
||||
return q, scales
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 1. is_quantized_kv_cache / get_kv_quant_mode
|
||||
# ===========================================================================
|
||||
class TestIsQuantizedKvCache:
|
||||
def test_fp8_variants(self):
|
||||
assert is_quantized_kv_cache("fp8")
|
||||
assert is_quantized_kv_cache("fp8_e4m3")
|
||||
assert is_quantized_kv_cache("fp8_e5m2")
|
||||
|
||||
def test_int4_per_token_head(self):
|
||||
assert is_quantized_kv_cache("int4_per_token_head")
|
||||
|
||||
def test_int8_per_token_head(self):
|
||||
assert is_quantized_kv_cache("int8_per_token_head")
|
||||
|
||||
def test_fp8_per_token_head(self):
|
||||
assert is_quantized_kv_cache("fp8_per_token_head")
|
||||
|
||||
def test_auto(self):
|
||||
assert not is_quantized_kv_cache("auto")
|
||||
|
||||
def test_bfloat16(self):
|
||||
assert not is_quantized_kv_cache("bfloat16")
|
||||
|
||||
def test_kv_quant_mode_int4(self):
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
assert (
|
||||
get_kv_quant_mode("int4_per_token_head") == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
)
|
||||
|
||||
def test_kv_quant_mode_int8(self):
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
assert (
|
||||
get_kv_quant_mode("int8_per_token_head") == KVQuantMode.INT8_PER_TOKEN_HEAD
|
||||
)
|
||||
|
||||
def test_kv_quant_mode_fp8(self):
|
||||
from vllm.v1.kv_cache_interface import get_kv_quant_mode
|
||||
|
||||
assert get_kv_quant_mode("fp8_per_token_head") == KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 2. Triton per-token-head kernel (reshape-and-cache)
|
||||
# ===========================================================================
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_KV_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_reshape_and_cache_per_token_head(
|
||||
qcfg: QuantConfig,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
seed: int,
|
||||
):
|
||||
"""Test triton_reshape_and_cache_flash_per_token_head_quant kernel."""
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_per_token_head_quant,
|
||||
)
|
||||
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 4
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
cache_head_size = head_size // 2 if is_int4 else head_size
|
||||
|
||||
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
|
||||
key_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
value_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
|
||||
num_slots = block_size * num_blocks
|
||||
slot_mapping = torch.tensor(
|
||||
random.sample(range(num_slots), num_tokens), dtype=torch.long
|
||||
)
|
||||
|
||||
triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
)
|
||||
|
||||
# INT4 (RHT + asymmetric), INT8/FP8 have different dequant paths. Only
|
||||
# INT8/FP8 can be compared to a PyTorch reference.
|
||||
if not is_int4:
|
||||
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, qcfg)
|
||||
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, qcfg)
|
||||
|
||||
for i, slot in enumerate(slot_mapping.tolist()):
|
||||
blk = slot // block_size
|
||||
off = slot % block_size
|
||||
|
||||
if is_int4:
|
||||
# Coarser quantization → wider tolerance.
|
||||
deq_atol = deq_rtol = 0.5
|
||||
for label, data, cache, sc in [
|
||||
("key", key, key_cache, k_scale_cache),
|
||||
("val", value, value_cache, v_scale_cache),
|
||||
]:
|
||||
packed_scale = sc[blk, off] # [num_heads] float32
|
||||
scale_bits = packed_scale.view(torch.int32)
|
||||
zp = (scale_bits & 0xF).to(torch.float32)
|
||||
clean_scale = (scale_bits & -16).view(torch.float32)
|
||||
|
||||
packed = cache[blk, off]
|
||||
lo = (packed & 0xF).to(torch.float32)
|
||||
hi = ((packed >> 4) & 0xF).to(torch.float32)
|
||||
full = torch.zeros(num_heads, head_size, dtype=torch.float32)
|
||||
full[:, 0::2] = lo
|
||||
full[:, 1::2] = hi
|
||||
# Asymmetric dequant in RHT domain, then IRHT/d → original
|
||||
deq_rht = (full - zp[:, None]) * clean_scale[:, None]
|
||||
deq = single_rht(deq_rht, inverse=True) / head_size
|
||||
ref_deq = data[i].float()
|
||||
torch.testing.assert_close(deq, ref_deq, atol=deq_atol, rtol=deq_rtol)
|
||||
else:
|
||||
actual_k_scale = k_scale_cache[blk, off] # [num_heads]
|
||||
k_deq = key_cache[blk, off].float() * actual_k_scale[:, None]
|
||||
k_ref_deq = key[i].float()
|
||||
torch.testing.assert_close(
|
||||
k_deq,
|
||||
k_ref_deq,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
)
|
||||
actual_v_scale = v_scale_cache[blk, off] # [num_heads]
|
||||
v_deq = value_cache[blk, off].float() * actual_v_scale[:, None]
|
||||
v_ref_deq = value[i].float()
|
||||
torch.testing.assert_close(
|
||||
v_deq,
|
||||
v_ref_deq,
|
||||
atol=0.1,
|
||||
rtol=0.1,
|
||||
)
|
||||
# Per-head scales: [num_heads]
|
||||
torch.testing.assert_close(
|
||||
k_scale_cache[blk, off], ref_k_scales[i], atol=1e-4, rtol=1e-3
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
v_scale_cache[blk, off], ref_v_scales[i], atol=1e-4, rtol=1e-3
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 3. Per-token-head round-trip accuracy (quantize -> dequantize)
|
||||
# ===========================================================================
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16])
|
||||
@pytest.mark.parametrize("num_heads", [4])
|
||||
@pytest.mark.parametrize("head_size", [128])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@torch.inference_mode()
|
||||
def test_per_token_head_round_trip_accuracy(
|
||||
qcfg: QuantConfig,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
):
|
||||
"""Verify per-token-head round-trip: kernel dequant matches reference.
|
||||
|
||||
INT8: round-to-nearest before int8 store.
|
||||
FP8: hardware cast (clamp then cast).
|
||||
"""
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_per_token_head_quant,
|
||||
)
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
set_random_seed(42)
|
||||
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 2
|
||||
cache_head_size = head_size // 2 if is_int4 else head_size
|
||||
|
||||
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
|
||||
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
|
||||
|
||||
key_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
value_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.long)
|
||||
|
||||
triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
)
|
||||
|
||||
rt_atol = 0.5 if is_int4 else 0.1
|
||||
|
||||
for i in range(num_tokens):
|
||||
blk = i // block_size
|
||||
off = i % block_size
|
||||
|
||||
for label, data, cache, sc in [
|
||||
("key", key, key_cache, k_scale_cache),
|
||||
("val", value, value_cache, v_scale_cache),
|
||||
]:
|
||||
for h in range(num_heads):
|
||||
orig = data[i, h].float()
|
||||
actual_sc = sc[blk, off, h]
|
||||
if is_int4:
|
||||
sc_bits = actual_sc.view(torch.int32)
|
||||
zp = (sc_bits & 0xF).to(torch.float32)
|
||||
clean_sc = (sc_bits & -16).view(torch.float32)
|
||||
packed = cache[blk, off, h]
|
||||
lo = (packed & 0xF).to(torch.float32)
|
||||
hi = ((packed >> 4) & 0xF).to(torch.float32)
|
||||
full = torch.zeros(head_size)
|
||||
full[0::2] = lo
|
||||
full[1::2] = hi
|
||||
deq_rht = (full - zp) * clean_sc
|
||||
actual_deq = (
|
||||
single_rht(deq_rht.unsqueeze(0), inverse=True).squeeze(0)
|
||||
/ head_size
|
||||
)
|
||||
else:
|
||||
actual_deq = cache[blk, off, h].float() * actual_sc
|
||||
torch.testing.assert_close(
|
||||
actual_deq,
|
||||
orig,
|
||||
atol=rt_atol,
|
||||
rtol=rt_atol,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_int8_per_token_head_raw_cache_matches_round_reference():
|
||||
"""INT8 cache writes should match round-to-nearest quantization exactly."""
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_per_token_head_quant,
|
||||
)
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
|
||||
head_size = 8
|
||||
block_size = 4
|
||||
|
||||
key = torch.tensor(
|
||||
[[[-127.0, -2.6, -2.4, -1.6, -1.4, -0.6, -0.4, 127.0]]],
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
value = -key
|
||||
|
||||
key_cache = torch.zeros(1, block_size, 1, head_size, dtype=torch.int8)
|
||||
value_cache = torch.zeros_like(key_cache)
|
||||
k_scale_cache = torch.ones(1, block_size, 1, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones_like(k_scale_cache)
|
||||
slot_mapping = torch.tensor([2], dtype=torch.long)
|
||||
|
||||
triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=INT8_CONFIG.kv_quant_mode,
|
||||
)
|
||||
|
||||
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, INT8_CONFIG)
|
||||
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, INT8_CONFIG)
|
||||
|
||||
slot = slot_mapping.item()
|
||||
blk = slot // block_size
|
||||
off = slot % block_size
|
||||
assert torch.equal(key_cache[blk, off], ref_k_quant[0])
|
||||
assert torch.equal(value_cache[blk, off], ref_v_quant[0])
|
||||
torch.testing.assert_close(k_scale_cache[blk, off], ref_k_scales[0])
|
||||
torch.testing.assert_close(v_scale_cache[blk, off], ref_v_scales[0])
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 4. Negative slot mapping (padding tokens should be skipped)
|
||||
# ===========================================================================
|
||||
@torch.inference_mode()
|
||||
def test_per_token_head_negative_slot_skipped(qcfg: QuantConfig):
|
||||
"""Tokens with slot_mapping=-1 should leave the cache unchanged."""
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_per_token_head_quant,
|
||||
)
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
num_tokens = 4
|
||||
num_heads = 2
|
||||
head_size = 64
|
||||
block_size = 16
|
||||
num_blocks = 2
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
cache_head_size = head_size // 2 if is_int4 else head_size
|
||||
|
||||
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
|
||||
|
||||
key_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
value_cache = torch.zeros(
|
||||
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
|
||||
)
|
||||
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
|
||||
|
||||
slot_mapping = torch.tensor([0, -1, 1, -1], dtype=torch.long)
|
||||
|
||||
key_cache_before = key_cache.clone()
|
||||
val_cache_before = value_cache.clone()
|
||||
|
||||
triton_reshape_and_cache_flash_per_token_head_quant(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
k_scale_cache,
|
||||
v_scale_cache,
|
||||
slot_mapping,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
)
|
||||
|
||||
# Slots 0 and 1 should have been written (tokens 0 and 2)
|
||||
assert not torch.equal(key_cache[0, 0], key_cache_before[0, 0])
|
||||
assert not torch.equal(key_cache[0, 1], key_cache_before[0, 1])
|
||||
assert not torch.equal(value_cache[0, 0], val_cache_before[0, 0])
|
||||
|
||||
# All other slots should be unchanged
|
||||
assert torch.equal(key_cache[0, 2:], key_cache_before[0, 2:])
|
||||
assert torch.equal(key_cache[1], key_cache_before[1])
|
||||
assert torch.equal(value_cache[0, 2:], val_cache_before[0, 2:])
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 5. process_weights_after_loading -- per-token-head early return
|
||||
# ===========================================================================
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
|
||||
)
|
||||
def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
|
||||
"""Per-token-head should set _k_scale=1.0, _v_scale=1.0
|
||||
and delete checkpoint attrs."""
|
||||
from vllm.model_executor.layers.quantization.kv_cache import (
|
||||
BaseKVCacheMethod,
|
||||
)
|
||||
|
||||
layer = MagicMock()
|
||||
layer.kv_cache_dtype = kv_cache_dtype
|
||||
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.prob_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer._k_scale = torch.tensor(0.0)
|
||||
layer._v_scale = torch.tensor(0.0)
|
||||
layer._k_scale_float = 0.0
|
||||
layer._v_scale_float = 0.0
|
||||
|
||||
method = BaseKVCacheMethod.__new__(BaseKVCacheMethod)
|
||||
method.quant_config = MagicMock()
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer._k_scale_float == 1.0
|
||||
assert layer._v_scale_float == 1.0
|
||||
assert not hasattr(layer, "k_scale")
|
||||
assert not hasattr(layer, "v_scale")
|
||||
assert not hasattr(layer, "q_scale")
|
||||
assert not hasattr(layer, "prob_scale")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 6. Triton unified_attention -- per-token-head scale cache (INT4/INT8/FP8)
|
||||
# ===========================================================================
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
[
|
||||
[(1, 128)],
|
||||
[(1, 64), (1, 32)],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", [(4, 4)])
|
||||
@pytest.mark.parametrize("head_size", [128])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@torch.inference_mode()
|
||||
def test_triton_unified_attention_per_token_head_scale(
|
||||
qcfg: QuantConfig,
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
):
|
||||
"""End-to-end: quantized KV with per-token-head scale caches."""
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
set_random_seed(0)
|
||||
|
||||
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
|
||||
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [s[0] for s in seq_lens]
|
||||
kv_lens = [s[1] for s in seq_lens]
|
||||
num_query_heads, num_kv_heads = num_heads
|
||||
max_query_len = max(query_lens)
|
||||
max_kv_len = max(kv_lens)
|
||||
scale = head_size**-0.5
|
||||
num_blocks = 2048
|
||||
|
||||
query = torch.randn(
|
||||
sum(query_lens), num_query_heads, head_size, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
key_cache_bf16 = torch.randn(
|
||||
num_blocks, block_size, num_kv_heads, head_size, dtype=torch.bfloat16
|
||||
)
|
||||
value_cache_bf16 = torch.randn_like(key_cache_bf16)
|
||||
|
||||
if is_int4:
|
||||
# Asymmetric quantization reference (matches the Triton kernel).
|
||||
kf = key_cache_bf16.float()
|
||||
vf = value_cache_bf16.float()
|
||||
k_min = kf.amin(dim=-1)
|
||||
k_max = kf.amax(dim=-1)
|
||||
v_min = vf.amin(dim=-1)
|
||||
v_max = vf.amax(dim=-1)
|
||||
k_scale_cache = ((k_max - k_min) / 15.0).clamp(min=1e-6).to(torch.float32)
|
||||
v_scale_cache = ((v_max - v_min) / 15.0).clamp(min=1e-6).to(torch.float32)
|
||||
k_zp = (-k_min / k_scale_cache).round().clamp(0, 15)
|
||||
v_zp = (-v_min / v_scale_cache).round().clamp(0, 15)
|
||||
|
||||
key_cache_q_full = (
|
||||
(kf / k_scale_cache[..., None] + k_zp[..., None]).round().clamp(0, 15)
|
||||
)
|
||||
value_cache_q_full = (
|
||||
(vf / v_scale_cache[..., None] + v_zp[..., None]).round().clamp(0, 15)
|
||||
)
|
||||
|
||||
# Dequantized reference: x_hat = (q - zp) * scale
|
||||
key_cache_deq = (key_cache_q_full - k_zp[..., None]) * k_scale_cache[..., None]
|
||||
value_cache_deq = (value_cache_q_full - v_zp[..., None]) * v_scale_cache[
|
||||
..., None
|
||||
]
|
||||
|
||||
# Pack two uint4 values into one byte
|
||||
def _pack_int4(data_float):
|
||||
u = data_float.to(torch.uint8)
|
||||
lo = u[..., 0::2]
|
||||
hi = u[..., 1::2]
|
||||
return (lo & 0xF) | ((hi & 0xF) << 4)
|
||||
|
||||
key_cache_q = _pack_int4(key_cache_q_full)
|
||||
value_cache_q = _pack_int4(value_cache_q_full)
|
||||
|
||||
# Steganography: pack zp into low 4 bits of scale
|
||||
k_zp_int = k_zp.to(torch.int32)
|
||||
k_bits = k_scale_cache.view(torch.int32)
|
||||
k_scale_cache = ((k_bits & -16) | (k_zp_int & 0xF)).view(torch.float32)
|
||||
v_zp_int = v_zp.to(torch.int32)
|
||||
v_bits = v_scale_cache.view(torch.int32)
|
||||
v_scale_cache = ((v_bits & -16) | (v_zp_int & 0xF)).view(torch.float32)
|
||||
else:
|
||||
# Symmetric quantization for int8/fp8.
|
||||
k_absmax = key_cache_bf16.float().abs().amax(dim=-1)
|
||||
v_absmax = value_cache_bf16.float().abs().amax(dim=-1)
|
||||
k_scale_cache = (k_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
|
||||
v_scale_cache = (v_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
|
||||
scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None]
|
||||
scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None]
|
||||
|
||||
key_cache_q_full = scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max)
|
||||
value_cache_q_full = scaled_v.round().clamp(qcfg.quant_min, qcfg.quant_max)
|
||||
|
||||
key_cache_deq = key_cache_q_full * k_scale_cache[:, :, :, None]
|
||||
value_cache_deq = value_cache_q_full * v_scale_cache[:, :, :, None]
|
||||
|
||||
if not is_int4 and qcfg.rounds_before_store:
|
||||
key_cache_q = key_cache_q_full.to(qcfg.cache_dtype)
|
||||
value_cache_q = value_cache_q_full.to(qcfg.cache_dtype)
|
||||
elif not is_int4:
|
||||
key_cache_q = scaled_k.clamp(qcfg.quant_min, qcfg.quant_max).to(
|
||||
qcfg.cache_dtype
|
||||
)
|
||||
value_cache_q = scaled_v.clamp(qcfg.quant_min, qcfg.quant_max).to(
|
||||
qcfg.cache_dtype
|
||||
)
|
||||
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
kv_lens_t = torch.tensor(kv_lens, dtype=torch.int32)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
seq_threshold_3D = 0
|
||||
num_par_softmax_segments = 16
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
output_q = torch.empty_like(query)
|
||||
unified_attention(
|
||||
q=query,
|
||||
k=key_cache_q,
|
||||
v=value_cache_q,
|
||||
out=output_q,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
seqused_k=kv_lens_t,
|
||||
max_seqlen_q=max_query_len,
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
block_table=block_tables,
|
||||
softcap=0,
|
||||
q_descale=None,
|
||||
k_descale=None,
|
||||
v_descale=None,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
num_par_softmax_segments=num_par_softmax_segments,
|
||||
softmax_segm_output=softmax_segm_output,
|
||||
softmax_segm_max=softmax_segm_max,
|
||||
softmax_segm_expsum=softmax_segm_expsum,
|
||||
kv_quant_mode=qcfg.kv_quant_mode,
|
||||
k_scale_cache=k_scale_cache,
|
||||
v_scale_cache=v_scale_cache,
|
||||
)
|
||||
|
||||
output_ref = torch.empty_like(query)
|
||||
unified_attention(
|
||||
q=query,
|
||||
k=key_cache_deq.to(torch.bfloat16),
|
||||
v=value_cache_deq.to(torch.bfloat16),
|
||||
out=output_ref,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
seqused_k=kv_lens_t,
|
||||
max_seqlen_q=max_query_len,
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
block_table=block_tables,
|
||||
softcap=0,
|
||||
q_descale=None,
|
||||
k_descale=None,
|
||||
v_descale=None,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
num_par_softmax_segments=num_par_softmax_segments,
|
||||
softmax_segm_output=softmax_segm_output,
|
||||
softmax_segm_max=softmax_segm_max,
|
||||
softmax_segm_expsum=softmax_segm_expsum,
|
||||
)
|
||||
|
||||
# Coarser quantization → wider tolerance.
|
||||
if is_int4:
|
||||
# Hopper's attention reduction order can move a few BF16 elements by
|
||||
# just over 1.0 after INT4 quantization.
|
||||
atol = 1.1 if current_platform.is_device_capability_family(90) else 0.5
|
||||
rtol = 0.5
|
||||
else:
|
||||
atol, rtol = 5e-2, 5e-2
|
||||
torch.testing.assert_close(output_q, output_ref, atol=atol, rtol=rtol)
|
||||
@@ -1,152 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests register custom quantization config.
|
||||
|
||||
See https://github.com/vllm-project/vllm/issues/11926 for more details.
|
||||
|
||||
Run `pytest tests/quantization/test_register_quantization_config.py`.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase, # noqa: E501
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationMethods,
|
||||
get_quantization_config,
|
||||
register_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig, # noqa: E501
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class FakeQuantLinearMethod(UnquantizedLinearMethod):
|
||||
"""Fake quantization linear method for per-token dynamic quantization."""
|
||||
|
||||
def __init__(self, num_bits: int = 8) -> None:
|
||||
"""Initialize the quantization method."""
|
||||
super().__init__()
|
||||
self.num_bits = num_bits
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Perform fake quantization before the linear layer."""
|
||||
# Calculate the scales dynamically
|
||||
max_val = torch.amax(x, dim=(0, -1), keepdims=True)
|
||||
min_val = torch.amin(x, dim=(0, -1), keepdims=True)
|
||||
scales = (max_val - min_val) / (2**self.num_bits - 1)
|
||||
|
||||
# Fake quantize the input
|
||||
quant_x = torch.clamp(
|
||||
torch.round(x / scales),
|
||||
-(2 ** (self.num_bits - 1)),
|
||||
2 ** (self.num_bits - 1) - 1,
|
||||
)
|
||||
dequant_x = quant_x * scales
|
||||
|
||||
return F.linear(dequant_x, layer.weight, bias)
|
||||
|
||||
|
||||
@register_quantization_config("custom_quant")
|
||||
class CustomQuantConfig(QuantizationConfig):
|
||||
"""Custom quantization config for per-token dynamic fake quantization."""
|
||||
|
||||
def __init__(self, num_bits: int = 8) -> None:
|
||||
"""Initialize the quantization config."""
|
||||
super().__init__()
|
||||
self.num_bits = num_bits
|
||||
|
||||
def get_name(self) -> QuantizationMethods:
|
||||
"""Name of the quantization method."""
|
||||
return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names.
|
||||
|
||||
def get_supported_act_dtypes(self) -> list[torch.dtype]:
|
||||
"""List of supported activation dtypes."""
|
||||
return [torch.float16, torch.bfloat16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
"""Minimum GPU capability to support the quantization method."""
|
||||
return -1
|
||||
|
||||
@staticmethod
|
||||
def get_config_filenames() -> list[str]:
|
||||
"""List of filenames to search for in the model directory."""
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "CustomQuantConfig":
|
||||
"""Create a config class from the model's quantization config."""
|
||||
return CustomQuantConfig(num_bits=config.get("num_bits", 8))
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> FakeQuantLinearMethod | None:
|
||||
"""Get the quantize method to use for the quantized layer."""
|
||||
if isinstance(layer, LinearBase):
|
||||
return FakeQuantLinearMethod(num_bits=self.num_bits)
|
||||
return None
|
||||
|
||||
|
||||
def test_register_quantization_config(caplog_vllm):
|
||||
"""Test register custom quantization config."""
|
||||
# The quantization method `custom_quant` should be registered.
|
||||
assert get_quantization_config("custom_quant") == CustomQuantConfig
|
||||
|
||||
# The quantization method `custom_quant` is already exists,
|
||||
# should raise a debug message when re-registering it.
|
||||
with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
|
||||
register_quantization_config("custom_quant")(CustomQuantConfig)
|
||||
|
||||
assert any(
|
||||
"The quantization method 'custom_quant' already exists" in message
|
||||
for message in caplog_vllm.messages
|
||||
), "Expected a debug message when re-registering custom_quant"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
argnames="model",
|
||||
argvalues=[
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
],
|
||||
)
|
||||
def test_custom_quant(model, monkeypatch, dist_init, workspace_init):
|
||||
"""Test infer with the custom quantization method."""
|
||||
vllm_model, vllm_config = load_model_without_vllm_runner(
|
||||
model,
|
||||
quantization="custom_quant",
|
||||
)
|
||||
layer = vllm_model.model.layers[0]
|
||||
qkv_proj = layer.self_attn.qkv_proj
|
||||
|
||||
# Check the quantization method is FakeQuantLinearMethod.
|
||||
assert isinstance(qkv_proj.quant_method, FakeQuantLinearMethod)
|
||||
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
device_type = current_platform.device_type
|
||||
input_ids = torch.tensor([1, 2, 3, 4], device=device_type)
|
||||
positions = torch.arange(input_ids.numel(), device=device_type)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
|
||||
):
|
||||
hidden_states = vllm_model(input_ids, positions, None)
|
||||
logits = vllm_model.compute_logits(hidden_states)
|
||||
assert torch.isfinite(logits).all()
|
||||
@@ -1,146 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm equivalents of the quantized-MoE model startup/generation contract.
|
||||
|
||||
Like test_blackwell_moe.py, these tests use reduced models and dummy weights.
|
||||
They validate model/backend integration, not trained-model accuracy.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not on_gfx950(),
|
||||
reason="ROCm FP8/MXFP4 model integration requires gfx950",
|
||||
)
|
||||
|
||||
COMMON_OVERRIDES = {
|
||||
"num_hidden_layers": 4,
|
||||
"hidden_size": 512,
|
||||
"intermediate_size": 1024,
|
||||
"num_attention_heads": 8,
|
||||
"num_key_value_heads": 2,
|
||||
}
|
||||
|
||||
CASES = [
|
||||
pytest.param(
|
||||
"RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
|
||||
"efe1ced428db63e7ccbcf367334596f77e9af140",
|
||||
"triton",
|
||||
"TRITON_MLA",
|
||||
{
|
||||
"n_routed_experts": 8,
|
||||
"n_shared_experts": 1,
|
||||
"num_experts_per_tok": 2,
|
||||
"moe_intermediate_size": 512,
|
||||
},
|
||||
["--code-revision", "e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11"],
|
||||
id="deepseek-fp8-per-tensor-triton",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-30B-A3B-FP8",
|
||||
"d206ba732169f29bb77fbf80fc2c4b81d4d30782",
|
||||
"triton",
|
||||
"TRITON_ATTN",
|
||||
{
|
||||
"num_experts": 8,
|
||||
"num_experts_per_tok": 2,
|
||||
"moe_intermediate_size": 512,
|
||||
"head_dim": 64,
|
||||
},
|
||||
[],
|
||||
id="qwen3-fp8-block-triton",
|
||||
),
|
||||
pytest.param(
|
||||
"openai/gpt-oss-20b",
|
||||
"6cee5e81ee83917806bbde320786a8fb61efebee",
|
||||
"aiter",
|
||||
"TRITON_ATTN",
|
||||
{
|
||||
"num_local_experts": 8,
|
||||
"num_experts_per_tok": 2,
|
||||
"experts_per_token": 2,
|
||||
"layer_types": ["sliding_attention", "full_attention"] * 2,
|
||||
},
|
||||
[],
|
||||
id="gptoss-mxfp4-aiter",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False], ids=["eager", "compiled"])
|
||||
@pytest.mark.parametrize(
|
||||
"model,revision,moe_backend,attention_backend,overrides,extra_args", CASES
|
||||
)
|
||||
def test_rocm_quantized_moe_generation(
|
||||
model,
|
||||
revision,
|
||||
moe_backend,
|
||||
attention_backend,
|
||||
overrides,
|
||||
extra_args,
|
||||
enforce_eager,
|
||||
):
|
||||
"""Quantized MoE models warm up and return finite logits through serving."""
|
||||
args = [
|
||||
"--revision",
|
||||
revision,
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--trust-remote-code",
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-batched-tokens",
|
||||
"256",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--gpu-memory-utilization",
|
||||
"0.02",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(256 * 1024 * 1024),
|
||||
"--moe-backend",
|
||||
moe_backend,
|
||||
"--attention-backend",
|
||||
attention_backend,
|
||||
*extra_args,
|
||||
]
|
||||
if enforce_eager:
|
||||
args.append("--enforce-eager")
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model,
|
||||
args,
|
||||
env_dict={"VLLM_ROCM_USE_AITER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"},
|
||||
override_hf_configs={**COMMON_OVERRIDES, **overrides},
|
||||
max_wait_seconds=1200,
|
||||
) as server:
|
||||
prompts = [[1, 2, 3], [1, 2, 3, 4] * 4 + [5]]
|
||||
completion = server.get_client().completions.create(
|
||||
model=model,
|
||||
prompt=prompts,
|
||||
temperature=0,
|
||||
max_tokens=2,
|
||||
logprobs=1,
|
||||
extra_body={"ignore_eos": True, "skip_special_tokens": False},
|
||||
)
|
||||
assert len(completion.choices) == len(prompts)
|
||||
assert completion.usage is not None
|
||||
assert completion.usage.prompt_tokens == sum(map(len, prompts))
|
||||
assert completion.usage.completion_tokens == 2 * len(prompts)
|
||||
for choice in completion.choices:
|
||||
assert choice.finish_reason == "length"
|
||||
assert choice.text
|
||||
assert choice.logprobs is not None
|
||||
assert choice.logprobs.token_logprobs is not None
|
||||
assert len(choice.logprobs.token_logprobs) == 2
|
||||
assert all(
|
||||
p is not None and math.isfinite(p)
|
||||
for p in choice.logprobs.token_logprobs
|
||||
)
|
||||
@@ -1,413 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from packaging import version
|
||||
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
else:
|
||||
|
||||
def on_gfx950() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
TORCHAO_AVAILABLE = importlib.util.find_spec("torchao") is not None
|
||||
TORCHAO_VERSION_0_18_AVAILABLE = TORCHAO_AVAILABLE and version.parse(
|
||||
importlib.metadata.version("torchao")
|
||||
) >= version.parse("0.18.0")
|
||||
|
||||
|
||||
# Fix for leaking `torch.set_float32_matmul_precision("high")`
|
||||
# https://github.com/pytorch/ao/blob/v0.17.0/torchao/quantization/utils.py#L696
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_float32_matmul_precision():
|
||||
original_precision = torch.get_float32_matmul_precision()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.set_float32_matmul_precision(original_precision)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm() and current_platform.is_fp8_fnuz(),
|
||||
reason="Only fp8_fnuz supported on CDNA3 architecture",
|
||||
)
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
def test_pre_quantized_model(monkeypatch, dist_init, workspace_init):
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
"torchao-testing/opt-125m-Float8WeightOnlyConfig-v2-0.15.0",
|
||||
quantization="torchao",
|
||||
dtype="bfloat16",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE_TYPE)
|
||||
positions = torch.arange(input_ids.numel(), device=DEVICE_TYPE)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
|
||||
):
|
||||
hidden_states = model(input_ids, positions, None)
|
||||
logits = model.compute_logits(hidden_states)
|
||||
assert torch.isfinite(logits).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skip(
|
||||
reason="since torchao nightly is only compatible with torch nightly"
|
||||
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
|
||||
"torchao tests that requires newer versions (0.14.0.dev+) for now"
|
||||
)
|
||||
def test_opt_125m_awq_int4wo_model_loading_with_params(vllm_runner):
|
||||
torch._dynamo.reset()
|
||||
model_name = "torchao-testing/opt-125m-AWQConfig-Int4WeightOnlyConfig-v2-0.14.0.dev"
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
quantization="torchao",
|
||||
dtype="bfloat16",
|
||||
pt_load_map_location=f"{DEVICE_TYPE}:0",
|
||||
) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skipif(
|
||||
on_gfx950() and not TORCHAO_VERSION_0_18_AVAILABLE,
|
||||
reason="requires torchao>=0.18.0 on gfx950",
|
||||
)
|
||||
def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
|
||||
"""Testing online quantization, load_weights integration point,
|
||||
with config dict serialized to json string
|
||||
"""
|
||||
torch._dynamo.reset()
|
||||
model_name = "facebook/opt-125m"
|
||||
|
||||
import json
|
||||
|
||||
from torchao.core.config import config_to_dict
|
||||
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
||||
|
||||
torchao_quant_config = Float8DynamicActivationFloat8WeightConfig(
|
||||
granularity=PerRow()
|
||||
)
|
||||
hf_overrides = {
|
||||
"quantization_config_dict_json": json.dumps(
|
||||
config_to_dict(torchao_quant_config)
|
||||
)
|
||||
}
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
dtype="bfloat16",
|
||||
pt_load_map_location=f"{DEVICE_TYPE}:0",
|
||||
quantization="torchao",
|
||||
hf_overrides=hf_overrides,
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
|
||||
load_config = llm.llm.llm_engine.vllm_config.load_config
|
||||
model_config = llm.llm.llm_engine.vllm_config.model_config
|
||||
|
||||
def load_weights(model):
|
||||
model_loader = get_model_loader(load_config)
|
||||
assert isinstance(model_loader, DefaultModelLoader)
|
||||
weights_iterator = model_loader.get_all_weights(model_config, model)
|
||||
model.load_weights(weights_iterator)
|
||||
|
||||
llm.apply_model(load_weights)
|
||||
|
||||
reload_output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
assert output[0][0] == reload_output[0][0]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skipif(
|
||||
on_gfx950() and not TORCHAO_VERSION_0_18_AVAILABLE,
|
||||
reason="requires torchao>=0.18.0 on gfx950",
|
||||
)
|
||||
def test_online_quant_config_file(vllm_runner):
|
||||
"""Testing on the fly quantization, load_weights integration point,
|
||||
with config file
|
||||
"""
|
||||
torch._dynamo.reset()
|
||||
model_name = "facebook/opt-125m"
|
||||
import json
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from torchao.core.config import config_to_dict
|
||||
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
||||
|
||||
config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
||||
|
||||
with NamedTemporaryFile(mode="w", delete=False) as f:
|
||||
f.write(json.dumps(config_to_dict(config)))
|
||||
# close the file to save it
|
||||
f.close()
|
||||
config_file_name = str(f.name)
|
||||
|
||||
hf_overrides = {"quantization_config_file": config_file_name}
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
dtype="bfloat16",
|
||||
pt_load_map_location=f"{DEVICE_TYPE}:0",
|
||||
quantization="torchao",
|
||||
hf_overrides=hf_overrides,
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skipif(
|
||||
on_gfx950() and not TORCHAO_VERSION_0_18_AVAILABLE,
|
||||
reason="requires torchao>=0.18.0 on gfx950",
|
||||
)
|
||||
def test_reload_weights():
|
||||
import json
|
||||
|
||||
from torchao.core.config import config_to_dict
|
||||
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
torchao_quant_config = Float8DynamicActivationFloat8WeightConfig(
|
||||
granularity=PerRow()
|
||||
)
|
||||
|
||||
hf_overrides = {
|
||||
"quantization_config_dict_json": json.dumps(
|
||||
config_to_dict(torchao_quant_config)
|
||||
)
|
||||
}
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
dtype="bfloat16",
|
||||
load_format="dummy",
|
||||
enforce_eager=True,
|
||||
quantization="torchao",
|
||||
hf_overrides=hf_overrides,
|
||||
)
|
||||
# Update load format from `dummy` to `auto`
|
||||
llm.collective_rpc(
|
||||
"update_config", args=({"load_config": {"load_format": "auto"}},)
|
||||
)
|
||||
# Now reload real weights inplace
|
||||
llm.collective_rpc("reload_weights")
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0, top_p=0.95)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
# make sure it runs
|
||||
for output in outputs:
|
||||
generated_text = output.outputs[0].text
|
||||
assert generated_text
|
||||
# can also uncomment locally to make sure the generated
|
||||
# output makes sense
|
||||
# prompt = output.prompt
|
||||
# print(f"Prompt: {prompt!r}")
|
||||
# print(f"Output: {generated_text!r}")
|
||||
# print("-" * 60)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skip(
|
||||
reason="since torchao nightly is only compatible with torch nightly"
|
||||
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
|
||||
"torchao tests that requires newer versions (0.15.0.dev+) for now"
|
||||
)
|
||||
def test_safetensors_model_loading_with_params(vllm_runner):
|
||||
torch._dynamo.reset()
|
||||
# using this model to test safetensors loading with file sharding
|
||||
model_name = "torchao-testing/Qwen3-8B-INT4-0.15.0dev-safetensors"
|
||||
with vllm_runner(model_name=model_name, dtype="bfloat16") as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skip(
|
||||
reason="since torchao nightly is only compatible with torch nightly"
|
||||
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
|
||||
"torchao tests that requires newer versions (0.14.0.dev+) for now"
|
||||
)
|
||||
def test_opt_125m_module_fqn_to_config_regex_model(vllm_runner):
|
||||
torch._dynamo.reset()
|
||||
model_name = "torchao-testing/opt-125m-ModuleFqnToConfig-v1-regex-0.14.0.dev"
|
||||
with vllm_runner(
|
||||
model_name=model_name, dtype="bfloat16", pt_load_map_location=f"{DEVICE_TYPE}:0"
|
||||
) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skip(
|
||||
reason="since torchao nightly is only compatible with torch nightly"
|
||||
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
|
||||
"torchao tests that requires newer versions (0.14.0.dev+) for now"
|
||||
)
|
||||
def test_opt_125m_int4wo_model_running_preshuffled_kernel(vllm_runner, monkeypatch):
|
||||
"""We load a model with Int4Tensor (plain format) linear weights
|
||||
and verify that the weight is updated to Int4PreshuffledTensor
|
||||
after loading in vllm
|
||||
"""
|
||||
from torchao.quantization import Int4PreshuffledTensor
|
||||
from torchao.utils import _is_fbgemm_gpu_genai_available, is_sm_at_least_90
|
||||
|
||||
torch._dynamo.reset()
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
model_name = "torchao-testing/opt-125m-Int4WeightOnlyConfig-v2-0.14.0.dev"
|
||||
# Note: using enforce_eager=True because the `bf16i4bf16_shuffled` doesn't
|
||||
# have meta kernel implemented yet, can remove this flag after that is implemented
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
quantization="torchao",
|
||||
dtype="bfloat16",
|
||||
pt_load_map_location=f"{DEVICE_TYPE}:0",
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
|
||||
def has_int4_preshuffled_tensor_weight(model):
|
||||
return isinstance(
|
||||
model.model.decoder.layers[0].self_attn.qkv_proj.weight,
|
||||
Int4PreshuffledTensor,
|
||||
)
|
||||
|
||||
def get_weight_attrs(model):
|
||||
weight = model.model.decoder.layers[0].self_attn.qkv_proj.weight
|
||||
return [
|
||||
weight.requires_grad,
|
||||
weight.input_dim,
|
||||
weight.output_dim,
|
||||
hasattr(weight, "weight_loader"),
|
||||
]
|
||||
|
||||
llm_engine = llm.get_llm().llm_engine
|
||||
has_int4_preshuffled_tensor = any(
|
||||
llm_engine.apply_model(has_int4_preshuffled_tensor_weight)
|
||||
)
|
||||
weight_attrs = llm_engine.apply_model(get_weight_attrs)[0]
|
||||
|
||||
# making sure we are using Int4PreshuffledTensor on H100 GPU, when
|
||||
# fbgemm_gpu_genai
|
||||
# library is installed, otherwise it should be using Int4Tensor
|
||||
if _is_fbgemm_gpu_genai_available() and is_sm_at_least_90():
|
||||
assert has_int4_preshuffled_tensor
|
||||
else:
|
||||
assert not has_int4_preshuffled_tensor
|
||||
|
||||
assert weight_attrs == [False, 1, 0, True]
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
|
||||
@pytest.mark.skip(
|
||||
reason="since torchao nightly is only compatible with torch nightly"
|
||||
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
|
||||
"torchao tests that requires newer versions (0.14.0.dev+) for now"
|
||||
)
|
||||
def test_opt_125m_int4wo_model_running_preshuffled_kernel_online_quant(
|
||||
vllm_runner, monkeypatch
|
||||
):
|
||||
"""We load a bf16 model and online quantize the model to int4, then verify that
|
||||
the weights are updated to Int4PreshuffledTensor after online quantization
|
||||
"""
|
||||
from torchao.quantization import Int4PreshuffledTensor
|
||||
from torchao.utils import _is_fbgemm_gpu_genai_available, is_sm_at_least_90
|
||||
|
||||
torch._dynamo.reset()
|
||||
model_name = "facebook/opt-125m"
|
||||
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
import json
|
||||
|
||||
from torchao.core.config import config_to_dict
|
||||
from torchao.quantization import Int4WeightOnlyConfig
|
||||
|
||||
torchao_quant_config = Int4WeightOnlyConfig(
|
||||
group_size=128, int4_packing_format="plain"
|
||||
)
|
||||
hf_overrides = {
|
||||
"quantization_config_dict_json": json.dumps(
|
||||
config_to_dict(torchao_quant_config)
|
||||
)
|
||||
}
|
||||
|
||||
# Note: using enforce_eager=True because the `bf16i4bf16_shuffled` doesn't
|
||||
# have meta kernel implemented yet, can remove this flag after that is implemented
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
quantization="torchao",
|
||||
dtype="bfloat16",
|
||||
pt_load_map_location=f"{DEVICE_TYPE}:0",
|
||||
hf_overrides=hf_overrides,
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
|
||||
def has_int4_preshuffled_tensor_weight(model):
|
||||
return isinstance(
|
||||
model.model.decoder.layers[0].self_attn.qkv_proj.weight,
|
||||
Int4PreshuffledTensor,
|
||||
)
|
||||
|
||||
def get_weight_attrs(model):
|
||||
weight = model.model.decoder.layers[0].self_attn.qkv_proj.weight
|
||||
return [
|
||||
weight.requires_grad,
|
||||
weight.input_dim,
|
||||
weight.output_dim,
|
||||
hasattr(weight, "weight_loader"),
|
||||
]
|
||||
|
||||
llm_engine = llm.get_llm().llm_engine
|
||||
has_int4_preshuffled_tensor = any(
|
||||
llm_engine.apply_model(has_int4_preshuffled_tensor_weight)
|
||||
)
|
||||
weight_attrs = llm_engine.apply_model(get_weight_attrs)[0]
|
||||
|
||||
# making sure we are using Int4PreshuffledTensor on H100 GPU, when
|
||||
# fbgemm_gpu_genai
|
||||
# library is installed, otherwise it should be using Int4Tensor
|
||||
if _is_fbgemm_gpu_genai_available() and is_sm_at_least_90():
|
||||
assert has_int4_preshuffled_tensor
|
||||
else:
|
||||
assert not has_int4_preshuffled_tensor
|
||||
|
||||
assert weight_attrs == [False, 1, 0, True]
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
|
||||
|
||||
assert output
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
@@ -1,160 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import NvFp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.utils import flashinfer_fp4_moe
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_hidden_dim_for_fi,
|
||||
align_fp4_moe_weights_for_fi,
|
||||
)
|
||||
|
||||
|
||||
def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch):
|
||||
monkeypatch.setattr(flashinfer_fp4_moe, "swizzle_blockscale", lambda x: x)
|
||||
|
||||
num_experts = 3
|
||||
layer = SimpleNamespace(
|
||||
activation=SimpleNamespace(is_gated=False),
|
||||
moe_config=SimpleNamespace(
|
||||
moe_parallel_config=SimpleNamespace(enable_eplb=False)
|
||||
),
|
||||
)
|
||||
w13 = torch.zeros((num_experts, 2, 1), dtype=torch.uint8)
|
||||
w2 = torch.zeros((num_experts, 2, 1), dtype=torch.uint8)
|
||||
w13_scale = torch.zeros((num_experts, 2, 1), dtype=torch.float8_e4m3fn)
|
||||
w2_scale = torch.zeros((num_experts, 2, 1), dtype=torch.float8_e4m3fn)
|
||||
weight_scale = torch.ones(num_experts)
|
||||
|
||||
outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
backend=NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
layer=layer,
|
||||
w13=w13,
|
||||
w13_scale=w13_scale,
|
||||
w13_scale_2=weight_scale,
|
||||
a13_scale=torch.tensor([1.0, 2.0, 3.0]),
|
||||
w2=w2,
|
||||
w2_scale=w2_scale,
|
||||
w2_scale_2=weight_scale,
|
||||
a2_scale=torch.tensor([4.0, 5.0, 6.0]),
|
||||
is_act_and_mul=False,
|
||||
)
|
||||
a13_scale, a2_scale = outputs[3], outputs[7]
|
||||
|
||||
torch.testing.assert_close(a13_scale, torch.full((num_experts,), 3.0))
|
||||
torch.testing.assert_close(a2_scale, torch.full((num_experts,), 6.0))
|
||||
distinct_values = torch.arange(num_experts, dtype=torch.float32)
|
||||
a13_scale.copy_(distinct_values)
|
||||
a2_scale.copy_(distinct_values)
|
||||
torch.testing.assert_close(a13_scale, distinct_values)
|
||||
torch.testing.assert_close(a2_scale, distinct_values)
|
||||
|
||||
|
||||
def test_align_trtllm_fp4_moe_hidden_dim_noop():
|
||||
w13 = torch.arange(2 * 8 * 256, dtype=torch.uint8).reshape(2, 8, 256)
|
||||
w13_scale = torch.arange(2 * 8 * 32, dtype=torch.uint8).reshape(2, 8, 32)
|
||||
w2 = torch.arange(2 * 512 * 4, dtype=torch.uint8).reshape(2, 512, 4)
|
||||
w2_scale = torch.arange(2 * 512 * 1, dtype=torch.uint8).reshape(2, 512, 1)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, padded_hidden = (
|
||||
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
)
|
||||
|
||||
assert padded_hidden == 512
|
||||
assert out_w13 is w13
|
||||
assert out_w13_scale is w13_scale
|
||||
assert out_w2 is w2
|
||||
assert out_w2_scale is w2_scale
|
||||
|
||||
|
||||
def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
|
||||
hidden_dim = 2688
|
||||
padded_hidden_dim = 2816
|
||||
|
||||
w13 = torch.arange(2 * 12 * (hidden_dim // 2), dtype=torch.uint8).reshape(
|
||||
2, 12, hidden_dim // 2
|
||||
)
|
||||
w13_scale = torch.arange(2 * 12 * (hidden_dim // 16), dtype=torch.uint8).reshape(
|
||||
2, 12, hidden_dim // 16
|
||||
)
|
||||
|
||||
w2 = torch.arange(2 * hidden_dim * 6, dtype=torch.uint8).reshape(2, hidden_dim, 6)
|
||||
w2_scale = torch.arange(2 * hidden_dim * 2, dtype=torch.uint8).reshape(
|
||||
2, hidden_dim, 2
|
||||
)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, out_hidden_dim = (
|
||||
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
)
|
||||
|
||||
assert out_hidden_dim == padded_hidden_dim
|
||||
assert out_w13.shape == (2, 12, padded_hidden_dim // 2)
|
||||
assert out_w13_scale.shape == (2, 12, padded_hidden_dim // 16)
|
||||
assert out_w2.shape == (2, padded_hidden_dim, 6)
|
||||
assert out_w2_scale.shape == (2, padded_hidden_dim, 2)
|
||||
|
||||
torch.testing.assert_close(out_w13[:, :, : hidden_dim // 2], w13)
|
||||
torch.testing.assert_close(out_w13_scale[:, :, : hidden_dim // 16], w13_scale)
|
||||
torch.testing.assert_close(out_w2[:, :hidden_dim, :], w2)
|
||||
torch.testing.assert_close(out_w2_scale[:, :hidden_dim, :], w2_scale)
|
||||
|
||||
assert torch.count_nonzero(out_w13[:, :, hidden_dim // 2 :]) == 0
|
||||
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
|
||||
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
|
||||
|
||||
|
||||
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
|
||||
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
|
||||
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
|
||||
intermediate = 160
|
||||
padded_intermediate = 192
|
||||
hidden_dim = 32
|
||||
|
||||
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
|
||||
up = torch.full_like(gate, 2)
|
||||
w13 = torch.cat((gate, up), dim=1)
|
||||
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
|
||||
up_scale = torch.full_like(gate_scale, 4)
|
||||
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
|
||||
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
|
||||
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
|
||||
align_fp4_moe_weights_for_fi(
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
is_act_and_mul=True,
|
||||
min_alignment=64,
|
||||
)
|
||||
)
|
||||
|
||||
assert out_intermediate == padded_intermediate
|
||||
torch.testing.assert_close(out_w13[:, :intermediate], gate)
|
||||
torch.testing.assert_close(
|
||||
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
|
||||
)
|
||||
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
|
||||
torch.testing.assert_close(
|
||||
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
|
||||
up_scale,
|
||||
)
|
||||
|
||||
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
|
||||
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
|
||||
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
|
||||
assert (
|
||||
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
|
||||
)
|
||||
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
|
||||
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
|
||||
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
|
||||
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
|
||||
@@ -1,199 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.quantization import (
|
||||
QuantizationConfigArgs,
|
||||
QuantSpec,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import get_quantization_config
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def quant_spec(**kwargs: Any) -> QuantSpec:
|
||||
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
|
||||
return TypeAdapter(QuantSpec).validate_python(kwargs)
|
||||
|
||||
|
||||
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
|
||||
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
|
||||
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
|
||||
|
||||
|
||||
def _limit_num_hidden_layers(
|
||||
model: torch.nn.Module, num_hidden_layers: int | None
|
||||
) -> None:
|
||||
if num_hidden_layers is None:
|
||||
return
|
||||
|
||||
original_load_weights = model.load_weights
|
||||
|
||||
def should_load_weight(name: str) -> bool:
|
||||
layers_prefix = "layers."
|
||||
if name.startswith(layers_prefix):
|
||||
layer_name = name.removeprefix(layers_prefix)
|
||||
elif ".layers." in name:
|
||||
layer_name = name.split(".layers.", maxsplit=1)[1]
|
||||
else:
|
||||
return True
|
||||
|
||||
layer_idx_str = layer_name.split(".", maxsplit=1)[0]
|
||||
if layer_idx_str.isdigit():
|
||||
return int(layer_idx_str) < num_hidden_layers
|
||||
return True
|
||||
|
||||
def load_weights(weights):
|
||||
weights = (
|
||||
(name, weight) for name, weight in weights if should_load_weight(name)
|
||||
)
|
||||
return original_load_weights(weights)
|
||||
|
||||
model.load_weights = load_weights
|
||||
|
||||
|
||||
def load_model_without_vllm_runner(
|
||||
model_path: str,
|
||||
*,
|
||||
dtype: str | torch.dtype = "bfloat16",
|
||||
quantization: str | None = None,
|
||||
model_config_kwargs: dict[str, Any] | None = None,
|
||||
vllm_config_kwargs: dict[str, Any] | None = None,
|
||||
model_loader_cls: type = DefaultModelLoader,
|
||||
) -> tuple[torch.nn.Module, VllmConfig]:
|
||||
"""Instantiate a model, load weights, and process them for inference."""
|
||||
model_config = ModelConfig(
|
||||
model=model_path,
|
||||
dtype=dtype,
|
||||
quantization=quantization,
|
||||
**(model_config_kwargs or {}),
|
||||
)
|
||||
model_config.quantization_config = resolve_quantization_config(
|
||||
model_config.quantization, model_config.quantization_config
|
||||
)
|
||||
vllm_config_args = dict(vllm_config_kwargs or {})
|
||||
vllm_config_args.setdefault(
|
||||
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
|
||||
)
|
||||
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
|
||||
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
|
||||
text_config_overrides = hf_overrides.get("text_config") or {}
|
||||
num_hidden_layers = text_config_overrides.get(
|
||||
"num_hidden_layers", hf_overrides.get("num_hidden_layers")
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
model_loader = model_loader_cls(vllm_config.load_config)
|
||||
if num_hidden_layers is not None:
|
||||
original_load_weights = model_loader.load_weights
|
||||
|
||||
def load_weights(model, model_config):
|
||||
_limit_num_hidden_layers(model, num_hidden_layers)
|
||||
original_load_weights(model, model_config)
|
||||
|
||||
model_loader.load_weights = load_weights
|
||||
model = model_loader.load_model(vllm_config, model_config)
|
||||
|
||||
return model, vllm_config
|
||||
|
||||
|
||||
def is_quant_method_supported(quant_method: str) -> bool:
|
||||
# Currently, quantization tests only run GPUs
|
||||
if current_platform.is_cpu():
|
||||
return False
|
||||
try:
|
||||
current_platform.verify_quantization(quant_method)
|
||||
except ValueError:
|
||||
return False
|
||||
if current_platform.is_xpu():
|
||||
return True
|
||||
capability = current_platform.get_device_capability()
|
||||
assert capability is not None
|
||||
|
||||
min_capability = get_quantization_config(quant_method).get_min_capability()
|
||||
|
||||
return capability.to_int() >= min_capability
|
||||
|
||||
|
||||
def _test_online_quant_peak_mem_impl(
|
||||
quantization_arg_value,
|
||||
vllm_runner,
|
||||
caplog_mp_spawn,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
# Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
|
||||
# 1. it covers both Linear and MoE paths
|
||||
# 2. it is already used by other tests in CI, so adding it here
|
||||
# does not increase disk space for CI runners
|
||||
# I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
|
||||
# which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
|
||||
# 1.3 GiB fp8), but could not as adding one more model makes CI
|
||||
# run out of disk space.
|
||||
model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
|
||||
# Force spawn to ensure caplog_mp_spawn works consistently
|
||||
# (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
with (
|
||||
caplog_mp_spawn(logging.DEBUG) as log_holder,
|
||||
vllm_runner(
|
||||
model_name,
|
||||
quantization=quantization_arg_value,
|
||||
enforce_eager=True,
|
||||
) as llm,
|
||||
):
|
||||
outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
|
||||
log_text = log_holder.text
|
||||
|
||||
# Parse memory usage from captured logs
|
||||
model_memory_gib = None
|
||||
peak_memory_gib = None
|
||||
for line in log_text.splitlines():
|
||||
if model_memory_gib is None:
|
||||
match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
|
||||
if match:
|
||||
model_memory_gib = float(match.group(1))
|
||||
if peak_memory_gib is None:
|
||||
match = re.search(
|
||||
r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
|
||||
)
|
||||
if match:
|
||||
peak_memory_gib = float(match.group(1))
|
||||
|
||||
assert model_memory_gib is not None, "Could not find model loading memory log"
|
||||
assert peak_memory_gib is not None, "Could not find peak memory log"
|
||||
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
|
||||
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
|
||||
|
||||
expected_model_memory_gib = 6.7
|
||||
|
||||
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
|
||||
# GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
|
||||
# number is expected as when we load and quantize weights in a streaming
|
||||
# fashion we need to have individual weights in bf16 + fp8 alive at the
|
||||
# same time.
|
||||
expected_peak_memory_gib = expected_model_memory_gib * 1.4
|
||||
|
||||
assert model_memory_gib < expected_model_memory_gib, (
|
||||
f"{model_memory_gib=} higher than {expected_model_memory_gib}"
|
||||
)
|
||||
assert peak_memory_gib < expected_peak_memory_gib, (
|
||||
f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
|
||||
)
|
||||
Reference in New Issue
Block a user