[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-239
View File
@@ -1,239 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for AutoAWQConfig behavior after unification.
These tests verify the bug fixes for:
1. CPU platform override conflict (auto_awq should not override on CPU)
2. MoE fallback compatibility (full_config["quant_method"] should be "awq")
3. Config attribute consistency
4. End-to-end quantization method loading (auto_awq loads and runs correctly)
Note: Tests that require importing the full auto_awq module (which has GPU-dependent
imports) should use subprocess or be run in a GPU environment.
"""
from __future__ import annotations
import pytest
import torch
from tests.quantization.utils import (
is_quant_method_supported,
load_model_without_vllm_runner,
)
from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.platforms import current_platform
def _get_auto_awq_config_source() -> str:
"""Read the AutoAWQConfig class source code for isolated testing."""
import inspect
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
return inspect.getsource(auto_awq_module.AutoAWQConfig)
class TestAutoAWQConfigFromConfig:
"""Tests for AutoAWQConfig.from_config behavior.
These tests require GPU environment to import the full module.
They are skipped on non-GPU platforms.
"""
def test_full_config_quant_method_is_awq_for_moe_fallback(self):
"""full_config should have quant_method='awq' for MoE fallback compatibility.
MoeWNA16Config only accepts 'gptq' or 'awq' as linear_quant_method.
If full_config has 'auto_awq', the MoE fallback will fail.
"""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
config = {
"w_bit": 4,
"q_group_size": 128,
"zero_point": True,
"lm_head": False,
}
awq_config = AutoAWQConfig.from_config(config)
# Verify quant_method is 'awq' for MoE fallback
assert awq_config.full_config["quant_method"] == "awq", (
f"Expected quant_method='awq', got {awq_config.full_config['quant_method']}"
)
def test_full_config_preserves_other_fields(self):
"""full_config should preserve all original config fields."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
config = {
"w_bit": 4,
"q_group_size": 128,
"zero_point": True,
"lm_head": False,
"custom_field": "custom_value",
}
awq_config = AutoAWQConfig.from_config(config)
assert awq_config.full_config["w_bit"] == 4
assert awq_config.full_config["q_group_size"] == 128
assert awq_config.full_config["zero_point"] is True
assert awq_config.full_config["lm_head"] is False
assert awq_config.full_config["custom_field"] == "custom_value"
def test_full_config_is_copy_not_original(self):
"""full_config should be a copy, not the original dict."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
config = {
"w_bit": 4,
"q_group_size": 128,
"zero_point": True,
"lm_head": False,
}
original_quant_method = config.get("quant_method")
AutoAWQConfig.from_config(config)
# Original config should not be modified
assert config.get("quant_method") == original_quant_method
class TestAutoAWQConfigAttributes:
"""Tests for AutoAWQConfig attribute consistency.
These tests require GPU environment to import the full module.
They are skipped on non-GPU platforms.
"""
def test_config_attributes_match_input(self):
"""Config attributes should match input values."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
awq_config = AutoAWQConfig(
weight_bits=4,
group_size=128,
zero_point=True,
lm_head_quantized=False,
modules_to_not_convert=["lm_head"],
)
assert awq_config.weight_bits == 4
assert awq_config.group_size == 128
assert awq_config.zero_point is True
assert awq_config.lm_head_quantized is False
assert awq_config.modules_to_not_convert == ["lm_head"]
def test_pack_factor_for_4bit(self):
"""Pack factor should be 8 for 4-bit quantization."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
awq_config = AutoAWQConfig(
weight_bits=4,
group_size=128,
zero_point=True,
lm_head_quantized=False,
)
assert awq_config.pack_factor == 8 # 32 // 4
class TestAutoAWQConfigOverrideLogic:
"""Tests for override logic by parsing source code (no GPU import required)."""
def _get_auto_awq_source(self) -> str:
"""Read the auto_awq.py source file."""
import inspect
import pathlib
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
source_path = inspect.getfile(auto_awq_module)
return pathlib.Path(source_path).read_text()
def test_cpu_check_in_override_method(self):
"""override_quantization_method should check current_platform.is_cpu()."""
source = self._get_auto_awq_source()
# Verify the CPU check exists in override method
assert "current_platform.is_cpu()" in source, (
"override_quantization_method should check is_cpu()"
)
assert "return None" in source, (
"override_quantization_method should return None on CPU"
)
def test_quant_method_normalization_in_from_config(self):
"""from_config should normalize quant_method to 'awq' for MoE fallback."""
source = self._get_auto_awq_source()
# Verify the normalization exists
assert (
'"quant_method"] = "awq"' in source or "'quant_method'] = 'awq'" in source
), "from_config should set quant_method='awq' in full_config"
# =============================================================================
# End-to-end integration tests (require GPU environment)
# =============================================================================
PROMPT = "On the surface of Mars, we found"
# Small AWQ model for testing - using Qwen2 1.5B which has official AWQ checkpoint
AWQ_MODELS = [
"Qwen/Qwen2-1.5B-Instruct-AWQ",
]
@pytest.mark.skipif(
not is_quant_method_supported("auto_awq"),
reason="auto_awq is not supported on this GPU type.",
)
@pytest.mark.parametrize("model_id", AWQ_MODELS)
def test_auto_awq_quantization_method(
model_id: str, monkeypatch, dist_init, workspace_init
):
"""Test that quantization='auto_awq' loads and runs correctly."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model, vllm_config = load_model_without_vllm_runner(
model_id,
dtype=torch.float16,
quantization="auto_awq",
model_config_kwargs={"max_model_len": 2048},
)
target_device = torch.device(current_platform.device_type)
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQLinearMethod,
AutoAWQMarlinLinearMethod,
)
qkv_proj = model.model.layers[0].self_attn.qkv_proj
assert isinstance(
qkv_proj.quant_method,
(AutoAWQLinearMethod, AutoAWQMarlinLinearMethod),
), (
"Expected AutoAWQLinearMethod or AutoAWQMarlinLinearMethod, "
f"got {type(qkv_proj.quant_method)}"
)
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
input_ids = torch.tensor([1, 2, 3, 4], device=target_device)
positions = torch.arange(input_ids.numel(), device=target_device)
with (
set_current_vllm_config(vllm_config),
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
):
hidden_states = model(input_ids, positions, None)
logits = model.compute_logits(hidden_states)
assert torch.isfinite(logits).all()
def test_auto_awq_config_get_name():
"""Test that AutoAWQConfig.get_name() returns 'auto_awq'."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
assert AutoAWQConfig.get_name() == "auto_awq"
-211
View File
@@ -1,211 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests that the auto_gptq quantization method works correctly.
Run `pytest tests/quantization/test_auto_gptq.py -v -s`.
"""
from types import SimpleNamespace
import pytest
import torch
from tests.quantization.utils import (
is_quant_method_supported,
load_model_without_vllm_runner,
)
from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
AutoGPTQLinearMethod,
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
is_layer_gptq_quantized,
override_config,
)
from vllm.platforms import current_platform
PROMPT = "On the surface of Mars, we found"
MODELS = [
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
]
@pytest.mark.skipif(
not is_quant_method_supported("auto_gptq"),
reason="auto_gptq is not supported on this GPU type.",
)
@pytest.mark.parametrize("model_id", MODELS)
def test_auto_gptq_quantization_method(
model_id: str, monkeypatch, dist_init, workspace_init
):
"""Test that quantization='auto_gptq' loads and runs correctly."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model, vllm_config = load_model_without_vllm_runner(
model_id,
dtype=torch.float16,
quantization="auto_gptq",
model_config_kwargs={"max_model_len": 2048},
)
qkv_proj = model.model.layers[0].self_attn.qkv_proj
assert isinstance(qkv_proj.quant_method, AutoGPTQLinearMethod)
target_device = torch.device(current_platform.device_type)
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
input_ids = torch.tensor([1, 2, 3, 4], device=target_device)
positions = torch.arange(input_ids.numel(), device=target_device)
with (
set_current_vllm_config(vllm_config),
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
):
hidden_states = model(input_ids, positions, None)
logits = model.compute_logits(hidden_states)
assert torch.isfinite(logits).all()
def test_auto_gptq_config_get_name():
"""Test that AutoGPTQConfig.get_name() returns 'auto_gptq'."""
assert AutoGPTQConfig.get_name() == "auto_gptq"
def test_auto_gptq_quantizes_every_layer_when_the_module_list_is_unknown():
"""An empty module list means the safetensors metadata could not be read,
not that the checkpoint holds no quantized layer."""
assert is_layer_gptq_quantized(
prefix="model.layers.0.mlp.down_proj",
quantized_layers=[],
)
assert not is_layer_gptq_quantized(
prefix="model.layers.0.mlp.down_proj",
quantized_layers=["self_attn.q_proj"],
)
@pytest.mark.parametrize(
"dynamic",
[
{},
{r"+:model\.layers\.0\..*": {"desc_act": True}},
{
r"+:model\.layers\.0\..*": {
"desc_act": True,
"group_size": 32,
}
},
],
)
def test_auto_gptq_rejects_group_activation_order(dynamic):
desc_act = not dynamic
with pytest.raises(ValueError, match="group activation ordering"):
AutoGPTQConfig(4, 128, desc_act, True, False, dynamic, {})
def test_auto_gptq_normalizes_channelwise_activation_order():
config = AutoGPTQConfig(4, -1, True, True, False, {}, {})
assert not config.desc_act
config = AutoGPTQConfig(
4,
128,
False,
True,
False,
{r"+:model\.layers\.0\..*": {"desc_act": True, "group_size": -1}},
{},
)
override_config(config, "model.layers.0.self_attn.q_proj")
assert config.group_size == -1
assert not config.desc_act
with pytest.raises(ValueError, match="group activation ordering"):
AutoGPTQConfig(
4,
-1,
True,
True,
False,
{r"+:model\.layers\.0\..*": {"group_size": 128}},
{},
)
def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
method = object.__new__(AutoGPTQMoEMethod)
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
method.input_dtype = None
method.experts_cls = None
method.moe = SimpleNamespace(w13_num_shards=2)
layer = torch.nn.Module()
method.create_weights(
layer=layer,
num_experts=2,
hidden_size=8,
intermediate_size_per_partition=4,
params_dtype=torch.float16,
weight_loader=lambda *args, **kwargs: None,
)
assert layer.w13_bias.shape == (2, 8)
assert layer.w2_bias.shape == (2, 8)
assert torch.count_nonzero(layer.w13_bias) == 0
assert torch.count_nonzero(layer.w2_bias) == 0
def test_routed_experts_loads_per_expert_biases():
class Loader:
quant_config = None
quant_method = object()
_fused_shared_expert_quantizer = None
moe_config = SimpleNamespace(
is_act_and_mul=True,
tp_rank=0,
moe_parallel_config=SimpleNamespace(tp_size=1),
)
_get_hidden_dim = staticmethod(RoutedExperts._get_hidden_dim)
_narrow_expert_data_for_padding = staticmethod(
RoutedExperts._narrow_expert_data_for_padding
)
_load_w13 = RoutedExperts._load_w13
_loaded_expert_biases = set()
@staticmethod
def _map_global_expert_id_to_local_expert_id(expert_id):
return expert_id
loader = Loader()
w13_bias = torch.nn.Parameter(torch.zeros(1, 8), requires_grad=False)
w2_bias = torch.nn.Parameter(torch.zeros(1, 4), requires_grad=False)
for shard_id, loaded in (
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
):
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
loader,
w13_bias,
loaded,
weight_name="model.layers.0.mlp.experts.w13_bias",
shard_id=shard_id,
expert_id=0,
return_success=True,
)
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
loader,
w2_bias,
torch.tensor([9.0, 10.0, 11.0, 12.0]),
weight_name="model.layers.0.mlp.experts.w2_bias",
shard_id="w2",
expert_id=0,
return_success=True,
)
assert torch.equal(w13_bias, torch.arange(1, 9, dtype=torch.float32).reshape(1, 8))
assert torch.equal(w2_bias, torch.arange(9, 13, dtype=torch.float32).reshape(1, 4))
assert loader._loaded_expert_biases == {"w13_bias", "w2_bias"}
File diff suppressed because it is too large Load Diff
-312
View File
@@ -1,312 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
from typing import Any
import pytest
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip(
"This test only runs on Blackwell GPUs (SM10x).", allow_module_level=True
)
@pytest.fixture(scope="module", autouse=True)
def set_test_environment():
"""Sets environment variables required for this test module."""
# Make sure TRTLLM attention is available
os.environ["VLLM_HAS_FLASHINFER_CUBIN"] = "1"
# Set compilation threads to 16 to speed up startup
os.environ["FLASHINFER_NVCC_THREADS"] = "16"
# Override the backbone layers to 4 for faster startup
HF_OVERRIDE_TEXT = {
"num_layers": 4,
"num_hidden_layers": 4,
}
HF_OVERRIDE_MM = {
"text_config": {"num_layers": 4, "num_hidden_layers": 4},
}
def can_initialize(
model: str,
hf_overrides: dict[str, Any] | None = None,
extra_args: list[str] | None = None,
):
# Server arguments
extra_args = extra_args if extra_args is not None else []
server_args = [
"--max-model-len",
"2048",
"--max-num-batched-tokens",
"256",
"--load-format",
"dummy",
"--trust-remote-code",
"--limit-mm-per-prompt",
json.dumps({"image": 0}),
*extra_args,
]
# Launch server and make a simple request
with RemoteOpenAIServer(
model,
server_args,
max_wait_seconds=1500, # Due to FlashInfer compile
override_hf_configs=hf_overrides,
) as server:
client = server.get_client()
# Make a simple request to verify the server works
completion = client.completions.create(
model=model,
prompt=["Hello, World!"],
temperature=0,
max_tokens=2,
)
print(completion)
assert completion.choices[0].text is not None
## Llama4 ##
@pytest.mark.skip(
reason=(
"RuntimeError: run_moe() Expected a value of type "
"'Optional[List[Tensor]]' for argument '_9' but instead found type "
"'list'."
)
)
def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
hf_overrides=HF_OVERRIDE_MM,
extra_args=["--moe-backend=flashinfer_cutlass"],
)
def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
hf_overrides=HF_OVERRIDE_MM,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
hf_overrides=HF_OVERRIDE_MM,
extra_args=["--moe-backend=flashinfer_cutlass"],
)
def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
hf_overrides=HF_OVERRIDE_MM,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
## DeepSeekV3 ##
def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"deepseek-ai/DeepSeek-V3.1",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=deep_gemm"],
)
def test_deepseek_fp8_block_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"deepseek-ai/DeepSeek-V3.1",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=triton"],
)
@pytest.mark.skip(
reason=(
"Known issue: lack of kernel support. "
"Expected failure: assert self.block_quant is None"
)
)
def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"deepseek-ai/DeepSeek-V3.1",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_cutlass"],
)
def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"deepseek-ai/DeepSeek-V3.1",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
def test_deepseek_nvfp4_moe_flashinfer_vllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/DeepSeek-R1-0528-FP4-v2",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=cutlass"],
)
def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/DeepSeek-R1-0528-FP4-v2",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_cutlass"],
)
def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/DeepSeek-R1-0528-FP4-v2",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
## GPT-OSS ##
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=[
"--moe-backend",
"flashinfer_cutlass",
"--quantization-config.moe.activation",
"mxfp8",
],
)
def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=[
"--quantization-config.moe.activation",
"mxfp8",
],
)
def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--enforce-eager"],
)
## Qwen3 Next ##
@pytest.mark.skip(
reason=(
"FLASHINFER TRTLLM MoE has a bug with all negative router logits "
"for models with RENORMALIZE. This will be re-enabled once the "
"issue is fixed in flashinfer."
)
)
def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
## NemoTron ##
def test_nemotron_fp8_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_cutlass"],
)
@pytest.mark.skip(
reason=(
"FP8 MoE backend FLASHINFER_TRTLLM does not support the "
"deployment configuration since kernel does not support "
"no act_and_mul MLP layer."
)
)
def test_nemotron_fp8_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
@pytest.mark.skip(
reason=(
"FP8 MoE backend TRITON does not support the "
"deployment configuration since kernel does not support "
"no act_and_mul MLP layer."
)
)
def test_nemotron_fp8_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=triton"],
)
def test_nemotron_fp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_cutlass"],
)
def test_nemotron_fp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
can_initialize(
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
def test_nemotron_nvfp4_per_token_moe_flashinfer_trtllm(
monkeypatch: pytest.MonkeyPatch,
):
can_initialize(
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=[
"--moe-backend=flashinfer_trtllm",
"--quantization-config.moe.activation=nvfp4_per_token",
],
)
File diff suppressed because it is too large Load Diff
-68
View File
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests whether Marlin models can be loaded from the autogptq config.
Run `pytest tests/quantization/test_configs.py --forked`.
"""
import pytest
from vllm.config import ModelConfig
from vllm.platforms import current_platform
# Model Id // Quantization Arg // Expected Type
MODEL_ARG_EXPTYPES = [
# AUTOGPTQ
# compat: autogptq <=0.7.1 is_marlin_format: bool
# Model Serialized in Exllama Format.
("TheBloke/Llama-2-7B-Chat-GPTQ", None, "auto_gptq"),
(
"TheBloke/Llama-2-7B-Chat-GPTQ",
"marlin",
"auto_gptq" if current_platform.is_cuda_alike() else "ERROR",
),
("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "auto_gptq"),
("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
# compat: autogptq >=0.8.0 use checkpoint_format: str
# Model Serialized in Exllama Format.
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "auto_gptq"),
(
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
"marlin",
"auto_gptq" if current_platform.is_cuda_alike() else "ERROR",
),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
# AUTOAWQ
# AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models
# when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq"
(
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
None,
"auto_awq",
),
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"),
(
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
"marlin",
"auto_awq" if current_platform.is_cuda_alike() else "ERROR",
),
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
]
@pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)
def test_auto_gptq(model_arg_exptype: tuple[str, None, str]) -> None:
model_path, quantization_arg, expected_type = model_arg_exptype
try:
model_config = ModelConfig(model_path, quantization=quantization_arg)
found_quantization_type = model_config.quantization
except ValueError:
found_quantization_type = "ERROR"
assert found_quantization_type == expected_type, (
f"Expected quant_type == {expected_type} for {model_path}, "
f"but found {found_quantization_type} "
f"for no --quantization {quantization_arg} case"
)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
MODELS = [
"RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8", # INT8 W8A8 MoE
]
DTYPE = ["bfloat16"]
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", DTYPE)
def test_cpu_w8a8(vllm_runner, model, dtype):
with vllm_runner(model, dtype=dtype) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
assert output
print(output)
-86
View File
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import CpuArchEnum, current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
from vllm._custom_ops import cpu_gemm_wna16 # noqa: E402
from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( # noqa: E402
_get_isa_hint,
)
# Isolated runs of cpu_gemm_wna16 can hang unless AMX tiles are initialized.
# Guard so non-AMX hosts still take the vec/rvv path.
if torch.cpu._is_amx_tile_supported():
torch.cpu._init_amx()
@pytest.mark.cpu_test
def test_cpu_gemm_wna16_3d_input():
"""cpu_gemm_wna16 flattens 3D [B, T, K] to 2D and reshapes the output back."""
if not hasattr(torch.ops._C, "cpu_gemm_wna16"):
pytest.skip("cpu_gemm_wna16 op is not available")
B, T, K, N = 2, 5, 64, 64
pack_factor = 8
group_size = 32
dtype = torch.bfloat16
q_weight = torch.zeros(N // 16, K * 16 // pack_factor, dtype=torch.int32)
scales = torch.ones(K // group_size, N, dtype=dtype)
x_3d = torch.randn(B, T, K, dtype=dtype)
x_2d = x_3d.reshape(-1, K)
isa_hint = _get_isa_hint(dtype)
kwargs = dict(
q_weight=q_weight,
scales=scales,
zeros=None,
bias=None,
pack_factor=pack_factor,
isa_hint=isa_hint,
)
out_3d = cpu_gemm_wna16(input=x_3d, **kwargs)
out_2d = cpu_gemm_wna16(input=x_2d, **kwargs)
assert out_3d.shape == (B, T, N)
assert out_2d.shape == (B * T, N)
torch.testing.assert_close(out_3d.reshape(-1, N), out_2d)
MODELS = [
"TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ",
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
"Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE
"openai/gpt-oss-20b", # MXFP4 W4A16
"QuixiAI/Qwen3-30B-A3B-AWQ", # AWQ W4A16 MoE
"Qwen/Qwen3-30B-A3B-GPTQ-Int4", # GPTQ W4A16 MoE
"RedHatAI/Qwen3-30B-A3B-quantized.w4a16", # compressed-tensors W4A16 MoE
]
DTYPE = ["bfloat16"]
@pytest.mark.skipif(
current_platform.get_cpu_architecture() != CpuArchEnum.POWERPC,
reason="PowerPC-only test",
)
def test_cpu_wna16_power_bf16_uses_vsx():
"""Use the POWER10 BF16 MMA microkernel instead of generic vectors."""
assert _get_isa_hint(torch.bfloat16) == "vsx"
assert _get_isa_hint(torch.float16) == "vec"
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", DTYPE)
def test_cpu_quant(vllm_runner, model, dtype):
with vllm_runner(model, dtype=dtype) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
assert output
print(output)
-180
View File
@@ -1,180 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Cutlass W4A16 (Machete) kernel on Hopper.
Verifies that W4A16 quantized models loaded through vllm select the
MacheteLinearKernel on sm_90 GPUs, that weights are correctly repacked,
and that inference produces valid output.
Run `pytest tests/quantization/test_cutlass_w4a16.py`.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.has_device_capability(90) or current_platform.is_rocm():
pytest.skip(
"Machete W4A16 requires Hopper (sm_90).",
allow_module_level=True,
)
from vllm.model_executor.kernels.linear import (
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
from vllm.model_executor.kernels.linear.mixed_precision import (
MacheteLinearKernel,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsLinearMethod,
CompressedTensorsWNA16,
)
from vllm.scalar_type import scalar_types
@pytest.fixture(scope="function", autouse=True)
def enable_pickle(monkeypatch):
"""`LLM.apply_model` requires pickling a function."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
@pytest.mark.parametrize(
"act_type,weight_type,group_size,zero_points",
[
(torch.float16, scalar_types.uint4b8, 128, False),
(torch.bfloat16, scalar_types.uint4b8, 128, False),
(torch.float16, scalar_types.uint4, 128, True),
(torch.float16, scalar_types.uint4b8, -1, False),
],
ids=[
"fp16-gptq-g128",
"bf16-gptq-g128",
"fp16-awq-g128",
"fp16-channelwise",
],
)
def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points):
"""Verify choose_mp_linear_kernel picks MacheteLinearKernel."""
config = MPLinearLayerConfig(
full_weight_shape=(4096, 4096),
partition_weight_shape=(4096, 4096),
act_type=act_type,
weight_type=weight_type,
group_size=group_size,
zero_points=zero_points,
)
kernel = choose_mp_linear_kernel(config)
assert kernel is MacheteLinearKernel, (
f"Expected MacheteLinearKernel, got {kernel.__name__}"
)
@pytest.mark.parametrize(
"full_shape,part_shape,weight_type,group_size,expected_reason",
[
(
(4096, 4096),
(4096, 4096),
scalar_types.float6_e3m2f,
128,
"Quant type",
),
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, "Group size"),
],
ids=["unsupported-quant-type", "unsupported-group-size"],
)
def test_machete_rejects_invalid_config(
full_shape, part_shape, weight_type, group_size, expected_reason
):
"""Verify Machete rejects unsupported configurations."""
config = MPLinearLayerConfig(
full_weight_shape=full_shape,
partition_weight_shape=part_shape,
act_type=torch.float16,
weight_type=weight_type,
group_size=group_size,
zero_points=False,
)
can_impl, reason = MacheteLinearKernel.can_implement(config)
assert not can_impl
assert reason is not None
assert expected_reason in reason
def test_kernel_selection_with_disabled_machete(monkeypatch):
"""Verify kernel selection falls back when Machete is disabled."""
monkeypatch.setattr("vllm.envs.VLLM_DISABLED_KERNELS", ["MacheteLinearKernel"])
config = MPLinearLayerConfig(
full_weight_shape=(4096, 4096),
partition_weight_shape=(4096, 4096),
act_type=torch.float16,
weight_type=scalar_types.uint4b8,
group_size=128,
zero_points=False,
)
kernel = choose_mp_linear_kernel(config)
assert kernel is not MacheteLinearKernel, "MacheteLinearKernel should be disabled"
@pytest.mark.parametrize(
"model_name",
[
"nm-testing/tinyllama-oneshot-w4a16-channel-v2",
],
)
def test_w4a16_machete_e2e(vllm_runner, model_name):
"""Load a W4A16 model, verify Machete kernel is used, and generate."""
with vllm_runner(model_name, enforce_eager=True, gpu_memory_utilization=0.5) as llm:
def check_model(model):
layer = model.model.layers[0]
qkv_proj = layer.self_attn.qkv_proj
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
assert isinstance(qkv_proj.scheme.kernel, MacheteLinearKernel), (
f"Expected MacheteLinearKernel on Hopper, "
f"got {type(qkv_proj.scheme.kernel).__name__}"
)
assert hasattr(qkv_proj, "weight_packed")
assert hasattr(qkv_proj, "weight_scale")
assert qkv_proj.weight_packed.dtype == torch.int32
llm.apply_model(check_model)
output = llm.generate_greedy("Hello my name is", max_tokens=10)
assert output
assert len(output[0][1]) > 0
def test_w4a16_machete_bfloat16_deterministic(vllm_runner):
"""Verify Machete works with bf16 activations and is deterministic."""
model_name = "nm-testing/tinyllama-oneshot-w4a16-channel-v2"
prompt = "The capital of France is"
with vllm_runner(
model_name,
enforce_eager=True,
dtype="bfloat16",
gpu_memory_utilization=0.5,
) as llm:
def check_kernel_type(model):
layer = model.model.layers[0]
scheme = layer.self_attn.qkv_proj.scheme
assert isinstance(scheme.kernel, MacheteLinearKernel), (
f"Expected MacheteLinearKernel with bf16, "
f"got {type(scheme.kernel).__name__}"
)
llm.apply_model(check_kernel_type)
out1 = llm.generate_greedy(prompt, max_tokens=10)
out2 = llm.generate_greedy(prompt, max_tokens=10)
assert out1[0][1] == out2[0][1], (
f"Non-deterministic: '{out1[0][1]}' vs '{out2[0][1]}'"
)
@@ -1,71 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FP8 per-channel online quantization.
Per-output-channel weight scale + dynamic per-token activation scale.
bf16/fp16 checkpoints are quantized at load time with one fp32 scale per
output channel for weights and one fp32 scale per token for activations
(computed dynamically inside the kernel). Run via
`pytest tests/quantization/test_fp8_per_channel.py --forked`.
"""
import pytest
import torch
from tests.quantization.utils import is_quant_method_supported
from vllm import _custom_ops as ops
from vllm.config.quantization import (
_ONLINE_SHORTHANDS,
QUANT_KEY_NAMES,
QuantizationConfigArgs,
)
from vllm.model_executor.layers.quantization.online.base import (
_ONLINE_LINEAR_METHODS,
_ONLINE_MOE_METHODS,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PtpcOnlineLinearMethod,
Fp8PtpcOnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticChannelSym,
)
from vllm.platforms import current_platform
def test_fp8_per_channel_shorthand_registered() -> None:
"""The `fp8_per_channel` CLI shorthand must resolve to a config that
dispatches the per-channel methods. Guards against regressions in
`_ONLINE_SHORTHANDS` / `_ONLINE_LINEAR_METHODS` / `_ONLINE_MOE_METHODS`
drifting out of sync.
"""
args = _ONLINE_SHORTHANDS["fp8_per_channel"]
assert isinstance(args, QuantizationConfigArgs)
assert args.linear is not None
assert args.moe is not None
assert args.linear.weight is kFp8StaticChannelSym
assert args.moe.weight is kFp8StaticChannelSym
assert _ONLINE_LINEAR_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineLinearMethod
assert _ONLINE_MOE_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineMoEMethod
assert QUANT_KEY_NAMES["fp8_per_channel_static"] is kFp8StaticChannelSym
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_scaled_fp8_quant_per_channel_shape() -> None:
"""Verify the kernel call per-channel quant depends on: passing a 2D
weight to `ops.scaled_fp8_quant` with `use_per_token_if_dynamic=True`
yields one scale per output row -- a [N, 1] fp32 tensor.
"""
x = (torch.randn(size=(96, 256), device=current_platform.device_type) * 13).to(
torch.bfloat16
)
y, s = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
assert y.shape == (96, 256)
assert y.dtype == current_platform.fp8_dtype()
assert s.shape == (96, 1)
assert s.dtype == torch.float32
-207
View File
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MXFP4 MoE oracle backend selection on mi355x (GFX950).
These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
"""
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_requires_qwen38_tep8_emulation,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.platforms import current_platform
ROCM_AVAILABLE = current_platform.is_rocm()
ROCM_GFX950 = False
ROCM_AITER_SUPPORTED = False
if ROCM_AVAILABLE:
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.platforms.rocm import on_gfx950
ROCM_GFX950 = on_gfx950()
ROCM_AITER_SUPPORTED = is_aiter_found_and_supported()
def set_rocm_aiter(monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None:
value = "1" if enabled else "0"
monkeypatch.setenv("VLLM_ROCM_USE_AITER", value)
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", value)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", enabled)
monkeypatch.setattr(rocm_aiter_ops, "_FMOE_ENABLED", enabled)
@pytest.fixture
def enable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
set_rocm_aiter(monkeypatch, True)
@pytest.fixture
def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
set_rocm_aiter(monkeypatch, False)
def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
num_experts=8,
experts_per_token=2,
hidden_dim=256,
intermediate_size=256,
num_local_experts=8,
num_logical_experts=8,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
moe_backend=moe_backend,
)
def _make_qwen38_tep8_moe_config() -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
num_experts=512,
experts_per_token=9,
hidden_dim=2560,
intermediate_size=640,
num_local_experts=64,
num_logical_experts=512,
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=1,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
)
def test_qwen38_tep8_requires_emulation_only_on_gfx950(monkeypatch):
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
config = _make_qwen38_tep8_moe_config()
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
assert _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
config.moe_parallel_config.ep_size = 4
config.num_local_experts = 128
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
config = _make_qwen38_tep8_moe_config()
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_rocm", lambda: False)
assert not _requires_qwen38_tep8_emulation(config, kMxfp4Dynamic)
@pytest.mark.parametrize(
"requested_backend,expected_backend",
[
("auto", Mxfp4MoeBackend.EMULATION),
("aiter", Mxfp4MoeBackend.AITER_MXFP4_MXFP4),
],
)
def test_qwen38_tep8_auto_fallback_respects_explicit_backend(
requested_backend,
expected_backend,
monkeypatch,
):
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
class SupportedExperts:
@staticmethod
def is_supported_config(*args, **kwargs):
return True, None
config = _make_qwen38_tep8_moe_config()
config.moe_backend = requested_backend
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
monkeypatch.setattr(
mxfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
)
backend, experts_cls = select_mxfp4_moe_backend(
config, activation_key=kMxfp4Dynamic
)
assert backend == expected_backend
assert experts_cls is SupportedExperts
@pytest.fixture
def mxfp4_oracle_config():
"""Stub the config the oracle reads (``model_config.quantization_config``)
so backend dispatch resolves without a real model / user override."""
from unittest.mock import patch
with patch(
"vllm.model_executor.layers.fused_moe.oracle.mxfp4.get_current_vllm_config"
) as mock_get_config:
mock_get_config.return_value.model_config.quantization_config = None
yield
@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
@pytest.mark.skipif(not ROCM_AITER_SUPPORTED, reason="Requires supported AITER")
def test_w4a4_dispatches_to_aiter(mxfp4_oracle_config, enable_rocm_aiter):
"""With AITER enabled + GFX950, W4A4 selects AITER_MXFP4_MXFP4."""
config = _make_w4a4_moe_config()
backend, experts_cls = select_mxfp4_moe_backend(
config, activation_key=kMxfp4Dynamic
)
assert backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4
assert experts_cls is not None
@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
def test_w4a4_falls_back_without_aiter(
mxfp4_oracle_config,
disable_rocm_aiter,
):
config = _make_w4a4_moe_config()
backend, experts_cls = select_mxfp4_moe_backend(
config, activation_key=kMxfp4Dynamic
)
assert backend == Mxfp4MoeBackend.EMULATION
assert experts_cls is not None
@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
def test_w4a4_dispatches_to_emulation_with_moe_backend(mxfp4_oracle_config):
"""With --moe-backend emulation, W4A4 selects EMULATION."""
config = _make_w4a4_moe_config(moe_backend="emulation")
backend, experts_cls = select_mxfp4_moe_backend(
config, activation_key=kMxfp4Dynamic
)
assert backend == Mxfp4MoeBackend.EMULATION
assert experts_cls is not None
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
pytestmark = pytest.mark.skipif(
not (has_humming() and current_platform.is_cuda()),
reason="requires humming and CUDA",
)
def test_humming_input_quant_compiles_fullgraph():
"""Humming's forward helpers look up ``DeviceInfo`` on every call; vLLM
traces them with ``fullgraph=True`` (vllm-project/humming#147)."""
import vllm.utils.humming as hm
weight_schema = hm.HummingWeightSchema(b_dtype="int4", weight_scale_group_size=128)
input_schema = hm.HummingInputSchema(a_dtype="int8")
from humming.transform import prepare_layer_config
config = prepare_layer_config(
shape_n=256,
shape_k=256,
weight_schema=weight_schema,
input_schema=input_schema,
torch_dtype=torch.bfloat16,
device=torch.device("cuda"),
)
x = torch.randn(16, 256, dtype=torch.bfloat16, device="cuda")
def quant(t: torch.Tensor):
return hm.may_process_input(config, t)
expected = quant(x)
actual = torch.compile(quant, fullgraph=True)(x)
for a, e in zip(actual, expected):
if e is None:
assert a is None
else:
torch.testing.assert_close(a, e)
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation.
The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a
grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8
`test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling
it in vLLM comes down to two things, both exercised here:
1. A grouped FP8 activation must classify to a float32-scaled key
(kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale
activations (group size 32) keep their e8m0 (uint8) scale.
2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE
(weight, activation) pair.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kMxfp4Static,
kMxfp8Dynamic,
)
from vllm.utils.import_utils import has_humming
# make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the
# config-plumbing asserts below never read them, so a placeholder satisfies the
# required arg without pulling in the humming package.
_PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None}
def test_mxfp4_weight_with_block_fp8_activation_is_supported():
"""MXFP4 weight + block-FP8 (group-128) activation is an allowed pair."""
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym)
def test_mxfp4_weight_with_per_token_fp8_activation_still_supported():
"""The pre-existing per-token FP8 pairing must keep working."""
assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym)
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
@pytest.mark.parametrize(
"group_size,expected_key",
[
# per-token FP8 -> float32 per-token scale
(0, kFp8DynamicTokenSym),
# block FP8 -> float32 group-128 scale (the newly-fixed case)
(128, kFp8Dynamic128Sym),
# MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved)
(32, kMxfp8Dynamic),
],
)
def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
from vllm.model_executor.layers.quantization.utils.humming import (
input_schema_to_quant_key,
)
from vllm.utils.humming import HummingInputSchema
from vllm.utils.humming import dtypes as humming_dtypes
schema = HummingInputSchema(
a_dtype=humming_dtypes.float8e4m3,
input_scale_group_size=group_size,
)
assert input_schema_to_quant_key(schema, param_dtype=torch.bfloat16) == expected_key
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
def test_humming_bf16_input_schema_is_unquantized():
from vllm.model_executor.layers.quantization.utils.humming import (
input_schema_to_quant_key,
)
from vllm.utils.humming import HummingInputSchema
# No a_dtype -> unquantized (bf16/fp16) inputs -> None.
assert (
input_schema_to_quant_key(HummingInputSchema(), param_dtype=torch.bfloat16)
is None
)
def test_block_fp8_activation_quant_config_is_block_quantized():
"""A block-FP8 activation group shape yields a block-quantized MoE config
(FP8 dtype, [1, 128] block, not per-act-token). This is what makes the
DeepEP prepare/finalize step quantize activations to block FP8 *before* the
all-to-all dispatch instead of deferring to Humming -- and what makes
HummingExpertsBase.expects_unquantized_inputs return False."""
from vllm.model_executor.layers.quantization.utils.humming import (
make_humming_moe_quant_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
qc = make_humming_moe_quant_config(
quant_dtype=current_platform.fp8_dtype(),
weight_dtype="float4e2m1",
activation_group_shape=GroupShape(row=1, col=128),
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
)
assert qc.is_block_quantized
assert qc.block_shape == [1, 128]
assert not qc.per_act_token_quant
assert qc.quant_dtype == current_platform.fp8_dtype()
def test_default_activation_quant_config_defers_to_humming():
"""Without a block activation shape the config stays per-token (the deferred
path): Humming quantizes internally, preserving pre-existing behavior for
every non-block-FP8 scheme."""
from vllm.model_executor.layers.quantization.utils.humming import (
make_humming_moe_quant_config,
)
from vllm.platforms import current_platform
qc = make_humming_moe_quant_config(
quant_dtype=current_platform.fp8_dtype(),
weight_dtype="float4e2m1",
humming_configs=_PLACEHOLDER_HUMMING_CONFIGS,
)
assert not qc.is_block_quantized
assert qc.per_act_token_quant
-137
View File
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for INT8 (W8A8) fused-MoE oracle backend selection.
These exercise ``select_int8_moe_backend`` only (no MoE kernels are launched),
so they run on any platform where the Triton INT8 MoE kernel is available —
CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950.
"""
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
Int8MoeBackend,
select_int8_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTensorSym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorSym,
)
from vllm.platforms import current_platform
# The Triton int8_w8a8 fused-MoE kernel is available on CUDA (Turing+), on
# ROCm CDNA GPUs and on XPU. Gate on that rather than on a specific arch.
INT8_MOE_SUPPORTED = (
(current_platform.is_cuda() and current_platform.has_device_capability((7, 5)))
or current_platform.is_rocm()
or current_platform.is_xpu()
)
requires_int8_moe = pytest.mark.skipif(
not INT8_MOE_SUPPORTED,
reason="Requires Triton INT8 MoE support (CUDA SM>=7.5, ROCm, or XPU)",
)
# So FusedMoEConfig.device names the device the test actually runs on.
DEVICE = current_platform.device_type
def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
num_experts=8,
experts_per_token=2,
hidden_dim=256,
intermediate_size=256,
num_local_experts=8,
num_logical_experts=8,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device=DEVICE,
routing_method=RoutingMethodType.Renormalize,
moe_backend=moe_backend,
)
@requires_int8_moe
@pytest.mark.parametrize(
"weight_key,activation_key",
[
pytest.param(
kInt8StaticChannelSym,
kInt8DynamicTokenSym,
id="per_channel_weight-per_token_act",
),
pytest.param(
kInt8StaticTensorSym,
kInt8DynamicTensorSym,
id="per_tensor_weight-per_tensor_act",
),
],
)
def test_int8_dynamic_schemes_dispatch_to_triton(weight_key, activation_key):
"""Both dynamic-activation INT8 MoE schemes select the Triton backend."""
config = _make_int8_moe_config()
backend, experts_cls = select_int8_moe_backend(
config, weight_key=weight_key, activation_key=activation_key
)
assert backend == Int8MoeBackend.TRITON
assert experts_cls is not None
@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU-only behaviour")
def test_scaled_int8_quant_is_available_on_xpu():
"""The per-tensor scheme's activation quantization works on XPU.
``_int8_quantize`` sends per-tensor activations through
``ops.scaled_int8_quant``, which has an XPU branch. This is what lets the
per-tensor scheme be enabled here alongside CUDA and ROCm; pin it so a
future change cannot silently remove that branch.
"""
from vllm import _custom_ops as ops
x = torch.randn(4, 16, device=DEVICE, dtype=torch.bfloat16)
scale = torch.full((1,), 0.05, device=DEVICE, dtype=torch.float32)
q, returned_scale, azp = ops.scaled_int8_quant(x, scale=scale)
assert q.dtype == torch.int8
assert q.shape == x.shape
assert azp is None
torch.testing.assert_close(returned_scale, scale)
@requires_int8_moe
def test_int8_explicit_moe_backend_triton():
"""An explicit --moe-backend triton selects the Triton INT8 backend."""
config = _make_int8_moe_config(moe_backend="triton")
backend, experts_cls = select_int8_moe_backend(
config,
weight_key=kInt8StaticChannelSym,
activation_key=kInt8DynamicTokenSym,
)
assert backend == Int8MoeBackend.TRITON
assert experts_cls is not None
@requires_int8_moe
def test_int8_unsupported_moe_backend_raises():
"""An unsupported --moe-backend for INT8 MoE raises a clear error."""
config = _make_int8_moe_config(moe_backend="cutlass")
with pytest.raises(ValueError, match="not supported for Int8 MoE"):
select_int8_moe_backend(
config,
weight_key=kInt8StaticChannelSym,
activation_key=kInt8DynamicTokenSym,
)
-122
View File
@@ -1,122 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests whether gptq models with quantized lm_head can be loaded.
Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
"""
from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import preset_name_to_scheme
from tests.quantization.utils import load_model_without_vllm_runner
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
)
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
PROMPT = "On the surface of Mars, we found"
MODELS_QUANT = [
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", False),
]
@pytest.mark.parametrize("model_id, lm_head_quantized", MODELS_QUANT)
def test_lm_head(
model_id: str,
lm_head_quantized: bool,
monkeypatch,
dist_init,
workspace_init,
) -> None:
# `LLM.apply_model` requires pickling a function.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model, _ = load_model_without_vllm_runner(
model_id,
dtype=torch.float16,
model_config_kwargs={
"max_model_len": 2048,
"hf_overrides": {"num_hidden_layers": 3},
},
)
lm_head_layer = model.lm_head
if lm_head_quantized:
assert isinstance(lm_head_layer.quant_method, AutoGPTQLinearMethod)
else:
assert isinstance(lm_head_layer.quant_method, UnquantizedEmbeddingMethod)
@pytest.mark.skipif(
not current_platform.is_cuda() or not has_humming(), reason="requires Humming/CUDA"
)
@pytest.mark.parametrize(
"preset,quant_format",
[
("FP8_DYNAMIC", "float-quantized"),
("NVFP4A16", "nvfp4-pack-quantized"),
("W4A16", "pack-quantized"),
("W4A16_NVFP4", None),
],
)
@pytest.mark.parametrize("bias", [False, True])
@torch.inference_mode()
def test_quantized_lm_head_matches_linear(
dist_init, default_vllm_config, preset, quant_format, bias
):
"""An explicitly quantized head must produce the same logits as a linear layer."""
default_vllm_config.model_config = SimpleNamespace(
dtype=torch.bfloat16, head_dtype=None
)
default_vllm_config.kernel_config.linear_backend = "humming"
quant_config: QuantizationConfig
if quant_format is None:
quant_config = ModelOptNvFp4Config(
quant_method=preset,
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
)
else:
scheme = preset_name_to_scheme(preset, targets=["Linear", "lm_head"])
quant_config = CompressedTensorsConfig.from_config(
{"config_groups": {"group_0": scheme.model_dump()}, "format": quant_format}
)
with torch.device("cuda"):
kwargs = dict(
bias=bias,
params_dtype=torch.bfloat16,
quant_config=quant_config,
disable_tp=True,
)
head = ParallelLMHead(500, 256, prefix="lm_head", **kwargs)
linear = ColumnParallelLinear(256, 512, prefix="proj", **kwargs)
for name, param in linear.named_parameters():
if "scale" in name:
param.fill_(1.0)
elif name == "weight_shape":
param.copy_(torch.tensor([512, 256]))
elif param.is_floating_point():
param.copy_(torch.randn(param.shape, dtype=torch.float32))
else:
param.random_(0, 127)
head.load_state_dict(linear.state_dict())
for layer in (head, linear):
layer.quant_method.process_weights_after_loading(layer)
x = torch.randn(8, 256, dtype=torch.bfloat16)
expected, _ = linear(x)
actual = LogitsProcessor(500)(head, x, head.bias)
assert torch.isfinite(actual).all()
torch.testing.assert_close(actual, expected[:, :500], rtol=0, atol=0)
-624
View File
@@ -1,624 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import (
QuantizationArgs,
QuantizationStrategy,
QuantizationType,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
_backend_incompatibility_reason,
_convert_moe_wna16_humming_tensors,
convert_to_wna16_moe_kernel_format,
map_wna16_backend,
)
from vllm.model_executor.layers.quantization import moe_wna16
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.platforms import current_platform
def test_map_wna16_backend_supports_triton():
assert map_wna16_backend("triton") == WNA16MoEBackend.TRITON
@pytest.mark.parametrize(
"config",
[
{"desc_act": True, "group_size": 128},
{
"desc_act": False,
"group_size": 128,
"dynamic": {r"+:model\.layers\.0\..*": {"desc_act": True}},
},
],
)
def test_moe_wna16_rejects_gptq_group_activation_order(config):
config.update({"quant_method": "gptq", "bits": 4, "sym": True})
with pytest.raises(ValueError, match="group activation ordering"):
MoeWNA16Config.from_config(config)
def test_moe_wna16_accepts_channelwise_gptq_activation_order():
config = {
"quant_method": "gptq",
"bits": 4,
"group_size": -1,
"desc_act": True,
"sym": True,
}
assert MoeWNA16Config.is_moe_wna16_compatible(config)
MoeWNA16Config.from_config(config)
@pytest.mark.parametrize(
("backend", "quant_config", "may_have_zp", "may_have_bias", "expected"),
[
(
WNA16MoEBackend.TRITON,
AutoAWQConfig(4, 128, True, False),
True,
False,
"AutoAWQ weight layout",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
False,
True,
"bias",
),
(
WNA16MoEBackend.MARLIN,
MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=4,
group_size=128,
has_zp=False,
lm_head_quantized=False,
modules_to_not_convert=None,
full_config={},
),
False,
False,
"MoeWNA16 checkpoint layout",
),
(
WNA16MoEBackend.RDNA3,
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
False,
False,
"compressed-tensors",
),
(
WNA16MoEBackend.RDNA3,
QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=False,
dynamic=False,
group_size=128,
),
True,
False,
"asymmetric",
),
(
WNA16MoEBackend.RDNA3,
QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.CHANNEL,
symmetric=True,
dynamic=False,
),
False,
False,
"group-wise scales",
),
],
)
def test_wna16_oracle_rejects_incompatible_quant_structures(
backend, quant_config, may_have_zp, may_have_bias, expected
):
from tests.kernels.moe.utils import make_dummy_moe_config
moe_config = make_dummy_moe_config()
reason = _backend_incompatibility_reason(
backend=backend,
moe_config=moe_config,
quant_config=quant_config,
may_have_zp=may_have_zp,
may_have_bias=may_have_bias,
allow_tile_padding=True,
)
assert reason is not None
assert expected in reason
def test_compressed_tensors_nfirst_weights_pass_through_for_triton():
"""CT provides N-first [E, N, K_packed] int32; Triton expects N-first uint8.
No transpose needed — just a view from int32 to uint8.
"""
quant_config = QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=32,
)
# N-first layout: [E, N, K_packed]
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2)
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2)
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4)
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3)
converted = convert_to_wna16_moe_kernel_format(
backend=WNA16MoEBackend.TRITON,
layer=torch.nn.Module(),
quant_config=quant_config,
input_dtype=None,
w13=w13,
w2=w2,
w13_scale=w13_scale,
w2_scale=w2_scale,
)
assert converted is not None
assert torch.equal(converted[0], w13.contiguous().view(torch.uint8))
assert torch.equal(converted[1], w2.contiguous().view(torch.uint8))
assert torch.equal(converted[2], w13_scale)
assert torch.equal(converted[3], w2_scale)
def test_gptq_nfirst_weights_pass_through_for_triton():
"""GPTQ frontend normalizes to N-first before calling oracle.
Verifies the oracle handles N-first GPTQ inputs the same way as CT.
"""
quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
# N-first layout (after GPTQ frontend transpose): [E, N, K_packed]
w13 = torch.arange(16, dtype=torch.int32).reshape(1, 8, 2)
w2 = torch.arange(12, dtype=torch.int32).reshape(1, 6, 2)
w13_scale = torch.arange(32, dtype=torch.float16).reshape(1, 8, 4)
w2_scale = torch.arange(18, dtype=torch.float16).reshape(1, 6, 3)
converted = convert_to_wna16_moe_kernel_format(
backend=WNA16MoEBackend.TRITON,
layer=torch.nn.Module(),
quant_config=quant_config,
input_dtype=None,
w13=w13,
w2=w2,
w13_scale=w13_scale,
w2_scale=w2_scale,
)
assert converted is not None
assert torch.equal(converted[0], w13.contiguous().view(torch.uint8))
assert torch.equal(converted[1], w2.contiguous().view(torch.uint8))
assert torch.equal(converted[2], w13_scale)
assert torch.equal(converted[3], w2_scale)
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
method = object.__new__(MoeWNA16Method)
method.experts_cls = object
method.wna16_backend = WNA16MoEBackend.HUMMING
method.moe = object()
quant_config = object()
monkeypatch.setattr(
method, "get_fused_moe_quant_config", lambda layer: quant_config
)
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
captured = {}
kernel = object()
def fake_make_wna16_moe_kernel(**kwargs):
captured.update(kwargs)
return kernel
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
method._setup_kernel(layer)
assert method.moe_kernel is kernel
assert captured["backend"] == WNA16MoEBackend.HUMMING
def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
qweight = torch.arange(32, dtype=torch.uint8).reshape(1, 4, 8)
scales = torch.arange(16, dtype=torch.float16).reshape(1, 4, 4)
qzeros = torch.arange(16, dtype=torch.uint8).reshape(1, 8, 2)
converted = _convert_moe_wna16_humming_tensors(
{"qweight": qweight, "scales": scales, "qzeros": qzeros},
has_zero_point=True,
)
assert torch.equal(converted["weight"], qweight.view(torch.int32))
assert converted["weight"].shape == (1, 4, 2)
assert torch.equal(converted["weight_scale"], scales)
expected_qzeros = (
qzeros.transpose(-1, -2)
.contiguous()
.view(torch.int32)
.transpose(-1, -2)
.contiguous()
)
assert torch.equal(converted["zero_point"], expected_qzeros)
assert converted["zero_point"].shape == (1, 2, 2)
def test_moe_wna16_uses_humming_quant_config(monkeypatch):
from vllm.model_executor.layers.quantization.utils import humming as humming_utils
method = object.__new__(MoeWNA16Method)
method.wna16_backend = WNA16MoEBackend.HUMMING
layer = object()
quant_config = object()
monkeypatch.setattr(
humming_utils,
"get_humming_moe_quant_config",
lambda actual_layer, *args, **kwargs: (
quant_config if actual_layer is layer else None
),
)
assert method.get_fused_moe_quant_config(layer) is quant_config
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
)
@pytest.mark.parametrize("num_bits", [3, 5, 6, 7])
def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
num_bits,
):
pytest.importorskip("humming")
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
quant_args = QuantizationArgs(
num_bits=num_bits,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
)
moe_config = make_dummy_moe_config(
num_experts=2,
hidden_dim=256,
intermediate_size=512,
)
moe_config.moe_backend = "humming"
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
layer = torch.nn.Module()
method.create_weights(
layer,
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=512,
params_dtype=torch.float16,
)
packed_hidden = (256 * num_bits + 31) // 32
packed_intermediate = (512 * num_bits + 31) // 32
assert method.wna16_backend == WNA16MoEBackend.HUMMING
assert layer.w13_weight_packed.shape == (2, 1024, packed_hidden)
assert layer.w2_weight_packed.shape == (2, 256, packed_intermediate)
assert layer.w13_weight_scale.shape == (2, 1024, 2)
assert layer.w2_weight_scale.shape == (2, 256, 4)
assert layer.w13_weight_packed.dtype is torch.int32
assert layer.w2_weight_scale.dtype is torch.float16
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
)
def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
pytest.importorskip("humming")
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
quant_args = QuantizationArgs(
num_bits=3,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
)
moe_config = make_dummy_moe_config(
num_experts=2,
hidden_dim=256,
intermediate_size=512,
)
moe_config.moe_backend = "humming"
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
layer = torch.nn.Module()
layer.moe_config = moe_config
layer.params_dtype = torch.bfloat16
layer.layer_name = "test.humming_moe"
layer._expert_routing_tables = lambda: (None, None, None)
method.create_weights(
layer,
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=512,
params_dtype=torch.bfloat16,
)
layer.cuda()
for parameter in layer.parameters():
parameter.data.zero_()
method.process_weights_after_loading(layer)
assert method.wna16_backend == WNA16MoEBackend.HUMMING
assert method.moe_kernel is not None
assert set(layer.weight_schemas) == {"w13", "w2"}
assert set(layer.humming_configs) == {"w13", "w2"}
assert not hasattr(layer, "w13_weight_packed")
assert not hasattr(layer, "w2_weight_packed")
assert layer.w13_weight.dtype is torch.int32
assert layer.w2_weight.dtype is torch.int32
def test_moe_wna16_forwards_packed_modules_mapping_to_linear_delegate(monkeypatch):
"""The linear delegate must receive packed_modules_mapping.
It is rebuilt from the raw HF quantization dict, which lists shard names and
never fused ones, so without the mapping a fused layer resolves to
`UnquantizedLinearMethod` and the checkpoint's qweight has nowhere to load.
"""
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
config = MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=4,
group_size=128,
has_zp=False,
lm_head_quantized=False,
modules_to_not_convert=None,
full_config={
"bits": 4,
"group_size": 128,
"desc_act": False,
"sym": True,
"quant_method": "gptq",
# As emitted by AutoGPTQ: shard names, never the fused name.
"modules_in_block_to_quantize": [["mlp.gate_proj", "mlp.up_proj"]],
},
)
config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
seen: dict[str, dict[str, list[str]]] = {}
monkeypatch.setattr(
AutoGPTQConfig,
"get_quant_method",
lambda self, layer, prefix: seen.setdefault(
"mapping", self.packed_modules_mapping
),
)
layer = ColumnParallelLinear.__new__(ColumnParallelLinear)
config.get_quant_method(layer, "model.layers.0.mlp.gate_up_proj")
assert seen["mapping"] == {"gate_up_proj": ["gate_proj", "up_proj"]}
def test_xpu_platform_supports_moe_wna16():
"""Regression guard for the XPU quantization allowlist."""
try:
from vllm.platforms.xpu import XPUPlatform
except ImportError:
pytest.skip("vllm_xpu_kernels not importable outside an XPU stack")
assert "moe_wna16" in XPUPlatform.supported_quantization
def _channelwise_int4_args() -> QuantizationArgs:
"""A per-channel int4 checkpoint, which leaves ``group_size`` unset."""
args = QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.CHANNEL,
symmetric=True,
dynamic=False,
)
assert args.group_size is None, "premise: CHANNEL leaves group_size unset"
return args
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="check_moe_marlin_supports_config rejects every config on ROCm",
)
@pytest.mark.parametrize("backend", [WNA16MoEBackend.MARLIN, WNA16MoEBackend.TRITON])
def test_wna16_oracle_accepts_unset_group_size(backend):
"""Both backends must *accept* a per-channel config, not just survive it.
-1 is a supported Marlin group size and the shapes below pass the Marlin
tiling checks, while Triton never reads group_size for QuantizationArgs.
A reason string from either backend would mean the unset group_size cost
the layer its preferred kernel instead of raising TypeError.
"""
from tests.kernels.moe.utils import make_dummy_moe_config
# hidden_dim % 128 and intermediate % 64 must hold, or the Marlin shape
# check rejects the config before group_size is ever read.
reason = _backend_incompatibility_reason(
backend=backend,
moe_config=make_dummy_moe_config(
num_experts=2, hidden_dim=256, intermediate_size=512
),
quant_config=_channelwise_int4_args(),
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is None
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Marlin is only a candidate WNA16 MoE backend on CUDA; elsewhere "
"__init__ takes the non-Marlin branch, which rejects channelwise",
)
def test_compressed_tensors_wna16_moe_marlin_prep_with_unset_group_size():
"""Load a per-channel checkpoint through the Marlin path end to end.
``__init__`` and Marlin weight prep read ``group_size`` independently, so
both have to normalise the unset value. The post-repack shapes prove prep
ran with the Marlin K/N rather than merely returning something.
"""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
num_experts, hidden_size, intermediate_size = 2, 256, 512
moe_config = make_dummy_moe_config(
num_experts=num_experts,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
)
moe_config.moe_backend = "marlin"
method = CompressedTensorsWNA16MoEMethod(_channelwise_int4_args(), None, moe_config)
assert method.wna16_backend == WNA16MoEBackend.MARLIN
assert method.group_size == -1
layer = torch.nn.Module()
layer.intermediate_size_per_partition = intermediate_size
layer._expert_routing_tables = lambda: (None, None, None)
method.create_weights(
layer,
num_experts=num_experts,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size,
intermediate_size_full=intermediate_size,
params_dtype=torch.bfloat16,
)
layer.cuda()
for parameter in layer.parameters():
parameter.data.zero_()
method.process_weights_after_loading(layer)
# gptq_marlin_moe_repack packs to (size_k // 16, size_n * 2) for int4; w13
# is repacked with size_k=hidden_size, size_n=2*intermediate_size, and w2
# the other way round. Channelwise keeps one scale group per channel.
assert layer.w13_weight_packed.shape == (
num_experts,
hidden_size // 16,
4 * intermediate_size,
)
assert layer.w2_weight_packed.shape == (
num_experts,
intermediate_size // 16,
2 * hidden_size,
)
assert layer.w13_weight_scale.shape == (num_experts, 1, 2 * intermediate_size)
assert layer.w2_weight_scale.shape == (num_experts, 1, hidden_size)
def test_moe_wna16_w13_qzeros_shard_split_int8_asym_gptq(monkeypatch):
"""8-bit asymmetric GPTQ MoE must split w13 zero-points by pack factor.
``w13_qzeros`` is packed along the intermediate dim by ``bit8_pack_factor``,
so the gate/up (w1/w3) boundary in the weight loader is
``shard_size // bit8_pack_factor``. Hardcoding ``shard_size // 2`` only
holds for 4-bit and raises a shape mismatch when loading an 8-bit
(unpacked) checkpoint.
"""
from tests.kernels.moe.utils import make_dummy_moe_config
monkeypatch.setattr(
moe_wna16, "get_tp_group", lambda: SimpleNamespace(device="cpu")
)
monkeypatch.setattr(moe_wna16, "get_tensor_model_parallel_rank", lambda: 0)
num_experts, hidden, intermediate, group_size = 2, 256, 512, 128
moe_config = make_dummy_moe_config(
num_experts=num_experts,
hidden_dim=hidden,
intermediate_size=intermediate,
)
quant_config = MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=8,
group_size=group_size,
has_zp=True,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
)
method = object.__new__(MoeWNA16Method)
method.quant_config = quant_config
method.moe = moe_config
layer = torch.nn.Module()
layer.intermediate_size_per_partition = intermediate
layer.moe_config = moe_config
method.create_weights(
layer,
num_experts=num_experts,
hidden_size=hidden,
intermediate_size_per_partition=intermediate,
params_dtype=torch.float16,
weight_loader=lambda *a, **k: True,
)
# 8-bit is unpacked along the intermediate dim: dim1 == 2 * intermediate.
assert layer.w13_qzeros.shape == (
num_experts,
2 * intermediate,
hidden // group_size,
)
loader = layer.w13_qzeros.weight_loader
# Raw GPTQ 8-bit qzeros per shard: int32 (num_groups, intermediate // 4);
# the loader views them as uint8 -> (num_groups, intermediate), then
# transposes and offsets by 1.
num_groups = hidden // group_size
raw_w1 = torch.arange(num_groups * (intermediate // 4), dtype=torch.int32).reshape(
num_groups, intermediate // 4
)
raw_w3 = raw_w1 + 1000
# Must not raise (old code raised a shape mismatch here for 8-bit).
loader(layer.w13_qzeros, raw_w1, "w13_qzeros", "w1", 0, False)
loader(layer.w13_qzeros, raw_w3, "w13_qzeros", "w3", 0, False)
expected_w1 = raw_w1.view(torch.uint8).T + 1
expected_w3 = raw_w3.view(torch.uint8).T + 1
assert torch.equal(layer.w13_qzeros[0, :intermediate].cpu(), expected_w1)
assert torch.equal(layer.w13_qzeros[0, intermediate:].cpu(), expected_w3)
-629
View File
@@ -1,629 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the online MXFP4 weight quantization backends.
Each backend (Triton, aiter, XPU) quantizes a synthetic bf16/fp16
tensor to MXFP4, the result is dequantized back to bf16/fp16, and compared
against the pure-torch reference in `reference_mxfp4.py`.
"""
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.config.model import ModelConfig
from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
AiterMxfp4LinearKernel,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.online.mxfp4 import (
Mxfp4OnlineLinearMethod,
Mxfp4OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.quark.quark_moe import (
QuarkOCP_MX_MoEMethod,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_ocp_mx import (
QuarkOCP_MX,
)
from vllm.model_executor.layers.quantization.quark.utils import (
quark_quantize_weight_to_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
downcast_to_mxfp,
mxfp4_quantize,
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
kMxfp4Static,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import PlatformEnum, current_platform
from .reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
def fix_negative_zeros(tensor: torch.Tensor) -> torch.Tensor:
"""Normalize FP4 e2m1 negative-zero codewords (0b1000) to positive zero
(0b0000) in a packed uint8 tensor (two e2m1 values per byte, low nibble
first).
-0.0 and +0.0 dequantize to the same float value, but different MXFP4
quantization backends disagree on which one they emit for values that
round to zero magnitude. This normalizes both packed representations so
tensors from different backends can be compared with `torch.equal`.
"""
assert tensor.dtype == torch.uint8, (
f"Expected a torch.uint8 tensor, got {tensor.dtype}"
)
low_nibble = tensor & 0x0F
high_nibble = tensor & 0xF0
low_nibble = torch.where(
low_nibble == 0x08, torch.zeros_like(low_nibble), low_nibble
)
high_nibble = torch.where(
high_nibble == 0x80, torch.zeros_like(high_nibble), high_nibble
)
return low_nibble | high_nibble
def assert_quantized_weights_equal(
checkpoint_layer: torch.nn.Module,
online_layer: torch.nn.Module,
keys: tuple[str, ...],
packed_weight_keys: tuple[str, ...],
) -> None:
"""Assert the checkpoint and online paths produced byte-identical weights.
`packed_weight_keys` lists the entries holding packed FP4 codewords, which
need negative-zero normalization before comparison.
"""
for key in keys:
checkpoint_tensor = getattr(checkpoint_layer, key).view(torch.uint8)
online_tensor = getattr(online_layer, key).view(torch.uint8)
# NOTE: AMD Quark checkpoints use exclusively **positive** zeros,
# while other mxfp4_quantize implementations from mxfp4_utils.py may not.
if key in packed_weight_keys:
checkpoint_tensor = fix_negative_zeros(checkpoint_tensor)
online_tensor = fix_negative_zeros(online_tensor)
assert checkpoint_tensor.shape == online_tensor.shape
assert checkpoint_tensor.dtype == online_tensor.dtype
num_mismatched = (checkpoint_tensor != online_tensor).sum().item()
total = checkpoint_tensor.numel()
assert torch.equal(checkpoint_tensor, online_tensor), (
f"{key}: {num_mismatched}/{total} "
f"({100 * num_mismatched / total:.4f}%) mismatched bytes"
)
def _skip_reason_if_unavailable(backend: str, dtype: torch.dtype) -> str | None:
"""Return a skip reason if `backend` cannot run on the current host."""
if backend == "triton":
if not (current_platform.is_cuda() or current_platform.is_rocm()):
return "Triton MXFP4 kernel requires a CUDA or ROCm GPU."
return None
if backend == "aiter":
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
return "aiter is not available/supported on this platform."
if dtype != torch.bfloat16:
return "aiter's dynamic_mxfp4_quant only supports bfloat16 input."
return None
if backend == "xpu":
if not current_platform.is_xpu():
return "not on XPU platform."
return None
if backend == "quark":
try:
import quark.torch.kernel.mx # noqa: F401
except ImportError:
return "amd-quark is not installed."
return None
raise ValueError(f"Unknown backend {backend}")
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("backend", ["triton", "aiter", "xpu", "quark"])
def test_mxfp4_quantization_correctness(backend: str, dtype: torch.dtype):
"""Tests that the different implementations of mxfp4_quantize
in mxfp4_utils.py all match.
"""
skip_reason = _skip_reason_if_unavailable(backend, dtype)
if skip_reason is not None:
pytest.skip(skip_reason)
torch.manual_seed(3)
num_rows = 64
hidden_size = 32 * 32 # multiple 32-element MXFP4 blocks
device = current_platform.device_type
x = (torch.rand(num_rows, hidden_size, dtype=dtype, device=device) - 0.5) * 2
# Vary the magnitude block-to-block so several scale exponents are
# exercised, rather than a single one for the whole tensor.
scalings = [2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]
for i in range(hidden_size // 32):
x[:, i * 32 : (i + 1) * 32] *= scalings[i % len(scalings)]
if backend == "triton":
x_fp4, x_scale, _ = downcast_to_mxfp(x, axis=-1)
elif backend == "aiter":
x_fp4, x_scale = quark_quantize_weight_to_mxfp4(x)
elif backend == "xpu":
# TODO: enable this test on XPU
pytest.skip("xpu mxfp4 quantization match to reference is untested")
# from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
# xpu_mxfp4_quantize,
# )
# x_fp4, x_scale = xpu_mxfp4_quantize(x)
elif backend == "quark":
result = quant_dequant_mxfp4(x, scale_calculation_mode="even")
else:
raise ValueError(f"Unknown backend {backend}")
if backend != "quark":
result = dq_mxfp4_torch(x_fp4, x_scale, x.dtype)
reference = qdq_mxfp4_torch(x, scale_calculation_mode="even")
assert torch.equal(result, reference)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only tested on ROCm/CUDA."
)
@pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int):
"""TP-aligned MXFP4 shards match slices of unsharded quantization."""
torch.manual_seed(3)
device = current_platform.device_type
hidden_size = 1024
intermediate_size = 512
dense_weight = torch.randn(
hidden_size, intermediate_size, dtype=torch.bfloat16, device=device
)
w13_weight = torch.randn(
2,
2 * intermediate_size,
hidden_size,
dtype=torch.bfloat16,
device=device,
)
w2_weight = torch.randn(
2,
hidden_size,
intermediate_size,
dtype=torch.bfloat16,
device=device,
)
dense_quantized, dense_scales = mxfp4_quantize(dense_weight)
w13_quantized, w13_scales = mxfp4_quantize(w13_weight)
w2_quantized, w2_scales = mxfp4_quantize(w2_weight)
for tp_rank in range(tp_size):
output_per_partition = hidden_size // tp_size
input_per_partition = intermediate_size // tp_size
output_start = tp_rank * output_per_partition
input_start = tp_rank * input_per_partition
column_quantized, column_scales = mxfp4_quantize(
dense_weight.narrow(0, output_start, output_per_partition)
)
assert torch.equal(
column_quantized,
dense_quantized.narrow(0, output_start, output_per_partition),
)
assert torch.equal(
column_scales,
dense_scales.narrow(0, output_start, output_per_partition),
)
row_quantized, row_scales = mxfp4_quantize(
dense_weight.narrow(1, input_start, input_per_partition)
)
assert torch.equal(
row_quantized,
dense_quantized.narrow(1, input_start // 2, input_per_partition // 2),
)
assert torch.equal(
row_scales,
dense_scales.narrow(1, input_start // 32, input_per_partition // 32),
)
w13_quantized_shard, w13_scales_shard = mxfp4_quantize(
w13_weight.narrow(1, 2 * input_start, 2 * input_per_partition)
)
assert torch.equal(
w13_quantized_shard,
w13_quantized.narrow(1, 2 * input_start, 2 * input_per_partition),
)
assert torch.equal(
w13_scales_shard,
w13_scales.narrow(1, 2 * input_start, 2 * input_per_partition),
)
w2_quantized_shard, w2_scales_shard = mxfp4_quantize(
w2_weight.narrow(2, input_start, input_per_partition)
)
assert torch.equal(
w2_quantized_shard,
w2_quantized.narrow(2, input_start // 2, input_per_partition // 2),
)
assert torch.equal(
w2_scales_shard,
w2_scales.narrow(2, input_start // 32, input_per_partition // 32),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only tested on ROCm/CUDA."
)
@pytest.mark.parametrize("moe_backend", ["aiter", "emulation"])
@pytest.mark.parametrize(
"unpadded_hidden_size,unpadded_intermediate_size",
[
pytest.param(256, 256, id="no_padding"),
# Not multiples of the MXFP4 block size (32), so that every backend --
# including emulation, whose round-up granularity is 32 -- has to pad.
pytest.param(240, 208, id="padding"),
],
)
def test_online_mxfp4_moe_matches_quark(
moe_backend: MoEBackend,
unpadded_hidden_size: int,
unpadded_intermediate_size: int,
default_vllm_config,
dist_init,
monkeypatch,
):
"""Ensures `Mxfp4OnlineMoEMethod` (online quantization)
and `QuarkOCP_MX_MoEMethod` (AMD Quark checkpoints) produce the same weights,
with same MOE backend used.
The `padding` case additionally covers `maybe_roundup_sizes`: the online
path allocates its bf16 weights with `torch.empty_strided` and the loader
only writes the unpadded slice, so the padding is filled with NaN here to
reproduce that uninitialized state. It must be zeroed before quantization
for the two paths to agree, since Quark allocates its padding with
`torch.zeros`.
"""
if moe_backend == "aiter":
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
rocm_aiter_ops.refresh_env_variables()
if current_platform.is_cuda():
pytest.skip(
"mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4 requires "
"rocm_aiter_ops in weight conversion, not compatible on cuda"
)
elif moe_backend == "emulation":
# `OCP_MXQuantizationEmulationTritonExperts.is_supported_config` gates
# on `has_quark()`, but its weight processing does not require it.
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.experts."
"ocp_mx_emulation_moe.has_quark",
lambda: True,
)
default_vllm_config.model_config = ModelConfig()
num_experts = 4
device = current_platform.device_type
def make_layer(prefix: str) -> RoutedExperts:
runner = FusedMoEFactory(
num_experts=num_experts,
top_k=2,
hidden_size=unpadded_hidden_size,
intermediate_size=unpadded_intermediate_size,
prefix=prefix,
)
layer = runner.routed_experts
layer.moe_config.moe_backend = moe_backend
return layer
# `create_weights` implementations use plain `torch.zeros`/`torch.randn`
# with no explicit device, so without this context they would default
# to CPU and diverge from the (GPU) tensors produced by
# `mxfp4_quantize`/`replace_parameter`.
with torch.device(device):
checkpoint_layer = make_layer("checkpoint_layer")
online_layer = make_layer("online_layer")
checkpoint_method = QuarkOCP_MX_MoEMethod(
moe=checkpoint_layer.moe_config,
weight_quant_key=kMxfp4Static,
activation_quant_key=kMxfp4Dynamic,
)
online_method = Mxfp4OnlineMoEMethod(moe=online_layer.moe_config)
# `RoutedExperts.__init__` applies this round-up in production; these
# layers are built without a quant config, so it is applied explicitly.
# Both methods must agree on the padded sizes.
roundup_arguments = dict(
hidden_size=unpadded_hidden_size,
intermediate_size_per_partition=unpadded_intermediate_size,
act_dtype=torch.bfloat16,
moe_parallel_config=online_layer.moe_config.moe_parallel_config,
)
hidden_size, intermediate_size = online_method.maybe_roundup_sizes(
**roundup_arguments
)
assert (
hidden_size,
intermediate_size,
) == checkpoint_method.maybe_roundup_sizes(**roundup_arguments)
if unpadded_hidden_size == 240:
# Padded case.
assert hidden_size > unpadded_hidden_size
assert intermediate_size > unpadded_intermediate_size
for layer in (checkpoint_layer, online_layer):
assert layer.moe_config.hidden_dim_unpadded == unpadded_hidden_size
assert (
layer.moe_config.intermediate_size_per_partition_unpadded
== unpadded_intermediate_size
)
layer.moe_config.hidden_dim = hidden_size
layer.moe_config.intermediate_size_per_partition = intermediate_size
gate_up_weight = torch.randn(
num_experts,
2 * unpadded_intermediate_size,
unpadded_hidden_size,
dtype=torch.bfloat16,
)
down_weight = torch.randn(
num_experts,
unpadded_hidden_size,
unpadded_intermediate_size,
dtype=torch.bfloat16,
)
scalings = [2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]
for i, start in enumerate(range(0, unpadded_hidden_size, 32)):
gate_up_weight[..., start : start + 32] *= scalings[i % len(scalings)]
for i, start in enumerate(range(0, unpadded_intermediate_size, 32)):
down_weight[..., start : start + 32] *= scalings[i % len(scalings)]
def scatter_into_padded(
gate_up_weight: torch.Tensor,
down_weight: torch.Tensor,
padding_value: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Place the source weights into buffers of the padded size, the
way the weight loader writes only the unpadded slice of a larger
allocation."""
w13 = torch.full(
(num_experts, 2 * intermediate_size, hidden_size),
padding_value,
dtype=torch.bfloat16,
)
w2 = torch.full(
(num_experts, hidden_size, intermediate_size),
padding_value,
dtype=torch.bfloat16,
)
# w13 stacks gate and up, each padded to `intermediate_size`.
w13[:, :unpadded_intermediate_size, :unpadded_hidden_size] = gate_up_weight[
:, :unpadded_intermediate_size, :
]
w13[
:,
intermediate_size : intermediate_size + unpadded_intermediate_size,
:unpadded_hidden_size,
] = gate_up_weight[:, unpadded_intermediate_size:, :]
w2[:, :unpadded_hidden_size, :unpadded_intermediate_size] = down_weight
return w13, w2
# Checkpoint path: pre-quantize the source weights to MXFP4 with the
# ~same RTN recipe a real Quark checkpoint, then feed the
# already-packed tensors into `QuarkOCP_MX_MoEMethod`. Quark's
# `create_weights` allocates with `torch.zeros`, so its padding is zero.
checkpoint_w13, checkpoint_w2 = scatter_into_padded(
gate_up_weight, down_weight, padding_value=0.0
)
checkpoint_w13, checkpoint_w13_scale = mxfp4_quantize(checkpoint_w13)
checkpoint_w2, checkpoint_w2_scale = mxfp4_quantize(checkpoint_w2)
checkpoint_method.create_weights(
layer=checkpoint_layer,
num_experts=num_experts,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size,
params_dtype=torch.bfloat16,
)
checkpoint_layer.w13_weight.data.copy_(checkpoint_w13)
checkpoint_layer.w2_weight.data.copy_(checkpoint_w2)
checkpoint_layer.w13_weight_scale.data.copy_(checkpoint_w13_scale)
checkpoint_layer.w2_weight_scale.data.copy_(checkpoint_w2_scale)
checkpoint_method.process_weights_after_loading(checkpoint_layer)
# Online path: feed the *same* raw bf16 source weights and let
# `Mxfp4OnlineMoEMethod` quantize them during
# `process_weights_after_loading`. The padding starts out
# uninitialized, standing in for the materialized meta tensor.
online_w13, online_w2 = scatter_into_padded(
gate_up_weight, down_weight, padding_value=float("nan")
)
online_method.create_weights(
layer=online_layer,
num_experts=num_experts,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size,
params_dtype=torch.bfloat16,
)
replace_parameter(online_layer, "w13_weight", online_w13)
replace_parameter(online_layer, "w2_weight", online_w2)
online_method.process_weights_after_loading(online_layer)
assert checkpoint_method.mxfp4_backend == online_method.mxfp4_backend
assert_quantized_weights_equal(
checkpoint_layer,
online_layer,
keys=("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale"),
packed_weight_keys=("w13_weight", "w2_weight"),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only tested on ROCm/CUDA."
)
@pytest.mark.parametrize("linear_backend", ["emulation", "aiter", "marlin"])
def test_online_mxfp4_dense_matches_quark(
linear_backend: str, default_vllm_config, dist_init, monkeypatch
):
"""Ensures `Mxfp4OnlineLinearMethod` (online quantization)
and `QuarkOCP_MX` (AMD Quark checkpoints) produce the same weights,
with same linear backend used.
"""
if linear_backend == "marlin" and not current_platform.is_cuda():
# `MarlinMxFp4LinearKernel.process_weights_after_loading` is CUDA-only
pytest.skip(
"MarlinMxFp4LinearKernel.process_weights_after_loading is CUDA-only."
)
if linear_backend == "aiter":
# `AiterMxfp4LinearKernel` is registered only for ROCm in
# `_POSSIBLE_MXFP4_KERNELS`, and `is_supported` gates on
# `current_platform.supports_mx()`. Force it onto the CUDA
# kernel list and bypass the platform gate so this backend can
# be exercised on CUDA hosts too.
monkeypatch.setattr(
AiterMxfp4LinearKernel,
"is_supported",
classmethod(lambda cls, compute_capability=None: (True, None)),
)
monkeypatch.setitem(
_POSSIBLE_MXFP4_KERNELS,
PlatformEnum.CUDA,
[
AiterMxfp4LinearKernel,
*_POSSIBLE_MXFP4_KERNELS.get(PlatformEnum.CUDA, []),
],
)
elif linear_backend == "emulation":
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
# EmulationMxfp4LinearKernel.process_weights_after_loading does not require it.
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
default_vllm_config.model_config = ModelConfig()
default_vllm_config.kernel_config.linear_backend = linear_backend
input_size = 256
output_size = 128
device = current_platform.device_type
# `create_weights` implementations use plain `torch.empty` with no explicit
# device, so without this context they would default to CPU and diverge
# from the (GPU) tensors produced by `mxfp4_quantize`/`replace_parameter`.
with torch.device(device):
checkpoint_layer = torch.nn.Module()
online_layer = torch.nn.Module()
for layer in (checkpoint_layer, online_layer):
# `LinearBase` subclasses normally set these attributes on the
# layer before `create_weights` is called; the marlin kernel's
# `process_weights_after_loading` reads them directly off the
# layer, so a bare `torch.nn.Module` needs them set explicitly.
layer.input_size = input_size
layer.output_size = output_size
layer.input_size_per_partition = input_size
layer.output_size_per_partition = output_size
layer.params_dtype = torch.bfloat16
weight = torch.randn(output_size, input_size, dtype=torch.bfloat16)
scalings = [2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]
for i in range(input_size // 32):
weight[:, i * 32 : (i + 1) * 32] *= scalings[i % len(scalings)]
# Checkpoint path: pre-quantize the source weight to MXFP4 with the
# ~same RTN recipe as a real Quark checkpoint, then feed the
# already-packed tensors into `QuarkOCP_MX`.
checkpoint_weight, checkpoint_weight_scale = mxfp4_quantize(weight)
checkpoint_scheme = QuarkOCP_MX(
weight_quant_key=kMxfp4Static,
activation_quant_key=kMxfp4Dynamic,
)
checkpoint_scheme.create_weights(
layer=checkpoint_layer,
output_partition_sizes=[output_size],
input_size_per_partition=input_size,
params_dtype=torch.bfloat16,
weight_loader=default_weight_loader,
)
checkpoint_layer.weight.data.copy_(checkpoint_weight)
checkpoint_layer.weight_scale.data.copy_(checkpoint_weight_scale)
checkpoint_scheme.process_weights_after_loading(checkpoint_layer)
# Online path: feed the *same* raw bf16 source weight and let
# `Mxfp4OnlineLinearMethod` quantize it during
# `process_weights_after_loading`.
online_method = Mxfp4OnlineLinearMethod()
online_method.create_weights(
layer=online_layer,
input_size_per_partition=input_size,
output_partition_sizes=[output_size],
input_size=input_size,
output_size=output_size,
params_dtype=torch.bfloat16,
weight_loader=default_weight_loader,
)
replace_parameter(online_layer, "weight", weight.clone())
online_method.process_weights_after_loading(online_layer)
assert type(checkpoint_scheme.ocp_mx_linear) is type(online_method.kernel)
assert_quantized_weights_equal(
checkpoint_layer,
online_layer,
keys=("weight", "weight_scale"),
packed_weight_keys=("weight",),
)
def test_quark_ocp_mx_dense_rejects_unaligned_input_partition():
scheme = object.__new__(QuarkOCP_MX)
layer = torch.nn.Module()
layer.prefix = "model.layers.0.mlp.shared_expert.down_proj"
with pytest.raises(
ValueError,
match=(
"input size per partition of 80, which must be divisible by "
"the OCP MX group size 32"
),
):
scheme.create_weights(
layer=layer,
output_partition_sizes=[2560],
input_size_per_partition=80,
params_dtype=torch.bfloat16,
weight_loader=lambda *_: None,
)
@@ -1,725 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for per-token-head KV cache quantization (INT4, INT8 and FP8).
Covers:
- Per-token-head Triton reshape-and-cache kernel
- Round-trip quantize/dequantize accuracy
- process_weights_after_loading early-return path
- End-to-end integration with Triton unified attention kernel
Run: pytest tests/quantization/test_per_token_kv_cache.py -v -s
"""
import random
from dataclasses import dataclass
from unittest.mock import MagicMock
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops.int4_per_token_head import single_rht
from vllm.v1.kv_cache_interface import KVQuantMode, is_quantized_kv_cache
DEVICE_TYPE = current_platform.device_type
# Skip entire module if no CUDA/ROCm GPU available
pytestmark = [
pytest.mark.skipif(
current_platform.is_cpu(),
reason="Per-token-head KV cache tests require GPU.",
),
]
# ---------------------------------------------------------------------------
# Test parameters
# ---------------------------------------------------------------------------
NUM_TOKENS = [1, 7, 42]
NUM_KV_HEADS = [1, 4, 8]
HEAD_SIZES = [64, 128]
BLOCK_SIZES = [16]
SEEDS = [0]
# Platform-dependent FP8 dtype and range
FP8_DTYPE = current_platform.fp8_dtype()
FP8_MIN, FP8_MAX = get_fp8_min_max()
# ---------------------------------------------------------------------------
# Per-dtype quantization config
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class QuantConfig:
"""Quantization parameters for a given cache dtype."""
cache_dtype: torch.dtype # torch.int8 or FP8_DTYPE
kv_cache_dtype_str: str # "int8_per_token_head" or "fp8_per_token_head"
quant_max: float
quant_min: float
kv_quant_mode: KVQuantMode
# INT8 rounds explicitly; FP8 relies on dtype cast rounding.
rounds_before_store: bool
INT8_CONFIG = QuantConfig(
cache_dtype=torch.int8,
kv_cache_dtype_str="int8_per_token_head",
quant_max=127.0,
quant_min=-128.0,
kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
rounds_before_store=True,
)
FP8_CONFIG = QuantConfig(
cache_dtype=FP8_DTYPE,
kv_cache_dtype_str="fp8_per_token_head",
quant_max=FP8_MAX,
quant_min=FP8_MIN,
kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
rounds_before_store=False,
)
INT4_CONFIG = QuantConfig(
cache_dtype=torch.uint8,
kv_cache_dtype_str="int4_per_token_head",
quant_max=7.0,
quant_min=-8.0,
kv_quant_mode=KVQuantMode.INT4_PER_TOKEN_HEAD,
# Unused for int4 (handled by its own rint path); kept for the dataclass.
rounds_before_store=False,
)
QUANT_CONFIGS = [INT4_CONFIG, INT8_CONFIG, FP8_CONFIG]
@pytest.fixture(params=QUANT_CONFIGS, ids=["int4", "int8", "fp8"])
def qcfg(request) -> QuantConfig:
return request.param
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _quantize_per_token_head_ref(
data: torch.Tensor, # [num_tokens, num_heads, head_size]
cfg: QuantConfig,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Reference per-token-head quantization (one scale per token per head).
Returns (quantized, scales) where scales is [num_tokens, num_heads].
"""
absmax = data.float().abs().amax(dim=2) # [num_tokens, num_heads]
scales = (absmax / cfg.quant_max).clamp(min=1e-6)
scaled = data.float() * (1.0 / scales[:, :, None])
if cfg.rounds_before_store:
q = scaled.round().clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
else:
q = scaled.clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
return q, scales
# ===========================================================================
# 1. is_quantized_kv_cache / get_kv_quant_mode
# ===========================================================================
class TestIsQuantizedKvCache:
def test_fp8_variants(self):
assert is_quantized_kv_cache("fp8")
assert is_quantized_kv_cache("fp8_e4m3")
assert is_quantized_kv_cache("fp8_e5m2")
def test_int4_per_token_head(self):
assert is_quantized_kv_cache("int4_per_token_head")
def test_int8_per_token_head(self):
assert is_quantized_kv_cache("int8_per_token_head")
def test_fp8_per_token_head(self):
assert is_quantized_kv_cache("fp8_per_token_head")
def test_auto(self):
assert not is_quantized_kv_cache("auto")
def test_bfloat16(self):
assert not is_quantized_kv_cache("bfloat16")
def test_kv_quant_mode_int4(self):
from vllm.v1.kv_cache_interface import get_kv_quant_mode
assert (
get_kv_quant_mode("int4_per_token_head") == KVQuantMode.INT4_PER_TOKEN_HEAD
)
def test_kv_quant_mode_int8(self):
from vllm.v1.kv_cache_interface import get_kv_quant_mode
assert (
get_kv_quant_mode("int8_per_token_head") == KVQuantMode.INT8_PER_TOKEN_HEAD
)
def test_kv_quant_mode_fp8(self):
from vllm.v1.kv_cache_interface import get_kv_quant_mode
assert get_kv_quant_mode("fp8_per_token_head") == KVQuantMode.FP8_PER_TOKEN_HEAD
# ===========================================================================
# 2. Triton per-token-head kernel (reshape-and-cache)
# ===========================================================================
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("num_heads", NUM_KV_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_reshape_and_cache_per_token_head(
qcfg: QuantConfig,
num_tokens: int,
num_heads: int,
head_size: int,
block_size: int,
seed: int,
):
"""Test triton_reshape_and_cache_flash_per_token_head_quant kernel."""
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash_per_token_head_quant,
)
set_random_seed(seed)
torch.set_default_device(DEVICE_TYPE)
num_blocks = (num_tokens + block_size - 1) // block_size + 4
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
cache_head_size = head_size // 2 if is_int4 else head_size
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
key_cache = torch.zeros(
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
)
value_cache = torch.zeros(
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
)
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
num_slots = block_size * num_blocks
slot_mapping = torch.tensor(
random.sample(range(num_slots), num_tokens), dtype=torch.long
)
triton_reshape_and_cache_flash_per_token_head_quant(
key,
value,
key_cache,
value_cache,
k_scale_cache,
v_scale_cache,
slot_mapping,
kv_quant_mode=qcfg.kv_quant_mode,
)
# INT4 (RHT + asymmetric), INT8/FP8 have different dequant paths. Only
# INT8/FP8 can be compared to a PyTorch reference.
if not is_int4:
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, qcfg)
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, qcfg)
for i, slot in enumerate(slot_mapping.tolist()):
blk = slot // block_size
off = slot % block_size
if is_int4:
# Coarser quantization → wider tolerance.
deq_atol = deq_rtol = 0.5
for label, data, cache, sc in [
("key", key, key_cache, k_scale_cache),
("val", value, value_cache, v_scale_cache),
]:
packed_scale = sc[blk, off] # [num_heads] float32
scale_bits = packed_scale.view(torch.int32)
zp = (scale_bits & 0xF).to(torch.float32)
clean_scale = (scale_bits & -16).view(torch.float32)
packed = cache[blk, off]
lo = (packed & 0xF).to(torch.float32)
hi = ((packed >> 4) & 0xF).to(torch.float32)
full = torch.zeros(num_heads, head_size, dtype=torch.float32)
full[:, 0::2] = lo
full[:, 1::2] = hi
# Asymmetric dequant in RHT domain, then IRHT/d → original
deq_rht = (full - zp[:, None]) * clean_scale[:, None]
deq = single_rht(deq_rht, inverse=True) / head_size
ref_deq = data[i].float()
torch.testing.assert_close(deq, ref_deq, atol=deq_atol, rtol=deq_rtol)
else:
actual_k_scale = k_scale_cache[blk, off] # [num_heads]
k_deq = key_cache[blk, off].float() * actual_k_scale[:, None]
k_ref_deq = key[i].float()
torch.testing.assert_close(
k_deq,
k_ref_deq,
atol=0.1,
rtol=0.1,
)
actual_v_scale = v_scale_cache[blk, off] # [num_heads]
v_deq = value_cache[blk, off].float() * actual_v_scale[:, None]
v_ref_deq = value[i].float()
torch.testing.assert_close(
v_deq,
v_ref_deq,
atol=0.1,
rtol=0.1,
)
# Per-head scales: [num_heads]
torch.testing.assert_close(
k_scale_cache[blk, off], ref_k_scales[i], atol=1e-4, rtol=1e-3
)
torch.testing.assert_close(
v_scale_cache[blk, off], ref_v_scales[i], atol=1e-4, rtol=1e-3
)
# ===========================================================================
# 3. Per-token-head round-trip accuracy (quantize -> dequantize)
# ===========================================================================
@pytest.mark.parametrize("num_tokens", [1, 16])
@pytest.mark.parametrize("num_heads", [4])
@pytest.mark.parametrize("head_size", [128])
@pytest.mark.parametrize("block_size", [16])
@torch.inference_mode()
def test_per_token_head_round_trip_accuracy(
qcfg: QuantConfig,
num_tokens: int,
num_heads: int,
head_size: int,
block_size: int,
):
"""Verify per-token-head round-trip: kernel dequant matches reference.
INT8: round-to-nearest before int8 store.
FP8: hardware cast (clamp then cast).
"""
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash_per_token_head_quant,
)
torch.set_default_device(DEVICE_TYPE)
set_random_seed(42)
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
num_blocks = (num_tokens + block_size - 1) // block_size + 2
cache_head_size = head_size // 2 if is_int4 else head_size
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
key_cache = torch.zeros(
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
)
value_cache = torch.zeros(
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
)
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
slot_mapping = torch.arange(num_tokens, dtype=torch.long)
triton_reshape_and_cache_flash_per_token_head_quant(
key,
value,
key_cache,
value_cache,
k_scale_cache,
v_scale_cache,
slot_mapping,
kv_quant_mode=qcfg.kv_quant_mode,
)
rt_atol = 0.5 if is_int4 else 0.1
for i in range(num_tokens):
blk = i // block_size
off = i % block_size
for label, data, cache, sc in [
("key", key, key_cache, k_scale_cache),
("val", value, value_cache, v_scale_cache),
]:
for h in range(num_heads):
orig = data[i, h].float()
actual_sc = sc[blk, off, h]
if is_int4:
sc_bits = actual_sc.view(torch.int32)
zp = (sc_bits & 0xF).to(torch.float32)
clean_sc = (sc_bits & -16).view(torch.float32)
packed = cache[blk, off, h]
lo = (packed & 0xF).to(torch.float32)
hi = ((packed >> 4) & 0xF).to(torch.float32)
full = torch.zeros(head_size)
full[0::2] = lo
full[1::2] = hi
deq_rht = (full - zp) * clean_sc
actual_deq = (
single_rht(deq_rht.unsqueeze(0), inverse=True).squeeze(0)
/ head_size
)
else:
actual_deq = cache[blk, off, h].float() * actual_sc
torch.testing.assert_close(
actual_deq,
orig,
atol=rt_atol,
rtol=rt_atol,
)
@torch.inference_mode()
def test_int8_per_token_head_raw_cache_matches_round_reference():
"""INT8 cache writes should match round-to-nearest quantization exactly."""
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash_per_token_head_quant,
)
torch.set_default_device(DEVICE_TYPE)
head_size = 8
block_size = 4
key = torch.tensor(
[[[-127.0, -2.6, -2.4, -1.6, -1.4, -0.6, -0.4, 127.0]]],
dtype=torch.bfloat16,
)
value = -key
key_cache = torch.zeros(1, block_size, 1, head_size, dtype=torch.int8)
value_cache = torch.zeros_like(key_cache)
k_scale_cache = torch.ones(1, block_size, 1, dtype=torch.float32)
v_scale_cache = torch.ones_like(k_scale_cache)
slot_mapping = torch.tensor([2], dtype=torch.long)
triton_reshape_and_cache_flash_per_token_head_quant(
key,
value,
key_cache,
value_cache,
k_scale_cache,
v_scale_cache,
slot_mapping,
kv_quant_mode=INT8_CONFIG.kv_quant_mode,
)
ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, INT8_CONFIG)
ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, INT8_CONFIG)
slot = slot_mapping.item()
blk = slot // block_size
off = slot % block_size
assert torch.equal(key_cache[blk, off], ref_k_quant[0])
assert torch.equal(value_cache[blk, off], ref_v_quant[0])
torch.testing.assert_close(k_scale_cache[blk, off], ref_k_scales[0])
torch.testing.assert_close(v_scale_cache[blk, off], ref_v_scales[0])
# ===========================================================================
# 4. Negative slot mapping (padding tokens should be skipped)
# ===========================================================================
@torch.inference_mode()
def test_per_token_head_negative_slot_skipped(qcfg: QuantConfig):
"""Tokens with slot_mapping=-1 should leave the cache unchanged."""
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash_per_token_head_quant,
)
torch.set_default_device(DEVICE_TYPE)
num_tokens = 4
num_heads = 2
head_size = 64
block_size = 16
num_blocks = 2
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
cache_head_size = head_size // 2 if is_int4 else head_size
key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
key_cache = torch.zeros(
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
)
value_cache = torch.zeros(
num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
)
k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
slot_mapping = torch.tensor([0, -1, 1, -1], dtype=torch.long)
key_cache_before = key_cache.clone()
val_cache_before = value_cache.clone()
triton_reshape_and_cache_flash_per_token_head_quant(
key,
value,
key_cache,
value_cache,
k_scale_cache,
v_scale_cache,
slot_mapping,
kv_quant_mode=qcfg.kv_quant_mode,
)
# Slots 0 and 1 should have been written (tokens 0 and 2)
assert not torch.equal(key_cache[0, 0], key_cache_before[0, 0])
assert not torch.equal(key_cache[0, 1], key_cache_before[0, 1])
assert not torch.equal(value_cache[0, 0], val_cache_before[0, 0])
# All other slots should be unchanged
assert torch.equal(key_cache[0, 2:], key_cache_before[0, 2:])
assert torch.equal(key_cache[1], key_cache_before[1])
assert torch.equal(value_cache[0, 2:], val_cache_before[0, 2:])
# ===========================================================================
# 5. process_weights_after_loading -- per-token-head early return
# ===========================================================================
@pytest.mark.parametrize(
"kv_cache_dtype",
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
)
def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
"""Per-token-head should set _k_scale=1.0, _v_scale=1.0
and delete checkpoint attrs."""
from vllm.model_executor.layers.quantization.kv_cache import (
BaseKVCacheMethod,
)
layer = MagicMock()
layer.kv_cache_dtype = kv_cache_dtype
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.prob_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer._k_scale = torch.tensor(0.0)
layer._v_scale = torch.tensor(0.0)
layer._k_scale_float = 0.0
layer._v_scale_float = 0.0
method = BaseKVCacheMethod.__new__(BaseKVCacheMethod)
method.quant_config = MagicMock()
method.process_weights_after_loading(layer)
assert layer._k_scale_float == 1.0
assert layer._v_scale_float == 1.0
assert not hasattr(layer, "k_scale")
assert not hasattr(layer, "v_scale")
assert not hasattr(layer, "q_scale")
assert not hasattr(layer, "prob_scale")
# ===========================================================================
# 6. Triton unified_attention -- per-token-head scale cache (INT4/INT8/FP8)
# ===========================================================================
@pytest.mark.parametrize(
"seq_lens",
[
[(1, 128)],
[(1, 64), (1, 32)],
],
)
@pytest.mark.parametrize("num_heads", [(4, 4)])
@pytest.mark.parametrize("head_size", [128])
@pytest.mark.parametrize("block_size", [16])
@torch.inference_mode()
def test_triton_unified_attention_per_token_head_scale(
qcfg: QuantConfig,
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_size: int,
block_size: int,
):
"""End-to-end: quantized KV with per-token-head scale caches."""
from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
torch.set_default_device(DEVICE_TYPE)
set_random_seed(0)
is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
num_seqs = len(seq_lens)
query_lens = [s[0] for s in seq_lens]
kv_lens = [s[1] for s in seq_lens]
num_query_heads, num_kv_heads = num_heads
max_query_len = max(query_lens)
max_kv_len = max(kv_lens)
scale = head_size**-0.5
num_blocks = 2048
query = torch.randn(
sum(query_lens), num_query_heads, head_size, dtype=torch.bfloat16
)
key_cache_bf16 = torch.randn(
num_blocks, block_size, num_kv_heads, head_size, dtype=torch.bfloat16
)
value_cache_bf16 = torch.randn_like(key_cache_bf16)
if is_int4:
# Asymmetric quantization reference (matches the Triton kernel).
kf = key_cache_bf16.float()
vf = value_cache_bf16.float()
k_min = kf.amin(dim=-1)
k_max = kf.amax(dim=-1)
v_min = vf.amin(dim=-1)
v_max = vf.amax(dim=-1)
k_scale_cache = ((k_max - k_min) / 15.0).clamp(min=1e-6).to(torch.float32)
v_scale_cache = ((v_max - v_min) / 15.0).clamp(min=1e-6).to(torch.float32)
k_zp = (-k_min / k_scale_cache).round().clamp(0, 15)
v_zp = (-v_min / v_scale_cache).round().clamp(0, 15)
key_cache_q_full = (
(kf / k_scale_cache[..., None] + k_zp[..., None]).round().clamp(0, 15)
)
value_cache_q_full = (
(vf / v_scale_cache[..., None] + v_zp[..., None]).round().clamp(0, 15)
)
# Dequantized reference: x_hat = (q - zp) * scale
key_cache_deq = (key_cache_q_full - k_zp[..., None]) * k_scale_cache[..., None]
value_cache_deq = (value_cache_q_full - v_zp[..., None]) * v_scale_cache[
..., None
]
# Pack two uint4 values into one byte
def _pack_int4(data_float):
u = data_float.to(torch.uint8)
lo = u[..., 0::2]
hi = u[..., 1::2]
return (lo & 0xF) | ((hi & 0xF) << 4)
key_cache_q = _pack_int4(key_cache_q_full)
value_cache_q = _pack_int4(value_cache_q_full)
# Steganography: pack zp into low 4 bits of scale
k_zp_int = k_zp.to(torch.int32)
k_bits = k_scale_cache.view(torch.int32)
k_scale_cache = ((k_bits & -16) | (k_zp_int & 0xF)).view(torch.float32)
v_zp_int = v_zp.to(torch.int32)
v_bits = v_scale_cache.view(torch.int32)
v_scale_cache = ((v_bits & -16) | (v_zp_int & 0xF)).view(torch.float32)
else:
# Symmetric quantization for int8/fp8.
k_absmax = key_cache_bf16.float().abs().amax(dim=-1)
v_absmax = value_cache_bf16.float().abs().amax(dim=-1)
k_scale_cache = (k_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
v_scale_cache = (v_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None]
scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None]
key_cache_q_full = scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max)
value_cache_q_full = scaled_v.round().clamp(qcfg.quant_min, qcfg.quant_max)
key_cache_deq = key_cache_q_full * k_scale_cache[:, :, :, None]
value_cache_deq = value_cache_q_full * v_scale_cache[:, :, :, None]
if not is_int4 and qcfg.rounds_before_store:
key_cache_q = key_cache_q_full.to(qcfg.cache_dtype)
value_cache_q = value_cache_q_full.to(qcfg.cache_dtype)
elif not is_int4:
key_cache_q = scaled_k.clamp(qcfg.quant_min, qcfg.quant_max).to(
qcfg.cache_dtype
)
value_cache_q = scaled_v.clamp(qcfg.quant_min, qcfg.quant_max).to(
qcfg.cache_dtype
)
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
)
kv_lens_t = torch.tensor(kv_lens, dtype=torch.int32)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
head_size_padded = next_power_of_2(head_size)
seq_threshold_3D = 0
num_par_softmax_segments = 16
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
output_q = torch.empty_like(query)
unified_attention(
q=query,
k=key_cache_q,
v=value_cache_q,
out=output_q,
cu_seqlens_q=cu_query_lens,
seqused_k=kv_lens_t,
max_seqlen_q=max_query_len,
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=(-1, -1),
block_table=block_tables,
softcap=0,
q_descale=None,
k_descale=None,
v_descale=None,
seq_threshold_3D=seq_threshold_3D,
num_par_softmax_segments=num_par_softmax_segments,
softmax_segm_output=softmax_segm_output,
softmax_segm_max=softmax_segm_max,
softmax_segm_expsum=softmax_segm_expsum,
kv_quant_mode=qcfg.kv_quant_mode,
k_scale_cache=k_scale_cache,
v_scale_cache=v_scale_cache,
)
output_ref = torch.empty_like(query)
unified_attention(
q=query,
k=key_cache_deq.to(torch.bfloat16),
v=value_cache_deq.to(torch.bfloat16),
out=output_ref,
cu_seqlens_q=cu_query_lens,
seqused_k=kv_lens_t,
max_seqlen_q=max_query_len,
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=(-1, -1),
block_table=block_tables,
softcap=0,
q_descale=None,
k_descale=None,
v_descale=None,
seq_threshold_3D=seq_threshold_3D,
num_par_softmax_segments=num_par_softmax_segments,
softmax_segm_output=softmax_segm_output,
softmax_segm_max=softmax_segm_max,
softmax_segm_expsum=softmax_segm_expsum,
)
# Coarser quantization → wider tolerance.
if is_int4:
# Hopper's attention reduction order can move a few BF16 elements by
# just over 1.0 after INT4 quantization.
atol = 1.1 if current_platform.is_device_capability_family(90) else 0.5
rtol = 0.5
else:
atol, rtol = 5e-2, 5e-2
torch.testing.assert_close(output_q, output_ref, atol=atol, rtol=rtol)
@@ -1,152 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests register custom quantization config.
See https://github.com/vllm-project/vllm/issues/11926 for more details.
Run `pytest tests/quantization/test_register_quantization_config.py`.
"""
import logging
from typing import Any
import pytest
import torch
import torch.nn.functional as F
from tests.quantization.utils import load_model_without_vllm_runner
from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
LinearBase, # noqa: E501
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import (
QuantizationMethods,
get_quantization_config,
register_quantization_config,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig, # noqa: E501
)
from vllm.platforms import current_platform
class FakeQuantLinearMethod(UnquantizedLinearMethod):
"""Fake quantization linear method for per-token dynamic quantization."""
def __init__(self, num_bits: int = 8) -> None:
"""Initialize the quantization method."""
super().__init__()
self.num_bits = num_bits
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Perform fake quantization before the linear layer."""
# Calculate the scales dynamically
max_val = torch.amax(x, dim=(0, -1), keepdims=True)
min_val = torch.amin(x, dim=(0, -1), keepdims=True)
scales = (max_val - min_val) / (2**self.num_bits - 1)
# Fake quantize the input
quant_x = torch.clamp(
torch.round(x / scales),
-(2 ** (self.num_bits - 1)),
2 ** (self.num_bits - 1) - 1,
)
dequant_x = quant_x * scales
return F.linear(dequant_x, layer.weight, bias)
@register_quantization_config("custom_quant")
class CustomQuantConfig(QuantizationConfig):
"""Custom quantization config for per-token dynamic fake quantization."""
def __init__(self, num_bits: int = 8) -> None:
"""Initialize the quantization config."""
super().__init__()
self.num_bits = num_bits
def get_name(self) -> QuantizationMethods:
"""Name of the quantization method."""
return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names.
def get_supported_act_dtypes(self) -> list[torch.dtype]:
"""List of supported activation dtypes."""
return [torch.float16, torch.bfloat16]
@classmethod
def get_min_capability(cls) -> int:
"""Minimum GPU capability to support the quantization method."""
return -1
@staticmethod
def get_config_filenames() -> list[str]:
"""List of filenames to search for in the model directory."""
return []
@classmethod
def from_config(cls, config: dict[str, Any]) -> "CustomQuantConfig":
"""Create a config class from the model's quantization config."""
return CustomQuantConfig(num_bits=config.get("num_bits", 8))
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> FakeQuantLinearMethod | None:
"""Get the quantize method to use for the quantized layer."""
if isinstance(layer, LinearBase):
return FakeQuantLinearMethod(num_bits=self.num_bits)
return None
def test_register_quantization_config(caplog_vllm):
"""Test register custom quantization config."""
# The quantization method `custom_quant` should be registered.
assert get_quantization_config("custom_quant") == CustomQuantConfig
# The quantization method `custom_quant` is already exists,
# should raise a debug message when re-registering it.
with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
register_quantization_config("custom_quant")(CustomQuantConfig)
assert any(
"The quantization method 'custom_quant' already exists" in message
for message in caplog_vllm.messages
), "Expected a debug message when re-registering custom_quant"
@pytest.mark.parametrize(
argnames="model",
argvalues=[
"meta-llama/Llama-3.2-1B-Instruct",
],
)
def test_custom_quant(model, monkeypatch, dist_init, workspace_init):
"""Test infer with the custom quantization method."""
vllm_model, vllm_config = load_model_without_vllm_runner(
model,
quantization="custom_quant",
)
layer = vllm_model.model.layers[0]
qkv_proj = layer.self_attn.qkv_proj
# Check the quantization method is FakeQuantLinearMethod.
assert isinstance(qkv_proj.quant_method, FakeQuantLinearMethod)
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
device_type = current_platform.device_type
input_ids = torch.tensor([1, 2, 3, 4], device=device_type)
positions = torch.arange(input_ids.numel(), device=device_type)
with (
set_current_vllm_config(vllm_config),
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
):
hidden_states = vllm_model(input_ids, positions, None)
logits = vllm_model.compute_logits(hidden_states)
assert torch.isfinite(logits).all()
-146
View File
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm equivalents of the quantized-MoE model startup/generation contract.
Like test_blackwell_moe.py, these tests use reduced models and dummy weights.
They validate model/backend integration, not trained-model accuracy.
"""
import math
import pytest
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm() or not on_gfx950(),
reason="ROCm FP8/MXFP4 model integration requires gfx950",
)
COMMON_OVERRIDES = {
"num_hidden_layers": 4,
"hidden_size": 512,
"intermediate_size": 1024,
"num_attention_heads": 8,
"num_key_value_heads": 2,
}
CASES = [
pytest.param(
"RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
"efe1ced428db63e7ccbcf367334596f77e9af140",
"triton",
"TRITON_MLA",
{
"n_routed_experts": 8,
"n_shared_experts": 1,
"num_experts_per_tok": 2,
"moe_intermediate_size": 512,
},
["--code-revision", "e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11"],
id="deepseek-fp8-per-tensor-triton",
),
pytest.param(
"Qwen/Qwen3-30B-A3B-FP8",
"d206ba732169f29bb77fbf80fc2c4b81d4d30782",
"triton",
"TRITON_ATTN",
{
"num_experts": 8,
"num_experts_per_tok": 2,
"moe_intermediate_size": 512,
"head_dim": 64,
},
[],
id="qwen3-fp8-block-triton",
),
pytest.param(
"openai/gpt-oss-20b",
"6cee5e81ee83917806bbde320786a8fb61efebee",
"aiter",
"TRITON_ATTN",
{
"num_local_experts": 8,
"num_experts_per_tok": 2,
"experts_per_token": 2,
"layer_types": ["sliding_attention", "full_attention"] * 2,
},
[],
id="gptoss-mxfp4-aiter",
),
]
@pytest.mark.parametrize("enforce_eager", [True, False], ids=["eager", "compiled"])
@pytest.mark.parametrize(
"model,revision,moe_backend,attention_backend,overrides,extra_args", CASES
)
def test_rocm_quantized_moe_generation(
model,
revision,
moe_backend,
attention_backend,
overrides,
extra_args,
enforce_eager,
):
"""Quantized MoE models warm up and return finite logits through serving."""
args = [
"--revision",
revision,
"--load-format",
"dummy",
"--trust-remote-code",
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-batched-tokens",
"256",
"--max-num-seqs",
"8",
"--gpu-memory-utilization",
"0.02",
"--kv-cache-memory-bytes",
str(256 * 1024 * 1024),
"--moe-backend",
moe_backend,
"--attention-backend",
attention_backend,
*extra_args,
]
if enforce_eager:
args.append("--enforce-eager")
with RemoteOpenAIServer(
model,
args,
env_dict={"VLLM_ROCM_USE_AITER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"},
override_hf_configs={**COMMON_OVERRIDES, **overrides},
max_wait_seconds=1200,
) as server:
prompts = [[1, 2, 3], [1, 2, 3, 4] * 4 + [5]]
completion = server.get_client().completions.create(
model=model,
prompt=prompts,
temperature=0,
max_tokens=2,
logprobs=1,
extra_body={"ignore_eos": True, "skip_special_tokens": False},
)
assert len(completion.choices) == len(prompts)
assert completion.usage is not None
assert completion.usage.prompt_tokens == sum(map(len, prompts))
assert completion.usage.completion_tokens == 2 * len(prompts)
for choice in completion.choices:
assert choice.finish_reason == "length"
assert choice.text
assert choice.logprobs is not None
assert choice.logprobs.token_logprobs is not None
assert len(choice.logprobs.token_logprobs) == 2
assert all(
p is not None and math.isfinite(p)
for p in choice.logprobs.token_logprobs
)
-413
View File
@@ -1,413 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.metadata
import importlib.util
import pytest
import torch
from packaging import version
from tests.quantization.utils import load_model_without_vllm_runner
from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
else:
def on_gfx950() -> bool:
return False
DEVICE_TYPE = current_platform.device_type
DTYPE = ["bfloat16"]
TORCHAO_AVAILABLE = importlib.util.find_spec("torchao") is not None
TORCHAO_VERSION_0_18_AVAILABLE = TORCHAO_AVAILABLE and version.parse(
importlib.metadata.version("torchao")
) >= version.parse("0.18.0")
# Fix for leaking `torch.set_float32_matmul_precision("high")`
# https://github.com/pytorch/ao/blob/v0.17.0/torchao/quantization/utils.py#L696
@pytest.fixture(autouse=True)
def restore_float32_matmul_precision():
original_precision = torch.get_float32_matmul_precision()
try:
yield
finally:
torch.set_float32_matmul_precision(original_precision)
@pytest.mark.skipif(
current_platform.is_rocm() and current_platform.is_fp8_fnuz(),
reason="Only fp8_fnuz supported on CDNA3 architecture",
)
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
def test_pre_quantized_model(monkeypatch, dist_init, workspace_init):
model, vllm_config = load_model_without_vllm_runner(
"torchao-testing/opt-125m-Float8WeightOnlyConfig-v2-0.15.0",
quantization="torchao",
dtype="bfloat16",
)
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE_TYPE)
positions = torch.arange(input_ids.numel(), device=DEVICE_TYPE)
with (
set_current_vllm_config(vllm_config),
set_forward_context(None, vllm_config, num_tokens=input_ids.numel()),
):
hidden_states = model(input_ids, positions, None)
logits = model.compute_logits(hidden_states)
assert torch.isfinite(logits).all()
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skip(
reason="since torchao nightly is only compatible with torch nightly"
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
"torchao tests that requires newer versions (0.14.0.dev+) for now"
)
def test_opt_125m_awq_int4wo_model_loading_with_params(vllm_runner):
torch._dynamo.reset()
model_name = "torchao-testing/opt-125m-AWQConfig-Int4WeightOnlyConfig-v2-0.14.0.dev"
with vllm_runner(
model_name=model_name,
quantization="torchao",
dtype="bfloat16",
pt_load_map_location=f"{DEVICE_TYPE}:0",
) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
assert output
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skipif(
on_gfx950() and not TORCHAO_VERSION_0_18_AVAILABLE,
reason="requires torchao>=0.18.0 on gfx950",
)
def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
"""Testing online quantization, load_weights integration point,
with config dict serialized to json string
"""
torch._dynamo.reset()
model_name = "facebook/opt-125m"
import json
from torchao.core.config import config_to_dict
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
torchao_quant_config = Float8DynamicActivationFloat8WeightConfig(
granularity=PerRow()
)
hf_overrides = {
"quantization_config_dict_json": json.dumps(
config_to_dict(torchao_quant_config)
)
}
with vllm_runner(
model_name=model_name,
dtype="bfloat16",
pt_load_map_location=f"{DEVICE_TYPE}:0",
quantization="torchao",
hf_overrides=hf_overrides,
enforce_eager=True,
) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
load_config = llm.llm.llm_engine.vllm_config.load_config
model_config = llm.llm.llm_engine.vllm_config.model_config
def load_weights(model):
model_loader = get_model_loader(load_config)
assert isinstance(model_loader, DefaultModelLoader)
weights_iterator = model_loader.get_all_weights(model_config, model)
model.load_weights(weights_iterator)
llm.apply_model(load_weights)
reload_output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
assert output[0][0] == reload_output[0][0]
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skipif(
on_gfx950() and not TORCHAO_VERSION_0_18_AVAILABLE,
reason="requires torchao>=0.18.0 on gfx950",
)
def test_online_quant_config_file(vllm_runner):
"""Testing on the fly quantization, load_weights integration point,
with config file
"""
torch._dynamo.reset()
model_name = "facebook/opt-125m"
import json
from tempfile import NamedTemporaryFile
from torchao.core.config import config_to_dict
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
with NamedTemporaryFile(mode="w", delete=False) as f:
f.write(json.dumps(config_to_dict(config)))
# close the file to save it
f.close()
config_file_name = str(f.name)
hf_overrides = {"quantization_config_file": config_file_name}
with vllm_runner(
model_name=model_name,
dtype="bfloat16",
pt_load_map_location=f"{DEVICE_TYPE}:0",
quantization="torchao",
hf_overrides=hf_overrides,
enforce_eager=True,
) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
assert output
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skipif(
on_gfx950() and not TORCHAO_VERSION_0_18_AVAILABLE,
reason="requires torchao>=0.18.0 on gfx950",
)
def test_reload_weights():
import json
from torchao.core.config import config_to_dict
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
from vllm import LLM, SamplingParams
torchao_quant_config = Float8DynamicActivationFloat8WeightConfig(
granularity=PerRow()
)
hf_overrides = {
"quantization_config_dict_json": json.dumps(
config_to_dict(torchao_quant_config)
)
}
llm = LLM(
model="Qwen/Qwen3-0.6B",
dtype="bfloat16",
load_format="dummy",
enforce_eager=True,
quantization="torchao",
hf_overrides=hf_overrides,
)
# Update load format from `dummy` to `auto`
llm.collective_rpc(
"update_config", args=({"load_config": {"load_format": "auto"}},)
)
# Now reload real weights inplace
llm.collective_rpc("reload_weights")
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0, top_p=0.95)
outputs = llm.generate(prompts, sampling_params)
# make sure it runs
for output in outputs:
generated_text = output.outputs[0].text
assert generated_text
# can also uncomment locally to make sure the generated
# output makes sense
# prompt = output.prompt
# print(f"Prompt: {prompt!r}")
# print(f"Output: {generated_text!r}")
# print("-" * 60)
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skip(
reason="since torchao nightly is only compatible with torch nightly"
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
"torchao tests that requires newer versions (0.15.0.dev+) for now"
)
def test_safetensors_model_loading_with_params(vllm_runner):
torch._dynamo.reset()
# using this model to test safetensors loading with file sharding
model_name = "torchao-testing/Qwen3-8B-INT4-0.15.0dev-safetensors"
with vllm_runner(model_name=model_name, dtype="bfloat16") as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
assert output
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skip(
reason="since torchao nightly is only compatible with torch nightly"
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
"torchao tests that requires newer versions (0.14.0.dev+) for now"
)
def test_opt_125m_module_fqn_to_config_regex_model(vllm_runner):
torch._dynamo.reset()
model_name = "torchao-testing/opt-125m-ModuleFqnToConfig-v1-regex-0.14.0.dev"
with vllm_runner(
model_name=model_name, dtype="bfloat16", pt_load_map_location=f"{DEVICE_TYPE}:0"
) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
assert output
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skip(
reason="since torchao nightly is only compatible with torch nightly"
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
"torchao tests that requires newer versions (0.14.0.dev+) for now"
)
def test_opt_125m_int4wo_model_running_preshuffled_kernel(vllm_runner, monkeypatch):
"""We load a model with Int4Tensor (plain format) linear weights
and verify that the weight is updated to Int4PreshuffledTensor
after loading in vllm
"""
from torchao.quantization import Int4PreshuffledTensor
from torchao.utils import _is_fbgemm_gpu_genai_available, is_sm_at_least_90
torch._dynamo.reset()
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model_name = "torchao-testing/opt-125m-Int4WeightOnlyConfig-v2-0.14.0.dev"
# Note: using enforce_eager=True because the `bf16i4bf16_shuffled` doesn't
# have meta kernel implemented yet, can remove this flag after that is implemented
with vllm_runner(
model_name=model_name,
quantization="torchao",
dtype="bfloat16",
pt_load_map_location=f"{DEVICE_TYPE}:0",
enforce_eager=True,
) as llm:
def has_int4_preshuffled_tensor_weight(model):
return isinstance(
model.model.decoder.layers[0].self_attn.qkv_proj.weight,
Int4PreshuffledTensor,
)
def get_weight_attrs(model):
weight = model.model.decoder.layers[0].self_attn.qkv_proj.weight
return [
weight.requires_grad,
weight.input_dim,
weight.output_dim,
hasattr(weight, "weight_loader"),
]
llm_engine = llm.get_llm().llm_engine
has_int4_preshuffled_tensor = any(
llm_engine.apply_model(has_int4_preshuffled_tensor_weight)
)
weight_attrs = llm_engine.apply_model(get_weight_attrs)[0]
# making sure we are using Int4PreshuffledTensor on H100 GPU, when
# fbgemm_gpu_genai
# library is installed, otherwise it should be using Int4Tensor
if _is_fbgemm_gpu_genai_available() and is_sm_at_least_90():
assert has_int4_preshuffled_tensor
else:
assert not has_int4_preshuffled_tensor
assert weight_attrs == [False, 1, 0, True]
output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
assert output
@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
@pytest.mark.skip(
reason="since torchao nightly is only compatible with torch nightly"
"currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
"torchao tests that requires newer versions (0.14.0.dev+) for now"
)
def test_opt_125m_int4wo_model_running_preshuffled_kernel_online_quant(
vllm_runner, monkeypatch
):
"""We load a bf16 model and online quantize the model to int4, then verify that
the weights are updated to Int4PreshuffledTensor after online quantization
"""
from torchao.quantization import Int4PreshuffledTensor
from torchao.utils import _is_fbgemm_gpu_genai_available, is_sm_at_least_90
torch._dynamo.reset()
model_name = "facebook/opt-125m"
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
import json
from torchao.core.config import config_to_dict
from torchao.quantization import Int4WeightOnlyConfig
torchao_quant_config = Int4WeightOnlyConfig(
group_size=128, int4_packing_format="plain"
)
hf_overrides = {
"quantization_config_dict_json": json.dumps(
config_to_dict(torchao_quant_config)
)
}
# Note: using enforce_eager=True because the `bf16i4bf16_shuffled` doesn't
# have meta kernel implemented yet, can remove this flag after that is implemented
with vllm_runner(
model_name=model_name,
quantization="torchao",
dtype="bfloat16",
pt_load_map_location=f"{DEVICE_TYPE}:0",
hf_overrides=hf_overrides,
enforce_eager=True,
) as llm:
def has_int4_preshuffled_tensor_weight(model):
return isinstance(
model.model.decoder.layers[0].self_attn.qkv_proj.weight,
Int4PreshuffledTensor,
)
def get_weight_attrs(model):
weight = model.model.decoder.layers[0].self_attn.qkv_proj.weight
return [
weight.requires_grad,
weight.input_dim,
weight.output_dim,
hasattr(weight, "weight_loader"),
]
llm_engine = llm.get_llm().llm_engine
has_int4_preshuffled_tensor = any(
llm_engine.apply_model(has_int4_preshuffled_tensor_weight)
)
weight_attrs = llm_engine.apply_model(get_weight_attrs)[0]
# making sure we are using Int4PreshuffledTensor on H100 GPU, when
# fbgemm_gpu_genai
# library is installed, otherwise it should be using Int4Tensor
if _is_fbgemm_gpu_genai_available() and is_sm_at_least_90():
assert has_int4_preshuffled_tensor
else:
assert not has_int4_preshuffled_tensor
assert weight_attrs == [False, 1, 0, True]
output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
assert output
if __name__ == "__main__":
pytest.main([__file__])
@@ -1,160 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import torch
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import NvFp4MoeBackend
from vllm.model_executor.layers.quantization.utils import flashinfer_fp4_moe
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_hidden_dim_for_fi,
align_fp4_moe_weights_for_fi,
)
def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch):
monkeypatch.setattr(flashinfer_fp4_moe, "swizzle_blockscale", lambda x: x)
num_experts = 3
layer = SimpleNamespace(
activation=SimpleNamespace(is_gated=False),
moe_config=SimpleNamespace(
moe_parallel_config=SimpleNamespace(enable_eplb=False)
),
)
w13 = torch.zeros((num_experts, 2, 1), dtype=torch.uint8)
w2 = torch.zeros((num_experts, 2, 1), dtype=torch.uint8)
w13_scale = torch.zeros((num_experts, 2, 1), dtype=torch.float8_e4m3fn)
w2_scale = torch.zeros((num_experts, 2, 1), dtype=torch.float8_e4m3fn)
weight_scale = torch.ones(num_experts)
outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass(
backend=NvFp4MoeBackend.FLASHINFER_CUTLASS,
layer=layer,
w13=w13,
w13_scale=w13_scale,
w13_scale_2=weight_scale,
a13_scale=torch.tensor([1.0, 2.0, 3.0]),
w2=w2,
w2_scale=w2_scale,
w2_scale_2=weight_scale,
a2_scale=torch.tensor([4.0, 5.0, 6.0]),
is_act_and_mul=False,
)
a13_scale, a2_scale = outputs[3], outputs[7]
torch.testing.assert_close(a13_scale, torch.full((num_experts,), 3.0))
torch.testing.assert_close(a2_scale, torch.full((num_experts,), 6.0))
distinct_values = torch.arange(num_experts, dtype=torch.float32)
a13_scale.copy_(distinct_values)
a2_scale.copy_(distinct_values)
torch.testing.assert_close(a13_scale, distinct_values)
torch.testing.assert_close(a2_scale, distinct_values)
def test_align_trtllm_fp4_moe_hidden_dim_noop():
w13 = torch.arange(2 * 8 * 256, dtype=torch.uint8).reshape(2, 8, 256)
w13_scale = torch.arange(2 * 8 * 32, dtype=torch.uint8).reshape(2, 8, 32)
w2 = torch.arange(2 * 512 * 4, dtype=torch.uint8).reshape(2, 512, 4)
w2_scale = torch.arange(2 * 512 * 1, dtype=torch.uint8).reshape(2, 512, 1)
out_w13, out_w13_scale, out_w2, out_w2_scale, padded_hidden = (
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
)
assert padded_hidden == 512
assert out_w13 is w13
assert out_w13_scale is w13_scale
assert out_w2 is w2
assert out_w2_scale is w2_scale
def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
hidden_dim = 2688
padded_hidden_dim = 2816
w13 = torch.arange(2 * 12 * (hidden_dim // 2), dtype=torch.uint8).reshape(
2, 12, hidden_dim // 2
)
w13_scale = torch.arange(2 * 12 * (hidden_dim // 16), dtype=torch.uint8).reshape(
2, 12, hidden_dim // 16
)
w2 = torch.arange(2 * hidden_dim * 6, dtype=torch.uint8).reshape(2, hidden_dim, 6)
w2_scale = torch.arange(2 * hidden_dim * 2, dtype=torch.uint8).reshape(
2, hidden_dim, 2
)
out_w13, out_w13_scale, out_w2, out_w2_scale, out_hidden_dim = (
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
)
assert out_hidden_dim == padded_hidden_dim
assert out_w13.shape == (2, 12, padded_hidden_dim // 2)
assert out_w13_scale.shape == (2, 12, padded_hidden_dim // 16)
assert out_w2.shape == (2, padded_hidden_dim, 6)
assert out_w2_scale.shape == (2, padded_hidden_dim, 2)
torch.testing.assert_close(out_w13[:, :, : hidden_dim // 2], w13)
torch.testing.assert_close(out_w13_scale[:, :, : hidden_dim // 16], w13_scale)
torch.testing.assert_close(out_w2[:, :hidden_dim, :], w2)
torch.testing.assert_close(out_w2_scale[:, :hidden_dim, :], w2_scale)
assert torch.count_nonzero(out_w13[:, :, hidden_dim // 2 :]) == 0
assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0
def test_align_trtllm_fp4_moe_intermediate_pads_gate_and_up_separately():
# Qwen3.8-Flash-Next has intermediate_size=640. At TP4 each rank owns
# 160 values, which TRTLLM pads to the next 64-value boundary (192).
intermediate = 160
padded_intermediate = 192
hidden_dim = 32
gate = torch.ones((1, intermediate, hidden_dim // 2), dtype=torch.uint8)
up = torch.full_like(gate, 2)
w13 = torch.cat((gate, up), dim=1)
gate_scale = torch.full((1, intermediate, hidden_dim // 16), 3, dtype=torch.uint8)
up_scale = torch.full_like(gate_scale, 4)
w13_scale = torch.cat((gate_scale, up_scale), dim=1)
w2 = torch.full((1, hidden_dim, intermediate // 2), 5, dtype=torch.uint8)
w2_scale = torch.full((1, hidden_dim, intermediate // 16), 6, dtype=torch.uint8)
out_w13, out_w13_scale, out_w2, out_w2_scale, out_intermediate = (
align_fp4_moe_weights_for_fi(
w13,
w13_scale,
w2,
w2_scale,
is_act_and_mul=True,
min_alignment=64,
)
)
assert out_intermediate == padded_intermediate
torch.testing.assert_close(out_w13[:, :intermediate], gate)
torch.testing.assert_close(
out_w13[:, padded_intermediate : padded_intermediate + intermediate], up
)
torch.testing.assert_close(out_w13_scale[:, :intermediate], gate_scale)
torch.testing.assert_close(
out_w13_scale[:, padded_intermediate : padded_intermediate + intermediate],
up_scale,
)
assert torch.count_nonzero(out_w13[:, intermediate:padded_intermediate]) == 0
assert torch.count_nonzero(out_w13[:, padded_intermediate + intermediate :]) == 0
assert torch.count_nonzero(out_w13_scale[:, intermediate:padded_intermediate]) == 0
assert (
torch.count_nonzero(out_w13_scale[:, padded_intermediate + intermediate :]) == 0
)
torch.testing.assert_close(out_w2[:, :, : intermediate // 2], w2)
torch.testing.assert_close(out_w2_scale[:, :, : intermediate // 16], w2_scale)
assert torch.count_nonzero(out_w2[:, :, intermediate // 2 :]) == 0
assert torch.count_nonzero(out_w2_scale[:, :, intermediate // 16 :]) == 0
-199
View File
@@ -1,199 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
from typing import Any
import regex as re
import torch
from pydantic import TypeAdapter
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.quantization import (
QuantizationConfigArgs,
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
def quant_spec(**kwargs: Any) -> QuantSpec:
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
return TypeAdapter(QuantSpec).validate_python(kwargs)
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
def _limit_num_hidden_layers(
model: torch.nn.Module, num_hidden_layers: int | None
) -> None:
if num_hidden_layers is None:
return
original_load_weights = model.load_weights
def should_load_weight(name: str) -> bool:
layers_prefix = "layers."
if name.startswith(layers_prefix):
layer_name = name.removeprefix(layers_prefix)
elif ".layers." in name:
layer_name = name.split(".layers.", maxsplit=1)[1]
else:
return True
layer_idx_str = layer_name.split(".", maxsplit=1)[0]
if layer_idx_str.isdigit():
return int(layer_idx_str) < num_hidden_layers
return True
def load_weights(weights):
weights = (
(name, weight) for name, weight in weights if should_load_weight(name)
)
return original_load_weights(weights)
model.load_weights = load_weights
def load_model_without_vllm_runner(
model_path: str,
*,
dtype: str | torch.dtype = "bfloat16",
quantization: str | None = None,
model_config_kwargs: dict[str, Any] | None = None,
vllm_config_kwargs: dict[str, Any] | None = None,
model_loader_cls: type = DefaultModelLoader,
) -> tuple[torch.nn.Module, VllmConfig]:
"""Instantiate a model, load weights, and process them for inference."""
model_config = ModelConfig(
model=model_path,
dtype=dtype,
quantization=quantization,
**(model_config_kwargs or {}),
)
model_config.quantization_config = resolve_quantization_config(
model_config.quantization, model_config.quantization_config
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault(
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
)
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
text_config_overrides = hf_overrides.get("text_config") or {}
num_hidden_layers = text_config_overrides.get(
"num_hidden_layers", hf_overrides.get("num_hidden_layers")
)
with set_current_vllm_config(vllm_config):
model_loader = model_loader_cls(vllm_config.load_config)
if num_hidden_layers is not None:
original_load_weights = model_loader.load_weights
def load_weights(model, model_config):
_limit_num_hidden_layers(model, num_hidden_layers)
original_load_weights(model, model_config)
model_loader.load_weights = load_weights
model = model_loader.load_model(vllm_config, model_config)
return model, vllm_config
def is_quant_method_supported(quant_method: str) -> bool:
# Currently, quantization tests only run GPUs
if current_platform.is_cpu():
return False
try:
current_platform.verify_quantization(quant_method)
except ValueError:
return False
if current_platform.is_xpu():
return True
capability = current_platform.get_device_capability()
assert capability is not None
min_capability = get_quantization_config(quant_method).get_min_capability()
return capability.to_int() >= min_capability
def _test_online_quant_peak_mem_impl(
quantization_arg_value,
vllm_runner,
caplog_mp_spawn,
monkeypatch,
) -> None:
# Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
# 1. it covers both Linear and MoE paths
# 2. it is already used by other tests in CI, so adding it here
# does not increase disk space for CI runners
# I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
# which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
# 1.3 GiB fp8), but could not as adding one more model makes CI
# run out of disk space.
model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
# Force spawn to ensure caplog_mp_spawn works consistently
# (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
with (
caplog_mp_spawn(logging.DEBUG) as log_holder,
vllm_runner(
model_name,
quantization=quantization_arg_value,
enforce_eager=True,
) as llm,
):
outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
print(outputs[0][1])
log_text = log_holder.text
# Parse memory usage from captured logs
model_memory_gib = None
peak_memory_gib = None
for line in log_text.splitlines():
if model_memory_gib is None:
match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
if match:
model_memory_gib = float(match.group(1))
if peak_memory_gib is None:
match = re.search(
r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
)
if match:
peak_memory_gib = float(match.group(1))
assert model_memory_gib is not None, "Could not find model loading memory log"
assert peak_memory_gib is not None, "Could not find peak memory log"
print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
expected_model_memory_gib = 6.7
# for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
# GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
# number is expected as when we load and quantize weights in a streaming
# fashion we need to have individual weights in bf16 + fp8 alive at the
# same time.
expected_peak_memory_gib = expected_model_memory_gib * 1.4
assert model_memory_gib < expected_model_memory_gib, (
f"{model_memory_gib=} higher than {expected_model_memory_gib}"
)
assert peak_memory_gib < expected_peak_memory_gib, (
f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
)