[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# flake8: noqa
|
||||
"""Tests fp8 models against ground truth generation
|
||||
Note: these tests will only pass on L4 GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_kv_cache_dtype
|
||||
from vllm.platforms import current_platform
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype,base_model,test_model",
|
||||
[
|
||||
# Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
|
||||
(
|
||||
"fp8_e4m3",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"nm-testing/Llama-3.2-1B-Instruct-FP8-KV",
|
||||
),
|
||||
# Test BF16 checkpoint w. fp8_e5m2 kv-cache.
|
||||
(
|
||||
"fp8_e5m2",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
# Test BF16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
|
||||
(
|
||||
"fp8_e4m3",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
],
|
||||
)
|
||||
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
@pytest.mark.parametrize("enforce_eager", [True])
|
||||
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
|
||||
# NOTE: Increasing this in this suite will fail CI because we currently cannot
|
||||
# reset distributed env properly. Use a value > 1 just when you test.
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_models(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
kv_cache_dtype: str,
|
||||
base_model: str,
|
||||
test_model: str,
|
||||
max_tokens: int,
|
||||
enforce_eager: bool,
|
||||
backend: str,
|
||||
tensor_parallel_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
Only checks log probs match to cover the discrepancy in
|
||||
numerical sensitive kernels.
|
||||
"""
|
||||
|
||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
|
||||
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
|
||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_cuda():
|
||||
pytest.skip(f"{kv_cache_dtype} is not supported by FLASH_ATTN on CUDA.")
|
||||
|
||||
if not flash_attn_supports_kv_cache_dtype(kv_cache_dtype):
|
||||
pytest.skip(
|
||||
f"{kv_cache_dtype} is not supported on this GPU type with {backend} attention."
|
||||
)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
with vllm_runner(
|
||||
base_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype="auto",
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
test_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="fp16_kv_cache",
|
||||
name_1="fp8_kv_cache",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="test for the CPU backend.")
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype,base_model,test_model",
|
||||
[
|
||||
# Test BF16 checkpoint w. fp8_e5m2 kv-cache.
|
||||
(
|
||||
"fp8_e5m2",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
],
|
||||
)
|
||||
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
def test_cpu_models(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
kv_cache_dtype: str,
|
||||
base_model: str,
|
||||
test_model: str,
|
||||
max_tokens: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
Only checks log probs match to cover the discrepancy in
|
||||
numerical sensitive kernels.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
with vllm_runner(
|
||||
base_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
dtype="bfloat16",
|
||||
kv_cache_dtype="auto",
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
test_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
dtype="bfloat16",
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16_kv_cache",
|
||||
name_1="fp8_kv_cache",
|
||||
)
|
||||
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end accuracy test for GPT-OSS model quantization.
|
||||
|
||||
Config:
|
||||
Task: gsm8k_platinum
|
||||
Filter: flexible-extract
|
||||
n-shot: 5
|
||||
Metric: exact_match
|
||||
|
||||
Run: pytest tests/models/quantization/test_gpt_oss.py
|
||||
"""
|
||||
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
from dataclasses import dataclass
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
mxfp4_w4a8_moe_quant_config,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
|
||||
aiter_triton_kernel_w4a8_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
aiter_triton_kernel_w4a16_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
else:
|
||||
|
||||
def on_gfx950() -> bool:
|
||||
return False
|
||||
|
||||
def on_gfx1250() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
MODEL_ACCURACIES = {
|
||||
# Full quantization: attention linears and MoE linears
|
||||
"amd/gpt-oss-20b-WFP8-AFP8-KVFP8": 0.89,
|
||||
# MoE linears only quantization
|
||||
"amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-FP8-KV-FP8": 0.89,
|
||||
# MoE linears only quantization
|
||||
# "amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-MXFP4-KV-FP8": 0.90,
|
||||
}
|
||||
|
||||
QUARK_MXFP4_AVAILABLE = importlib.util.find_spec("quark") is not None and version.parse(
|
||||
importlib.metadata.version("amd-quark")
|
||||
) >= version.parse("0.9.0")
|
||||
|
||||
|
||||
def has_huggingface_access(repo):
|
||||
try:
|
||||
hf_api().list_repo_refs(repo)
|
||||
return True
|
||||
except huggingface_hub.errors.RepositoryNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
HF_HUB_AMD_ORG_ACCESS = all(
|
||||
[has_huggingface_access(model_name) for model_name in MODEL_ACCURACIES]
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelCase:
|
||||
model_id: str
|
||||
tp: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationConfig:
|
||||
model_name: str
|
||||
|
||||
def get_model_args(self, tp_size: int):
|
||||
return {
|
||||
"pretrained": self.model_name,
|
||||
"chat_template_args": {"reasoning_effort": "low"},
|
||||
"enable_thinking": True,
|
||||
"think_end_token": "200008",
|
||||
"tensor_parallel_size": tp_size,
|
||||
"dtype": "auto",
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"trust_remote_code": False,
|
||||
"enable_prefix_caching": False,
|
||||
"enforce_eager": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not QUARK_MXFP4_AVAILABLE, reason="amd-quark>=0.9 is not available")
|
||||
@pytest.mark.skipif(
|
||||
not HF_HUB_AMD_ORG_ACCESS,
|
||||
reason="Read access to huggingface.co/amd is required for this test.",
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
|
||||
@pytest.mark.parametrize("model_name, expected_accuracy", MODEL_ACCURACIES.items())
|
||||
def test_gpt_oss_attention_quantization(
|
||||
model_name: str,
|
||||
tp_size: int,
|
||||
expected_accuracy: float,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if tp_size > current_platform.device_count():
|
||||
pytest.skip("Not enough GPUs to run this test case")
|
||||
|
||||
if "amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-FP8-KV-FP8" in model_name and on_gfx950():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
model_args = EvaluationConfig(model_name).get_model_args(tp_size)
|
||||
|
||||
extra_run_kwargs = {
|
||||
"gen_kwargs": {"max_gen_toks": 8000},
|
||||
"apply_chat_template": True,
|
||||
"fewshot_as_multiturn": True,
|
||||
"num_fewshot": 5,
|
||||
}
|
||||
|
||||
lm_eval_out = lm_eval.simple_evaluate(
|
||||
model="vllm",
|
||||
model_args=model_args,
|
||||
tasks="gsm8k_platinum",
|
||||
batch_size="auto",
|
||||
**extra_run_kwargs,
|
||||
)
|
||||
measured_accuracy = float(
|
||||
lm_eval_out["results"]["gsm8k_platinum"]["exact_match,flexible-extract"]
|
||||
)
|
||||
|
||||
rtol = 0.02
|
||||
assert measured_accuracy >= expected_accuracy - rtol, (
|
||||
f"Accuracy {measured_accuracy:.4f} is below threshold "
|
||||
f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (on_gfx950() or on_gfx1250()),
|
||||
reason="AITER MXFP4 MoE requires gfx950 or gfx1250",
|
||||
)
|
||||
@pytest.mark.skipif(not is_aiter_found_and_supported(), reason="aiter is not installed")
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() == 0, reason="no gpu available")
|
||||
@pytest.mark.parametrize(
|
||||
"mxfp4_backend",
|
||||
[Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_BF16],
|
||||
ids=["w4a8", "w4a16"],
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [32, 64, 128])
|
||||
def test_aiter_mxfp4_moe_ignores_padded_rows(
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
num_experts: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dist_init,
|
||||
) -> None:
|
||||
"""Garbage in cudagraph padding rows must not reach the unpadded outputs.
|
||||
|
||||
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
|
||||
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
|
||||
-inf/inf/nan.
|
||||
"""
|
||||
TOKENS_PADDED = 8
|
||||
TOKENS_UNPADDED = 7
|
||||
|
||||
# `rocm_aiter_ops` snapshots the environment at import time, so setting the
|
||||
# env var alone is not enough.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
set_random_seed(0)
|
||||
device = torch.device("cuda")
|
||||
hidden_size = 2048
|
||||
intermediate_size = 512
|
||||
|
||||
num_experts = 32
|
||||
topk = 4
|
||||
is_w4a8 = mxfp4_backend is Mxfp4MoeBackend.AITER_MXFP4_FP8
|
||||
|
||||
# TODO: clean this in oracle/mxfp4.
|
||||
# `convert_gpt_oss_weight_to_mxfp4_moe_kernel_format` maps
|
||||
# AITER_MXFP4_BF16 onto the CK layout (`shuffle_weight_a16w4`), which the
|
||||
# triton monolithic cannot consume -- it wants the `_swizzle_mxfp4`
|
||||
# PrecisionConfig that the TRITON branch of the same converter produces.
|
||||
weight_backend = (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8 if is_w4a8 else Mxfp4MoeBackend.TRITON
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.w13_weight = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 2),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
layer.w2_weight = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_experts, hidden_size, intermediate_size // 2),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Keep the exponents near 127 (2**0) so the dequantized weights stay in a sane
|
||||
# range and 255 (e8m0 NaN) is never hit.
|
||||
layer.w13_weight_scale = torch.randint(
|
||||
124,
|
||||
131,
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
layer.w2_weight_scale = torch.randint(
|
||||
124,
|
||||
131,
|
||||
(num_experts, hidden_size, intermediate_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
layer.w13_bias = torch.randn(
|
||||
(num_experts, 2 * intermediate_size), dtype=torch.float32, device=device
|
||||
)
|
||||
layer.w2_bias = torch.randn(
|
||||
(num_experts, hidden_size), dtype=torch.float32, device=device
|
||||
)
|
||||
# Read off `layer`, not off the arguments, by the AITER_MXFP4_FP8 branch.
|
||||
layer.w13_input_scale = torch.full(
|
||||
(num_experts,), 0.1, dtype=torch.float32, device=device
|
||||
)
|
||||
layer.w2_input_scale = torch.full(
|
||||
(num_experts,), 0.1, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
weight_backend,
|
||||
layer,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w2_weight_scale,
|
||||
w13_bias=layer.w13_bias,
|
||||
w2_bias=layer.w2_bias,
|
||||
)
|
||||
|
||||
quant_config_factory = (
|
||||
mxfp4_w4a8_moe_quant_config if is_w4a8 else mxfp4_w4a16_moe_quant_config
|
||||
)
|
||||
quant_config = quant_config_factory(
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
w1_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
)
|
||||
|
||||
hidden_states = torch.randn(
|
||||
(TOKENS_PADDED, hidden_size), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
gating_output = torch.randn(
|
||||
(TOKENS_PADDED, num_experts), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
|
||||
hidden_states[TOKENS_UNPADDED:, 0] = float("inf")
|
||||
hidden_states[TOKENS_UNPADDED:, 1] = float("-inf")
|
||||
hidden_states[TOKENS_UNPADDED:, 2] = float("nan")
|
||||
|
||||
gating_output[TOKENS_UNPADDED:, :] = float("-inf")
|
||||
gating_output[TOKENS_UNPADDED:, 0] = float("inf")
|
||||
gating_output[TOKENS_UNPADDED:, 1] = float("nan")
|
||||
gating_output[TOKENS_UNPADDED:, 2] = -float("nan")
|
||||
|
||||
is_padding = torch.zeros(TOKENS_PADDED, dtype=torch.bool, device=device)
|
||||
is_padding[TOKENS_UNPADDED:] = True
|
||||
|
||||
# Poison memory: after `del blocks`, further `torch.empty` in routing with wrongful
|
||||
# expert ids may read into these memory sections resulting in
|
||||
# potential memory access fault.
|
||||
blocks = []
|
||||
for numel in (16, 32, 64, 128, 256, 512, 1024, 4096, 1 << 14, 1 << 16, 1 << 20):
|
||||
blocks.append(torch.full((numel,), 100e7, dtype=torch.int32, device=device))
|
||||
# return blocks
|
||||
del blocks
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
with set_forward_context(None, get_current_vllm_config(), is_padding=is_padding):
|
||||
if is_w4a8:
|
||||
# AITER_MXFP4_FP8 routes through the triton monolithic expert, which
|
||||
# does its own top-k inside `aiter.ops.triton.moe.moe_routing`.
|
||||
output = aiter_triton_kernel_w4a8_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
w1=w13_weight,
|
||||
w2=w2_weight,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
global_num_experts=num_experts,
|
||||
unpadded_N_w1=2 * intermediate_size,
|
||||
unpadded_K_w1=hidden_size,
|
||||
unpadded_N_w2=hidden_size,
|
||||
unpadded_K_w2=intermediate_size,
|
||||
)
|
||||
else:
|
||||
output = aiter_triton_kernel_w4a16_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
w1=w13_weight,
|
||||
w2=w2_weight,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
global_num_experts=num_experts,
|
||||
unpadded_N_w1=2 * intermediate_size,
|
||||
unpadded_K_w1=hidden_size,
|
||||
unpadded_N_w2=hidden_size,
|
||||
unpadded_K_w2=intermediate_size,
|
||||
)
|
||||
|
||||
assert torch.isfinite(output[:TOKENS_UNPADDED]).all()
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests AutoGPTQ (GPTQ with Marlin kernels) output correctness.
|
||||
|
||||
Note: Marlin internally uses locks to synchronize the threads. This can
|
||||
result in very slight nondeterminism for Marlin. As a result, we re-run the test
|
||||
up to 3 times to see if we pass.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = [
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "main"),
|
||||
# desc_act=True is a no-op for channelwise quantization.
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
|
||||
("TechxGenus/gemma-2b-it-GPTQ", "main"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("auto_gptq")
|
||||
or current_platform.is_rocm()
|
||||
or not current_platform.is_cuda(),
|
||||
reason="auto_gptq is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
model_name, revision = model
|
||||
|
||||
# Run marlin.
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
revision=revision,
|
||||
dtype=dtype,
|
||||
quantization="marlin",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=1,
|
||||
) as gptq_marlin_model:
|
||||
gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
|
||||
example_prompts[:-1], max_tokens, num_logprobs
|
||||
)
|
||||
_ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
|
||||
|
||||
# Run gptq.
|
||||
# The naive gptq kernel doesn't support bf16 yet.
|
||||
# Here we always compare fp16/bf16 gpt marlin kernel
|
||||
# to fp16 gptq kernel.
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
revision=revision,
|
||||
dtype="half",
|
||||
quantization="gptq",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=1,
|
||||
) as gptq_model:
|
||||
gptq_outputs = gptq_model.generate_greedy_logprobs(
|
||||
example_prompts[:-1], max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=gptq_outputs,
|
||||
outputs_1_lst=gptq_marlin_outputs,
|
||||
name_0="gptq",
|
||||
name_1="gptq_marlin",
|
||||
)
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# flake8: noqa
|
||||
"""Tests Model Optimizer fp8 models against ground truth generation
|
||||
Note: these tests will only pass on H100
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = ["nvidia/Llama-3.1-8B-Instruct-FP8"]
|
||||
|
||||
EXPECTED_STRS_MAP = {
|
||||
"nvidia/Llama-3.1-8B-Instruct-FP8": [
|
||||
"You're referring to VLLM, a high-performance Large Language Model (LLM) inference and",
|
||||
"Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ",
|
||||
"The comparison between artificial intelligence (AI) and human intelligence in terms of processing information is a complex and",
|
||||
'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
|
||||
"**The Spark of Imagination**\n\nZeta-5, a sleek and efficient robot, whir",
|
||||
"The COVID-19 pandemic has had a profound impact on global economic structures and business models, leading to",
|
||||
"The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of",
|
||||
"Here are the translations:\n\n**Japanese:** 「早起きは早く獲物をとる",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# This test compares against golden strings for exact match since
|
||||
# there is no baseline implementation to compare against
|
||||
# and is unstable w.r.t specifics of the fp8 implementation or
|
||||
# the hardware being run on.
|
||||
# Disabled to prevent it from breaking the build
|
||||
@pytest.mark.skip(
|
||||
reason="Prevent unstable test based on golden strings from breaking the build."
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_models(example_prompts, model_name) -> None:
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
quantization="modelopt",
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
formatted_prompts = [
|
||||
tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
for prompt in example_prompts
|
||||
]
|
||||
params = SamplingParams(max_tokens=20, temperature=0)
|
||||
generations: list[str] = []
|
||||
# Note: these need to be run 1 at a time due to numerical precision,
|
||||
# since the expected strs were generated this way.
|
||||
for prompt in formatted_prompts:
|
||||
outputs = llm.generate(prompt, params)
|
||||
generations.append(outputs[0].outputs[0].text)
|
||||
del llm
|
||||
|
||||
print(model_name, generations)
|
||||
expected_strs = EXPECTED_STRS_MAP[model_name]
|
||||
for i in range(len(example_prompts)):
|
||||
generated_str = generations[i]
|
||||
expected_str = expected_strs[i]
|
||||
assert expected_str == generated_str, (
|
||||
f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}"
|
||||
)
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# flake8: noqa
|
||||
"""Tests Quark mxfp4 models against ground truth generation"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
MODELS = ["amd/Llama-2-7b-chat-hf-wmxfp4-amxfp4-kvfp8-scale-uint8"]
|
||||
|
||||
EXPECTED_STRS_MAP = {
|
||||
"amd/Llama-2-7b-chat-hf-wmxfp4-amxfp4-kvfp8-scale-uint8": [
|
||||
"\n### Key Features\n\n* **High-throughput Inference**: vLL",
|
||||
"\nArtificial intelligence (AI) has evolved significantly since its inception in the 1",
|
||||
"Artificial intelligence (AI) and human intelligence (HI) are two distinct concepts that have been",
|
||||
"A neural network is a machine learning model inspired by the structure of the human brain. It consists of",
|
||||
"\nTitle: The Dreaming Robot\n\nAs the sun set on the bustling metropol",
|
||||
"\nThe COVID-19 pandemic has had a profound impact on global economic structures and business",
|
||||
"The Mona Lisa painting, created by Leonardo da Vinci in the early 16th",
|
||||
" everybody knows this proverbial saying, but did you know that it's not entirely accurate?",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Model to be released in the future")
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_models(example_prompts, model_name) -> None:
|
||||
sampling_params = SamplingParams(max_tokens=20, temperature=0)
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
kv_cache_dtype="fp8",
|
||||
quantization="quark",
|
||||
)
|
||||
outputs = llm.generate(example_prompts, sampling_params)
|
||||
for i, output in enumerate(outputs):
|
||||
output_str = output.outputs[0].text
|
||||
expected_str = EXPECTED_STRS_MAP[model_name][i]
|
||||
assert expected_str == output_str, (
|
||||
f"Expected: {expected_str!r}\nvLLM: {output_str!r}"
|
||||
)
|
||||
@@ -1,255 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for online MXFP8 quantization.
|
||||
|
||||
Loads a BF16 model with ``--quantization mxfp8`` (online quantization) and
|
||||
compares log-probabilities against the same model served in BF16 without
|
||||
quantization. This exercises the full pipeline: config parsing,
|
||||
``Mxfp8OnlineLinearMethod``, ``Mxfp8OnlineMoEMethod``, weight loading,
|
||||
online quantization / shuffling, and inference through ``apply_monolithic``.
|
||||
|
||||
Layer skipping (``modules_to_not_convert``) is configured in the model's
|
||||
``config.json`` under ``quantization_config`` and is not tested here.
|
||||
|
||||
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
|
||||
diverse prompts from ``tests/prompts/example.txt``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
# A small MoE model that fits on a single GPU and has both linear + MoE layers.
|
||||
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
# A small dense model (no MoE) to validate the linear-only path.
|
||||
DENSE_MODEL = "Qwen/Qwen3-0.6B"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_TOKENS = 4
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_mxfp8_logprobs(
|
||||
vllm_runner,
|
||||
quant_baseline_logprobs,
|
||||
example_prompts,
|
||||
model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare BF16 baseline logprobs against online MXFP8-quantized model.
|
||||
|
||||
Runs the same model twice -- once in BF16 (baseline) and once with
|
||||
online MXFP8 quantization -- then checks that the top log-probabilities
|
||||
are close. Only 4 tokens are generated to keep the test fast while
|
||||
still catching numerical divergence.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
baseline_outputs = quant_baseline_logprobs(
|
||||
model,
|
||||
example_prompts,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_tokens=MAX_TOKENS,
|
||||
num_logprobs=NUM_LOG_PROBS,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
quantization="mxfp8",
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16",
|
||||
name_1="mxfp8",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AITER MXFP8 MoE backend is ROCm-only.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
def test_mxfp8_aiter_requires_swigluoai_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_mxfp8_moe.AiterMxfp8Experts,
|
||||
"_supports_current_device",
|
||||
staticmethod(lambda: True),
|
||||
)
|
||||
|
||||
config = FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend="aiter",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="requires activation=swigluoai_uninterleave"):
|
||||
select_mxfp8_moe_backend(config)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AITER MXFP8 MoE backend is ROCm-only.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
def test_mxfp8_aiter_requires_swigluoai_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_mxfp8_moe.AiterMxfp8Experts,
|
||||
"_supports_current_device",
|
||||
staticmethod(lambda: True),
|
||||
)
|
||||
|
||||
config = FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend="aiter",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="hardcodes SwiGLU-OAI"):
|
||||
select_mxfp8_moe_backend(config)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AITER MXFP8 MoE backend is ROCm-only.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
def test_mxfp8_aiter_accepts_swigluoai_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_mxfp8_moe.AiterMxfp8Experts,
|
||||
"_supports_current_device",
|
||||
staticmethod(lambda: True),
|
||||
)
|
||||
|
||||
config = FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend="aiter",
|
||||
swiglu_alpha=aiter_mxfp8_moe._AITER_SWIGLU_ALPHA,
|
||||
swiglu_beta=aiter_mxfp8_moe._AITER_SWIGLU_BETA,
|
||||
)
|
||||
|
||||
backend, experts_cls = select_mxfp8_moe_backend(config)
|
||||
|
||||
assert backend == Fp8MoeBackend.AITER_MXFP8
|
||||
assert experts_cls is aiter_mxfp8_moe.AiterMxfp8Experts
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_mxfp8_generation(vllm_runner, model: str) -> None:
|
||||
"""Smoke test: verify online MXFP8 model generates coherent text."""
|
||||
prompt = "1 2 3 4 5"
|
||||
with vllm_runner(
|
||||
model,
|
||||
enforce_eager=True,
|
||||
quantization="mxfp8",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
) as vllm_model:
|
||||
output = vllm_model.generate_greedy([prompt], max_tokens=5)
|
||||
|
||||
generated = output[0][1]
|
||||
assert len(generated) > len(prompt), (
|
||||
f"MXFP8 model produced no new tokens. Output: {generated!r}"
|
||||
)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# flake8: noqa
|
||||
"""Tests Model Optimizer nvfp4 models against ground truth generation
|
||||
Note: these tests will only pass on B200
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = ["nvidia/Llama-3.3-70B-Instruct-FP4"]
|
||||
|
||||
EXPECTED_STRS_MAP = {
|
||||
"nvidia/Llama-3.3-70B-Instruct-FP4": [
|
||||
"vLLM (Vectorized Large Language Model) is indeed a high-throughput and memory-efficient inference",
|
||||
"Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ",
|
||||
"Artificial intelligence (AI) and human intelligence (HI) are two distinct forms of intelligence that process",
|
||||
"A neural network is a type of machine learning model inspired by the structure and function of the human brain",
|
||||
"In the heart of a cutting-edge robotics lab, a team of engineers had been working tirelessly to push",
|
||||
"The COVID-19 pandemic has had a profound impact on global economic structures and future business models, leading",
|
||||
"The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of",
|
||||
"Here are the translations:\n\n* Japanese: (Sasuga no tori ga miwa o ts",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# This test compares against golden strings for exact match since
|
||||
# there is no baseline implementation to compare against
|
||||
# and is unstable w.r.t specifics of the fp4 implementation or
|
||||
# the hardware being run on.
|
||||
# Disabled to prevent it from breaking the build
|
||||
@pytest.mark.skip(
|
||||
reason="Prevent unstable test based on golden strings from breaking the build "
|
||||
" and test input model being too large and hanging the system."
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("modelopt_fp4"),
|
||||
reason="modelopt_fp4 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_models(example_prompts, model_name) -> None:
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
quantization="modelopt_fp4",
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
formatted_prompts = [
|
||||
tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
for prompt in example_prompts
|
||||
]
|
||||
params = SamplingParams(max_tokens=20, temperature=0)
|
||||
generations: List[str] = []
|
||||
# Note: these need to be run 1 at a time due to numerical precision,
|
||||
# since the expected strs were generated this way.
|
||||
for prompt in formatted_prompts:
|
||||
outputs = llm.generate(prompt, params)
|
||||
generations.append(outputs[0].outputs[0].text)
|
||||
del llm
|
||||
|
||||
print(model_name, generations)
|
||||
expected_strs = EXPECTED_STRS_MAP[model_name]
|
||||
for i in range(len(example_prompts)):
|
||||
generated_str = generations[i]
|
||||
expected_str = expected_strs[i]
|
||||
assert expected_str == generated_str, (
|
||||
f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}"
|
||||
)
|
||||
|
||||
|
||||
EAGER = [True, False]
|
||||
|
||||
SM_100_NVFP4_BACKENDS = [
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cutlass",
|
||||
"torch",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["nvidia/Llama-3.1-8B-Instruct-NVFP4"])
|
||||
@pytest.mark.parametrize("eager", EAGER)
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
"emulation",
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
|
||||
"flashinfer_cutlass",
|
||||
"torch",
|
||||
],
|
||||
)
|
||||
def test_nvfp4(vllm_runner, model, eager, backend):
|
||||
if backend == "flashinfer_cutedsl" and not (
|
||||
current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("The flashinfer_cutedsl backend is only supported on SM10x")
|
||||
|
||||
if (
|
||||
not current_platform.has_device_capability(100)
|
||||
and backend in SM_100_NVFP4_BACKENDS
|
||||
):
|
||||
pytest.skip(
|
||||
f"The backend {backend} is not supported with current_platform.has_device_capability(100) == False"
|
||||
)
|
||||
|
||||
with vllm_runner(model, enforce_eager=eager, linear_backend=backend) as llm:
|
||||
output = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
|
||||
assert output[0][1] == "1 2 3 4 5 6"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"nvidia/Qwen3-30B-A3B-NVFP4",
|
||||
"RedHatAI/Qwen3-30B-A3B-NVFP4",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("backend", ["emulation"])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
|
||||
)
|
||||
def test_nvfp4_moe(vllm_runner, model, backend):
|
||||
with vllm_runner(
|
||||
model,
|
||||
moe_backend=backend,
|
||||
linear_backend=backend,
|
||||
load_format="dummy",
|
||||
hf_overrides={"num_hidden_layers": 2},
|
||||
) as llm:
|
||||
_ = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""End-to-end accuracy tests for per-token-head KV cache quantization.
|
||||
|
||||
Compares logprobs between a baseline bf16 model and the same model with
|
||||
per-token-head quantized KV cache (int8 or fp8) using the Triton attention
|
||||
backend.
|
||||
|
||||
Run: pytest tests/models/quantization/test_per_token_kv_cache.py -v -s
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Per-token-head KV cache requires CUDA or ROCm GPU.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,test_model",
|
||||
[
|
||||
(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
@pytest.mark.parametrize("enforce_eager", [True])
|
||||
@pytest.mark.parametrize("backend", ["TRITON_ATTN"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_per_token_head_kv_cache_accuracy(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
base_model: str,
|
||||
test_model: str,
|
||||
kv_cache_dtype: str,
|
||||
max_tokens: int,
|
||||
enforce_eager: bool,
|
||||
backend: str,
|
||||
tensor_parallel_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare logprobs between bf16 baseline and per-token-head quantized KV
|
||||
cache.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
with vllm_runner(
|
||||
base_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype="auto",
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
test_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16_kv_cache",
|
||||
name_1=f"{kv_cache_dtype}_kv_cache",
|
||||
)
|
||||
Reference in New Issue
Block a user