Remove VLLM_TEST_FORCE_FP8_MARLIN to replace with linear_backend/moe_backend (#52182)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
88b2bff2c6
commit
bca7bea240
@@ -371,6 +371,10 @@ def test_fusion_rmsnorm_quant(
|
|||||||
use_aiter_fusion=False,
|
use_aiter_fusion=False,
|
||||||
use_aiter_quant=False,
|
use_aiter_quant=False,
|
||||||
)
|
)
|
||||||
|
if any(
|
||||||
|
type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers
|
||||||
|
):
|
||||||
|
pytest.skip(f"{force_kernel.__name__} is not supported on this platform")
|
||||||
|
|
||||||
backend, _ = _run_fusion_test(
|
backend, _ = _run_fusion_test(
|
||||||
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
|
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
|
||||||
|
|||||||
@@ -481,6 +481,12 @@ def test_mla_attention_quant_pattern(
|
|||||||
device=device,
|
device=device,
|
||||||
vllm_config=vllm_config_unfused,
|
vllm_config=vllm_config_unfused,
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
model_class is TestMLAAttentionFp8GroupQuantPatternModel
|
||||||
|
and type(model_unfused.block_fp8_linear.kernel)
|
||||||
|
is not CutlassFp8BlockScaledMMKernel
|
||||||
|
):
|
||||||
|
pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")
|
||||||
model_unfused = model_unfused.to(device)
|
model_unfused = model_unfused.to(device)
|
||||||
# HACK: See #131044
|
# HACK: See #131044
|
||||||
result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841
|
result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841
|
||||||
|
|||||||
@@ -2,6 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
|
|||||||
accuracy_threshold: 0.92
|
accuracy_threshold: 0.92
|
||||||
num_questions: 1319
|
num_questions: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_TEST_FORCE_FP8_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -2,6 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
|||||||
accuracy_threshold: 0.88
|
accuracy_threshold: 0.88
|
||||||
num_questions: 1319
|
num_questions: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_TEST_FORCE_FP8_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
|||||||
accuracy_threshold: 0.85
|
accuracy_threshold: 0.85
|
||||||
num_questions: 1319
|
num_questions: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_TEST_FORCE_FP8_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic"
|
|||||||
accuracy_threshold: 0.85
|
accuracy_threshold: 0.85
|
||||||
num_questions: 1319
|
num_questions: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_TEST_FORCE_FP8_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
|
|||||||
accuracy_threshold: 0.88
|
accuracy_threshold: 0.88
|
||||||
num_questions: 1319
|
num_questions: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_TEST_FORCE_FP8_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -2,6 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
|
|||||||
accuracy_threshold: 0.88
|
accuracy_threshold: 0.88
|
||||||
num_questions: 1319
|
num_questions: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_TEST_FORCE_FP8_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -64,10 +64,12 @@ def test_model_load_and_run(
|
|||||||
if use_rocm_aiter:
|
if use_rocm_aiter:
|
||||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||||
|
|
||||||
|
kwargs = {}
|
||||||
if force_marlin:
|
if force_marlin:
|
||||||
monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
|
kwargs["linear_backend"] = "marlin"
|
||||||
|
kwargs["moe_backend"] = "marlin"
|
||||||
|
|
||||||
with vllm_runner(model_id, enforce_eager=True) as llm:
|
with vllm_runner(model_id, enforce_eager=True, **kwargs) as llm:
|
||||||
# note: this does not test accuracy, just that we can run through
|
# note: this does not test accuracy, just that we can run through
|
||||||
# see lm-eval tests for accuracy
|
# see lm-eval tests for accuracy
|
||||||
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
|
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
|
||||||
@@ -98,8 +100,10 @@ def test_online_quantization(
|
|||||||
# `LLM.apply_model` requires pickling a function.
|
# `LLM.apply_model` requires pickling a function.
|
||||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||||
|
|
||||||
|
kwargs = {}
|
||||||
if force_marlin:
|
if force_marlin:
|
||||||
monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
|
kwargs["linear_backend"] = "marlin"
|
||||||
|
kwargs["moe_backend"] = "marlin"
|
||||||
|
|
||||||
model_dtype = "auto"
|
model_dtype = "auto"
|
||||||
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
|
||||||
@@ -112,6 +116,7 @@ def test_online_quantization(
|
|||||||
dtype=model_dtype,
|
dtype=model_dtype,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
kv_cache_dtype=kv_cache_dtype,
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
**kwargs,
|
||||||
) as llm:
|
) as llm:
|
||||||
|
|
||||||
def check_model(model):
|
def check_model(model):
|
||||||
@@ -340,7 +345,7 @@ def test_scaled_fp8_quant(dtype) -> None:
|
|||||||
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
|
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
|
||||||
# FP8 weight reloading does not support online quantization
|
# FP8 weight reloading does not support online quantization
|
||||||
@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False
|
@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False
|
||||||
@pytest.mark.parametrize("weight_block_size", [None, [1, 1]])
|
@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
|
||||||
# any postprocessing that is applied to the weights such as padding and repacking
|
# any postprocessing that is applied to the weights such as padding and repacking
|
||||||
# (excluding device sharding) must also be applied to the reloaded weights
|
# (excluding device sharding) must also be applied to the reloaded weights
|
||||||
#
|
#
|
||||||
@@ -371,6 +376,8 @@ def test_fp8_reloading(
|
|||||||
|
|
||||||
# Set model config as model_config.dtype is required in Fp8LinearMethod.
|
# Set model config as model_config.dtype is required in Fp8LinearMethod.
|
||||||
default_vllm_config.model_config = ModelConfig()
|
default_vllm_config.model_config = ModelConfig()
|
||||||
|
default_vllm_config.kernel_config.moe_backend = "triton"
|
||||||
|
layer_size = 128 if weight_block_size is not None else 1
|
||||||
with torch.device(f"{DEVICE_TYPE}:0"):
|
with torch.device(f"{DEVICE_TYPE}:0"):
|
||||||
config = Fp8Config(
|
config = Fp8Config(
|
||||||
is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
|
is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
|
||||||
@@ -378,14 +385,14 @@ def test_fp8_reloading(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if method_cls is Fp8LinearMethod:
|
if method_cls is Fp8LinearMethod:
|
||||||
layer = torch.nn.Linear(1, 1)
|
layer = torch.nn.Linear(layer_size, layer_size)
|
||||||
method = method_cls(config)
|
method = method_cls(config)
|
||||||
method.create_weights(
|
method.create_weights(
|
||||||
layer=layer,
|
layer=layer,
|
||||||
input_size_per_partition=1,
|
input_size_per_partition=layer_size,
|
||||||
output_partition_sizes=[1],
|
output_partition_sizes=[layer_size],
|
||||||
input_size=1,
|
input_size=layer_size,
|
||||||
output_size=1,
|
output_size=layer_size,
|
||||||
params_dtype=torch.bfloat16,
|
params_dtype=torch.bfloat16,
|
||||||
weight_loader=default_weight_loader,
|
weight_loader=default_weight_loader,
|
||||||
)
|
)
|
||||||
@@ -395,16 +402,16 @@ def test_fp8_reloading(
|
|||||||
layer = FusedMoEFactory(
|
layer = FusedMoEFactory(
|
||||||
num_experts=1,
|
num_experts=1,
|
||||||
top_k=1,
|
top_k=1,
|
||||||
hidden_size=1,
|
hidden_size=layer_size,
|
||||||
intermediate_size=1,
|
intermediate_size=layer_size,
|
||||||
)
|
)
|
||||||
layer = layer.routed_experts
|
layer = layer.routed_experts
|
||||||
method = method_cls(config, layer)
|
method = method_cls(config, layer)
|
||||||
method.create_weights(
|
method.create_weights(
|
||||||
layer=layer,
|
layer=layer,
|
||||||
num_experts=1,
|
num_experts=1,
|
||||||
hidden_size=1,
|
hidden_size=layer_size,
|
||||||
intermediate_size_per_partition=1,
|
intermediate_size_per_partition=layer_size,
|
||||||
params_dtype=torch.bfloat16,
|
params_dtype=torch.bfloat16,
|
||||||
weight_loader=default_weight_loader,
|
weight_loader=default_weight_loader,
|
||||||
)
|
)
|
||||||
|
|||||||
+8
-3
@@ -2339,6 +2339,10 @@ class TestFP8Layer(torch.nn.Module):
|
|||||||
force_kernel: type[_KernelT] | None = None,
|
force_kernel: type[_KernelT] | None = None,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
self.input_size_per_partition = weight_shape[1]
|
||||||
|
self.output_size_per_partition = weight_shape[0]
|
||||||
|
self.logical_widths = [self.output_size_per_partition]
|
||||||
|
self.orig_dtype = input_dtype
|
||||||
act_scale_desc = activation_quant_key.scale
|
act_scale_desc = activation_quant_key.scale
|
||||||
weight_scale_desc = weight_quant_key.scale
|
weight_scale_desc = weight_quant_key.scale
|
||||||
is_block_wise = act_scale_desc.group_shape.is_per_group()
|
is_block_wise = act_scale_desc.group_shape.is_per_group()
|
||||||
@@ -2346,11 +2350,12 @@ class TestFP8Layer(torch.nn.Module):
|
|||||||
block_size = weight_scale_desc.group_shape.col
|
block_size = weight_scale_desc.group_shape.col
|
||||||
weight_scale_shape = weight_shape[0] // block_size
|
weight_scale_shape = weight_shape[0] // block_size
|
||||||
self.weight_scale_inv = torch.rand(
|
self.weight_scale_inv = torch.rand(
|
||||||
(weight_scale_shape, weight_scale_shape), dtype=torch.float32
|
(weight_scale_shape, weight_scale_shape),
|
||||||
|
dtype=torch.float32,
|
||||||
|
device=device,
|
||||||
)
|
)
|
||||||
self.weight = torch.rand(weight_shape).to(dtype=FP8_DTYPE)
|
self.weight = torch.rand(weight_shape, device=device).to(dtype=FP8_DTYPE)
|
||||||
self.input_scale = None
|
self.input_scale = None
|
||||||
self.weight_scale = None
|
|
||||||
self.weight_block_size = [block_size, block_size]
|
self.weight_block_size = [block_size, block_size]
|
||||||
if transpose_weights:
|
if transpose_weights:
|
||||||
self.weight = self.weight.t()
|
self.weight = self.weight.t()
|
||||||
|
|||||||
@@ -1099,12 +1099,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower()
|
os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower()
|
||||||
in ("1", "true")
|
in ("1", "true")
|
||||||
),
|
),
|
||||||
# If set, forces FP8 Marlin to be used for FP8 quantization regardless
|
|
||||||
# of the hardware support for FP8 compute.
|
|
||||||
"VLLM_TEST_FORCE_FP8_MARLIN": lambda: (
|
|
||||||
os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower()
|
|
||||||
in ("1", "true")
|
|
||||||
),
|
|
||||||
"VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv(
|
"VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv(
|
||||||
"VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"
|
"VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -394,12 +394,12 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
|
|||||||
# in priority/performance order (when available)
|
# in priority/performance order (when available)
|
||||||
_POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
_POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||||
PlatformEnum.CUDA: [
|
PlatformEnum.CUDA: [
|
||||||
MarlinFP8ScaledMMLinearKernel,
|
|
||||||
FlashInferFP8ScaledMMLinearKernel,
|
FlashInferFP8ScaledMMLinearKernel,
|
||||||
CutlassFP8ScaledMMLinearKernel,
|
CutlassFP8ScaledMMLinearKernel,
|
||||||
B12xTensorFP8ScaledMMLinearKernel,
|
B12xTensorFP8ScaledMMLinearKernel,
|
||||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||||
|
MarlinFP8ScaledMMLinearKernel,
|
||||||
HummingFP8ScaledMMLinearKernel,
|
HummingFP8ScaledMMLinearKernel,
|
||||||
],
|
],
|
||||||
PlatformEnum.ROCM: [
|
PlatformEnum.ROCM: [
|
||||||
@@ -434,8 +434,8 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
|||||||
CutlassFp8BlockScaledMMKernel,
|
CutlassFp8BlockScaledMMKernel,
|
||||||
B12xFp8BlockScaledMMKernel,
|
B12xFp8BlockScaledMMKernel,
|
||||||
MarlinFP8ScaledMMLinearKernel,
|
MarlinFP8ScaledMMLinearKernel,
|
||||||
TritonFp8BlockScaledMMKernel,
|
|
||||||
HummingFP8ScaledMMLinearKernel,
|
HummingFP8ScaledMMLinearKernel,
|
||||||
|
TritonFp8BlockScaledMMKernel,
|
||||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||||
],
|
],
|
||||||
PlatformEnum.ROCM: [
|
PlatformEnum.ROCM: [
|
||||||
|
|||||||
@@ -43,16 +43,6 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
|||||||
return False, "FP8 Marlin requires compute capability 7.5 or higher"
|
return False, "FP8 Marlin requires compute capability 7.5 or higher"
|
||||||
if envs.VLLM_BATCH_INVARIANT:
|
if envs.VLLM_BATCH_INVARIANT:
|
||||||
return False, "FP8 Marlin not supported for batch invariant execution."
|
return False, "FP8 Marlin not supported for batch invariant execution."
|
||||||
if (
|
|
||||||
compute_capability is not None
|
|
||||||
and compute_capability >= 89
|
|
||||||
and not envs.VLLM_TEST_FORCE_FP8_MARLIN
|
|
||||||
):
|
|
||||||
return (
|
|
||||||
False,
|
|
||||||
"To apply FP8 Marlin on high-capability GPUs, please set "
|
|
||||||
"VLLM_TEST_FORCE_FP8_MARLIN=1",
|
|
||||||
)
|
|
||||||
return True, None
|
return True, None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -370,13 +370,6 @@ def select_fp8_moe_backend(
|
|||||||
backend, config, weight_key, activation_key, activation_format
|
backend, config, weight_key, activation_key, activation_format
|
||||||
)
|
)
|
||||||
|
|
||||||
# Handle explicit MARLIN FP8 configuration.
|
|
||||||
if envs.VLLM_TEST_FORCE_FP8_MARLIN:
|
|
||||||
backend = Fp8MoeBackend.MARLIN
|
|
||||||
return _return_or_raise(
|
|
||||||
backend, config, weight_key, activation_key, activation_format
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle explicit AITER FP8 configuration.
|
# Handle explicit AITER FP8 configuration.
|
||||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||||
skip_aiter_moe = (
|
skip_aiter_moe = (
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ from enum import Enum
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.envs as envs
|
|
||||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||||
from vllm.config.kernel import MoEBackend
|
from vllm.config.kernel import MoEBackend
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -271,12 +270,6 @@ def select_nvfp4_moe_backend(
|
|||||||
requested_backend, config, weight_key, activation_key, activation_format
|
requested_backend, config, weight_key, activation_key, activation_format
|
||||||
)
|
)
|
||||||
|
|
||||||
if envs.VLLM_TEST_FORCE_FP8_MARLIN:
|
|
||||||
backend = NvFp4MoeBackend.MARLIN
|
|
||||||
return _return_or_raise(
|
|
||||||
backend, config, weight_key, activation_key, activation_format
|
|
||||||
)
|
|
||||||
|
|
||||||
# Select kernels in order of backend.
|
# Select kernels in order of backend.
|
||||||
for backend in AVAILABLE_BACKENDS:
|
for backend in AVAILABLE_BACKENDS:
|
||||||
for k_cls in backend_to_kernel_cls(backend):
|
for k_cls in backend_to_kernel_cls(backend):
|
||||||
|
|||||||
Reference in New Issue
Block a user