Remove VLLM_TEST_FORCE_FP8_MARLIN to replace with linear_backend/moe_backend (#52182)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Misha Goin
2026-08-18 12:58:04 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 88b2bff2c6
commit bca7bea240
15 changed files with 46 additions and 66 deletions
+4
View File
@@ -371,6 +371,10 @@ def test_fusion_rmsnorm_quant(
use_aiter_fusion=False,
use_aiter_quant=False,
)
if any(
type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers
):
pytest.skip(f"{force_kernel.__name__} is not supported on this platform")
backend, _ = _run_fusion_test(
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
@@ -481,6 +481,12 @@ def test_mla_attention_quant_pattern(
device=device,
vllm_config=vllm_config_unfused,
)
if (
model_class is TestMLAAttentionFp8GroupQuantPatternModel
and type(model_unfused.block_fp8_linear.kernel)
is not CutlassFp8BlockScaledMMKernel
):
pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")
model_unfused = model_unfused.to(device)
# HACK: See #131044
result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841
@@ -2,6 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
accuracy_threshold: 0.92
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
env:
VLLM_TEST_FORCE_FP8_MARLIN: "1"
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
@@ -2,6 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
accuracy_threshold: 0.88
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
env:
VLLM_TEST_FORCE_FP8_MARLIN: "1"
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.85
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
env:
VLLM_TEST_FORCE_FP8_MARLIN: "1"
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic"
accuracy_threshold: 0.85
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
env:
VLLM_TEST_FORCE_FP8_MARLIN: "1"
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
accuracy_threshold: 0.88
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
env:
VLLM_TEST_FORCE_FP8_MARLIN: "1"
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
@@ -2,6 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
accuracy_threshold: 0.88
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
env:
VLLM_TEST_FORCE_FP8_MARLIN: "1"
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"
+20 -13
View File
@@ -64,10 +64,12 @@ def test_model_load_and_run(
if use_rocm_aiter:
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
kwargs = {}
if force_marlin:
monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
kwargs["linear_backend"] = "marlin"
kwargs["moe_backend"] = "marlin"
with vllm_runner(model_id, enforce_eager=True) as llm:
with vllm_runner(model_id, enforce_eager=True, **kwargs) as llm:
# note: this does not test accuracy, just that we can run through
# see lm-eval tests for accuracy
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
@@ -98,8 +100,10 @@ def test_online_quantization(
# `LLM.apply_model` requires pickling a function.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kwargs = {}
if force_marlin:
monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
kwargs["linear_backend"] = "marlin"
kwargs["moe_backend"] = "marlin"
model_dtype = "auto"
if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
@@ -112,6 +116,7 @@ def test_online_quantization(
dtype=model_dtype,
enforce_eager=True,
kv_cache_dtype=kv_cache_dtype,
**kwargs,
) as llm:
def check_model(model):
@@ -340,7 +345,7 @@ def test_scaled_fp8_quant(dtype) -> None:
@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
# FP8 weight reloading does not support online quantization
@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False
@pytest.mark.parametrize("weight_block_size", [None, [1, 1]])
@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
# any postprocessing that is applied to the weights such as padding and repacking
# (excluding device sharding) must also be applied to the reloaded weights
#
@@ -371,6 +376,8 @@ def test_fp8_reloading(
# Set model config as model_config.dtype is required in Fp8LinearMethod.
default_vllm_config.model_config = ModelConfig()
default_vllm_config.kernel_config.moe_backend = "triton"
layer_size = 128 if weight_block_size is not None else 1
with torch.device(f"{DEVICE_TYPE}:0"):
config = Fp8Config(
is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
@@ -378,14 +385,14 @@ def test_fp8_reloading(
)
if method_cls is Fp8LinearMethod:
layer = torch.nn.Linear(1, 1)
layer = torch.nn.Linear(layer_size, layer_size)
method = method_cls(config)
method.create_weights(
layer=layer,
input_size_per_partition=1,
output_partition_sizes=[1],
input_size=1,
output_size=1,
input_size_per_partition=layer_size,
output_partition_sizes=[layer_size],
input_size=layer_size,
output_size=layer_size,
params_dtype=torch.bfloat16,
weight_loader=default_weight_loader,
)
@@ -395,16 +402,16 @@ def test_fp8_reloading(
layer = FusedMoEFactory(
num_experts=1,
top_k=1,
hidden_size=1,
intermediate_size=1,
hidden_size=layer_size,
intermediate_size=layer_size,
)
layer = layer.routed_experts
method = method_cls(config, layer)
method.create_weights(
layer=layer,
num_experts=1,
hidden_size=1,
intermediate_size_per_partition=1,
hidden_size=layer_size,
intermediate_size_per_partition=layer_size,
params_dtype=torch.bfloat16,
weight_loader=default_weight_loader,
)
+8 -3
View File
@@ -2339,6 +2339,10 @@ class TestFP8Layer(torch.nn.Module):
force_kernel: type[_KernelT] | None = None,
):
super().__init__()
self.input_size_per_partition = weight_shape[1]
self.output_size_per_partition = weight_shape[0]
self.logical_widths = [self.output_size_per_partition]
self.orig_dtype = input_dtype
act_scale_desc = activation_quant_key.scale
weight_scale_desc = weight_quant_key.scale
is_block_wise = act_scale_desc.group_shape.is_per_group()
@@ -2346,11 +2350,12 @@ class TestFP8Layer(torch.nn.Module):
block_size = weight_scale_desc.group_shape.col
weight_scale_shape = weight_shape[0] // block_size
self.weight_scale_inv = torch.rand(
(weight_scale_shape, weight_scale_shape), dtype=torch.float32
(weight_scale_shape, weight_scale_shape),
dtype=torch.float32,
device=device,
)
self.weight = torch.rand(weight_shape).to(dtype=FP8_DTYPE)
self.weight = torch.rand(weight_shape, device=device).to(dtype=FP8_DTYPE)
self.input_scale = None
self.weight_scale = None
self.weight_block_size = [block_size, block_size]
if transpose_weights:
self.weight = self.weight.t()