Remove VLLM_TEST_FORCE_FP8_MARLIN to replace with linear_backend/moe_backend (#52182)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>

tests/compile/passes/test_fusion.py

@@ -371,6 +371,10 @@ def test_fusion_rmsnorm_quant(
             use_aiter_fusion=False,
             use_aiter_quant=False,
         )
+        if any(
+            type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers
+        ):
+            pytest.skip(f"{force_kernel.__name__} is not supported on this platform")
 
         backend, _ = _run_fusion_test(
             model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens

tests/compile/passes/test_mla_attn_quant_fusion.py

@@ -481,6 +481,12 @@ def test_mla_attention_quant_pattern(
             device=device,
             vllm_config=vllm_config_unfused,
         )
+        if (
+            model_class is TestMLAAttentionFp8GroupQuantPatternModel
+            and type(model_unfused.block_fp8_linear.kernel)
+            is not CutlassFp8BlockScaledMMKernel
+        ):
+            pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")
         model_unfused = model_unfused.to(device)
         # HACK: See #131044
         result_unfused_0 = model_unfused(q, kv_c_normed, k_pe)  # noqa: F841

tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml

@@ -2,6 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
 accuracy_threshold: 0.92
 num_questions: 1319
 num_fewshot: 5
-server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
-env:
-  VLLM_TEST_FORCE_FP8_MARLIN: "1"
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml

@@ -2,6 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
 accuracy_threshold: 0.88
 num_questions: 1319
 num_fewshot: 5
-server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
-env:
-  VLLM_TEST_FORCE_FP8_MARLIN: "1"
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
 accuracy_threshold: 0.85
 num_questions: 1319
 num_fewshot: 5
-server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
-env:
-  VLLM_TEST_FORCE_FP8_MARLIN: "1"
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic"
 accuracy_threshold: 0.85
 num_questions: 1319
 num_fewshot: 5
-server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
-env:
-  VLLM_TEST_FORCE_FP8_MARLIN: "1"
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
 accuracy_threshold: 0.88
 num_questions: 1319
 num_fewshot: 5
-server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
-env:
-  VLLM_TEST_FORCE_FP8_MARLIN: "1"
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml

@@ -2,6 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
 accuracy_threshold: 0.88
 num_questions: 1319
 num_fewshot: 5
-server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
-env:
-  VLLM_TEST_FORCE_FP8_MARLIN: "1"
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/quantization/test_fp8.py

@@ -64,10 +64,12 @@ def test_model_load_and_run(
     if use_rocm_aiter:
         monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
 
+    kwargs = {}
     if force_marlin:
-        monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
+        kwargs["linear_backend"] = "marlin"
+        kwargs["moe_backend"] = "marlin"
 
-    with vllm_runner(model_id, enforce_eager=True) as llm:
+    with vllm_runner(model_id, enforce_eager=True, **kwargs) as llm:
         # note: this does not test accuracy, just that we can run through
         # see lm-eval tests for accuracy
         outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
@@ -98,8 +100,10 @@ def test_online_quantization(
     # `LLM.apply_model` requires pickling a function.
     monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
 
+    kwargs = {}
     if force_marlin:
-        monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
+        kwargs["linear_backend"] = "marlin"
+        kwargs["moe_backend"] = "marlin"
 
     model_dtype = "auto"
     if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):
@@ -112,6 +116,7 @@ def test_online_quantization(
         dtype=model_dtype,
         enforce_eager=True,
         kv_cache_dtype=kv_cache_dtype,
+        **kwargs,
     ) as llm:
 
         def check_model(model):
@@ -340,7 +345,7 @@ def test_scaled_fp8_quant(dtype) -> None:
 @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
 # FP8 weight reloading does not support online quantization
 @pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True])  # skip False
-@pytest.mark.parametrize("weight_block_size", [None, [1, 1]])
+@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])
 # any postprocessing that is applied to the weights such as padding and repacking
 # (excluding device sharding) must also be applied to the reloaded weights
 #
@@ -371,6 +376,8 @@ def test_fp8_reloading(
 
     # Set model config as model_config.dtype is required in Fp8LinearMethod.
     default_vllm_config.model_config = ModelConfig()
+    default_vllm_config.kernel_config.moe_backend = "triton"
+    layer_size = 128 if weight_block_size is not None else 1
     with torch.device(f"{DEVICE_TYPE}:0"):
         config = Fp8Config(
             is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
@@ -378,14 +385,14 @@ def test_fp8_reloading(
         )
 
         if method_cls is Fp8LinearMethod:
-            layer = torch.nn.Linear(1, 1)
+            layer = torch.nn.Linear(layer_size, layer_size)
             method = method_cls(config)
             method.create_weights(
                 layer=layer,
-                input_size_per_partition=1,
-                output_partition_sizes=[1],
-                input_size=1,
-                output_size=1,
+                input_size_per_partition=layer_size,
+                output_partition_sizes=[layer_size],
+                input_size=layer_size,
+                output_size=layer_size,
                 params_dtype=torch.bfloat16,
                 weight_loader=default_weight_loader,
             )
@@ -395,16 +402,16 @@ def test_fp8_reloading(
             layer = FusedMoEFactory(
                 num_experts=1,
                 top_k=1,
-                hidden_size=1,
-                intermediate_size=1,
+                hidden_size=layer_size,
+                intermediate_size=layer_size,
             )
             layer = layer.routed_experts
             method = method_cls(config, layer)
             method.create_weights(
                 layer=layer,
                 num_experts=1,
-                hidden_size=1,
-                intermediate_size_per_partition=1,
+                hidden_size=layer_size,
+                intermediate_size_per_partition=layer_size,
                 params_dtype=torch.bfloat16,
                 weight_loader=default_weight_loader,
             )

tests/utils.py

@@ -2339,6 +2339,10 @@ class TestFP8Layer(torch.nn.Module):
         force_kernel: type[_KernelT] | None = None,
     ):
         super().__init__()
+        self.input_size_per_partition = weight_shape[1]
+        self.output_size_per_partition = weight_shape[0]
+        self.logical_widths = [self.output_size_per_partition]
+        self.orig_dtype = input_dtype
         act_scale_desc = activation_quant_key.scale
         weight_scale_desc = weight_quant_key.scale
         is_block_wise = act_scale_desc.group_shape.is_per_group()
@@ -2346,11 +2350,12 @@ class TestFP8Layer(torch.nn.Module):
             block_size = weight_scale_desc.group_shape.col
             weight_scale_shape = weight_shape[0] // block_size
             self.weight_scale_inv = torch.rand(
-                (weight_scale_shape, weight_scale_shape), dtype=torch.float32
+                (weight_scale_shape, weight_scale_shape),
+                dtype=torch.float32,
+                device=device,
             )
-            self.weight = torch.rand(weight_shape).to(dtype=FP8_DTYPE)
+            self.weight = torch.rand(weight_shape, device=device).to(dtype=FP8_DTYPE)
             self.input_scale = None
-            self.weight_scale = None
             self.weight_block_size = [block_size, block_size]
             if transpose_weights:
                 self.weight = self.weight.t()

vllm/envs.py

@@ -1099,12 +1099,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
         os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower()
         in ("1", "true")
     ),
-    # If set, forces FP8 Marlin to be used for FP8 quantization regardless
-    # of the hardware support for FP8 compute.
-    "VLLM_TEST_FORCE_FP8_MARLIN": lambda: (
-        os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower()
-        in ("1", "true")
-    ),
     "VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv(
         "VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"
     ),

vllm/model_executor/kernels/linear/__init__.py

@@ -394,12 +394,12 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
 # in priority/performance order (when available)
 _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
     PlatformEnum.CUDA: [
-        MarlinFP8ScaledMMLinearKernel,
         FlashInferFP8ScaledMMLinearKernel,
         CutlassFP8ScaledMMLinearKernel,
         B12xTensorFP8ScaledMMLinearKernel,
         PerTensorTorchFP8ScaledMMLinearKernel,
         ChannelWiseTorchFP8ScaledMMLinearKernel,
+        MarlinFP8ScaledMMLinearKernel,
         HummingFP8ScaledMMLinearKernel,
     ],
     PlatformEnum.ROCM: [
@@ -434,8 +434,8 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
         CutlassFp8BlockScaledMMKernel,
         B12xFp8BlockScaledMMKernel,
         MarlinFP8ScaledMMLinearKernel,
-        TritonFp8BlockScaledMMKernel,
         HummingFP8ScaledMMLinearKernel,
+        TritonFp8BlockScaledMMKernel,
         BlockWiseTorchFP8ScaledMMLinearKernel,
     ],
     PlatformEnum.ROCM: [

vllm/model_executor/kernels/linear/scaled_mm/marlin.py

@@ -43,16 +43,6 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
             return False, "FP8 Marlin requires compute capability 7.5 or higher"
         if envs.VLLM_BATCH_INVARIANT:
             return False, "FP8 Marlin not supported for batch invariant execution."
-        if (
-            compute_capability is not None
-            and compute_capability >= 89
-            and not envs.VLLM_TEST_FORCE_FP8_MARLIN
-        ):
-            return (
-                False,
-                "To apply FP8 Marlin on high-capability GPUs, please set "
-                "VLLM_TEST_FORCE_FP8_MARLIN=1",
-            )
         return True, None
 
     @classmethod

vllm/model_executor/layers/fused_moe/oracle/fp8.py

@@ -370,13 +370,6 @@ def select_fp8_moe_backend(
                 backend, config, weight_key, activation_key, activation_format
             )
 
-    # Handle explicit MARLIN FP8 configuration.
-    if envs.VLLM_TEST_FORCE_FP8_MARLIN:
-        backend = Fp8MoeBackend.MARLIN
-        return _return_or_raise(
-            backend, config, weight_key, activation_key, activation_format
-        )
-
     # Handle explicit AITER FP8 configuration.
     if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
         skip_aiter_moe = (

vllm/model_executor/layers/fused_moe/oracle/nvfp4.py

@@ -4,7 +4,6 @@ from enum import Enum
 
 import torch
 
-import vllm.envs as envs
 import vllm.model_executor.layers.fused_moe.modular_kernel as mk
 from vllm.config.kernel import MoEBackend
 from vllm.logger import init_logger
@@ -271,12 +270,6 @@ def select_nvfp4_moe_backend(
             requested_backend, config, weight_key, activation_key, activation_format
         )
 
-    if envs.VLLM_TEST_FORCE_FP8_MARLIN:
-        backend = NvFp4MoeBackend.MARLIN
-        return _return_or_raise(
-            backend, config, weight_key, activation_key, activation_format
-        )
-
     # Select kernels in order of backend.
     for backend in AVAILABLE_BACKENDS:
         for k_cls in backend_to_kernel_cls(backend):