From bca7bea2405127bd5291bb6fffa679bdcd8f6dd9 Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Tue, 18 Aug 2026 12:58:04 -0400 Subject: [PATCH] Remove VLLM_TEST_FORCE_FP8_MARLIN to replace with linear_backend/moe_backend (#52182) Signed-off-by: mgoin Co-authored-by: OpenAI Codex --- tests/compile/passes/test_fusion.py | 4 +++ .../passes/test_mla_attn_quant_fusion.py | 6 ++++ .../Llama-4-Scout-Fp8-ModelOpt-marlin.yaml | 4 +-- .../Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml | 4 +-- .../Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml | 4 +-- .../Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml | 4 +-- .../Qwen3-30B-A3B-NvFp4-CT-marlin.yaml | 4 +-- .../Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml | 4 +-- tests/quantization/test_fp8.py | 33 +++++++++++-------- tests/utils.py | 11 +++++-- vllm/envs.py | 6 ---- .../model_executor/kernels/linear/__init__.py | 4 +-- .../kernels/linear/scaled_mm/marlin.py | 10 ------ .../layers/fused_moe/oracle/fp8.py | 7 ---- .../layers/fused_moe/oracle/nvfp4.py | 7 ---- 15 files changed, 46 insertions(+), 66 deletions(-) diff --git a/tests/compile/passes/test_fusion.py b/tests/compile/passes/test_fusion.py index 591b014d9e..50c957fceb 100644 --- a/tests/compile/passes/test_fusion.py +++ b/tests/compile/passes/test_fusion.py @@ -371,6 +371,10 @@ def test_fusion_rmsnorm_quant( use_aiter_fusion=False, use_aiter_quant=False, ) + if any( + type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers + ): + pytest.skip(f"{force_kernel.__name__} is not supported on this platform") backend, _ = _run_fusion_test( model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens diff --git a/tests/compile/passes/test_mla_attn_quant_fusion.py b/tests/compile/passes/test_mla_attn_quant_fusion.py index 5b2ef5bfd9..106bb8e80a 100644 --- a/tests/compile/passes/test_mla_attn_quant_fusion.py +++ b/tests/compile/passes/test_mla_attn_quant_fusion.py @@ -481,6 +481,12 @@ def test_mla_attention_quant_pattern( device=device, vllm_config=vllm_config_unfused, ) + if ( + model_class is TestMLAAttentionFp8GroupQuantPatternModel + and type(model_unfused.block_fp8_linear.kernel) + is not CutlassFp8BlockScaledMMKernel + ): + pytest.skip("CUTLASS FP8 block kernel is not supported on this platform") model_unfused = model_unfused.to(device) # HACK: See #131044 result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841 diff --git a/tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml b/tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml index be8192f2a8..23ceed6803 100644 --- a/tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml @@ -2,6 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8" accuracy_threshold: 0.92 num_questions: 1319 num_fewshot: 5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2" -env: - VLLM_TEST_FORCE_FP8_MARLIN: "1" +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin" diff --git a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml index c3d86e6bfb..0b8c90a37e 100644 --- a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml @@ -2,6 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2" -env: - VLLM_TEST_FORCE_FP8_MARLIN: "1" +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin" diff --git a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml index 46eee74213..d22ee2086d 100644 --- a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml @@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block" accuracy_threshold: 0.85 num_questions: 1319 num_fewshot: 5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2" -env: - VLLM_TEST_FORCE_FP8_MARLIN: "1" +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin" diff --git a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml index 8ed6410c36..dd86ee528b 100644 --- a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml @@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic" accuracy_threshold: 0.85 num_questions: 1319 num_fewshot: 5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2" -env: - VLLM_TEST_FORCE_FP8_MARLIN: "1" +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin" diff --git a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml index 8199e65634..ffe6829ccb 100644 --- a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml @@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2" -env: - VLLM_TEST_FORCE_FP8_MARLIN: "1" +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin" diff --git a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml index 4156cec897..b6b71f4d3d 100644 --- a/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml +++ b/tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml @@ -2,6 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4" accuracy_threshold: 0.88 num_questions: 1319 num_fewshot: 5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2" -env: - VLLM_TEST_FORCE_FP8_MARLIN: "1" +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin" diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index a1c74259f8..5ad3b666c2 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -64,10 +64,12 @@ def test_model_load_and_run( if use_rocm_aiter: monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + kwargs = {} if force_marlin: - monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1") + kwargs["linear_backend"] = "marlin" + kwargs["moe_backend"] = "marlin" - with vllm_runner(model_id, enforce_eager=True) as llm: + with vllm_runner(model_id, enforce_eager=True, **kwargs) as llm: # note: this does not test accuracy, just that we can run through # see lm-eval tests for accuracy outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4) @@ -98,8 +100,10 @@ def test_online_quantization( # `LLM.apply_model` requires pickling a function. monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + kwargs = {} if force_marlin: - monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1") + kwargs["linear_backend"] = "marlin" + kwargs["moe_backend"] = "marlin" model_dtype = "auto" if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90): @@ -112,6 +116,7 @@ def test_online_quantization( dtype=model_dtype, enforce_eager=True, kv_cache_dtype=kv_cache_dtype, + **kwargs, ) as llm: def check_model(model): @@ -340,7 +345,7 @@ def test_scaled_fp8_quant(dtype) -> None: @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod]) # FP8 weight reloading does not support online quantization @pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False -@pytest.mark.parametrize("weight_block_size", [None, [1, 1]]) +@pytest.mark.parametrize("weight_block_size", [None, [128, 128]]) # any postprocessing that is applied to the weights such as padding and repacking # (excluding device sharding) must also be applied to the reloaded weights # @@ -371,6 +376,8 @@ def test_fp8_reloading( # Set model config as model_config.dtype is required in Fp8LinearMethod. default_vllm_config.model_config = ModelConfig() + default_vllm_config.kernel_config.moe_backend = "triton" + layer_size = 128 if weight_block_size is not None else 1 with torch.device(f"{DEVICE_TYPE}:0"): config = Fp8Config( is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, @@ -378,14 +385,14 @@ def test_fp8_reloading( ) if method_cls is Fp8LinearMethod: - layer = torch.nn.Linear(1, 1) + layer = torch.nn.Linear(layer_size, layer_size) method = method_cls(config) method.create_weights( layer=layer, - input_size_per_partition=1, - output_partition_sizes=[1], - input_size=1, - output_size=1, + input_size_per_partition=layer_size, + output_partition_sizes=[layer_size], + input_size=layer_size, + output_size=layer_size, params_dtype=torch.bfloat16, weight_loader=default_weight_loader, ) @@ -395,16 +402,16 @@ def test_fp8_reloading( layer = FusedMoEFactory( num_experts=1, top_k=1, - hidden_size=1, - intermediate_size=1, + hidden_size=layer_size, + intermediate_size=layer_size, ) layer = layer.routed_experts method = method_cls(config, layer) method.create_weights( layer=layer, num_experts=1, - hidden_size=1, - intermediate_size_per_partition=1, + hidden_size=layer_size, + intermediate_size_per_partition=layer_size, params_dtype=torch.bfloat16, weight_loader=default_weight_loader, ) diff --git a/tests/utils.py b/tests/utils.py index 86774bf4bf..07601b74e4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2339,6 +2339,10 @@ class TestFP8Layer(torch.nn.Module): force_kernel: type[_KernelT] | None = None, ): super().__init__() + self.input_size_per_partition = weight_shape[1] + self.output_size_per_partition = weight_shape[0] + self.logical_widths = [self.output_size_per_partition] + self.orig_dtype = input_dtype act_scale_desc = activation_quant_key.scale weight_scale_desc = weight_quant_key.scale is_block_wise = act_scale_desc.group_shape.is_per_group() @@ -2346,11 +2350,12 @@ class TestFP8Layer(torch.nn.Module): block_size = weight_scale_desc.group_shape.col weight_scale_shape = weight_shape[0] // block_size self.weight_scale_inv = torch.rand( - (weight_scale_shape, weight_scale_shape), dtype=torch.float32 + (weight_scale_shape, weight_scale_shape), + dtype=torch.float32, + device=device, ) - self.weight = torch.rand(weight_shape).to(dtype=FP8_DTYPE) + self.weight = torch.rand(weight_shape, device=device).to(dtype=FP8_DTYPE) self.input_scale = None - self.weight_scale = None self.weight_block_size = [block_size, block_size] if transpose_weights: self.weight = self.weight.t() diff --git a/vllm/envs.py b/vllm/envs.py index 1a7458802a..f80c51207f 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1099,12 +1099,6 @@ environment_variables: dict[str, Callable[[], Any]] = { os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower() in ("1", "true") ), - # If set, forces FP8 Marlin to be used for FP8 quantization regardless - # of the hardware support for FP8 compute. - "VLLM_TEST_FORCE_FP8_MARLIN": lambda: ( - os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() - in ("1", "true") - ), "VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv( "VLLM_TEST_FORCE_LOAD_FORMAT", "dummy" ), diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 3ba499155b..b94c8f7d58 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -394,12 +394,12 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]] # in priority/performance order (when available) _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = { PlatformEnum.CUDA: [ - MarlinFP8ScaledMMLinearKernel, FlashInferFP8ScaledMMLinearKernel, CutlassFP8ScaledMMLinearKernel, B12xTensorFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, ChannelWiseTorchFP8ScaledMMLinearKernel, + MarlinFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ @@ -434,8 +434,8 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ CutlassFp8BlockScaledMMKernel, B12xFp8BlockScaledMMKernel, MarlinFP8ScaledMMLinearKernel, - TritonFp8BlockScaledMMKernel, HummingFP8ScaledMMLinearKernel, + TritonFp8BlockScaledMMKernel, BlockWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ diff --git a/vllm/model_executor/kernels/linear/scaled_mm/marlin.py b/vllm/model_executor/kernels/linear/scaled_mm/marlin.py index 66a03b4d20..53bdc724bb 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/marlin.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/marlin.py @@ -43,16 +43,6 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): return False, "FP8 Marlin requires compute capability 7.5 or higher" if envs.VLLM_BATCH_INVARIANT: return False, "FP8 Marlin not supported for batch invariant execution." - if ( - compute_capability is not None - and compute_capability >= 89 - and not envs.VLLM_TEST_FORCE_FP8_MARLIN - ): - return ( - False, - "To apply FP8 Marlin on high-capability GPUs, please set " - "VLLM_TEST_FORCE_FP8_MARLIN=1", - ) return True, None @classmethod diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 6153ed28c1..b2d76e4ef9 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -370,13 +370,6 @@ def select_fp8_moe_backend( backend, config, weight_key, activation_key, activation_format ) - # Handle explicit MARLIN FP8 configuration. - if envs.VLLM_TEST_FORCE_FP8_MARLIN: - backend = Fp8MoeBackend.MARLIN - return _return_or_raise( - backend, config, weight_key, activation_key, activation_format - ) - # Handle explicit AITER FP8 configuration. if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"): skip_aiter_moe = ( diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index f48e79e13d..3f23ecd619 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -4,7 +4,6 @@ from enum import Enum import torch -import vllm.envs as envs import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.config.kernel import MoEBackend from vllm.logger import init_logger @@ -271,12 +270,6 @@ def select_nvfp4_moe_backend( requested_backend, config, weight_key, activation_key, activation_format ) - if envs.VLLM_TEST_FORCE_FP8_MARLIN: - backend = NvFp4MoeBackend.MARLIN - return _return_or_raise( - backend, config, weight_key, activation_key, activation_format - ) - # Select kernels in order of backend. for backend in AVAILABLE_BACKENDS: for k_cls in backend_to_kernel_cls(backend):