Deprecations for v0.23 and v0.24 (#44992)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-11 14:35:38 +00:00
committed by GitHub
parent 55911db580
commit 03878d1c22
35 changed files with 100 additions and 674 deletions
@@ -6,9 +6,7 @@ tasks:
value: 0.7142 value: 0.7142
- name: "exact_match,flexible-extract" - name: "exact_match,flexible-extract"
value: 0.4579 value: 0.4579
env_vars: moe_backend: "flashinfer_cutlass"
VLLM_USE_FLASHINFER_MOE_FP8: "1"
VLLM_FLASHINFER_MOE_BACKEND: "throughput"
limit: 1319 limit: 1319
num_fewshot: 5 num_fewshot: 5
max_model_len: 262144 max_model_len: 262144
@@ -68,6 +68,10 @@ def launch_lm_eval(eval_config, tp_size):
if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]: if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
model_args += "attention_backend=TRITON_ATTN" model_args += "attention_backend=TRITON_ATTN"
moe_backend = eval_config.get("moe_backend", None)
if moe_backend is not None:
model_args += f"moe_backend={moe_backend},"
env_vars = eval_config.get("env_vars", None) env_vars = eval_config.get("env_vars", None)
with scoped_env_vars(env_vars): with scoped_env_vars(env_vars):
results = lm_eval.simple_evaluate( results = lm_eval.simple_evaluate(
+1 -1
View File
@@ -42,7 +42,7 @@ th {
1. All types: mxfp4, nvfp4, int4, int8, fp8 1. All types: mxfp4, nvfp4, int4, int8, fp8
2. A,T quantization occurs after dispatch. 2. A,T quantization occurs after dispatch.
3. All quantization happens after dispatch. 3. All quantization happens after dispatch.
4. Controlled by different env vars (`VLLM_FLASHINFER_MOE_BACKEND` "throughput" or "latency") 4. Controlled by `--moe-backend` (`flashinfer_cutlass` or `flashinfer_trtllm`)
5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API. 5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API.
6. This depends on the experts implementation. 6. This depends on the experts implementation.
+1 -1
View File
@@ -143,4 +143,4 @@ More examples can be found here: [examples/pooling/reward](../../../examples/poo
### `LLM.reward` ### `LLM.reward`
`llm.reward` api is deprecated and will be removed in v0.23. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead. `llm.reward` API is deprecated and was removed in v0.24. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
@@ -102,7 +102,7 @@ def test_async_tp_pass_correctness(
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch): def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
if ( if (
not current_platform.is_cuda() not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100) or not current_platform.is_device_capability_family(100)
@@ -111,8 +111,6 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
if not has_flashinfer(): if not has_flashinfer():
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path") pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass")
tp_size = 2 tp_size = 2
if num_gpus_available < tp_size: if num_gpus_available < tp_size:
pytest.skip(f"Need at least {tp_size} GPUs") pytest.skip(f"Need at least {tp_size} GPUs")
@@ -126,6 +124,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
"8", "8",
"--load-format", "--load-format",
"dummy", "dummy",
"--linear-backend",
"flashinfer_cutlass",
"--hf-overrides", "--hf-overrides",
json.dumps(NVFP4_HF_OVERRIDES), json.dumps(NVFP4_HF_OVERRIDES),
] ]
-4
View File
@@ -1226,10 +1226,6 @@ class VllmRunner:
req_outputs = self.llm.encode(prompts, pooling_task="token_classify") req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
return [req_output.outputs.data for req_output in req_outputs] return [req_output.outputs.data for req_output in req_outputs]
def reward(self, prompts: list[str]) -> list[list[float]]:
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
return [req_output.outputs.data for req_output in req_outputs]
def score( def score(
self, self,
text_1: list[str] | str, text_1: list[str] | str,
@@ -37,6 +37,7 @@ class TestConfig:
hidden_size: int hidden_size: int
intermediate_size: int intermediate_size: int
num_tokens: int num_tokens: int
moe_backend: str
def make_fused_moe_layer( def make_fused_moe_layer(
@@ -114,6 +115,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
vllm_config = VllmConfig() vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = test_config.moe_backend
with set_current_vllm_config(vllm_config): with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized( ensure_model_parallel_initialized(
@@ -250,7 +252,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
@pytest.mark.parametrize("hidden_size", [256]) @pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("intermediate_size", [256]) @pytest.mark.parametrize("intermediate_size", [256])
@pytest.mark.parametrize("num_tokens", [256]) @pytest.mark.parametrize("num_tokens", [256])
@pytest.mark.parametrize("backend", ["latency", "throughput"]) @pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
def test_eplb_fml( def test_eplb_fml(
world_size: int, world_size: int,
num_layers: int, num_layers: int,
@@ -258,12 +260,8 @@ def test_eplb_fml(
hidden_size: int, hidden_size: int,
intermediate_size: int, intermediate_size: int,
num_tokens: int, num_tokens: int,
backend: str, moe_backend: str,
monkeypatch,
): ):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", backend)
if torch.accelerator.device_count() < world_size: if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test") pytest.skip(f"Need at least {world_size} GPUs to run the test")
@@ -278,6 +276,7 @@ def test_eplb_fml(
hidden_size=hidden_size, hidden_size=hidden_size,
intermediate_size=intermediate_size, intermediate_size=intermediate_size,
num_tokens=num_tokens, num_tokens=num_tokens,
moe_backend=moe_backend,
) )
distributed_run( distributed_run(
@@ -45,9 +45,10 @@ def test_config(llm: LLM):
def test_pooling_params(llm: LLM): def test_pooling_params(llm: LLM):
def get_outputs(use_activation): def get_outputs(use_activation):
outputs = llm.reward( outputs = llm.encode(
prompts, prompts,
pooling_params=PoolingParams(use_activation=use_activation), pooling_params=PoolingParams(use_activation=use_activation),
pooling_task="token_classify",
use_tqdm=False, use_tqdm=False,
) )
return torch.cat([x.outputs.data for x in outputs]) return torch.cat([x.outputs.data for x in outputs])
@@ -3,6 +3,4 @@
model_name: "openai/gpt-oss-20b" model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568 metric_threshold: 0.568
reasoning_effort: "low" reasoning_effort: "low"
server_args: "--tensor-parallel-size 2" server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1"
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_trtllm"
@@ -3,6 +3,4 @@
model_name: "openai/gpt-oss-20b" model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568 metric_threshold: 0.568
reasoning_effort: "low" reasoning_effort: "low"
server_args: "--tensor-parallel-size 2" server_args: "--tensor-parallel-size 2 --moe-backend marlin --linear-backend marlin"
env:
VLLM_MXFP4_USE_MARLIN: "1"
+1 -1
View File
@@ -1,5 +1,5 @@
# B200 model configurations for GPQA evaluation # B200 model configurations for GPQA evaluation
# Tests different environment variable combinations # Tests different environment variable combinations
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml
gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
+1 -1
View File
@@ -1,5 +1,5 @@
# H100 model configurations for GPQA evaluation # H100 model configurations for GPQA evaluation
# Tests different environment variable combinations # Tests different environment variable combinations
gpt-oss-20b-baseline.yaml gpt-oss-20b-baseline.yaml
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml
gpt-oss-20b-marlin.yaml gpt-oss-20b-marlin.yaml
+2 -2
View File
@@ -30,9 +30,9 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct"
accuracy_threshold: 0.54 # Minimum expected accuracy accuracy_threshold: 0.54 # Minimum expected accuracy
num_questions: 1319 # Number of questions (default: full test set) num_questions: 1319 # Number of questions (default: full test set)
num_fewshot: 5 # Few-shot examples from train set num_fewshot: 5 # Few-shot examples from train set
server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments server_args: "--max-model-len 4096 --tensor-parallel-size 2 --moe-backend flashinfer_cutlass" # Server arguments
env: # Environment variables (optional) env: # Environment variables (optional)
VLLM_USE_FLASHINFER_MOE_FP4: "1" VLLM_LOGGING_LEVEL: "DEBUG"
``` ```
The `server_args` field accepts any arguments that can be passed to `vllm serve`. The `server_args` field accepts any arguments that can be passed to `vllm serve`.
+1 -3
View File
@@ -1585,7 +1585,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
e: int, e: int,
topk: int, topk: int,
dtype: torch.dtype, dtype: torch.dtype,
monkeypatch,
workspace_init, workspace_init,
): ):
""" """
@@ -1593,8 +1592,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
""" """
set_random_seed(7) set_random_seed(7)
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig, FusedMoEConfig,
FusedMoEParallelConfig, FusedMoEParallelConfig,
@@ -1626,6 +1623,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
in_dtype=dtype, in_dtype=dtype,
routing_method=RoutingMethodType.Renormalize, routing_method=RoutingMethodType.Renormalize,
max_num_tokens=next_power_of_2(m), max_num_tokens=next_power_of_2(m),
moe_backend="flashinfer_trtllm",
) )
with set_current_vllm_config(vllm_config): with set_current_vllm_config(vllm_config):
+3 -6
View File
@@ -1804,12 +1804,9 @@ def test_moe_layer(
if os.environ.get("VLLM_LOGGING_LEVEL") is None: if os.environ.get("VLLM_LOGGING_LEVEL") is None:
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR") monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
# TODO # TODO: cover FlashInfer MoE backends via moe_backend, e.g.
# VLLM_FLASHINFER_MOE_BACKEND=latency # moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl
# VLLM_USE_FLASHINFER_MOE_FP16=1 # (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1.
# VLLM_USE_FLASHINFER_MOE_FP8
# VLLM_USE_FLASHINFER_MOE_FP4
# VLLM_USE_FLASHINFER_MOE_INT4
parallel_config = ParallelConfig( parallel_config = ParallelConfig(
pipeline_parallel_size=1, pipeline_parallel_size=1,
@@ -123,7 +123,7 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
@pytest.mark.skipif( @pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms." not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
) )
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeypatch): def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled.""" """Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
with ( with (
patch.object(current_platform, "is_cuda", return_value=True), patch.object(current_platform, "is_cuda", return_value=True),
@@ -134,9 +134,8 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp
patch.object(current_platform, "is_out_of_tree", return_value=False), patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True), patch.object(current_platform, "has_device_capability", return_value=True),
): ):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
moe_config = make_dummy_moe_config() moe_config = make_dummy_moe_config()
moe_config.moe_backend = "flashinfer_trtllm"
# TRTLLM requires EP and does not support DP # TRTLLM requires EP and does not support DP
moe_config.moe_parallel_config.use_ep = True moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False moe_config.moe_parallel_config.use_dp = False
@@ -168,7 +167,6 @@ def test_select_cuda_flashinfer_cutlass_backend(
mock_has_flashinfer, mock_has_flashinfer,
mock_is_supported_trtllm, mock_is_supported_trtllm,
mock_is_supported_cutlass, mock_is_supported_cutlass,
monkeypatch,
): ):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available """Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available.""" and FlashInfer CUTLASS is available."""
@@ -181,10 +179,9 @@ def test_select_cuda_flashinfer_cutlass_backend(
patch.object(current_platform, "is_out_of_tree", return_value=False), patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True), patch.object(current_platform, "has_device_capability", return_value=True),
): ):
# Enable FlashInfer via env var
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
moe_config = make_dummy_moe_config() moe_config = make_dummy_moe_config()
# Select FlashInfer CUTLASS explicitly
moe_config.moe_backend = "flashinfer_cutlass"
# CUTLASS requires EP and does not support DP # CUTLASS requires EP and does not support DP
moe_config.moe_parallel_config.use_ep = True moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False moe_config.moe_parallel_config.use_dp = False
@@ -241,37 +238,3 @@ def test_select_explicit_triton_backend(is_lora_enabled):
assert selected_backend == UnquantizedMoeBackend.TRITON assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None assert experts_cls is not None
@skipif_not_cuda_rocm
def test_select_explicit_triton_ignores_flashinfer_env(monkeypatch):
"""Explicit triton backend should override FlashInfer env selection."""
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = False
moe_config.moe_backend = "triton"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
def test_select_lora_ignores_flashinfer_env(monkeypatch):
"""LoRA path should still choose Triton even if FlashInfer env is on."""
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = True
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
+36 -38
View File
@@ -83,57 +83,55 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False]) @pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
@pytest.mark.parametrize("specialize_active_lora", [True, False]) @pytest.mark.parametrize("specialize_active_lora", [True, False])
def test_gpt_oss_lora( def test_gpt_oss_lora(
monkeypatch: pytest.MonkeyPatch,
gptoss20b_lora_files, gptoss20b_lora_files,
mxfp4_use_marlin, mxfp4_use_marlin,
specialize_active_lora, specialize_active_lora,
): ):
with monkeypatch.context() as m: llm = vllm.LLM(
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0") MODEL_PATH,
llm = vllm.LLM( max_model_len=1024,
MODEL_PATH, enable_lora=True,
max_model_len=1024, max_loras=4,
enable_lora=True, max_lora_rank=8,
max_loras=4, max_num_seqs=2,
max_lora_rank=8, max_num_batched_tokens=2048,
max_num_seqs=2, specialize_active_lora=specialize_active_lora,
max_num_batched_tokens=2048, moe_backend="marlin" if mxfp4_use_marlin else "auto",
specialize_active_lora=specialize_active_lora, linear_backend="marlin" if mxfp4_use_marlin else "auto",
compilation_config=vllm.config.CompilationConfig( # Avoid OOM compilation_config=vllm.config.CompilationConfig( # Avoid OOM
cudagraph_specialize_lora=False, cudagraph_specialize_lora=False,
), ),
) )
generate_and_test(llm, gptoss20b_lora_files, lora_id=1) generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
generate_and_test(llm, gptoss20b_lora_files, lora_id=2) generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
@multi_gpu_test(num_gpus=2) @multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("fully_sharded_loras", [False, True]) @pytest.mark.parametrize("fully_sharded_loras", [False, True])
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False]) @pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
def test_gpt_oss_lora_tp2( def test_gpt_oss_lora_tp2(
monkeypatch: pytest.MonkeyPatch,
gptoss20b_lora_files, gptoss20b_lora_files,
fully_sharded_loras, fully_sharded_loras,
mxfp4_use_marlin, mxfp4_use_marlin,
): ):
with monkeypatch.context() as m: llm = vllm.LLM(
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0") MODEL_PATH,
llm = vllm.LLM( max_model_len=1024,
MODEL_PATH, enable_lora=True,
max_model_len=1024, max_loras=2,
enable_lora=True, max_num_seqs=2,
max_loras=2, max_num_batched_tokens=2048,
max_num_seqs=2, tensor_parallel_size=2,
max_num_batched_tokens=2048, gpu_memory_utilization=0.8,
tensor_parallel_size=2, fully_sharded_loras=fully_sharded_loras,
gpu_memory_utilization=0.8, enable_expert_parallel=not fully_sharded_loras,
fully_sharded_loras=fully_sharded_loras, moe_backend="marlin" if mxfp4_use_marlin else "auto",
enable_expert_parallel=not fully_sharded_loras, linear_backend="marlin" if mxfp4_use_marlin else "auto",
compilation_config=vllm.config.CompilationConfig( compilation_config=vllm.config.CompilationConfig(
cudagraph_specialize_lora=False, cudagraph_specialize_lora=False,
), ),
) )
generate_and_test(llm, gptoss20b_lora_files, lora_id=1) generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
generate_and_test(llm, gptoss20b_lora_files, lora_id=2) generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
@@ -106,7 +106,7 @@ def test_reward_models_using_activation(
dtype=dtype, dtype=dtype,
pooler_config=PoolerConfig(use_activation=False), pooler_config=PoolerConfig(use_activation=False),
) as vllm_model: ) as vllm_model:
wo_activation = vllm_model.reward(example_prompts) wo_activation = vllm_model.token_classify(example_prompts)
with vllm_runner( with vllm_runner(
model, model,
@@ -114,7 +114,7 @@ def test_reward_models_using_activation(
dtype=dtype, dtype=dtype,
pooler_config=PoolerConfig(use_activation=True), pooler_config=PoolerConfig(use_activation=True),
) as vllm_model: ) as vllm_model:
w_activation = vllm_model.reward(example_prompts) w_activation = vllm_model.token_classify(example_prompts)
for wo, w in zip(wo_activation, w_activation): for wo, w in zip(wo_activation, w_activation):
wo = torch.tensor(wo) wo = torch.tensor(wo)
+2 -2
View File
@@ -107,7 +107,7 @@ def test_prm_models(
pytest.skip("CPU only supports V1") pytest.skip("CPU only supports V1")
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model: with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.reward(math_step_prompts) vllm_outputs = vllm_model.token_classify(math_step_prompts)
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model: with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
hf_model = step_reward_patch_hf_model(hf_model) hf_model = step_reward_patch_hf_model(hf_model)
@@ -146,7 +146,7 @@ def test_prm_models_with_golden_outputs(
pytest.skip(f"No available golden outputs for {model}.") pytest.skip(f"No available golden outputs for {model}.")
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model: with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.reward(math_step_prompts) vllm_outputs = vllm_model.token_classify(math_step_prompts)
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model]) golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
+2 -2
View File
@@ -133,11 +133,11 @@ def test_nvfp4(vllm_runner, model, eager, backend):
not current_platform.is_rocm(), not current_platform.is_rocm(),
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx", reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
) )
def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch): def test_nvfp4_moe(vllm_runner, model, backend):
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)
with vllm_runner( with vllm_runner(
model, model,
moe_backend=backend, moe_backend=backend,
linear_backend=backend,
load_format="dummy", load_format="dummy",
hf_overrides={"num_hidden_layers": 2}, hf_overrides={"num_hidden_layers": 2},
) as llm: ) as llm:
+5 -2
View File
@@ -185,8 +185,11 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch): def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1") can_initialize(
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT) "openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+1 -7
View File
@@ -1617,7 +1617,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"custom", "custom",
"custom_audio", "custom_audio",
"custom_image", "custom_image",
"custom_mm",
"prefix_repetition", "prefix_repetition",
"spec_bench", "spec_bench",
"speed_bench", "speed_bench",
@@ -2106,12 +2105,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample, no_oversample=args.no_oversample,
) )
elif args.dataset_name in ("custom_image", "custom_mm"): elif args.dataset_name == "custom_image":
if args.dataset_name == "custom_mm":
logger.warning(
"Dataset name 'custom_mm' is deprecated and will be removed in v0.24. "
"Use '--dataset-name custom_image' instead."
)
dataset = CustomImageDataset( dataset = CustomImageDataset(
dataset_path=args.dataset_path, dataset_path=args.dataset_path,
disable_shuffle=args.disable_shuffle, disable_shuffle=args.disable_shuffle,
-11
View File
@@ -134,10 +134,6 @@ class PassConfig:
"""Enable async TP.""" """Enable async TP."""
fuse_allreduce_rms: bool = None # type: ignore[assignment] fuse_allreduce_rms: bool = None # type: ignore[assignment]
"""Enable flashinfer allreduce fusion.""" """Enable flashinfer allreduce fusion."""
fuse_minimax_qk_norm: bool = None # type: ignore[assignment]
"""Deprecated. The MiniMax QK norm fusion is now applied automatically at
runtime (see `MiniMaxText01RMSNormTP.forward_qkv`). This flag is kept for
backward compatibility and has no effect; it will be removed in v0.23."""
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment] enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
"""Enable fused Q/K RMSNorm + RoPE pass.""" """Enable fused Q/K RMSNorm + RoPE pass."""
fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment] fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment]
@@ -296,13 +292,6 @@ class PassConfig:
"current platform is not CUDA or ROCm. The fusion will be disabled." "current platform is not CUDA or ROCm. The fusion will be disabled."
) )
self.fuse_rope_kvcache_cat_mla = False self.fuse_rope_kvcache_cat_mla = False
if self.fuse_minimax_qk_norm is not None:
logger.warning_once(
"`fuse_minimax_qk_norm` is deprecated and has no effect; "
"the MiniMax QK norm fusion is now applied automatically at "
"runtime when its conditions are met. This flag will be "
"removed in v0.23."
)
def log_enabled_passes(self) -> None: def log_enabled_passes(self) -> None:
""" """
+2
View File
@@ -142,6 +142,7 @@ LinearBackend = Literal[
"flashinfer_cutlass", "flashinfer_cutlass",
"flashinfer_trtllm", "flashinfer_trtllm",
"flashinfer_cudnn", "flashinfer_cudnn",
"flashinfer_b12x",
"marlin", "marlin",
"triton", "triton",
"deep_gemm", "deep_gemm",
@@ -197,6 +198,7 @@ class KernelConfig:
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels - "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels - "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
- "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
- "marlin": Use Marlin kernels - "marlin": Use Marlin kernels
- "triton": Use Triton-based kernels - "triton": Use Triton-based kernels
- "deep_gemm": Use DeepGEMM kernels - "deep_gemm": Use DeepGEMM kernels
-44
View File
@@ -286,50 +286,6 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
return [ClassificationRequestOutput.from_base(item) for item in items] return [ClassificationRequestOutput.from_base(item) for item in items]
def reward(
self,
prompts: PromptType | Sequence[PromptType],
/,
*,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[PoolingRequestOutput]:
"""
Generate rewards for each prompt.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `PoolingRequestOutput` objects containing the
pooled hidden states in the same order as the input prompts.
"""
logger.warning_once(
"`llm.reward` api is deprecated and will be removed in v0.23. "
'Please use `LLM.encode` with `pooling_task="classify"` or '
'`pooling_task="token_classify"` instead.'
)
return self.encode(
prompts,
use_tqdm=use_tqdm,
lora_request=lora_request,
pooling_params=pooling_params,
pooling_task="token_classify",
tokenization_kwargs=tokenization_kwargs,
)
def score( def score(
self, self,
data_1: ScoreInput | list[ScoreInput], data_1: ScoreInput | list[ScoreInput],
-168
View File
@@ -8,7 +8,6 @@ import os
import sys import sys
import tempfile import tempfile
import uuid import uuid
import warnings
from collections.abc import Callable from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Literal from typing import TYPE_CHECKING, Any, Literal
@@ -167,7 +166,6 @@ if TYPE_CHECKING:
VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None
VLLM_HUMMING_USE_F16_ACCUM: bool = False VLLM_HUMMING_USE_F16_ACCUM: bool = False
VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None
VLLM_MXFP4_USE_MARLIN: bool | None = None
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
VLLM_V1_USE_OUTLINES_CACHE: bool = False VLLM_V1_USE_OUTLINES_CACHE: bool = False
VLLM_TPU_BUCKET_PADDING_GAP: int = 0 VLLM_TPU_BUCKET_PADDING_GAP: int = 0
@@ -184,13 +182,7 @@ if TYPE_CHECKING:
] = "relax" ] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_FP16: bool = False
VLLM_USE_FLASHINFER_MOE_FP8: bool = False
VLLM_USE_FLASHINFER_MOE_FP4: bool = False
VLLM_USE_FLASHINFER_MOE_INT4: bool = False VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = (
"latency"
)
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto" VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024 VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
@@ -212,7 +204,6 @@ if TYPE_CHECKING:
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
VLLM_USE_NVFP4_CT_EMULATIONS: bool = False
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[ VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
"FP", "INT8", "INT6", "INT4", "NONE" "FP", "INT8", "INT6", "INT4", "NONE"
] = "NONE" ] = "NONE"
@@ -225,12 +216,8 @@ if TYPE_CHECKING:
VLLM_LOOPBACK_IP: str = "" VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
VLLM_ENABLE_RESPONSES_API_STORE: bool = False VLLM_ENABLE_RESPONSES_API_STORE: bool = False
VLLM_NVFP4_GEMM_BACKEND: str | None = None
VLLM_HAS_FLASHINFER_CUBIN: bool = False VLLM_HAS_FLASHINFER_CUBIN: bool = False
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
VLLM_TUNED_CONFIG_FOLDER: str | None = None VLLM_TUNED_CONFIG_FOLDER: str | None = None
@@ -257,7 +244,6 @@ if TYPE_CHECKING:
VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
VLLM_USE_NCCL_SYMM_MEM: bool = False VLLM_USE_NCCL_SYMM_MEM: bool = False
VLLM_NCCL_INCLUDE_PATH: str | None = None VLLM_NCCL_INCLUDE_PATH: str | None = None
VLLM_USE_FBGEMM: bool = False
VLLM_GC_DEBUG: str = "" VLLM_GC_DEBUG: str = ""
VLLM_DEBUG_WORKSPACE: bool = False VLLM_DEBUG_WORKSPACE: bool = False
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
@@ -350,27 +336,6 @@ def use_mega_aot_artifact():
return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1" return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1"
def deprecated_env(
env_name: str,
removal_version: str,
replacement: str,
getter: Callable[[], Any],
) -> Callable[[], Any]:
"""Wrap an env-var getter to emit a FutureWarning when the var is set."""
def _read() -> Any:
if env_name in os.environ:
warnings.warn(
f"{env_name} is deprecated and will be removed in "
f"{removal_version}. {replacement}",
FutureWarning,
stacklevel=2,
)
return getter()
return _read
def env_with_choices( def env_with_choices(
env_name: str, env_name: str,
default: str | None, default: str | None,
@@ -1371,15 +1336,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MARLIN_USE_ATOMIC_ADD": lambda: ( "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: (
os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1" os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1"
), ),
# Whether to use marlin kernel in mxfp4 quantization method
# Deprecated: use --moe-backend marlin (MoE) or --linear-backend marlin
# (linear) instead.
"VLLM_MXFP4_USE_MARLIN": deprecated_env(
"VLLM_MXFP4_USE_MARLIN",
"v0.23",
"Use --moe-backend marlin or --linear-backend marlin.",
lambda: maybe_convert_bool(os.environ.get("VLLM_MXFP4_USE_MARLIN", None)),
),
# The activation dtype for marlin kernel # The activation dtype for marlin kernel
"VLLM_MARLIN_INPUT_DTYPE": env_with_choices( "VLLM_MARLIN_INPUT_DTYPE": env_with_choices(
"VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"] "VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"]
@@ -1472,68 +1428,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool( "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1")) int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
), ),
# Allow use of FlashInfer BF16 MoE kernels for fused moe ops.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_FP16": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_FP16",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))),
),
# Allow use of FlashInfer FP8 MoE kernels for fused moe ops.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_FP8": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_FP8",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP8", "0"))),
),
# Allow use of FlashInfer NVFP4 MoE kernels for fused moe ops.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_FP4": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_FP4",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass, "
"flashinfer_cutedsl).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP4", "0"))),
),
# Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops. # Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops.
"VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool( "VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool(
int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0")) int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0"))
), ),
# If set to 1, use the FlashInfer
# MXFP8 (activation) x MXFP4 (weight) MoE backend.
# Deprecated: use --moe-backend flashinfer_trtllm combined with
# --quantization_config.moe.activation mxfp8.
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8",
"v0.23",
"Use --moe-backend flashinfer_trtllm with "
"--quantization_config.moe.activation mxfp8.",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))),
),
# If set to 1, use the FlashInfer CUTLASS backend for
# MXFP8 (activation) x MXFP4 (weight) MoE.
# Deprecated: use --moe-backend flashinfer_cutlass combined with
# --quantization_config.moe.activation mxfp8.
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS",
"v0.23",
"Use --moe-backend flashinfer_cutlass with "
"--quantization_config.moe.activation mxfp8.",
lambda: bool(
int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0"))
),
),
# If set to 1, use the FlashInfer
# BF16 (activation) x MXFP4 (weight) MoE backend.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "0"))),
),
# Control the cache sized used by the xgrammar compiler. The default # Control the cache sized used by the xgrammar compiler. The default
# of 512 MB should be enough for roughly 1000 JSON schemas. # of 512 MB should be enough for roughly 1000 JSON schemas.
# It can be changed with this variable if needed for some reason. # It can be changed with this variable if needed for some reason.
@@ -1585,25 +1483,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv( "MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME" "MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
), ),
# Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
# Both require compute capability 10.0 or above.
# Available options:
# - "throughput": [default]
# Uses CUTLASS kernels optimized for high-throughput batch inference.
# - "latency":
# Uses TensorRT-LLM kernels optimized for low-latency inference.
# Deprecated: pass --moe-backend flashinfer_{trtllm,cutlass,cutedsl} directly.
"VLLM_FLASHINFER_MOE_BACKEND": deprecated_env(
"VLLM_FLASHINFER_MOE_BACKEND",
"v0.23",
"Use --moe-backend flashinfer_trtllm, flashinfer_cutlass, or "
"flashinfer_cutedsl.",
env_with_choices(
"VLLM_FLASHINFER_MOE_BACKEND",
"latency",
["throughput", "latency", "masked_gemm"],
),
),
# Override the directory for the FlashInfer autotune config cache. # Override the directory for the FlashInfer autotune config cache.
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv( "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
@@ -1681,16 +1560,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool( "VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0")) int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
), ),
# Controls whether or not emulations are used for NVFP4
# generations on machines < 100 for compressed-tensors
# models
# Deprecated: use --linear-backend emulation instead.
"VLLM_USE_NVFP4_CT_EMULATIONS": deprecated_env(
"VLLM_USE_NVFP4_CT_EMULATIONS",
"v0.23",
"Use --linear-backend emulation.",
lambda: bool(int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0"))),
),
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup. # Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int( "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480") os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
@@ -1700,35 +1569,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool( "VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0")) int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0"))
), ),
# Supported options:
# - "flashinfer-cudnn": use flashinfer cudnn GEMM backend
# - "flashinfer-trtllm": use flashinfer trtllm GEMM backend
# - "flashinfer-cutlass": use flashinfer cutlass GEMM backend
# - "marlin": use marlin GEMM backend (for GPUs without native FP4 support)
# - "emulation":
# use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations.
# This is only meant for research purposes to run on devices where NVFP4
# GEMM kernels are not available.
# - <none>: automatically pick an available backend
# Deprecated: use --linear-backend instead.
"VLLM_NVFP4_GEMM_BACKEND": deprecated_env(
"VLLM_NVFP4_GEMM_BACKEND",
"v0.23",
"Use --linear-backend.",
env_with_choices(
"VLLM_NVFP4_GEMM_BACKEND",
None,
[
"flashinfer-b12x",
"flashinfer-cudnn",
"flashinfer-trtllm",
"flashinfer-cutlass",
"cutlass",
"marlin",
"emulation",
],
),
),
# Controls garbage collection during CUDA graph capture. # Controls garbage collection during CUDA graph capture.
# If set to 0 (default), enables GC freezing to speed up capture time. # If set to 0 (default), enables GC freezing to speed up capture time.
# If set to 1, allows GC to run during capture. # If set to 1, allows GC to run during capture.
@@ -1892,14 +1732,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
), ),
# NCCL header path # NCCL header path
"VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None), "VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
# Flag to enable FBGemm kernels on model execution
# Deprecated: use --linear-backend fbgemm instead.
"VLLM_USE_FBGEMM": deprecated_env(
"VLLM_USE_FBGEMM",
"v0.23",
"Use --linear-backend fbgemm.",
lambda: bool(int(os.getenv("VLLM_USE_FBGEMM", "0"))),
),
# GC debug config # GC debug config
# - VLLM_GC_DEBUG=0: disable GC debugger # - VLLM_GC_DEBUG=0: disable GC debugger
# - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times # - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
+8 -47
View File
@@ -212,6 +212,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"flashinfer_cudnn": { "flashinfer_cudnn": {
FlashInferCudnnNvFp4LinearKernel, FlashInferCudnnNvFp4LinearKernel,
}, },
"flashinfer_b12x": {
FlashInferB12xNvFp4LinearKernel,
},
"marlin": { "marlin": {
MarlinFP8ScaledMMLinearKernel, MarlinFP8ScaledMMLinearKernel,
MarlinLinearKernel, MarlinLinearKernel,
@@ -392,7 +395,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
PlatformEnum.CUDA: [ PlatformEnum.CUDA: [
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until # FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
# upstream CUTLASS SM121 MMA op guard is resolved; use # upstream CUTLASS SM121 MMA op guard is resolved; use
# VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly. # --linear-backend flashinfer_b12x to opt in explicitly.
FlashInferCutlassNvFp4LinearKernel, FlashInferCutlassNvFp4LinearKernel,
CutlassNvFp4LinearKernel, CutlassNvFp4LinearKernel,
MarlinNvFp4LinearKernel, MarlinNvFp4LinearKernel,
@@ -752,20 +755,6 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
current platform.""" current platform."""
linear_backend = _get_linear_backend() linear_backend = _get_linear_backend()
force_kernel: type[MxFp4LinearKernel] | None = None
if linear_backend == "auto" and envs.VLLM_MXFP4_USE_MARLIN:
force_kernel = MarlinMxFp4LinearKernel
if force_kernel is not None:
is_supported, reason = force_kernel.is_supported()
if not is_supported:
raise ValueError(
f"Forced MXFP4 kernel {force_kernel.__name__} is not "
f"supported: {reason}"
)
logger.info_once("Using %s for MXFP4 GEMM", force_kernel.__name__)
return force_kernel(MxFp4LinearLayerConfig())
platform = current_platform._enum platform = current_platform._enum
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, [])) possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
@@ -836,18 +825,6 @@ def init_wfp8_a16_linear_kernel(
) )
# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
"flashinfer-b12x": FlashInferB12xNvFp4LinearKernel,
"flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
"cutlass": CutlassNvFp4LinearKernel,
"marlin": MarlinNvFp4LinearKernel,
"flashinfer-trtllm": FlashInferTrtllmNvFp4LinearKernel,
"flashinfer-cudnn": FlashInferCudnnNvFp4LinearKernel,
"emulation": EmulationNvFp4LinearKernel,
}
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel: def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
"""Select and instantiate the best NVFP4 linear kernel for the """Select and instantiate the best NVFP4 linear kernel for the
current platform.""" current platform."""
@@ -855,8 +832,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the # VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
# batch-invariant CUTLASS implementation when available, otherwise fall # batch-invariant CUTLASS implementation when available, otherwise fall
# back to emulation. It overrides both --linear-backend and the deprecated # back to emulation. It overrides --linear-backend.
# env vars below.
force_kernel: type[NvFp4LinearKernel] | None = None force_kernel: type[NvFp4LinearKernel] | None = None
linear_backend = _get_linear_backend() linear_backend = _get_linear_backend()
if envs.VLLM_BATCH_INVARIANT: if envs.VLLM_BATCH_INVARIANT:
@@ -888,24 +864,9 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
reason, reason,
) )
force_kernel = EmulationNvFp4LinearKernel force_kernel = EmulationNvFp4LinearKernel
elif linear_backend == "auto": elif linear_backend == "auto" and use_a16:
# Deprecated env-var overrides — only honoured when --linear-backend # Force a16 (Marlin) when running weight-only quantization.
# is "auto". Deprecation warnings are emitted from vllm/envs.py. force_kernel = MarlinNvFp4LinearKernel
if use_a16: # force a16 if running weight-only quantization
force_kernel = MarlinNvFp4LinearKernel
elif envs.VLLM_USE_FBGEMM:
force_kernel = FbgemmNvFp4LinearKernel
elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
force_kernel = EmulationNvFp4LinearKernel
elif envs.VLLM_NVFP4_GEMM_BACKEND is not None:
backend_name = envs.VLLM_NVFP4_GEMM_BACKEND
force_kernel = _NVFP4_BACKEND_TO_KERNEL.get(backend_name)
if force_kernel is None:
raise ValueError(
f"Unknown VLLM_NVFP4_GEMM_BACKEND={backend_name!r}. "
f"Valid choices: "
f"{list(_NVFP4_BACKEND_TO_KERNEL.keys())}"
)
if force_kernel is not None: if force_kernel is not None:
is_supported, reason = force_kernel.is_supported() is_supported, reason = force_kernel.is_supported()
@@ -20,8 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import (
) )
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
get_flashinfer_moe_backend,
prepare_fp8_moe_layer_for_fi, prepare_fp8_moe_layer_for_fi,
) )
from vllm.model_executor.layers.quantization.utils.fp8_utils import ( from vllm.model_executor.layers.quantization.utils.fp8_utils import (
@@ -321,54 +319,6 @@ def select_fp8_moe_backend(
requested_backend, config, weight_key, activation_key, activation_format requested_backend, config, weight_key, activation_key, activation_format
) )
# Handle explicit FlashInfer FP8 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"):
if not envs.VLLM_USE_FLASHINFER_MOE_FP8:
# If the user rejects FlashInfer remove those backends.
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_TRTLLM)
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_CUTLASS)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
fi_backend = get_flashinfer_moe_backend()
if fi_backend == FlashinferMoeBackend.CUTLASS:
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
else:
raise ValueError(
f"FlashInfer MOE backend {fi_backend} does not support FP8 MoE."
)
k_cls = backend_to_kernel_cls(backend)[0]
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
else:
# If the user is not explicit about the backend, try both.
for backend in [
Fp8MoeBackend.FLASHINFER_TRTLLM,
Fp8MoeBackend.FLASHINFER_CUTLASS,
]:
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls,
config,
weight_key,
activation_key,
activation_format,
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no "
"FlashInfer FP8 MoE backend supports the configuration."
)
# Handle explicit DeepGEMM FP8 configuration. # Handle explicit DeepGEMM FP8 configuration.
if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"): if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"):
if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM: if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM:
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Literal, Union
import torch import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend from vllm.config.kernel import MoEBackend
from vllm.config.quantization import QuantizationConfigArgs from vllm.config.quantization import QuantizationConfigArgs
@@ -465,74 +464,6 @@ def select_mxfp4_moe_backend(
_get_priority_backends_for_gpt_oss(), requested_activation_key _get_priority_backends_for_gpt_oss(), requested_activation_key
) )
# Handle explicit FlashInfer MXFP4 BF16 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
for _b in (
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
):
if _b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(_b)
else:
if current_platform.is_device_capability(90):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
config,
kMxfp4Static,
None,
activation_format,
)
if current_platform.is_device_capability_family(100):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
config,
kMxfp4Static,
None,
activation_format,
)
raise ValueError(
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16=1 is set but the "
"current device capability is not supported. "
"Only SM90 (CUTLASS) and SM100+ (TRTLLM) are supported."
)
# Handle explicit FlashInfer MXFP4 MXFP8 TRTLLM configuration.
if (
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8
):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
config,
kMxfp4Static,
kMxfp8Dynamic,
activation_format,
)
# Handle explicit FlashInfer MXFP4 MXFP8 CUTLASS configuration.
if (
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS")
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS
):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
config,
kMxfp4Static,
kMxfp8Dynamic,
activation_format,
)
# Handle explicit Marlin MXFP4 configuration.
if envs.is_set("VLLM_MXFP4_USE_MARLIN") and envs.VLLM_MXFP4_USE_MARLIN:
return _return_or_raise(
Mxfp4MoeBackend.MARLIN,
config,
kMxfp4Static,
None,
activation_format,
)
for backend in AVAILABLE_BACKENDS: for backend in AVAILABLE_BACKENDS:
# Use requested_activation_key if provided, otherwise use backend default # Use requested_activation_key if provided, otherwise use backend default
act_key = ( act_key = (
@@ -22,10 +22,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass, prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl, prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
) )
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
get_flashinfer_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_nvfp4_moe_layer_for_marlin, prepare_nvfp4_moe_layer_for_marlin,
) )
@@ -58,12 +54,6 @@ FLASHINFER_NVFP4_MOE_BACKENDS = [
NvFp4MoeBackend.FLASHINFER_B12X, NvFp4MoeBackend.FLASHINFER_B12X,
] ]
fi_2_vllm_backend_map: dict[FlashinferMoeBackend, NvFp4MoeBackend] = {
FlashinferMoeBackend.CUTLASS: NvFp4MoeBackend.FLASHINFER_CUTLASS,
FlashinferMoeBackend.TENSORRT_LLM: NvFp4MoeBackend.FLASHINFER_TRTLLM,
FlashinferMoeBackend.CUTEDSL: NvFp4MoeBackend.FLASHINFER_CUTEDSL,
}
def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool: def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool:
# Checks whether `backend` supports quantizing with scaling factors # Checks whether `backend` supports quantizing with scaling factors
@@ -258,55 +248,6 @@ def select_nvfp4_moe_backend(
requested_backend, config, weight_key, activation_key, activation_format requested_backend, config, weight_key, activation_key, activation_format
) )
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"):
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
# If the user rejects FlashInfer remove those backends.
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
if b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(b)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
if (
config.swiglu_limit is not None
and backend not in NVFP4_BACKENDS_WITH_CLAMP
):
raise ValueError(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
f"({backend.value}) does not apply the SwiGLU clamp."
)
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
else:
# If the user is not explicit about the backend, try each.
fi_backends = [
b
for b in FLASHINFER_NVFP4_MOE_BACKENDS
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
]
for backend in fi_backends:
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls,
config,
weight_key,
activation_key,
activation_format,
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no "
"FlashInfer NVFP4 MoE backend supports the configuration."
)
if envs.VLLM_TEST_FORCE_FP8_MARLIN: if envs.VLLM_TEST_FORCE_FP8_MARLIN:
backend = NvFp4MoeBackend.MARLIN backend = NvFp4MoeBackend.MARLIN
return _return_or_raise( return _return_or_raise(
@@ -19,9 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig, FusedMoEQuantConfig,
) )
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
convert_moe_weights_to_flashinfer_trtllm_block_layout, convert_moe_weights_to_flashinfer_trtllm_block_layout,
get_flashinfer_moe_backend,
swap_w13_to_w31, swap_w13_to_w31,
) )
from vllm.platforms import current_platform from vllm.platforms import current_platform
@@ -230,49 +228,6 @@ def select_unquantized_moe_backend(
return _return_or_raise(requested_backend, moe_config, activation_format) return _return_or_raise(requested_backend, moe_config, activation_format)
# Handle explicit FlashInfer FP16 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP16"):
if not envs.VLLM_USE_FLASHINFER_MOE_FP16:
if UnquantizedMoeBackend.FLASHINFER_TRTLLM in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_TRTLLM)
if UnquantizedMoeBackend.FLASHINFER_CUTLASS in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_CUTLASS)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
fi_backend = get_flashinfer_moe_backend()
if fi_backend == FlashinferMoeBackend.CUTLASS:
backend = UnquantizedMoeBackend.FLASHINFER_CUTLASS
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
else:
raise ValueError(
f"FlashInfer MOE backend {fi_backend} "
"does not support unquantized MoE."
)
k_cls = backend_to_kernel_cls(backend)
return _return_or_raise(backend, moe_config, activation_format)
else:
# If the user is not explicit about the backend, try both.
for backend in [
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
UnquantizedMoeBackend.FLASHINFER_CUTLASS,
]:
k_cls = backend_to_kernel_cls(backend)
supported, reason = k_cls.is_supported_config(
k_cls, moe_config, None, None, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no "
"FlashInfer unquantized MoE backend supports the configuration."
)
# Handle explicit AITER FP8 configuration. # Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"): if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE: if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
@@ -6,7 +6,7 @@ import nixl_ep
import torch import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs from vllm.config import get_current_vllm_config
from vllm.distributed import get_ep_group from vllm.distributed import get_ep_group
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -192,10 +192,11 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
x = x.view((-1, hidden_dim)) x = x.view((-1, hidden_dim))
q_dtype = quant_config.quant_dtype q_dtype = quant_config.quant_dtype
if envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm": moe_backend = get_current_vllm_config().kernel_config.moe_backend
if moe_backend == "flashinfer_cutedsl":
logger.info_once( logger.info_once(
"Skip quantization when using FlashInfer CUTEDSL(masked_gemm) " "Skip quantization when using FlashInfer CUTEDSL "
"for ModelOptNvFp4FusedMoE." "(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE."
) )
q_dtype = None q_dtype = None
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
import torch import torch
import vllm.envs as envs
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_weights_for_fi, align_fp4_moe_weights_for_fi,
@@ -15,10 +14,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale, swizzle_blockscale,
) )
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
has_flashinfer_cutlass_fused_moe,
)
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -34,16 +29,6 @@ __all__ = [
] ]
def is_flashinfer_fp4_cutlass_moe_available() -> bool:
"""Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used."""
return (
envs.VLLM_USE_FLASHINFER_MOE_FP4
and has_flashinfer_cutlass_fused_moe()
and current_platform.is_cuda()
and current_platform.has_device_capability(100)
)
def reorder_w1w3_to_w3w1( def reorder_w1w3_to_w3w1(
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2 weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
@@ -5,10 +5,8 @@ from typing import TYPE_CHECKING
import torch import torch
from vllm import envs
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up from vllm.utils.math_utils import round_up
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -95,34 +93,6 @@ def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
) )
def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
backend_map = {
"throughput": FlashinferMoeBackend.CUTLASS,
"latency": FlashinferMoeBackend.TENSORRT_LLM,
"masked_gemm": FlashinferMoeBackend.CUTEDSL,
}
flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND
if flashinfer_moe_backend in backend_map:
if (
flashinfer_moe_backend == "latency"
and not current_platform.is_device_capability_family(100)
):
logger.info_once(
"Flashinfer TRTLLM MOE backend is only supported on "
"SM100 and later, using CUTLASS backend instead",
)
return FlashinferMoeBackend.CUTLASS
return backend_map[flashinfer_moe_backend]
elif current_platform.is_device_capability(90):
return FlashinferMoeBackend.CUTLASS
raise ValueError(
f"Unknown flashinfer moe backend: {flashinfer_moe_backend!r}. "
f"Expected one of {list(backend_map.keys())}."
)
def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool: def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool:
# TODO(shuw@nvidia): Update when new backends are added. # TODO(shuw@nvidia): Update when new backends are added.
backends_supporting_global_sf = ( backends_supporting_global_sf = (