Deprecations for v0.23 and v0.24 (#44992)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -6,9 +6,7 @@ tasks:
|
|||||||
value: 0.7142
|
value: 0.7142
|
||||||
- name: "exact_match,flexible-extract"
|
- name: "exact_match,flexible-extract"
|
||||||
value: 0.4579
|
value: 0.4579
|
||||||
env_vars:
|
moe_backend: "flashinfer_cutlass"
|
||||||
VLLM_USE_FLASHINFER_MOE_FP8: "1"
|
|
||||||
VLLM_FLASHINFER_MOE_BACKEND: "throughput"
|
|
||||||
limit: 1319
|
limit: 1319
|
||||||
num_fewshot: 5
|
num_fewshot: 5
|
||||||
max_model_len: 262144
|
max_model_len: 262144
|
||||||
|
|||||||
@@ -68,6 +68,10 @@ def launch_lm_eval(eval_config, tp_size):
|
|||||||
if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
|
if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
|
||||||
model_args += "attention_backend=TRITON_ATTN"
|
model_args += "attention_backend=TRITON_ATTN"
|
||||||
|
|
||||||
|
moe_backend = eval_config.get("moe_backend", None)
|
||||||
|
if moe_backend is not None:
|
||||||
|
model_args += f"moe_backend={moe_backend},"
|
||||||
|
|
||||||
env_vars = eval_config.get("env_vars", None)
|
env_vars = eval_config.get("env_vars", None)
|
||||||
with scoped_env_vars(env_vars):
|
with scoped_env_vars(env_vars):
|
||||||
results = lm_eval.simple_evaluate(
|
results = lm_eval.simple_evaluate(
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ th {
|
|||||||
1. All types: mxfp4, nvfp4, int4, int8, fp8
|
1. All types: mxfp4, nvfp4, int4, int8, fp8
|
||||||
2. A,T quantization occurs after dispatch.
|
2. A,T quantization occurs after dispatch.
|
||||||
3. All quantization happens after dispatch.
|
3. All quantization happens after dispatch.
|
||||||
4. Controlled by different env vars (`VLLM_FLASHINFER_MOE_BACKEND` "throughput" or "latency")
|
4. Controlled by `--moe-backend` (`flashinfer_cutlass` or `flashinfer_trtllm`)
|
||||||
5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API.
|
5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API.
|
||||||
6. This depends on the experts implementation.
|
6. This depends on the experts implementation.
|
||||||
|
|
||||||
|
|||||||
@@ -143,4 +143,4 @@ More examples can be found here: [examples/pooling/reward](../../../examples/poo
|
|||||||
|
|
||||||
### `LLM.reward`
|
### `LLM.reward`
|
||||||
|
|
||||||
`llm.reward` api is deprecated and will be removed in v0.23. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
|
`llm.reward` API is deprecated and was removed in v0.24. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ def test_async_tp_pass_correctness(
|
|||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
|
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
|
||||||
if (
|
if (
|
||||||
not current_platform.is_cuda()
|
not current_platform.is_cuda()
|
||||||
or not current_platform.is_device_capability_family(100)
|
or not current_platform.is_device_capability_family(100)
|
||||||
@@ -111,8 +111,6 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
|
|||||||
if not has_flashinfer():
|
if not has_flashinfer():
|
||||||
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
|
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
|
||||||
|
|
||||||
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass")
|
|
||||||
|
|
||||||
tp_size = 2
|
tp_size = 2
|
||||||
if num_gpus_available < tp_size:
|
if num_gpus_available < tp_size:
|
||||||
pytest.skip(f"Need at least {tp_size} GPUs")
|
pytest.skip(f"Need at least {tp_size} GPUs")
|
||||||
@@ -126,6 +124,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
|
|||||||
"8",
|
"8",
|
||||||
"--load-format",
|
"--load-format",
|
||||||
"dummy",
|
"dummy",
|
||||||
|
"--linear-backend",
|
||||||
|
"flashinfer_cutlass",
|
||||||
"--hf-overrides",
|
"--hf-overrides",
|
||||||
json.dumps(NVFP4_HF_OVERRIDES),
|
json.dumps(NVFP4_HF_OVERRIDES),
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1226,10 +1226,6 @@ class VllmRunner:
|
|||||||
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
|
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
|
||||||
return [req_output.outputs.data for req_output in req_outputs]
|
return [req_output.outputs.data for req_output in req_outputs]
|
||||||
|
|
||||||
def reward(self, prompts: list[str]) -> list[list[float]]:
|
|
||||||
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
|
|
||||||
return [req_output.outputs.data for req_output in req_outputs]
|
|
||||||
|
|
||||||
def score(
|
def score(
|
||||||
self,
|
self,
|
||||||
text_1: list[str] | str,
|
text_1: list[str] | str,
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ class TestConfig:
|
|||||||
hidden_size: int
|
hidden_size: int
|
||||||
intermediate_size: int
|
intermediate_size: int
|
||||||
num_tokens: int
|
num_tokens: int
|
||||||
|
moe_backend: str
|
||||||
|
|
||||||
|
|
||||||
def make_fused_moe_layer(
|
def make_fused_moe_layer(
|
||||||
@@ -114,6 +115,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
|||||||
vllm_config = VllmConfig()
|
vllm_config = VllmConfig()
|
||||||
vllm_config.parallel_config.data_parallel_size = world_size
|
vllm_config.parallel_config.data_parallel_size = world_size
|
||||||
vllm_config.parallel_config.enable_expert_parallel = True
|
vllm_config.parallel_config.enable_expert_parallel = True
|
||||||
|
vllm_config.kernel_config.moe_backend = test_config.moe_backend
|
||||||
|
|
||||||
with set_current_vllm_config(vllm_config):
|
with set_current_vllm_config(vllm_config):
|
||||||
ensure_model_parallel_initialized(
|
ensure_model_parallel_initialized(
|
||||||
@@ -250,7 +252,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
|||||||
@pytest.mark.parametrize("hidden_size", [256])
|
@pytest.mark.parametrize("hidden_size", [256])
|
||||||
@pytest.mark.parametrize("intermediate_size", [256])
|
@pytest.mark.parametrize("intermediate_size", [256])
|
||||||
@pytest.mark.parametrize("num_tokens", [256])
|
@pytest.mark.parametrize("num_tokens", [256])
|
||||||
@pytest.mark.parametrize("backend", ["latency", "throughput"])
|
@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
|
||||||
def test_eplb_fml(
|
def test_eplb_fml(
|
||||||
world_size: int,
|
world_size: int,
|
||||||
num_layers: int,
|
num_layers: int,
|
||||||
@@ -258,12 +260,8 @@ def test_eplb_fml(
|
|||||||
hidden_size: int,
|
hidden_size: int,
|
||||||
intermediate_size: int,
|
intermediate_size: int,
|
||||||
num_tokens: int,
|
num_tokens: int,
|
||||||
backend: str,
|
moe_backend: str,
|
||||||
monkeypatch,
|
|
||||||
):
|
):
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
|
|
||||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", backend)
|
|
||||||
|
|
||||||
if torch.accelerator.device_count() < world_size:
|
if torch.accelerator.device_count() < world_size:
|
||||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||||
|
|
||||||
@@ -278,6 +276,7 @@ def test_eplb_fml(
|
|||||||
hidden_size=hidden_size,
|
hidden_size=hidden_size,
|
||||||
intermediate_size=intermediate_size,
|
intermediate_size=intermediate_size,
|
||||||
num_tokens=num_tokens,
|
num_tokens=num_tokens,
|
||||||
|
moe_backend=moe_backend,
|
||||||
)
|
)
|
||||||
|
|
||||||
distributed_run(
|
distributed_run(
|
||||||
|
|||||||
@@ -45,9 +45,10 @@ def test_config(llm: LLM):
|
|||||||
|
|
||||||
def test_pooling_params(llm: LLM):
|
def test_pooling_params(llm: LLM):
|
||||||
def get_outputs(use_activation):
|
def get_outputs(use_activation):
|
||||||
outputs = llm.reward(
|
outputs = llm.encode(
|
||||||
prompts,
|
prompts,
|
||||||
pooling_params=PoolingParams(use_activation=use_activation),
|
pooling_params=PoolingParams(use_activation=use_activation),
|
||||||
|
pooling_task="token_classify",
|
||||||
use_tqdm=False,
|
use_tqdm=False,
|
||||||
)
|
)
|
||||||
return torch.cat([x.outputs.data for x in outputs])
|
return torch.cat([x.outputs.data for x in outputs])
|
||||||
|
|||||||
+1
-3
@@ -3,6 +3,4 @@
|
|||||||
model_name: "openai/gpt-oss-20b"
|
model_name: "openai/gpt-oss-20b"
|
||||||
metric_threshold: 0.568
|
metric_threshold: 0.568
|
||||||
reasoning_effort: "low"
|
reasoning_effort: "low"
|
||||||
server_args: "--tensor-parallel-size 2"
|
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass"
|
||||||
env:
|
|
||||||
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1"
|
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
model_name: "openai/gpt-oss-20b"
|
||||||
|
metric_threshold: 0.568
|
||||||
|
reasoning_effort: "low"
|
||||||
|
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_trtllm"
|
||||||
@@ -3,6 +3,4 @@
|
|||||||
model_name: "openai/gpt-oss-20b"
|
model_name: "openai/gpt-oss-20b"
|
||||||
metric_threshold: 0.568
|
metric_threshold: 0.568
|
||||||
reasoning_effort: "low"
|
reasoning_effort: "low"
|
||||||
server_args: "--tensor-parallel-size 2"
|
server_args: "--tensor-parallel-size 2 --moe-backend marlin --linear-backend marlin"
|
||||||
env:
|
|
||||||
VLLM_MXFP4_USE_MARLIN: "1"
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# B200 model configurations for GPQA evaluation
|
# B200 model configurations for GPQA evaluation
|
||||||
# Tests different environment variable combinations
|
# Tests different environment variable combinations
|
||||||
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
|
gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml
|
||||||
gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
|
gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
|
||||||
gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
|
gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
# H100 model configurations for GPQA evaluation
|
# H100 model configurations for GPQA evaluation
|
||||||
# Tests different environment variable combinations
|
# Tests different environment variable combinations
|
||||||
gpt-oss-20b-baseline.yaml
|
gpt-oss-20b-baseline.yaml
|
||||||
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
|
gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml
|
||||||
gpt-oss-20b-marlin.yaml
|
gpt-oss-20b-marlin.yaml
|
||||||
|
|||||||
@@ -30,9 +30,9 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct"
|
|||||||
accuracy_threshold: 0.54 # Minimum expected accuracy
|
accuracy_threshold: 0.54 # Minimum expected accuracy
|
||||||
num_questions: 1319 # Number of questions (default: full test set)
|
num_questions: 1319 # Number of questions (default: full test set)
|
||||||
num_fewshot: 5 # Few-shot examples from train set
|
num_fewshot: 5 # Few-shot examples from train set
|
||||||
server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments
|
server_args: "--max-model-len 4096 --tensor-parallel-size 2 --moe-backend flashinfer_cutlass" # Server arguments
|
||||||
env: # Environment variables (optional)
|
env: # Environment variables (optional)
|
||||||
VLLM_USE_FLASHINFER_MOE_FP4: "1"
|
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||||
```
|
```
|
||||||
|
|
||||||
The `server_args` field accepts any arguments that can be passed to `vllm serve`.
|
The `server_args` field accepts any arguments that can be passed to `vllm serve`.
|
||||||
|
|||||||
@@ -1585,7 +1585,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
|||||||
e: int,
|
e: int,
|
||||||
topk: int,
|
topk: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
monkeypatch,
|
|
||||||
workspace_init,
|
workspace_init,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
@@ -1593,8 +1592,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
|||||||
"""
|
"""
|
||||||
set_random_seed(7)
|
set_random_seed(7)
|
||||||
|
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
|
||||||
|
|
||||||
from vllm.model_executor.layers.fused_moe.config import (
|
from vllm.model_executor.layers.fused_moe.config import (
|
||||||
FusedMoEConfig,
|
FusedMoEConfig,
|
||||||
FusedMoEParallelConfig,
|
FusedMoEParallelConfig,
|
||||||
@@ -1626,6 +1623,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
|||||||
in_dtype=dtype,
|
in_dtype=dtype,
|
||||||
routing_method=RoutingMethodType.Renormalize,
|
routing_method=RoutingMethodType.Renormalize,
|
||||||
max_num_tokens=next_power_of_2(m),
|
max_num_tokens=next_power_of_2(m),
|
||||||
|
moe_backend="flashinfer_trtllm",
|
||||||
)
|
)
|
||||||
|
|
||||||
with set_current_vllm_config(vllm_config):
|
with set_current_vllm_config(vllm_config):
|
||||||
|
|||||||
@@ -1804,12 +1804,9 @@ def test_moe_layer(
|
|||||||
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
|
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
|
||||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
|
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
|
||||||
|
|
||||||
# TODO
|
# TODO: cover FlashInfer MoE backends via moe_backend, e.g.
|
||||||
# VLLM_FLASHINFER_MOE_BACKEND=latency
|
# moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl
|
||||||
# VLLM_USE_FLASHINFER_MOE_FP16=1
|
# (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1.
|
||||||
# VLLM_USE_FLASHINFER_MOE_FP8
|
|
||||||
# VLLM_USE_FLASHINFER_MOE_FP4
|
|
||||||
# VLLM_USE_FLASHINFER_MOE_INT4
|
|
||||||
|
|
||||||
parallel_config = ParallelConfig(
|
parallel_config = ParallelConfig(
|
||||||
pipeline_parallel_size=1,
|
pipeline_parallel_size=1,
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
|
|||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||||
)
|
)
|
||||||
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeypatch):
|
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
|
||||||
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
|
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
|
||||||
with (
|
with (
|
||||||
patch.object(current_platform, "is_cuda", return_value=True),
|
patch.object(current_platform, "is_cuda", return_value=True),
|
||||||
@@ -134,9 +134,8 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp
|
|||||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||||
):
|
):
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
|
||||||
|
|
||||||
moe_config = make_dummy_moe_config()
|
moe_config = make_dummy_moe_config()
|
||||||
|
moe_config.moe_backend = "flashinfer_trtllm"
|
||||||
# TRTLLM requires EP and does not support DP
|
# TRTLLM requires EP and does not support DP
|
||||||
moe_config.moe_parallel_config.use_ep = True
|
moe_config.moe_parallel_config.use_ep = True
|
||||||
moe_config.moe_parallel_config.use_dp = False
|
moe_config.moe_parallel_config.use_dp = False
|
||||||
@@ -168,7 +167,6 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
|||||||
mock_has_flashinfer,
|
mock_has_flashinfer,
|
||||||
mock_is_supported_trtllm,
|
mock_is_supported_trtllm,
|
||||||
mock_is_supported_cutlass,
|
mock_is_supported_cutlass,
|
||||||
monkeypatch,
|
|
||||||
):
|
):
|
||||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||||
and FlashInfer CUTLASS is available."""
|
and FlashInfer CUTLASS is available."""
|
||||||
@@ -181,10 +179,9 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
|||||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||||
):
|
):
|
||||||
# Enable FlashInfer via env var
|
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
|
||||||
|
|
||||||
moe_config = make_dummy_moe_config()
|
moe_config = make_dummy_moe_config()
|
||||||
|
# Select FlashInfer CUTLASS explicitly
|
||||||
|
moe_config.moe_backend = "flashinfer_cutlass"
|
||||||
# CUTLASS requires EP and does not support DP
|
# CUTLASS requires EP and does not support DP
|
||||||
moe_config.moe_parallel_config.use_ep = True
|
moe_config.moe_parallel_config.use_ep = True
|
||||||
moe_config.moe_parallel_config.use_dp = False
|
moe_config.moe_parallel_config.use_dp = False
|
||||||
@@ -241,37 +238,3 @@ def test_select_explicit_triton_backend(is_lora_enabled):
|
|||||||
|
|
||||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||||
assert experts_cls is not None
|
assert experts_cls is not None
|
||||||
|
|
||||||
|
|
||||||
@skipif_not_cuda_rocm
|
|
||||||
def test_select_explicit_triton_ignores_flashinfer_env(monkeypatch):
|
|
||||||
"""Explicit triton backend should override FlashInfer env selection."""
|
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
|
||||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
|
|
||||||
|
|
||||||
moe_config = make_dummy_moe_config()
|
|
||||||
moe_config.is_lora_enabled = False
|
|
||||||
moe_config.moe_backend = "triton"
|
|
||||||
|
|
||||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
|
||||||
moe_config=moe_config
|
|
||||||
)
|
|
||||||
|
|
||||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
|
||||||
assert experts_cls is not None
|
|
||||||
|
|
||||||
|
|
||||||
@skipif_not_cuda_rocm
|
|
||||||
def test_select_lora_ignores_flashinfer_env(monkeypatch):
|
|
||||||
"""LoRA path should still choose Triton even if FlashInfer env is on."""
|
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
|
||||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
|
|
||||||
|
|
||||||
moe_config = make_dummy_moe_config()
|
|
||||||
moe_config.is_lora_enabled = True
|
|
||||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
|
||||||
moe_config=moe_config
|
|
||||||
)
|
|
||||||
|
|
||||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
|
||||||
assert experts_cls is not None
|
|
||||||
|
|||||||
@@ -83,57 +83,55 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
|
|||||||
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
||||||
@pytest.mark.parametrize("specialize_active_lora", [True, False])
|
@pytest.mark.parametrize("specialize_active_lora", [True, False])
|
||||||
def test_gpt_oss_lora(
|
def test_gpt_oss_lora(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
gptoss20b_lora_files,
|
gptoss20b_lora_files,
|
||||||
mxfp4_use_marlin,
|
mxfp4_use_marlin,
|
||||||
specialize_active_lora,
|
specialize_active_lora,
|
||||||
):
|
):
|
||||||
with monkeypatch.context() as m:
|
llm = vllm.LLM(
|
||||||
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
|
MODEL_PATH,
|
||||||
llm = vllm.LLM(
|
max_model_len=1024,
|
||||||
MODEL_PATH,
|
enable_lora=True,
|
||||||
max_model_len=1024,
|
max_loras=4,
|
||||||
enable_lora=True,
|
max_lora_rank=8,
|
||||||
max_loras=4,
|
max_num_seqs=2,
|
||||||
max_lora_rank=8,
|
max_num_batched_tokens=2048,
|
||||||
max_num_seqs=2,
|
specialize_active_lora=specialize_active_lora,
|
||||||
max_num_batched_tokens=2048,
|
moe_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||||
specialize_active_lora=specialize_active_lora,
|
linear_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||||
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
|
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
|
||||||
cudagraph_specialize_lora=False,
|
cudagraph_specialize_lora=False,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
||||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
||||||
|
|
||||||
|
|
||||||
@multi_gpu_test(num_gpus=2)
|
@multi_gpu_test(num_gpus=2)
|
||||||
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
|
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
|
||||||
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
||||||
def test_gpt_oss_lora_tp2(
|
def test_gpt_oss_lora_tp2(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
|
||||||
gptoss20b_lora_files,
|
gptoss20b_lora_files,
|
||||||
fully_sharded_loras,
|
fully_sharded_loras,
|
||||||
mxfp4_use_marlin,
|
mxfp4_use_marlin,
|
||||||
):
|
):
|
||||||
with monkeypatch.context() as m:
|
llm = vllm.LLM(
|
||||||
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
|
MODEL_PATH,
|
||||||
llm = vllm.LLM(
|
max_model_len=1024,
|
||||||
MODEL_PATH,
|
enable_lora=True,
|
||||||
max_model_len=1024,
|
max_loras=2,
|
||||||
enable_lora=True,
|
max_num_seqs=2,
|
||||||
max_loras=2,
|
max_num_batched_tokens=2048,
|
||||||
max_num_seqs=2,
|
tensor_parallel_size=2,
|
||||||
max_num_batched_tokens=2048,
|
gpu_memory_utilization=0.8,
|
||||||
tensor_parallel_size=2,
|
fully_sharded_loras=fully_sharded_loras,
|
||||||
gpu_memory_utilization=0.8,
|
enable_expert_parallel=not fully_sharded_loras,
|
||||||
fully_sharded_loras=fully_sharded_loras,
|
moe_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||||
enable_expert_parallel=not fully_sharded_loras,
|
linear_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||||
compilation_config=vllm.config.CompilationConfig(
|
compilation_config=vllm.config.CompilationConfig(
|
||||||
cudagraph_specialize_lora=False,
|
cudagraph_specialize_lora=False,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
||||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ def test_reward_models_using_activation(
|
|||||||
dtype=dtype,
|
dtype=dtype,
|
||||||
pooler_config=PoolerConfig(use_activation=False),
|
pooler_config=PoolerConfig(use_activation=False),
|
||||||
) as vllm_model:
|
) as vllm_model:
|
||||||
wo_activation = vllm_model.reward(example_prompts)
|
wo_activation = vllm_model.token_classify(example_prompts)
|
||||||
|
|
||||||
with vllm_runner(
|
with vllm_runner(
|
||||||
model,
|
model,
|
||||||
@@ -114,7 +114,7 @@ def test_reward_models_using_activation(
|
|||||||
dtype=dtype,
|
dtype=dtype,
|
||||||
pooler_config=PoolerConfig(use_activation=True),
|
pooler_config=PoolerConfig(use_activation=True),
|
||||||
) as vllm_model:
|
) as vllm_model:
|
||||||
w_activation = vllm_model.reward(example_prompts)
|
w_activation = vllm_model.token_classify(example_prompts)
|
||||||
|
|
||||||
for wo, w in zip(wo_activation, w_activation):
|
for wo, w in zip(wo_activation, w_activation):
|
||||||
wo = torch.tensor(wo)
|
wo = torch.tensor(wo)
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ def test_prm_models(
|
|||||||
pytest.skip("CPU only supports V1")
|
pytest.skip("CPU only supports V1")
|
||||||
|
|
||||||
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
||||||
vllm_outputs = vllm_model.reward(math_step_prompts)
|
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||||
|
|
||||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||||
hf_model = step_reward_patch_hf_model(hf_model)
|
hf_model = step_reward_patch_hf_model(hf_model)
|
||||||
@@ -146,7 +146,7 @@ def test_prm_models_with_golden_outputs(
|
|||||||
pytest.skip(f"No available golden outputs for {model}.")
|
pytest.skip(f"No available golden outputs for {model}.")
|
||||||
|
|
||||||
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
||||||
vllm_outputs = vllm_model.reward(math_step_prompts)
|
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||||
|
|
||||||
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
||||||
|
|
||||||
|
|||||||
@@ -133,11 +133,11 @@ def test_nvfp4(vllm_runner, model, eager, backend):
|
|||||||
not current_platform.is_rocm(),
|
not current_platform.is_rocm(),
|
||||||
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
|
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
|
||||||
)
|
)
|
||||||
def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch):
|
def test_nvfp4_moe(vllm_runner, model, backend):
|
||||||
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)
|
|
||||||
with vllm_runner(
|
with vllm_runner(
|
||||||
model,
|
model,
|
||||||
moe_backend=backend,
|
moe_backend=backend,
|
||||||
|
linear_backend=backend,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
hf_overrides={"num_hidden_layers": 2},
|
hf_overrides={"num_hidden_layers": 2},
|
||||||
) as llm:
|
) as llm:
|
||||||
|
|||||||
@@ -185,8 +185,11 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1")
|
can_initialize(
|
||||||
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
|
"openai/gpt-oss-20b",
|
||||||
|
hf_overrides=HF_OVERRIDE_TEXT,
|
||||||
|
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||||
|
|||||||
@@ -1617,7 +1617,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
|
|||||||
"custom",
|
"custom",
|
||||||
"custom_audio",
|
"custom_audio",
|
||||||
"custom_image",
|
"custom_image",
|
||||||
"custom_mm",
|
|
||||||
"prefix_repetition",
|
"prefix_repetition",
|
||||||
"spec_bench",
|
"spec_bench",
|
||||||
"speed_bench",
|
"speed_bench",
|
||||||
@@ -2106,12 +2105,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
|
|||||||
no_oversample=args.no_oversample,
|
no_oversample=args.no_oversample,
|
||||||
)
|
)
|
||||||
|
|
||||||
elif args.dataset_name in ("custom_image", "custom_mm"):
|
elif args.dataset_name == "custom_image":
|
||||||
if args.dataset_name == "custom_mm":
|
|
||||||
logger.warning(
|
|
||||||
"Dataset name 'custom_mm' is deprecated and will be removed in v0.24. "
|
|
||||||
"Use '--dataset-name custom_image' instead."
|
|
||||||
)
|
|
||||||
dataset = CustomImageDataset(
|
dataset = CustomImageDataset(
|
||||||
dataset_path=args.dataset_path,
|
dataset_path=args.dataset_path,
|
||||||
disable_shuffle=args.disable_shuffle,
|
disable_shuffle=args.disable_shuffle,
|
||||||
|
|||||||
@@ -134,10 +134,6 @@ class PassConfig:
|
|||||||
"""Enable async TP."""
|
"""Enable async TP."""
|
||||||
fuse_allreduce_rms: bool = None # type: ignore[assignment]
|
fuse_allreduce_rms: bool = None # type: ignore[assignment]
|
||||||
"""Enable flashinfer allreduce fusion."""
|
"""Enable flashinfer allreduce fusion."""
|
||||||
fuse_minimax_qk_norm: bool = None # type: ignore[assignment]
|
|
||||||
"""Deprecated. The MiniMax QK norm fusion is now applied automatically at
|
|
||||||
runtime (see `MiniMaxText01RMSNormTP.forward_qkv`). This flag is kept for
|
|
||||||
backward compatibility and has no effect; it will be removed in v0.23."""
|
|
||||||
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
|
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
|
||||||
"""Enable fused Q/K RMSNorm + RoPE pass."""
|
"""Enable fused Q/K RMSNorm + RoPE pass."""
|
||||||
fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment]
|
fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment]
|
||||||
@@ -296,13 +292,6 @@ class PassConfig:
|
|||||||
"current platform is not CUDA or ROCm. The fusion will be disabled."
|
"current platform is not CUDA or ROCm. The fusion will be disabled."
|
||||||
)
|
)
|
||||||
self.fuse_rope_kvcache_cat_mla = False
|
self.fuse_rope_kvcache_cat_mla = False
|
||||||
if self.fuse_minimax_qk_norm is not None:
|
|
||||||
logger.warning_once(
|
|
||||||
"`fuse_minimax_qk_norm` is deprecated and has no effect; "
|
|
||||||
"the MiniMax QK norm fusion is now applied automatically at "
|
|
||||||
"runtime when its conditions are met. This flag will be "
|
|
||||||
"removed in v0.23."
|
|
||||||
)
|
|
||||||
|
|
||||||
def log_enabled_passes(self) -> None:
|
def log_enabled_passes(self) -> None:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -142,6 +142,7 @@ LinearBackend = Literal[
|
|||||||
"flashinfer_cutlass",
|
"flashinfer_cutlass",
|
||||||
"flashinfer_trtllm",
|
"flashinfer_trtllm",
|
||||||
"flashinfer_cudnn",
|
"flashinfer_cudnn",
|
||||||
|
"flashinfer_b12x",
|
||||||
"marlin",
|
"marlin",
|
||||||
"triton",
|
"triton",
|
||||||
"deep_gemm",
|
"deep_gemm",
|
||||||
@@ -197,6 +198,7 @@ class KernelConfig:
|
|||||||
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
|
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
|
||||||
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
|
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
|
||||||
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
|
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
|
||||||
|
- "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
|
||||||
- "marlin": Use Marlin kernels
|
- "marlin": Use Marlin kernels
|
||||||
- "triton": Use Triton-based kernels
|
- "triton": Use Triton-based kernels
|
||||||
- "deep_gemm": Use DeepGEMM kernels
|
- "deep_gemm": Use DeepGEMM kernels
|
||||||
|
|||||||
@@ -286,50 +286,6 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
|||||||
|
|
||||||
return [ClassificationRequestOutput.from_base(item) for item in items]
|
return [ClassificationRequestOutput.from_base(item) for item in items]
|
||||||
|
|
||||||
def reward(
|
|
||||||
self,
|
|
||||||
prompts: PromptType | Sequence[PromptType],
|
|
||||||
/,
|
|
||||||
*,
|
|
||||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
|
||||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
|
||||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
|
||||||
tokenization_kwargs: dict[str, Any] | None = None,
|
|
||||||
) -> list[PoolingRequestOutput]:
|
|
||||||
"""
|
|
||||||
Generate rewards for each prompt.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
|
||||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
|
||||||
for more details about the format of each prompt.
|
|
||||||
pooling_params: The pooling parameters for pooling. If None, we
|
|
||||||
use the default pooling parameters.
|
|
||||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
|
||||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
|
||||||
it is used to create the progress bar.
|
|
||||||
If `False`, no progress bar is created.
|
|
||||||
lora_request: LoRA request to use for generation, if any.
|
|
||||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
A list of `PoolingRequestOutput` objects containing the
|
|
||||||
pooled hidden states in the same order as the input prompts.
|
|
||||||
"""
|
|
||||||
logger.warning_once(
|
|
||||||
"`llm.reward` api is deprecated and will be removed in v0.23. "
|
|
||||||
'Please use `LLM.encode` with `pooling_task="classify"` or '
|
|
||||||
'`pooling_task="token_classify"` instead.'
|
|
||||||
)
|
|
||||||
return self.encode(
|
|
||||||
prompts,
|
|
||||||
use_tqdm=use_tqdm,
|
|
||||||
lora_request=lora_request,
|
|
||||||
pooling_params=pooling_params,
|
|
||||||
pooling_task="token_classify",
|
|
||||||
tokenization_kwargs=tokenization_kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def score(
|
def score(
|
||||||
self,
|
self,
|
||||||
data_1: ScoreInput | list[ScoreInput],
|
data_1: ScoreInput | list[ScoreInput],
|
||||||
|
|||||||
-168
@@ -8,7 +8,6 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import uuid
|
import uuid
|
||||||
import warnings
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import TYPE_CHECKING, Any, Literal
|
from typing import TYPE_CHECKING, Any, Literal
|
||||||
|
|
||||||
@@ -167,7 +166,6 @@ if TYPE_CHECKING:
|
|||||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None
|
VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None
|
||||||
VLLM_HUMMING_USE_F16_ACCUM: bool = False
|
VLLM_HUMMING_USE_F16_ACCUM: bool = False
|
||||||
VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None
|
VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None
|
||||||
VLLM_MXFP4_USE_MARLIN: bool | None = None
|
|
||||||
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
|
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
|
||||||
VLLM_V1_USE_OUTLINES_CACHE: bool = False
|
VLLM_V1_USE_OUTLINES_CACHE: bool = False
|
||||||
VLLM_TPU_BUCKET_PADDING_GAP: int = 0
|
VLLM_TPU_BUCKET_PADDING_GAP: int = 0
|
||||||
@@ -184,13 +182,7 @@ if TYPE_CHECKING:
|
|||||||
] = "relax"
|
] = "relax"
|
||||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||||
VLLM_USE_FLASHINFER_MOE_FP16: bool = False
|
|
||||||
VLLM_USE_FLASHINFER_MOE_FP8: bool = False
|
|
||||||
VLLM_USE_FLASHINFER_MOE_FP4: bool = False
|
|
||||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||||
VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = (
|
|
||||||
"latency"
|
|
||||||
)
|
|
||||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||||
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
|
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
|
||||||
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
|
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
|
||||||
@@ -212,7 +204,6 @@ if TYPE_CHECKING:
|
|||||||
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
|
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
|
||||||
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
|
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
|
||||||
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
|
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
|
||||||
VLLM_USE_NVFP4_CT_EMULATIONS: bool = False
|
|
||||||
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
|
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
|
||||||
"FP", "INT8", "INT6", "INT4", "NONE"
|
"FP", "INT8", "INT6", "INT4", "NONE"
|
||||||
] = "NONE"
|
] = "NONE"
|
||||||
@@ -225,12 +216,8 @@ if TYPE_CHECKING:
|
|||||||
VLLM_LOOPBACK_IP: str = ""
|
VLLM_LOOPBACK_IP: str = ""
|
||||||
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
||||||
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
|
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
|
||||||
VLLM_NVFP4_GEMM_BACKEND: str | None = None
|
|
||||||
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
||||||
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False
|
|
||||||
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
|
|
||||||
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
||||||
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False
|
|
||||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||||
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
|
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
|
||||||
VLLM_TUNED_CONFIG_FOLDER: str | None = None
|
VLLM_TUNED_CONFIG_FOLDER: str | None = None
|
||||||
@@ -257,7 +244,6 @@ if TYPE_CHECKING:
|
|||||||
VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
|
VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
|
||||||
VLLM_USE_NCCL_SYMM_MEM: bool = False
|
VLLM_USE_NCCL_SYMM_MEM: bool = False
|
||||||
VLLM_NCCL_INCLUDE_PATH: str | None = None
|
VLLM_NCCL_INCLUDE_PATH: str | None = None
|
||||||
VLLM_USE_FBGEMM: bool = False
|
|
||||||
VLLM_GC_DEBUG: str = ""
|
VLLM_GC_DEBUG: str = ""
|
||||||
VLLM_DEBUG_WORKSPACE: bool = False
|
VLLM_DEBUG_WORKSPACE: bool = False
|
||||||
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
|
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
|
||||||
@@ -350,27 +336,6 @@ def use_mega_aot_artifact():
|
|||||||
return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1"
|
return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1"
|
||||||
|
|
||||||
|
|
||||||
def deprecated_env(
|
|
||||||
env_name: str,
|
|
||||||
removal_version: str,
|
|
||||||
replacement: str,
|
|
||||||
getter: Callable[[], Any],
|
|
||||||
) -> Callable[[], Any]:
|
|
||||||
"""Wrap an env-var getter to emit a FutureWarning when the var is set."""
|
|
||||||
|
|
||||||
def _read() -> Any:
|
|
||||||
if env_name in os.environ:
|
|
||||||
warnings.warn(
|
|
||||||
f"{env_name} is deprecated and will be removed in "
|
|
||||||
f"{removal_version}. {replacement}",
|
|
||||||
FutureWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return getter()
|
|
||||||
|
|
||||||
return _read
|
|
||||||
|
|
||||||
|
|
||||||
def env_with_choices(
|
def env_with_choices(
|
||||||
env_name: str,
|
env_name: str,
|
||||||
default: str | None,
|
default: str | None,
|
||||||
@@ -1371,15 +1336,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_MARLIN_USE_ATOMIC_ADD": lambda: (
|
"VLLM_MARLIN_USE_ATOMIC_ADD": lambda: (
|
||||||
os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1"
|
os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1"
|
||||||
),
|
),
|
||||||
# Whether to use marlin kernel in mxfp4 quantization method
|
|
||||||
# Deprecated: use --moe-backend marlin (MoE) or --linear-backend marlin
|
|
||||||
# (linear) instead.
|
|
||||||
"VLLM_MXFP4_USE_MARLIN": deprecated_env(
|
|
||||||
"VLLM_MXFP4_USE_MARLIN",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend marlin or --linear-backend marlin.",
|
|
||||||
lambda: maybe_convert_bool(os.environ.get("VLLM_MXFP4_USE_MARLIN", None)),
|
|
||||||
),
|
|
||||||
# The activation dtype for marlin kernel
|
# The activation dtype for marlin kernel
|
||||||
"VLLM_MARLIN_INPUT_DTYPE": env_with_choices(
|
"VLLM_MARLIN_INPUT_DTYPE": env_with_choices(
|
||||||
"VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"]
|
"VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"]
|
||||||
@@ -1472,68 +1428,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||||
int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
|
int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
|
||||||
),
|
),
|
||||||
# Allow use of FlashInfer BF16 MoE kernels for fused moe ops.
|
|
||||||
# Deprecated: use --moe-backend to select a kernel explicitly.
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_FP16": deprecated_env(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_FP16",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))),
|
|
||||||
),
|
|
||||||
# Allow use of FlashInfer FP8 MoE kernels for fused moe ops.
|
|
||||||
# Deprecated: use --moe-backend to select a kernel explicitly.
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_FP8": deprecated_env(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_FP8",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP8", "0"))),
|
|
||||||
),
|
|
||||||
# Allow use of FlashInfer NVFP4 MoE kernels for fused moe ops.
|
|
||||||
# Deprecated: use --moe-backend to select a kernel explicitly.
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_FP4": deprecated_env(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_FP4",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass, "
|
|
||||||
"flashinfer_cutedsl).",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP4", "0"))),
|
|
||||||
),
|
|
||||||
# Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops.
|
# Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops.
|
||||||
"VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool(
|
"VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool(
|
||||||
int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0"))
|
int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0"))
|
||||||
),
|
),
|
||||||
# If set to 1, use the FlashInfer
|
|
||||||
# MXFP8 (activation) x MXFP4 (weight) MoE backend.
|
|
||||||
# Deprecated: use --moe-backend flashinfer_trtllm combined with
|
|
||||||
# --quantization_config.moe.activation mxfp8.
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": deprecated_env(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend flashinfer_trtllm with "
|
|
||||||
"--quantization_config.moe.activation mxfp8.",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))),
|
|
||||||
),
|
|
||||||
# If set to 1, use the FlashInfer CUTLASS backend for
|
|
||||||
# MXFP8 (activation) x MXFP4 (weight) MoE.
|
|
||||||
# Deprecated: use --moe-backend flashinfer_cutlass combined with
|
|
||||||
# --quantization_config.moe.activation mxfp8.
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": deprecated_env(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend flashinfer_cutlass with "
|
|
||||||
"--quantization_config.moe.activation mxfp8.",
|
|
||||||
lambda: bool(
|
|
||||||
int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0"))
|
|
||||||
),
|
|
||||||
),
|
|
||||||
# If set to 1, use the FlashInfer
|
|
||||||
# BF16 (activation) x MXFP4 (weight) MoE backend.
|
|
||||||
# Deprecated: use --moe-backend to select a kernel explicitly.
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": deprecated_env(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "0"))),
|
|
||||||
),
|
|
||||||
# Control the cache sized used by the xgrammar compiler. The default
|
# Control the cache sized used by the xgrammar compiler. The default
|
||||||
# of 512 MB should be enough for roughly 1000 JSON schemas.
|
# of 512 MB should be enough for roughly 1000 JSON schemas.
|
||||||
# It can be changed with this variable if needed for some reason.
|
# It can be changed with this variable if needed for some reason.
|
||||||
@@ -1585,25 +1483,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
|
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
|
||||||
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
|
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
|
||||||
),
|
),
|
||||||
# Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
|
|
||||||
# Both require compute capability 10.0 or above.
|
|
||||||
# Available options:
|
|
||||||
# - "throughput": [default]
|
|
||||||
# Uses CUTLASS kernels optimized for high-throughput batch inference.
|
|
||||||
# - "latency":
|
|
||||||
# Uses TensorRT-LLM kernels optimized for low-latency inference.
|
|
||||||
# Deprecated: pass --moe-backend flashinfer_{trtllm,cutlass,cutedsl} directly.
|
|
||||||
"VLLM_FLASHINFER_MOE_BACKEND": deprecated_env(
|
|
||||||
"VLLM_FLASHINFER_MOE_BACKEND",
|
|
||||||
"v0.23",
|
|
||||||
"Use --moe-backend flashinfer_trtllm, flashinfer_cutlass, or "
|
|
||||||
"flashinfer_cutedsl.",
|
|
||||||
env_with_choices(
|
|
||||||
"VLLM_FLASHINFER_MOE_BACKEND",
|
|
||||||
"latency",
|
|
||||||
["throughput", "latency", "masked_gemm"],
|
|
||||||
),
|
|
||||||
),
|
|
||||||
# Override the directory for the FlashInfer autotune config cache.
|
# Override the directory for the FlashInfer autotune config cache.
|
||||||
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
|
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
|
||||||
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
|
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
|
||||||
@@ -1681,16 +1560,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
|
"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
|
||||||
int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
|
int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
|
||||||
),
|
),
|
||||||
# Controls whether or not emulations are used for NVFP4
|
|
||||||
# generations on machines < 100 for compressed-tensors
|
|
||||||
# models
|
|
||||||
# Deprecated: use --linear-backend emulation instead.
|
|
||||||
"VLLM_USE_NVFP4_CT_EMULATIONS": deprecated_env(
|
|
||||||
"VLLM_USE_NVFP4_CT_EMULATIONS",
|
|
||||||
"v0.23",
|
|
||||||
"Use --linear-backend emulation.",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0"))),
|
|
||||||
),
|
|
||||||
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
|
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
|
||||||
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
|
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
|
||||||
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
|
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
|
||||||
@@ -1700,35 +1569,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
|
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
|
||||||
int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0"))
|
int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0"))
|
||||||
),
|
),
|
||||||
# Supported options:
|
|
||||||
# - "flashinfer-cudnn": use flashinfer cudnn GEMM backend
|
|
||||||
# - "flashinfer-trtllm": use flashinfer trtllm GEMM backend
|
|
||||||
# - "flashinfer-cutlass": use flashinfer cutlass GEMM backend
|
|
||||||
# - "marlin": use marlin GEMM backend (for GPUs without native FP4 support)
|
|
||||||
# - "emulation":
|
|
||||||
# use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations.
|
|
||||||
# This is only meant for research purposes to run on devices where NVFP4
|
|
||||||
# GEMM kernels are not available.
|
|
||||||
# - <none>: automatically pick an available backend
|
|
||||||
# Deprecated: use --linear-backend instead.
|
|
||||||
"VLLM_NVFP4_GEMM_BACKEND": deprecated_env(
|
|
||||||
"VLLM_NVFP4_GEMM_BACKEND",
|
|
||||||
"v0.23",
|
|
||||||
"Use --linear-backend.",
|
|
||||||
env_with_choices(
|
|
||||||
"VLLM_NVFP4_GEMM_BACKEND",
|
|
||||||
None,
|
|
||||||
[
|
|
||||||
"flashinfer-b12x",
|
|
||||||
"flashinfer-cudnn",
|
|
||||||
"flashinfer-trtllm",
|
|
||||||
"flashinfer-cutlass",
|
|
||||||
"cutlass",
|
|
||||||
"marlin",
|
|
||||||
"emulation",
|
|
||||||
],
|
|
||||||
),
|
|
||||||
),
|
|
||||||
# Controls garbage collection during CUDA graph capture.
|
# Controls garbage collection during CUDA graph capture.
|
||||||
# If set to 0 (default), enables GC freezing to speed up capture time.
|
# If set to 0 (default), enables GC freezing to speed up capture time.
|
||||||
# If set to 1, allows GC to run during capture.
|
# If set to 1, allows GC to run during capture.
|
||||||
@@ -1892,14 +1732,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
),
|
),
|
||||||
# NCCL header path
|
# NCCL header path
|
||||||
"VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
|
"VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
|
||||||
# Flag to enable FBGemm kernels on model execution
|
|
||||||
# Deprecated: use --linear-backend fbgemm instead.
|
|
||||||
"VLLM_USE_FBGEMM": deprecated_env(
|
|
||||||
"VLLM_USE_FBGEMM",
|
|
||||||
"v0.23",
|
|
||||||
"Use --linear-backend fbgemm.",
|
|
||||||
lambda: bool(int(os.getenv("VLLM_USE_FBGEMM", "0"))),
|
|
||||||
),
|
|
||||||
# GC debug config
|
# GC debug config
|
||||||
# - VLLM_GC_DEBUG=0: disable GC debugger
|
# - VLLM_GC_DEBUG=0: disable GC debugger
|
||||||
# - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
|
# - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
|
||||||
|
|||||||
@@ -212,6 +212,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
|||||||
"flashinfer_cudnn": {
|
"flashinfer_cudnn": {
|
||||||
FlashInferCudnnNvFp4LinearKernel,
|
FlashInferCudnnNvFp4LinearKernel,
|
||||||
},
|
},
|
||||||
|
"flashinfer_b12x": {
|
||||||
|
FlashInferB12xNvFp4LinearKernel,
|
||||||
|
},
|
||||||
"marlin": {
|
"marlin": {
|
||||||
MarlinFP8ScaledMMLinearKernel,
|
MarlinFP8ScaledMMLinearKernel,
|
||||||
MarlinLinearKernel,
|
MarlinLinearKernel,
|
||||||
@@ -392,7 +395,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
|||||||
PlatformEnum.CUDA: [
|
PlatformEnum.CUDA: [
|
||||||
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
|
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
|
||||||
# upstream CUTLASS SM121 MMA op guard is resolved; use
|
# upstream CUTLASS SM121 MMA op guard is resolved; use
|
||||||
# VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly.
|
# --linear-backend flashinfer_b12x to opt in explicitly.
|
||||||
FlashInferCutlassNvFp4LinearKernel,
|
FlashInferCutlassNvFp4LinearKernel,
|
||||||
CutlassNvFp4LinearKernel,
|
CutlassNvFp4LinearKernel,
|
||||||
MarlinNvFp4LinearKernel,
|
MarlinNvFp4LinearKernel,
|
||||||
@@ -752,20 +755,6 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
|
|||||||
current platform."""
|
current platform."""
|
||||||
linear_backend = _get_linear_backend()
|
linear_backend = _get_linear_backend()
|
||||||
|
|
||||||
force_kernel: type[MxFp4LinearKernel] | None = None
|
|
||||||
if linear_backend == "auto" and envs.VLLM_MXFP4_USE_MARLIN:
|
|
||||||
force_kernel = MarlinMxFp4LinearKernel
|
|
||||||
|
|
||||||
if force_kernel is not None:
|
|
||||||
is_supported, reason = force_kernel.is_supported()
|
|
||||||
if not is_supported:
|
|
||||||
raise ValueError(
|
|
||||||
f"Forced MXFP4 kernel {force_kernel.__name__} is not "
|
|
||||||
f"supported: {reason}"
|
|
||||||
)
|
|
||||||
logger.info_once("Using %s for MXFP4 GEMM", force_kernel.__name__)
|
|
||||||
return force_kernel(MxFp4LinearLayerConfig())
|
|
||||||
|
|
||||||
platform = current_platform._enum
|
platform = current_platform._enum
|
||||||
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
|
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
|
||||||
|
|
||||||
@@ -836,18 +825,6 @@ def init_wfp8_a16_linear_kernel(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
|
|
||||||
_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
|
|
||||||
"flashinfer-b12x": FlashInferB12xNvFp4LinearKernel,
|
|
||||||
"flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
|
|
||||||
"cutlass": CutlassNvFp4LinearKernel,
|
|
||||||
"marlin": MarlinNvFp4LinearKernel,
|
|
||||||
"flashinfer-trtllm": FlashInferTrtllmNvFp4LinearKernel,
|
|
||||||
"flashinfer-cudnn": FlashInferCudnnNvFp4LinearKernel,
|
|
||||||
"emulation": EmulationNvFp4LinearKernel,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||||
"""Select and instantiate the best NVFP4 linear kernel for the
|
"""Select and instantiate the best NVFP4 linear kernel for the
|
||||||
current platform."""
|
current platform."""
|
||||||
@@ -855,8 +832,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
|||||||
|
|
||||||
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
|
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
|
||||||
# batch-invariant CUTLASS implementation when available, otherwise fall
|
# batch-invariant CUTLASS implementation when available, otherwise fall
|
||||||
# back to emulation. It overrides both --linear-backend and the deprecated
|
# back to emulation. It overrides --linear-backend.
|
||||||
# env vars below.
|
|
||||||
force_kernel: type[NvFp4LinearKernel] | None = None
|
force_kernel: type[NvFp4LinearKernel] | None = None
|
||||||
linear_backend = _get_linear_backend()
|
linear_backend = _get_linear_backend()
|
||||||
if envs.VLLM_BATCH_INVARIANT:
|
if envs.VLLM_BATCH_INVARIANT:
|
||||||
@@ -888,24 +864,9 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
|||||||
reason,
|
reason,
|
||||||
)
|
)
|
||||||
force_kernel = EmulationNvFp4LinearKernel
|
force_kernel = EmulationNvFp4LinearKernel
|
||||||
elif linear_backend == "auto":
|
elif linear_backend == "auto" and use_a16:
|
||||||
# Deprecated env-var overrides — only honoured when --linear-backend
|
# Force a16 (Marlin) when running weight-only quantization.
|
||||||
# is "auto". Deprecation warnings are emitted from vllm/envs.py.
|
force_kernel = MarlinNvFp4LinearKernel
|
||||||
if use_a16: # force a16 if running weight-only quantization
|
|
||||||
force_kernel = MarlinNvFp4LinearKernel
|
|
||||||
elif envs.VLLM_USE_FBGEMM:
|
|
||||||
force_kernel = FbgemmNvFp4LinearKernel
|
|
||||||
elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
|
|
||||||
force_kernel = EmulationNvFp4LinearKernel
|
|
||||||
elif envs.VLLM_NVFP4_GEMM_BACKEND is not None:
|
|
||||||
backend_name = envs.VLLM_NVFP4_GEMM_BACKEND
|
|
||||||
force_kernel = _NVFP4_BACKEND_TO_KERNEL.get(backend_name)
|
|
||||||
if force_kernel is None:
|
|
||||||
raise ValueError(
|
|
||||||
f"Unknown VLLM_NVFP4_GEMM_BACKEND={backend_name!r}. "
|
|
||||||
f"Valid choices: "
|
|
||||||
f"{list(_NVFP4_BACKEND_TO_KERNEL.keys())}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if force_kernel is not None:
|
if force_kernel is not None:
|
||||||
is_supported, reason = force_kernel.is_supported()
|
is_supported, reason = force_kernel.is_supported()
|
||||||
|
|||||||
@@ -20,8 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||||
FlashinferMoeBackend,
|
|
||||||
get_flashinfer_moe_backend,
|
|
||||||
prepare_fp8_moe_layer_for_fi,
|
prepare_fp8_moe_layer_for_fi,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||||
@@ -321,54 +319,6 @@ def select_fp8_moe_backend(
|
|||||||
requested_backend, config, weight_key, activation_key, activation_format
|
requested_backend, config, weight_key, activation_key, activation_format
|
||||||
)
|
)
|
||||||
|
|
||||||
# Handle explicit FlashInfer FP8 configuration.
|
|
||||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"):
|
|
||||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP8:
|
|
||||||
# If the user rejects FlashInfer remove those backends.
|
|
||||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_TRTLLM)
|
|
||||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_CUTLASS)
|
|
||||||
|
|
||||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
|
||||||
# If user is explicit about backend, validate it.
|
|
||||||
fi_backend = get_flashinfer_moe_backend()
|
|
||||||
if fi_backend == FlashinferMoeBackend.CUTLASS:
|
|
||||||
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
|
|
||||||
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
|
|
||||||
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
f"FlashInfer MOE backend {fi_backend} does not support FP8 MoE."
|
|
||||||
)
|
|
||||||
k_cls = backend_to_kernel_cls(backend)[0]
|
|
||||||
return _return_or_raise(
|
|
||||||
backend, config, weight_key, activation_key, activation_format
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# If the user is not explicit about the backend, try both.
|
|
||||||
for backend in [
|
|
||||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
|
||||||
Fp8MoeBackend.FLASHINFER_CUTLASS,
|
|
||||||
]:
|
|
||||||
for k_cls in backend_to_kernel_cls(backend):
|
|
||||||
supported, reason = k_cls.is_supported_config(
|
|
||||||
k_cls,
|
|
||||||
config,
|
|
||||||
weight_key,
|
|
||||||
activation_key,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
|
|
||||||
if supported:
|
|
||||||
logger.info_once(_make_log_backend(backend))
|
|
||||||
return backend, k_cls
|
|
||||||
else:
|
|
||||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
|
||||||
|
|
||||||
raise NotImplementedError(
|
|
||||||
"Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no "
|
|
||||||
"FlashInfer FP8 MoE backend supports the configuration."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle explicit DeepGEMM FP8 configuration.
|
# Handle explicit DeepGEMM FP8 configuration.
|
||||||
if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"):
|
if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"):
|
||||||
if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM:
|
if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM:
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Literal, Union
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||||
from vllm import envs
|
|
||||||
from vllm.config import get_current_vllm_config
|
from vllm.config import get_current_vllm_config
|
||||||
from vllm.config.kernel import MoEBackend
|
from vllm.config.kernel import MoEBackend
|
||||||
from vllm.config.quantization import QuantizationConfigArgs
|
from vllm.config.quantization import QuantizationConfigArgs
|
||||||
@@ -465,74 +464,6 @@ def select_mxfp4_moe_backend(
|
|||||||
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
||||||
)
|
)
|
||||||
|
|
||||||
# Handle explicit FlashInfer MXFP4 BF16 configuration.
|
|
||||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
|
|
||||||
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
|
|
||||||
for _b in (
|
|
||||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
|
||||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
|
||||||
):
|
|
||||||
if _b in AVAILABLE_BACKENDS:
|
|
||||||
AVAILABLE_BACKENDS.remove(_b)
|
|
||||||
else:
|
|
||||||
if current_platform.is_device_capability(90):
|
|
||||||
return _return_or_raise(
|
|
||||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
|
||||||
config,
|
|
||||||
kMxfp4Static,
|
|
||||||
None,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
if current_platform.is_device_capability_family(100):
|
|
||||||
return _return_or_raise(
|
|
||||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
|
||||||
config,
|
|
||||||
kMxfp4Static,
|
|
||||||
None,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
raise ValueError(
|
|
||||||
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16=1 is set but the "
|
|
||||||
"current device capability is not supported. "
|
|
||||||
"Only SM90 (CUTLASS) and SM100+ (TRTLLM) are supported."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle explicit FlashInfer MXFP4 MXFP8 TRTLLM configuration.
|
|
||||||
if (
|
|
||||||
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")
|
|
||||||
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8
|
|
||||||
):
|
|
||||||
return _return_or_raise(
|
|
||||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
|
||||||
config,
|
|
||||||
kMxfp4Static,
|
|
||||||
kMxfp8Dynamic,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle explicit FlashInfer MXFP4 MXFP8 CUTLASS configuration.
|
|
||||||
if (
|
|
||||||
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS")
|
|
||||||
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS
|
|
||||||
):
|
|
||||||
return _return_or_raise(
|
|
||||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
|
||||||
config,
|
|
||||||
kMxfp4Static,
|
|
||||||
kMxfp8Dynamic,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle explicit Marlin MXFP4 configuration.
|
|
||||||
if envs.is_set("VLLM_MXFP4_USE_MARLIN") and envs.VLLM_MXFP4_USE_MARLIN:
|
|
||||||
return _return_or_raise(
|
|
||||||
Mxfp4MoeBackend.MARLIN,
|
|
||||||
config,
|
|
||||||
kMxfp4Static,
|
|
||||||
None,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
|
|
||||||
for backend in AVAILABLE_BACKENDS:
|
for backend in AVAILABLE_BACKENDS:
|
||||||
# Use requested_activation_key if provided, otherwise use backend default
|
# Use requested_activation_key if provided, otherwise use backend default
|
||||||
act_key = (
|
act_key = (
|
||||||
|
|||||||
@@ -22,10 +22,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
|||||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
|
||||||
FlashinferMoeBackend,
|
|
||||||
get_flashinfer_moe_backend,
|
|
||||||
)
|
|
||||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||||
prepare_nvfp4_moe_layer_for_marlin,
|
prepare_nvfp4_moe_layer_for_marlin,
|
||||||
)
|
)
|
||||||
@@ -58,12 +54,6 @@ FLASHINFER_NVFP4_MOE_BACKENDS = [
|
|||||||
NvFp4MoeBackend.FLASHINFER_B12X,
|
NvFp4MoeBackend.FLASHINFER_B12X,
|
||||||
]
|
]
|
||||||
|
|
||||||
fi_2_vllm_backend_map: dict[FlashinferMoeBackend, NvFp4MoeBackend] = {
|
|
||||||
FlashinferMoeBackend.CUTLASS: NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
|
||||||
FlashinferMoeBackend.TENSORRT_LLM: NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
|
||||||
FlashinferMoeBackend.CUTEDSL: NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool:
|
def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool:
|
||||||
# Checks whether `backend` supports quantizing with scaling factors
|
# Checks whether `backend` supports quantizing with scaling factors
|
||||||
@@ -258,55 +248,6 @@ def select_nvfp4_moe_backend(
|
|||||||
requested_backend, config, weight_key, activation_key, activation_format
|
requested_backend, config, weight_key, activation_key, activation_format
|
||||||
)
|
)
|
||||||
|
|
||||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"):
|
|
||||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
|
|
||||||
# If the user rejects FlashInfer remove those backends.
|
|
||||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
|
|
||||||
if b in AVAILABLE_BACKENDS:
|
|
||||||
AVAILABLE_BACKENDS.remove(b)
|
|
||||||
|
|
||||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
|
||||||
# If user is explicit about backend, validate it.
|
|
||||||
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
|
|
||||||
if (
|
|
||||||
config.swiglu_limit is not None
|
|
||||||
and backend not in NVFP4_BACKENDS_WITH_CLAMP
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
|
||||||
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
|
|
||||||
f"({backend.value}) does not apply the SwiGLU clamp."
|
|
||||||
)
|
|
||||||
return _return_or_raise(
|
|
||||||
backend, config, weight_key, activation_key, activation_format
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# If the user is not explicit about the backend, try each.
|
|
||||||
fi_backends = [
|
|
||||||
b
|
|
||||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS
|
|
||||||
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
|
|
||||||
]
|
|
||||||
for backend in fi_backends:
|
|
||||||
for k_cls in backend_to_kernel_cls(backend):
|
|
||||||
supported, reason = k_cls.is_supported_config(
|
|
||||||
k_cls,
|
|
||||||
config,
|
|
||||||
weight_key,
|
|
||||||
activation_key,
|
|
||||||
activation_format,
|
|
||||||
)
|
|
||||||
if supported:
|
|
||||||
logger.info_once(_make_log_backend(backend))
|
|
||||||
return backend, k_cls
|
|
||||||
else:
|
|
||||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
|
||||||
|
|
||||||
raise NotImplementedError(
|
|
||||||
"Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no "
|
|
||||||
"FlashInfer NVFP4 MoE backend supports the configuration."
|
|
||||||
)
|
|
||||||
|
|
||||||
if envs.VLLM_TEST_FORCE_FP8_MARLIN:
|
if envs.VLLM_TEST_FORCE_FP8_MARLIN:
|
||||||
backend = NvFp4MoeBackend.MARLIN
|
backend = NvFp4MoeBackend.MARLIN
|
||||||
return _return_or_raise(
|
return _return_or_raise(
|
||||||
|
|||||||
@@ -19,9 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
|||||||
FusedMoEQuantConfig,
|
FusedMoEQuantConfig,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||||
FlashinferMoeBackend,
|
|
||||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||||
get_flashinfer_moe_backend,
|
|
||||||
swap_w13_to_w31,
|
swap_w13_to_w31,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
@@ -230,49 +228,6 @@ def select_unquantized_moe_backend(
|
|||||||
|
|
||||||
return _return_or_raise(requested_backend, moe_config, activation_format)
|
return _return_or_raise(requested_backend, moe_config, activation_format)
|
||||||
|
|
||||||
# Handle explicit FlashInfer FP16 configuration.
|
|
||||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP16"):
|
|
||||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP16:
|
|
||||||
if UnquantizedMoeBackend.FLASHINFER_TRTLLM in AVAILABLE_BACKENDS:
|
|
||||||
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_TRTLLM)
|
|
||||||
if UnquantizedMoeBackend.FLASHINFER_CUTLASS in AVAILABLE_BACKENDS:
|
|
||||||
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_CUTLASS)
|
|
||||||
|
|
||||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
|
||||||
# If user is explicit about backend, validate it.
|
|
||||||
fi_backend = get_flashinfer_moe_backend()
|
|
||||||
if fi_backend == FlashinferMoeBackend.CUTLASS:
|
|
||||||
backend = UnquantizedMoeBackend.FLASHINFER_CUTLASS
|
|
||||||
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
|
|
||||||
backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
f"FlashInfer MOE backend {fi_backend} "
|
|
||||||
"does not support unquantized MoE."
|
|
||||||
)
|
|
||||||
k_cls = backend_to_kernel_cls(backend)
|
|
||||||
return _return_or_raise(backend, moe_config, activation_format)
|
|
||||||
else:
|
|
||||||
# If the user is not explicit about the backend, try both.
|
|
||||||
for backend in [
|
|
||||||
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
|
||||||
UnquantizedMoeBackend.FLASHINFER_CUTLASS,
|
|
||||||
]:
|
|
||||||
k_cls = backend_to_kernel_cls(backend)
|
|
||||||
supported, reason = k_cls.is_supported_config(
|
|
||||||
k_cls, moe_config, None, None, activation_format
|
|
||||||
)
|
|
||||||
if supported:
|
|
||||||
logger.info_once(_make_log_backend(backend))
|
|
||||||
return backend, k_cls
|
|
||||||
else:
|
|
||||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
|
||||||
|
|
||||||
raise NotImplementedError(
|
|
||||||
"Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no "
|
|
||||||
"FlashInfer unquantized MoE backend supports the configuration."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Handle explicit AITER FP8 configuration.
|
# Handle explicit AITER FP8 configuration.
|
||||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||||
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import nixl_ep
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||||
from vllm import envs
|
from vllm.config import get_current_vllm_config
|
||||||
from vllm.distributed import get_ep_group
|
from vllm.distributed import get_ep_group
|
||||||
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
|
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -192,10 +192,11 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
|||||||
x = x.view((-1, hidden_dim))
|
x = x.view((-1, hidden_dim))
|
||||||
q_dtype = quant_config.quant_dtype
|
q_dtype = quant_config.quant_dtype
|
||||||
|
|
||||||
if envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm":
|
moe_backend = get_current_vllm_config().kernel_config.moe_backend
|
||||||
|
if moe_backend == "flashinfer_cutedsl":
|
||||||
logger.info_once(
|
logger.info_once(
|
||||||
"Skip quantization when using FlashInfer CUTEDSL(masked_gemm) "
|
"Skip quantization when using FlashInfer CUTEDSL "
|
||||||
"for ModelOptNvFp4FusedMoE."
|
"(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE."
|
||||||
)
|
)
|
||||||
q_dtype = None
|
q_dtype = None
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.envs as envs
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||||
align_fp4_moe_weights_for_fi,
|
align_fp4_moe_weights_for_fi,
|
||||||
@@ -15,10 +14,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
|||||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||||
swizzle_blockscale,
|
swizzle_blockscale,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
|
||||||
from vllm.utils.flashinfer import (
|
|
||||||
has_flashinfer_cutlass_fused_moe,
|
|
||||||
)
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||||
@@ -34,16 +29,6 @@ __all__ = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def is_flashinfer_fp4_cutlass_moe_available() -> bool:
|
|
||||||
"""Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used."""
|
|
||||||
return (
|
|
||||||
envs.VLLM_USE_FLASHINFER_MOE_FP4
|
|
||||||
and has_flashinfer_cutlass_fused_moe()
|
|
||||||
and current_platform.is_cuda()
|
|
||||||
and current_platform.has_device_capability(100)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def reorder_w1w3_to_w3w1(
|
def reorder_w1w3_to_w3w1(
|
||||||
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
|
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
|||||||
@@ -5,10 +5,8 @@ from typing import TYPE_CHECKING
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import envs
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||||
from vllm.platforms import current_platform
|
|
||||||
from vllm.utils.math_utils import round_up
|
from vllm.utils.math_utils import round_up
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -95,34 +93,6 @@ def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
|
|
||||||
backend_map = {
|
|
||||||
"throughput": FlashinferMoeBackend.CUTLASS,
|
|
||||||
"latency": FlashinferMoeBackend.TENSORRT_LLM,
|
|
||||||
"masked_gemm": FlashinferMoeBackend.CUTEDSL,
|
|
||||||
}
|
|
||||||
|
|
||||||
flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND
|
|
||||||
if flashinfer_moe_backend in backend_map:
|
|
||||||
if (
|
|
||||||
flashinfer_moe_backend == "latency"
|
|
||||||
and not current_platform.is_device_capability_family(100)
|
|
||||||
):
|
|
||||||
logger.info_once(
|
|
||||||
"Flashinfer TRTLLM MOE backend is only supported on "
|
|
||||||
"SM100 and later, using CUTLASS backend instead",
|
|
||||||
)
|
|
||||||
return FlashinferMoeBackend.CUTLASS
|
|
||||||
return backend_map[flashinfer_moe_backend]
|
|
||||||
elif current_platform.is_device_capability(90):
|
|
||||||
return FlashinferMoeBackend.CUTLASS
|
|
||||||
|
|
||||||
raise ValueError(
|
|
||||||
f"Unknown flashinfer moe backend: {flashinfer_moe_backend!r}. "
|
|
||||||
f"Expected one of {list(backend_map.keys())}."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool:
|
def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool:
|
||||||
# TODO(shuw@nvidia): Update when new backends are added.
|
# TODO(shuw@nvidia): Update when new backends are added.
|
||||||
backends_supporting_global_sf = (
|
backends_supporting_global_sf = (
|
||||||
|
|||||||
Reference in New Issue
Block a user