Deprecations for v0.23 and v0.24 (#44992)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -102,7 +102,7 @@ def test_async_tp_pass_correctness(
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
|
||||
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
|
||||
if (
|
||||
not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100)
|
||||
@@ -111,8 +111,6 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
|
||||
if not has_flashinfer():
|
||||
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
|
||||
|
||||
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass")
|
||||
|
||||
tp_size = 2
|
||||
if num_gpus_available < tp_size:
|
||||
pytest.skip(f"Need at least {tp_size} GPUs")
|
||||
@@ -126,6 +124,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
|
||||
"8",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--linear-backend",
|
||||
"flashinfer_cutlass",
|
||||
"--hf-overrides",
|
||||
json.dumps(NVFP4_HF_OVERRIDES),
|
||||
]
|
||||
|
||||
@@ -1226,10 +1226,6 @@ class VllmRunner:
|
||||
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
|
||||
return [req_output.outputs.data for req_output in req_outputs]
|
||||
|
||||
def reward(self, prompts: list[str]) -> list[list[float]]:
|
||||
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
|
||||
return [req_output.outputs.data for req_output in req_outputs]
|
||||
|
||||
def score(
|
||||
self,
|
||||
text_1: list[str] | str,
|
||||
|
||||
@@ -37,6 +37,7 @@ class TestConfig:
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_tokens: int
|
||||
moe_backend: str
|
||||
|
||||
|
||||
def make_fused_moe_layer(
|
||||
@@ -114,6 +115,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.data_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.kernel_config.moe_backend = test_config.moe_backend
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
@@ -250,7 +252,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
@pytest.mark.parametrize("intermediate_size", [256])
|
||||
@pytest.mark.parametrize("num_tokens", [256])
|
||||
@pytest.mark.parametrize("backend", ["latency", "throughput"])
|
||||
@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
|
||||
def test_eplb_fml(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
@@ -258,12 +260,8 @@ def test_eplb_fml(
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
num_tokens: int,
|
||||
backend: str,
|
||||
monkeypatch,
|
||||
moe_backend: str,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", backend)
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
|
||||
@@ -278,6 +276,7 @@ def test_eplb_fml(
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_tokens=num_tokens,
|
||||
moe_backend=moe_backend,
|
||||
)
|
||||
|
||||
distributed_run(
|
||||
|
||||
@@ -45,9 +45,10 @@ def test_config(llm: LLM):
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.reward(
|
||||
outputs = llm.encode(
|
||||
prompts,
|
||||
pooling_params=PoolingParams(use_activation=use_activation),
|
||||
pooling_task="token_classify",
|
||||
use_tqdm=False,
|
||||
)
|
||||
return torch.cat([x.outputs.data for x in outputs])
|
||||
|
||||
+1
-3
@@ -3,6 +3,4 @@
|
||||
model_name: "openai/gpt-oss-20b"
|
||||
metric_threshold: 0.568
|
||||
reasoning_effort: "low"
|
||||
server_args: "--tensor-parallel-size 2"
|
||||
env:
|
||||
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1"
|
||||
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass"
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
model_name: "openai/gpt-oss-20b"
|
||||
metric_threshold: 0.568
|
||||
reasoning_effort: "low"
|
||||
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_trtllm"
|
||||
@@ -3,6 +3,4 @@
|
||||
model_name: "openai/gpt-oss-20b"
|
||||
metric_threshold: 0.568
|
||||
reasoning_effort: "low"
|
||||
server_args: "--tensor-parallel-size 2"
|
||||
env:
|
||||
VLLM_MXFP4_USE_MARLIN: "1"
|
||||
server_args: "--tensor-parallel-size 2 --moe-backend marlin --linear-backend marlin"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# B200 model configurations for GPQA evaluation
|
||||
# Tests different environment variable combinations
|
||||
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
|
||||
gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml
|
||||
gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
|
||||
gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
|
||||
@@ -1,5 +1,5 @@
|
||||
# H100 model configurations for GPQA evaluation
|
||||
# Tests different environment variable combinations
|
||||
gpt-oss-20b-baseline.yaml
|
||||
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
|
||||
gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml
|
||||
gpt-oss-20b-marlin.yaml
|
||||
|
||||
@@ -30,9 +30,9 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
accuracy_threshold: 0.54 # Minimum expected accuracy
|
||||
num_questions: 1319 # Number of questions (default: full test set)
|
||||
num_fewshot: 5 # Few-shot examples from train set
|
||||
server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments
|
||||
server_args: "--max-model-len 4096 --tensor-parallel-size 2 --moe-backend flashinfer_cutlass" # Server arguments
|
||||
env: # Environment variables (optional)
|
||||
VLLM_USE_FLASHINFER_MOE_FP4: "1"
|
||||
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||
```
|
||||
|
||||
The `server_args` field accepts any arguments that can be passed to `vllm serve`.
|
||||
|
||||
@@ -1585,7 +1585,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
monkeypatch,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
@@ -1593,8 +1592,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
||||
"""
|
||||
set_random_seed(7)
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -1626,6 +1623,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
moe_backend="flashinfer_trtllm",
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
|
||||
@@ -1804,12 +1804,9 @@ def test_moe_layer(
|
||||
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
|
||||
|
||||
# TODO
|
||||
# VLLM_FLASHINFER_MOE_BACKEND=latency
|
||||
# VLLM_USE_FLASHINFER_MOE_FP16=1
|
||||
# VLLM_USE_FLASHINFER_MOE_FP8
|
||||
# VLLM_USE_FLASHINFER_MOE_FP4
|
||||
# VLLM_USE_FLASHINFER_MOE_INT4
|
||||
# TODO: cover FlashInfer MoE backends via moe_backend, e.g.
|
||||
# moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl
|
||||
# (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1.
|
||||
|
||||
parallel_config = ParallelConfig(
|
||||
pipeline_parallel_size=1,
|
||||
|
||||
@@ -123,7 +123,7 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeypatch):
|
||||
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
@@ -134,9 +134,8 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
# TRTLLM requires EP and does not support DP
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.use_dp = False
|
||||
@@ -168,7 +167,6 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
||||
mock_has_flashinfer,
|
||||
mock_is_supported_trtllm,
|
||||
mock_is_supported_cutlass,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||
and FlashInfer CUTLASS is available."""
|
||||
@@ -181,10 +179,9 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||
):
|
||||
# Enable FlashInfer via env var
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
# Select FlashInfer CUTLASS explicitly
|
||||
moe_config.moe_backend = "flashinfer_cutlass"
|
||||
# CUTLASS requires EP and does not support DP
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.use_dp = False
|
||||
@@ -241,37 +238,3 @@ def test_select_explicit_triton_backend(is_lora_enabled):
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
def test_select_explicit_triton_ignores_flashinfer_env(monkeypatch):
|
||||
"""Explicit triton backend should override FlashInfer env selection."""
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.is_lora_enabled = False
|
||||
moe_config.moe_backend = "triton"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
def test_select_lora_ignores_flashinfer_env(monkeypatch):
|
||||
"""LoRA path should still choose Triton even if FlashInfer env is on."""
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.is_lora_enabled = True
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
@@ -83,57 +83,55 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
|
||||
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
||||
@pytest.mark.parametrize("specialize_active_lora", [True, False])
|
||||
def test_gpt_oss_lora(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
gptoss20b_lora_files,
|
||||
mxfp4_use_marlin,
|
||||
specialize_active_lora,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enable_lora=True,
|
||||
max_loras=4,
|
||||
max_lora_rank=8,
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=2048,
|
||||
specialize_active_lora=specialize_active_lora,
|
||||
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
|
||||
cudagraph_specialize_lora=False,
|
||||
),
|
||||
)
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enable_lora=True,
|
||||
max_loras=4,
|
||||
max_lora_rank=8,
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=2048,
|
||||
specialize_active_lora=specialize_active_lora,
|
||||
moe_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||
linear_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
|
||||
cudagraph_specialize_lora=False,
|
||||
),
|
||||
)
|
||||
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
|
||||
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
||||
def test_gpt_oss_lora_tp2(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
gptoss20b_lora_files,
|
||||
fully_sharded_loras,
|
||||
mxfp4_use_marlin,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enable_lora=True,
|
||||
max_loras=2,
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=2048,
|
||||
tensor_parallel_size=2,
|
||||
gpu_memory_utilization=0.8,
|
||||
fully_sharded_loras=fully_sharded_loras,
|
||||
enable_expert_parallel=not fully_sharded_loras,
|
||||
compilation_config=vllm.config.CompilationConfig(
|
||||
cudagraph_specialize_lora=False,
|
||||
),
|
||||
)
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
enable_lora=True,
|
||||
max_loras=2,
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=2048,
|
||||
tensor_parallel_size=2,
|
||||
gpu_memory_utilization=0.8,
|
||||
fully_sharded_loras=fully_sharded_loras,
|
||||
enable_expert_parallel=not fully_sharded_loras,
|
||||
moe_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||
linear_backend="marlin" if mxfp4_use_marlin else "auto",
|
||||
compilation_config=vllm.config.CompilationConfig(
|
||||
cudagraph_specialize_lora=False,
|
||||
),
|
||||
)
|
||||
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
|
||||
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
|
||||
|
||||
@@ -106,7 +106,7 @@ def test_reward_models_using_activation(
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_activation = vllm_model.reward(example_prompts)
|
||||
wo_activation = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
@@ -114,7 +114,7 @@ def test_reward_models_using_activation(
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_activation = vllm_model.reward(example_prompts)
|
||||
w_activation = vllm_model.token_classify(example_prompts)
|
||||
|
||||
for wo, w in zip(wo_activation, w_activation):
|
||||
wo = torch.tensor(wo)
|
||||
|
||||
@@ -107,7 +107,7 @@ def test_prm_models(
|
||||
pytest.skip("CPU only supports V1")
|
||||
|
||||
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.reward(math_step_prompts)
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
hf_model = step_reward_patch_hf_model(hf_model)
|
||||
@@ -146,7 +146,7 @@ def test_prm_models_with_golden_outputs(
|
||||
pytest.skip(f"No available golden outputs for {model}.")
|
||||
|
||||
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.reward(math_step_prompts)
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
||||
|
||||
|
||||
@@ -133,11 +133,11 @@ def test_nvfp4(vllm_runner, model, eager, backend):
|
||||
not current_platform.is_rocm(),
|
||||
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
|
||||
)
|
||||
def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)
|
||||
def test_nvfp4_moe(vllm_runner, model, backend):
|
||||
with vllm_runner(
|
||||
model,
|
||||
moe_backend=backend,
|
||||
linear_backend=backend,
|
||||
load_format="dummy",
|
||||
hf_overrides={"num_hidden_layers": 2},
|
||||
) as llm:
|
||||
|
||||
@@ -185,8 +185,11 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1")
|
||||
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=["--moe-backend=flashinfer_trtllm"],
|
||||
)
|
||||
|
||||
|
||||
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
Reference in New Issue
Block a user