[Compile] Remove torch.compile-based sequence parallelism and AsyncTP (#60517)

This commit is contained in:
Woosuk Kwon
2026-10-08 08:28:23 -07:00
committed by GitHub
parent 072d49bb6e
commit 194bd72be7
34 changed files with 32 additions and 4797 deletions
@@ -1,270 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
from contextlib import nullcontext
import pytest
from tests.compile.fusions_e2e.common import FUSION_LOG_PATTERNS
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.utils import (
compare_two_settings,
create_new_process_for_each_test,
multi_gpu_test,
)
from vllm.config import (
CompilationMode,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.flashinfer import has_flashinfer
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
NVFP4_HF_OVERRIDES = {
"num_hidden_layers": 4,
"hidden_size": 512,
"intermediate_size": 800,
"num_attention_heads": 4,
"num_key_value_heads": 1,
}
@multi_gpu_test(num_gpus=2)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
def test_rocm_async_tp_bf16_output_correctness(caplog_mp_spawn):
"""Trained-model outputs must agree with unfused TP, including padded decode."""
# A snapshot path pins both servers and the comparison helper's tokenizer.
model = hf_api().snapshot_download(
"Qwen/Qwen3-0.6B", revision="c1899de289a04d12100db370d81485cdf75e47ca"
)
common_args = [
"--dtype",
"bfloat16",
"--load-format",
"auto",
"--tensor-parallel-size",
"2",
"--distributed-executor-backend",
"mp",
"--max-model-len",
"512",
"--max-num-batched-tokens",
"512",
"--max-num-seqs",
"8",
"--kv-cache-memory-bytes",
str(64 * 1024**2),
"--gpu-memory-utilization",
"0.01",
"--attention-backend",
"TRITON_ATTN",
"--disable-custom-all-reduce",
]
def settings(enabled):
return [
*common_args,
"--compilation-config",
json.dumps(
{
"mode": CompilationMode.VLLM_COMPILE,
"cudagraph_mode": "NONE",
"splitting_ops": [],
"inductor_compile_config": {"force_disable_caches": True},
"pass_config": {
"enable_sp": enabled,
"fuse_gemm_comms": enabled,
"fuse_allreduce_rms": False,
"sp_min_token_num": 1,
},
}
),
]
env = {"VLLM_ROCM_USE_AITER": "0", "VLLM_DISABLE_COMPILE_CACHE": "1"}
with caplog_mp_spawn(logging.DEBUG) as logs:
compare_two_settings(
model,
settings(False),
settings(True),
env1=env,
env2=env,
method="generate",
force_v1_runner=True,
)
matches = [
int(count) for count in FUSION_LOG_PATTERNS["async_tp"].findall(logs.text)
]
print(f"AsyncTP rewrite counts: {matches}")
assert len(matches) >= 2 and all(count > 0 for count in matches), (
f"Expected positive AsyncTP rewrites on both TP workers, found {matches}"
)
@create_new_process_for_each_test()
@pytest.mark.parametrize(
"model_id",
["meta-llama/Llama-3.2-1B-Instruct", "RedHatAI/Llama-3.2-1B-Instruct-FP8"],
)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("async_tp_enabled", [True])
@pytest.mark.parametrize("distributed_backend", ["mp"])
def test_async_tp_pass_correctness(
model_id: str,
tp_size: int,
async_tp_enabled: bool,
distributed_backend: str,
num_gpus_available: int,
monkeypatch,
):
# Disable FlashInfer FP8 scaled_mm kernel as it is incompatible with
# async TP patterns. No-op on H100 (kernel requires CC >= 100).
monkeypatch.setenv("VLLM_DISABLED_KERNELS", "FlashInferFP8ScaledMMLinearKernel")
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_transformers_version(on_fail="skip")
model_info.check_available_online(on_fail="skip")
pp_size = 1
if num_gpus_available < tp_size:
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
common_args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"8",
]
compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
"compile_sizes": [2, 4, 8],
"splitting_ops": [],
"pass_config": {"fuse_gemm_comms": async_tp_enabled},
}
async_tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
distributed_backend,
"--compilation_config",
json.dumps(compilation_config),
]
tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
"mp",
]
compare_two_settings(
model_id,
async_tp_args,
tp_args,
method="generate",
force_v1_runner=True,
)
@create_new_process_for_each_test()
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, caplog_mp_spawn):
is_blackwell = current_platform.is_cuda() and (
current_platform.is_device_capability_family(100)
)
is_rocm_cdna = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_cdna
is_rocm_cdna = on_cdna()
if not (is_blackwell or is_rocm_cdna):
pytest.skip("NVFP4 AsyncTP requires Blackwell or ROCm CDNA")
if is_blackwell and not has_flashinfer():
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
tp_size = 2
if num_gpus_available < tp_size:
pytest.skip(f"Need at least {tp_size} GPUs")
common_args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"8",
"--load-format",
"dummy",
*(["--linear-backend", "flashinfer_cutlass"] if is_blackwell else []),
"--hf-overrides",
json.dumps(NVFP4_HF_OVERRIDES),
]
compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
"compile_sizes": [2, 4, 8],
"splitting_ops": [],
"pass_config": {
"enable_sp": True,
"fuse_gemm_comms": True,
"fuse_allreduce_rms": False,
"sp_min_token_num": 1,
},
}
if is_rocm_cdna:
compilation_config["inductor_compile_config"] = {"force_disable_caches": True}
async_tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
"mp",
"--compilation_config",
json.dumps(compilation_config),
]
tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
"mp",
]
env = {"VLLM_DISABLE_COMPILE_CACHE": "1"} if is_rocm_cdna else None
log_context = caplog_mp_spawn(logging.DEBUG) if is_rocm_cdna else nullcontext()
with log_context as logs:
compare_two_settings(
NVFP4_MODEL_ID,
async_tp_args,
tp_args,
env1=env,
env2=env,
method="generate",
force_v1_runner=True,
)
if is_rocm_cdna:
assert logs is not None
matches_by_rank = {
rank: [
int(count)
for line in logs.text.splitlines()
if line.startswith(f"(Worker_TP{rank} pid=")
for count in FUSION_LOG_PATTERNS["async_tp"].findall(line)
]
for rank in range(tp_size)
}
assert all(
any(count > 0 for count in matches) for matches in matches_by_rank.values()
), (
"Expected positive NVFP4 AsyncTP rewrites on both TP workers, "
f"found {matches_by_rank}"
)
@@ -1,414 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""
import json
import os
from dataclasses import dataclass
from typing import Literal, NamedTuple
import pytest
from vllm.config.compilation import CompilationMode
from vllm.config.model import RunnerOption
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from ...models.registry import HF_EXAMPLE_MODELS, _HfExamplesInfo
from ...utils import compare_all_settings, create_new_process_for_each_test
logger = init_logger("test_sequence_parallel")
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
NVFP4_MODEL_INFO = _HfExamplesInfo(NVFP4_MODEL_ID)
class ParallelSetup(NamedTuple):
tp_size: int
pp_size: int
fuse_norm_quant: bool
fuse_act_quant: bool
eager_mode: bool
class SPTestOptions(NamedTuple):
multi_node_only: bool
load_format: str | None = None
model_info: _HfExamplesInfo | None = None
@dataclass
class SPTestSettings:
parallel_setups: list[ParallelSetup]
distributed_backends: list[str]
runner: RunnerOption
test_options: SPTestOptions
@staticmethod
def fast(
*,
tp_base: int = 2,
pp_base: int = 1,
runner: RunnerOption = "auto",
multi_node_only: bool = False,
load_format: str | None = None,
):
parallel_setups = [
ParallelSetup(
tp_size=tp_base,
pp_size=pp_multiplier * pp_base,
fuse_norm_quant=False,
fuse_act_quant=False,
eager_mode=False,
)
for pp_multiplier in [1, 2]
]
return SPTestSettings(
parallel_setups=parallel_setups,
distributed_backends=["mp", "ray"],
runner=runner,
test_options=SPTestOptions(
multi_node_only=multi_node_only, load_format=load_format
),
)
@staticmethod
def fp8_quant(
*,
tp_base: int = 2,
pp_base: int = 1,
runner: RunnerOption = "auto",
multi_node_only: bool = False,
load_format: str | None = None,
):
parallel_setups = []
for fusion_val in [False, True]:
parallel_setups.append(
ParallelSetup(
tp_size=tp_base,
pp_size=pp_base,
fuse_norm_quant=fusion_val,
fuse_act_quant=fusion_val,
eager_mode=True,
)
)
return SPTestSettings(
parallel_setups=parallel_setups,
distributed_backends=["mp", "ray"],
runner=runner,
test_options=SPTestOptions(
multi_node_only=multi_node_only, load_format=load_format
),
)
def iter_params(self, model_id: str):
opts = self.test_options
for parallel_setup in self.parallel_setups:
for backend in self.distributed_backends:
yield (
model_id,
parallel_setup,
backend,
self.runner,
opts,
)
def _build_sp_args(
model_id: str,
parallel_setup: ParallelSetup,
distributed_backend: str,
runner: RunnerOption,
test_options: SPTestOptions,
num_gpus_available: int,
use_inductor_graph_partition: bool,
fuse_gemm_comms: bool,
enable_prompt_embeds: bool,
*,
is_multimodal: bool,
) -> tuple[list[str], list[str]] | None:
(
tp_size,
pp_size,
fuse_norm_quant,
fuse_act_quant,
eager_mode,
) = parallel_setup
multi_node_only = test_options.multi_node_only
load_format = test_options.load_format
model_info = test_options.model_info or HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_transformers_version(on_fail="skip")
trust_remote_code = model_info.trust_remote_code
tokenizer_mode = model_info.tokenizer_mode
hf_overrides = dict(model_info.hf_overrides)
require_embed_inputs = model_info.require_embed_inputs
if load_format == "dummy":
# Avoid OOM
text_overrides = {
"num_hidden_layers": 4,
"hidden_size": 512,
"intermediate_size": 800,
"num_attention_heads": 4,
"num_key_value_heads": 1,
}
if is_multimodal:
hf_overrides.update({"text_config": text_overrides})
else:
hf_overrides.update(text_overrides)
else:
model_info.check_available_online(on_fail="skip")
if num_gpus_available < tp_size * pp_size:
return None
if VLLM_MULTI_NODE and distributed_backend == "mp":
return None
if multi_node_only and not VLLM_MULTI_NODE:
return None
common_args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"8",
]
if eager_mode:
common_args.append("-cc.cudagraph_mode=none")
if runner != "auto":
common_args.extend(["--runner", runner])
if trust_remote_code:
common_args.append("--trust-remote-code")
if tokenizer_mode:
common_args.extend(["--tokenizer-mode", tokenizer_mode])
if load_format:
common_args.extend(["--load-format", load_format])
if hf_overrides:
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
if require_embed_inputs:
common_args.extend(
[
"--skip-tokenizer-init",
"--enable-prompt-embeds",
"--enable-mm-embeds",
]
)
elif enable_prompt_embeds:
common_args.append("--enable-prompt-embeds")
compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
"compile_sizes": [4, 8],
"pass_config": {
"enable_sp": True,
"fuse_gemm_comms": fuse_gemm_comms,
"fuse_norm_quant": fuse_norm_quant,
"fuse_act_quant": fuse_act_quant,
"fuse_allreduce_rms": False,
"eliminate_noops": True,
"sp_min_token_num": 0,
},
"use_inductor_graph_partition": use_inductor_graph_partition,
}
if not use_inductor_graph_partition:
compilation_config["splitting_ops"] = []
tp_sp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--pipeline-parallel-size",
str(pp_size),
"--distributed-executor-backend",
distributed_backend,
"--compilation_config",
json.dumps(compilation_config),
]
tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
"mp",
]
return tp_args, tp_sp_args
def _compare_sp_settings(
model_id: str,
settings: list[tuple[list[str], list[str]]],
*,
method: Literal["generate", "encode"],
) -> None:
if not settings:
pytest.skip("No supported sequence-parallel configurations")
settings_by_tp_args: dict[tuple[str, ...], list[list[str]]] = {}
for tp_args, tp_sp_args in settings:
settings_by_tp_args.setdefault(tuple(tp_args), []).append(tp_sp_args)
for tp_args_key, grouped_tp_sp_args in settings_by_tp_args.items():
all_args = [list(tp_args_key), *grouped_tp_sp_args]
compare_all_settings(
model_id,
all_args,
[None] * len(all_args),
method=method,
force_v1_runner=True,
)
SP_TEXT_GENERATION_MODELS = {
# [Decoder-only]
"hmellor/tiny-random-LlamaForCausalLM": SPTestSettings.fast(),
"RedHatAI/Llama-3.2-1B-Instruct-FP8": SPTestSettings.fp8_quant(),
}
@pytest.mark.parametrize(
("model_id", "settings"),
[
pytest.param(model_id, settings, id=model_id)
for model_id, settings in SP_TEXT_GENERATION_MODELS.items()
],
)
@create_new_process_for_each_test()
def test_tp_sp_generation(
model_id: str,
settings: SPTestSettings,
num_gpus_available,
):
# Skip FP8 SP-only test on sm89 (compute capability 8.9).
# An unknown capability (None) is left to fail in the test body rather than
# being silently skipped here.
capability = current_platform.get_device_capability()
if "fp8" in model_id.lower() and capability is not None and capability < (9, 0):
pytest.skip("FP8 reduction support begins with sm90 capable devices.")
graph_partition_options = [False]
if is_torch_equal_or_newer("2.9.0.dev"):
graph_partition_options.append(True)
comparisons = []
for _, parallel_setup, backend, runner, test_options in settings.iter_params(
model_id
):
for use_inductor_graph_partition in graph_partition_options:
comparison = _build_sp_args(
model_id,
parallel_setup,
backend,
runner,
test_options,
num_gpus_available,
use_inductor_graph_partition,
fuse_gemm_comms=False, # TODO: enable async TP
enable_prompt_embeds=False,
is_multimodal=False,
)
if comparison is not None:
comparisons.append(comparison)
_compare_sp_settings(model_id, comparisons, method="generate")
# Focused regression test for the SP + prompt_embeds graph-rewrite path.
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
# we don't double the matrix of `test_tp_sp_generation` above.
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
ParallelSetup(
tp_size=2,
pp_size=pp_size,
fuse_norm_quant=False,
fuse_act_quant=False,
eager_mode=False,
)
for pp_size in [1, 2]
]
@create_new_process_for_each_test()
def test_tp_sp_generation_prompt_embeds(
num_gpus_available,
):
model_id = "hmellor/tiny-random-LlamaForCausalLM"
graph_partition_options = [False]
if is_torch_equal_or_newer("2.9.0.dev"):
graph_partition_options.append(True)
comparisons = []
for parallel_setup in SP_PROMPT_EMBEDS_PARALLEL_SETUPS:
for use_inductor_graph_partition in graph_partition_options:
comparison = _build_sp_args(
model_id,
parallel_setup,
distributed_backend="mp",
runner="auto",
test_options=SPTestOptions(multi_node_only=False, load_format=None),
num_gpus_available=num_gpus_available,
use_inductor_graph_partition=use_inductor_graph_partition,
fuse_gemm_comms=False,
enable_prompt_embeds=True,
is_multimodal=False,
)
if comparison is not None:
comparisons.append(comparison)
_compare_sp_settings(model_id, comparisons, method="generate")
@create_new_process_for_each_test()
def test_tp_sp_nvfp4_generation(num_gpus_available: int):
is_blackwell = current_platform.is_cuda() and (
current_platform.is_device_capability_family(100)
)
is_rocm_cdna = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_cdna
is_rocm_cdna = on_cdna()
if not (is_blackwell or is_rocm_cdna):
pytest.skip("NVFP4 sequence parallelism requires Blackwell or ROCm CDNA")
comparison = _build_sp_args(
NVFP4_MODEL_ID,
ParallelSetup(
tp_size=2,
pp_size=1,
fuse_norm_quant=True,
fuse_act_quant=True,
eager_mode=True,
),
"mp",
"auto",
SPTestOptions(
multi_node_only=False,
load_format="dummy",
model_info=NVFP4_MODEL_INFO,
),
num_gpus_available,
use_inductor_graph_partition=False,
fuse_gemm_comms=False,
enable_prompt_embeds=False,
is_multimodal=False,
)
_compare_sp_settings(
NVFP4_MODEL_ID,
[] if comparison is None else [comparison],
method="generate",
)
-6
View File
@@ -21,8 +21,6 @@ class Matches(NamedTuple):
# distributed
ar_rms_fusion: int = 0
aiter_ar_rms_fusion: int = 0
sequence_parallel: int = 0
async_tp: int = 0
class ModelFusionInfo(NamedTuple):
@@ -143,8 +141,4 @@ FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
"aiter_ar_rms_fusion": re.compile(
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
),
"sequence_parallel": re.compile(
r"sequence_parallelism.py:\d+] Replaced (\d+) patterns"
),
"async_tp": re.compile(r"collective_fusion.py:\d+] Replaced (\d+) patterns"),
}
+2 -71
View File
@@ -170,26 +170,8 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
for match_name in matches_check:
log_matches = list(int(ms) for ms in log_matches_dict[match_name])
# AR+RMS skips the largest range; SP skips the smallest.
# When both are enabled, AR+RMS activation count is
# model-dependent (hidden_size affects threshold), so derive
# from log data.
if (
match_name == "ar_rms_fusion"
and "sequence_parallel" in matches_check
and num_compile_ranges >= 2
):
assert (
len(log_matches) >= tp_size and len(log_matches) % tp_size == 0
), (
f"Expected multiple of {tp_size} ar_rms log entries, "
f"found {len(log_matches)}"
)
num_ranges_activated = len(log_matches) // tp_size
elif (
match_name in ("ar_rms_fusion", "sequence_parallel")
and num_compile_ranges >= 2
):
# AR+RMS skips the largest range.
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
num_ranges_activated = num_compile_ranges - 1
else:
num_ranges_activated = num_compile_ranges
@@ -233,44 +215,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
f"Expecting at least {expected_matches - matches.ar_rms_fusion} "
f"where ar+rms+quant was activated"
)
elif (
match_name == "async_tp"
and "sequence_parallel" in matches_check
and num_compile_ranges >= 2
):
# AsyncTP only finds patterns on ranges where SP ran.
n_sp_ranges = num_compile_ranges - 1
assert (
sum(m == expected_matches for m in log_matches)
== tp_size * n_sp_ranges
), (
f"Expecting {expected_matches} async_tp on "
f"{tp_size * n_sp_ranges} SP-range entries, "
f"found: {log_matches}"
)
assert sum(m == 0 for m in log_matches) == tp_size, (
f"Expecting 0 async_tp on {tp_size} small-range entries "
f"(no SP), found: {log_matches}"
)
elif (
match_name == "ar_rms_fusion"
and "sequence_parallel" in matches_check
and num_compile_ranges >= 2
):
# SP consumes allreduce patterns first, so AR+RMS finds
# full matches only on the smallest range (no SP).
assert sum(m == expected_matches for m in log_matches) == tp_size, (
f"Expecting {expected_matches} ar_rms on "
f"{tp_size} small-range entries, found: {log_matches}"
)
assert sum(m == 0 for m in log_matches) == tp_size * (
num_ranges_activated - 1
), (
f"Expecting 0 ar_rms on "
f"{tp_size * (num_ranges_activated - 1)} large-range "
f"entries (SP took precedence), found: {log_matches}"
)
elif match_name == "act_quant_fusion":
actual_match = match_table.get("activation_quant_fusion_pass", 0)
assert actual_match == expected_matches * n_expected, (
@@ -305,17 +249,4 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
f"(found {len(log_matches)}) in:\n {log_holder.text}"
)
if match_name == "sequence_parallel" and num_compile_ranges >= 2:
log_matches = re.findall(
r"pass_manager.py:\d+] Skipping "
r".*SequenceParallelismPass.* with compile range",
log_holder.text,
)
n_expected = tp_size * (num_compile_ranges - num_ranges_activated)
assert len(log_matches) == n_expected, (
f'Could not find {n_expected} "Skipping SequenceParallelismPass" '
f"(found {len(log_matches)}) in:\n {log_holder.text}"
)
return run
-19
View File
@@ -73,8 +73,6 @@ llama3_8b = ModelFusionInfo(
matches=lambda n_layers: Matches(
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 4,
),
)
@@ -85,8 +83,6 @@ llama3_8b_fp8 = ModelFusionInfo(
act_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 4,
),
)
@@ -96,8 +92,6 @@ llama3_8b_fp4 = ModelFusionInfo(
act_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 4,
),
)
@@ -127,8 +121,6 @@ llama4_scout_fp8 = ModelFusionInfo(
rms_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2,
sequence_parallel=n_layers * 2,
async_tp=n_layers * 2 - 1,
),
)
@@ -138,8 +130,6 @@ llama4_scout_fp4 = ModelFusionInfo(
matches=lambda n_layers: Matches(
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2,
sequence_parallel=n_layers * 2,
async_tp=n_layers * 2 - 1,
),
)
@@ -149,8 +139,6 @@ qwen3_a3b = ModelFusionInfo(
norm_rope_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 2,
),
)
@@ -161,8 +149,6 @@ qwen3_a3b_fp8 = ModelFusionInfo(
norm_rope_fusion=n_layers,
attn_quant_fusion=0, # attn + group quant not supported
ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 2,
),
)
@@ -192,9 +178,6 @@ deepseek_v3_fp8 = ModelFusionInfo(
# MLA attn + per-group FP8 quant
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
# TODO
# sequence_parallel= n_layers * 2 + 1,
# async_tp=n_layers * 2,
),
)
@@ -213,8 +196,6 @@ gpt_oss_20b = ModelFusionInfo(
matches=lambda n_layers: Matches(
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 2,
),
model_kwargs=(
{"quantization_config": {"moe": {"activation": "mxfp8"}}}
@@ -1,339 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
from vllm.config import PassConfig
from vllm.platforms import current_platform
from ...utils import multi_gpu_test
from .common import (
INDUCTOR_GRAPH_PARTITION,
AttentionBackendCase,
Matches,
custom_ops_combos,
is_blackwell,
nvfp4_kernel_exposes_input_quant_key,
)
from .models import (
FLASHINFER_ATTN,
TRITON_ATTN,
llama3_8b,
llama3_8b_fp4,
llama3_8b_fp8,
llama4_scout_fp8,
qwen3_a3b,
)
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp8, llama4_scout_fp8],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
def test_tp2_async_tp_fp8_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
matches = matches_fn(n_layers)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_norm_quant=True,
fuse_act_quant=False,
fuse_attn_quant=True,
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=False,
# Override threshold for testing (models have small hidden_size)
sp_min_token_num=512,
),
)
matches_check = [
"rms_quant_fusion",
"norm_rope_fusion",
"attn_quant_fusion",
"sequence_parallel",
"async_tp",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp4],
)
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
def test_tp2_async_tp_nvfp4_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
if nvfp4_kernel_exposes_input_quant_key():
pytest.skip(
"NVFP4 kernel exposes input_quant_key; manual fusion fires "
"instead of compiler pass-based fusion"
)
# NVFP4 currently wires the all-gather + GEMM path only.
matches = matches_fn(n_layers)._replace(async_tp=n_layers * 2)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_act_quant=False,
fuse_attn_quant=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=False,
# Override threshold for testing (models have small hidden_size)
sp_min_token_num=512,
),
)
matches_check = [
"attn_quant_fusion",
"sequence_parallel",
"async_tp",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b, qwen3_a3b],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
def test_tp2_async_tp_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
matches = matches_fn(n_layers)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=False,
# Override threshold for testing (models have small hidden_size)
sp_min_token_num=512,
),
)
matches_check = [
"norm_rope_fusion",
"sequence_parallel",
"async_tp",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp8, llama4_scout_fp8],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
def test_tp2_sp_ar_rms_fp8_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
matches = matches_fn(n_layers)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_norm_quant=True,
fuse_act_quant=False,
fuse_attn_quant=True,
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=True,
# Override threshold for testing (models have small hidden_size)
sp_min_token_num=512,
),
)
matches_check = [
"rms_quant_fusion",
"norm_rope_fusion",
"attn_quant_fusion",
"ar_rms_fusion",
"sequence_parallel",
"async_tp",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b, qwen3_a3b],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
def test_tp2_sp_ar_rms_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
matches = matches_fn(n_layers)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=True,
# Override threshold for testing (models have small hidden_size)
sp_min_token_num=512,
),
)
matches_check = [
"norm_rope_fusion",
"ar_rms_fusion",
"sequence_parallel",
"async_tp",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
)
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bounded ROCm model coverage for actual sequence-parallel AsyncTP rewrites."""
import pytest
from tests.utils import multi_gpu_test
from vllm.config import PassConfig
from vllm.platforms import current_platform
from .common import INDUCTOR_GRAPH_PARTITION, custom_ops_combos
from .models import TRITON_ATTN, llama3_8b
@multi_gpu_test(num_gpus=2)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.parametrize("custom_ops", tuple(custom_ops_combos("rms_norm")))
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
def test_tp2_rocm_async_tp_bf16_fusions(
inductor_graph_partition, custom_ops, attn_backend, run_e2e_fusion_test
):
n_layers = 4
model_kwargs = dict(
hf_overrides=dict(
num_hidden_layers=n_layers,
hidden_size=512,
intermediate_size=1024,
num_attention_heads=8,
num_key_value_heads=2,
head_dim=64,
),
load_format="dummy",
max_model_len=1024,
kv_cache_memory_bytes=64 * 1024**2,
gpu_memory_utilization=0.01,
max_num_batched_tokens=1024,
max_num_seqs=32,
)
run_e2e_fusion_test(
llama3_8b.model_name,
llama3_8b.matches(n_layers),
model_kwargs,
attn_backend,
dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=False,
sp_min_token_num=512,
),
),
["norm_rope_fusion", "sequence_parallel", "async_tp"],
tp_size=2,
)
@@ -1,60 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm MoE coverage for the TP2 GEMM/collective fusion contract."""
import pytest
from tests.utils import multi_gpu_test
from vllm.config import PassConfig
from vllm.platforms import current_platform
from .common import INDUCTOR_GRAPH_PARTITION
from .models import TRITON_ATTN, qwen3_a3b
@multi_gpu_test(num_gpus=2)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION[:1])
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
def test_tp2_rocm_moe_async_tp_bf16_fusions(
inductor_graph_partition, attn_backend, run_e2e_fusion_test
):
"""MoE dense projections retain the Qwen SP/AsyncTP rewrite counts."""
n_layers = 4
run_e2e_fusion_test(
qwen3_a3b.model_name,
qwen3_a3b.matches(n_layers),
dict(
hf_overrides=dict(
num_hidden_layers=n_layers,
hidden_size=512,
intermediate_size=1024,
moe_intermediate_size=128,
num_attention_heads=8,
num_key_value_heads=2,
head_dim=64,
num_experts=8,
num_experts_per_tok=2,
),
load_format="dummy",
max_model_len=1024,
kv_cache_memory_bytes=64 * 1024**2,
gpu_memory_utilization=0.01,
max_num_batched_tokens=1024,
max_num_seqs=32,
),
attn_backend,
dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=["-rms_norm"],
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
enable_sp=True,
fuse_gemm_comms=True,
fuse_allreduce_rms=False,
sp_min_token_num=512,
),
),
["norm_rope_fusion", "sequence_parallel", "async_tp"],
tp_size=2,
)
@@ -1,394 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.envs as envs
from tests.compile.backend import TestBackend
from tests.utils import (
multi_gpu_test,
)
from vllm.compilation.passes.fusion.collective_fusion import AsyncTPPass
from vllm.config import (
CompilationConfig,
DeviceConfig,
ModelConfig,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.utils import Range
from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_file_store_init_method
from vllm.utils.system_utils import update_environment_variables
from vllm.utils.torch_utils import set_random_seed
DEVICE_TYPE = current_platform.device_type
FP8_DTYPE = current_platform.fp8_dtype()
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
class TestMMRSModel(torch.nn.Module):
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
super().__init__()
self.hidden_size = hidden_size
self.dtype = dtype
self.gate_proj = torch.nn.Parameter(
torch.empty((self.hidden_size * 2, hidden_size)), requires_grad=False
)
# Initialize weights
torch.nn.init.normal_(self.gate_proj, std=0.02)
def forward(self, hidden_states):
"""Forward pass implementing the mm + reduce scatter in the FX graph."""
# Reshape input
view = hidden_states.reshape(-1, self.hidden_size)
# matrix multiplication
permute = self.gate_proj.permute(1, 0)
mm = torch.mm(view, permute)
reduce_scatter = tensor_model_parallel_reduce_scatter(mm, dim=0)
return reduce_scatter
def ops_in_model_before(self):
return [torch.ops.vllm.reduce_scatter.default]
def ops_in_model_after(self):
return [torch.ops.symm_mem.fused_matmul_reduce_scatter.default]
class TestAGMMModel(torch.nn.Module):
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
super().__init__()
self.hidden_size = hidden_size
self.dtype = dtype
self.weight = torch.nn.Parameter(
torch.empty((hidden_size, hidden_size)), requires_grad=False
)
# Initialize weights
torch.nn.init.normal_(self.weight, std=0.02)
def forward(self, hidden_states):
"""Forward pass implementing the mm + all gather in the FX graph."""
# Reshape input
view = hidden_states.reshape(-1, self.hidden_size)
all_gather = tensor_model_parallel_all_gather(view, dim=0)
permute = self.weight.permute(1, 0)
mm = torch.mm(all_gather, permute)
return mm
def ops_in_model_before(self):
return [torch.ops.vllm.all_gather.default]
def ops_in_model_after(self):
return [torch.ops.symm_mem.fused_all_gather_matmul.default]
class _BaseScaledMMModel(torch.nn.Module):
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
super().__init__()
self.hidden_size = hidden_size
self.dtype = dtype
self.weight = (
torch.empty([hidden_size, hidden_size], dtype=FP8_DTYPE)
.contiguous()
.transpose(0, 1)
)
# Initialize scale_b for _scaled_mm.
self.scale_b = torch.ones(1, self.hidden_size, dtype=torch.float32)
class TestScaledMMRSModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""Forward pass implementing the scaled_mm + reduce scatter in the FX graph."""
fp8_input = input.to(FP8_DTYPE)
scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32)
scaled_mm = torch._scaled_mm(
fp8_input,
self.weight,
scale_a=scale_a,
scale_b=self.scale_b,
out_dtype=self.dtype,
)
reduce_scatter = tensor_model_parallel_reduce_scatter(scaled_mm, dim=0)
return reduce_scatter
def ops_in_model_before(self):
return [torch.ops.vllm.reduce_scatter.default]
def ops_in_model_after(self):
return [torch.ops.vllm.patched_fused_scaled_matmul_reduce_scatter.default]
class TestAGScaledMMModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""Forward pass implementing the all gather + scaled_mm in the FX graph."""
# Reshape input
fp8_input = input.to(FP8_DTYPE)
all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0)
scale_a = torch.ones(all_gather.shape[0], 1, dtype=torch.float32)
scaled_mm = torch._scaled_mm(
all_gather,
self.weight,
scale_a=scale_a,
scale_b=self.scale_b,
out_dtype=self.dtype,
)
return scaled_mm
def ops_in_model_before(self):
return [torch.ops.vllm.all_gather.default]
def ops_in_model_after(self):
return [torch.ops.symm_mem.fused_all_gather_scaled_matmul.default]
class TestCutlassScaledMMRSModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""Forward pass implementing the cutlass_scaled_mm + reduce scatter
in the FX graph
"""
fp8_input = input.to(FP8_DTYPE)
scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32)
mm_out = torch.empty(
(fp8_input.shape[0], self.weight.shape[1]),
dtype=self.dtype,
device=input.device,
)
torch.ops._C.cutlass_scaled_mm(
mm_out, fp8_input, self.weight, scale_a, self.scale_b, None
)
reduce_scatter = tensor_model_parallel_reduce_scatter(mm_out, dim=0)
return reduce_scatter
def ops_in_model_before(self):
return [torch.ops.vllm.reduce_scatter.default]
def ops_in_model_after(self):
return [torch.ops.vllm.patched_fused_scaled_matmul_reduce_scatter.default]
class TestAGCutlassScaledMMModel(_BaseScaledMMModel):
def forward(self, input: torch.Tensor):
"""Forward pass implementing the all gather + cutlass_scaled_mm
in the FX graph
"""
# Reshape input
fp8_input = input.to(FP8_DTYPE)
all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0)
scale_a = torch.ones(all_gather.shape[0], 1, dtype=torch.float32)
mm_out = torch.empty(
(all_gather.shape[0], self.weight.shape[1]),
dtype=self.dtype,
device=all_gather.device,
)
torch.ops._C.cutlass_scaled_mm(
mm_out, all_gather, self.weight, scale_a, self.scale_b, None
)
return mm_out
def ops_in_model_before(self):
return [torch.ops.vllm.all_gather.default]
def ops_in_model_after(self):
return [torch.ops.symm_mem.fused_all_gather_scaled_matmul.default]
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"test_model",
[
TestMMRSModel,
TestAGMMModel,
TestScaledMMRSModel,
TestAGScaledMMModel,
pytest.param(
TestCutlassScaledMMRSModel,
marks=pytest.mark.skipif(
not hasattr(torch.ops._C, "cutlass_scaled_mm"),
reason="Requires cutlass_scaled_mm",
),
),
pytest.param(
TestAGCutlassScaledMMModel,
marks=pytest.mark.skipif(
not hasattr(torch.ops._C, "cutlass_scaled_mm"),
reason="Requires cutlass_scaled_mm",
),
),
],
)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [16])
@pytest.mark.parametrize("hidden_size", [16])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("dynamic", [True, False])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_async_tp_pass_replace(
test_model: str,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
dynamic: bool,
):
num_processes = 2
distributed_init_method = get_file_store_init_method()
def run_torch_spawn(fn, nprocs):
# need to use torch.mp.spawn otherwise will have problems with
# torch.distributed and cuda
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
test_model,
batch_size,
seq_len,
hidden_size,
dtype,
dynamic,
distributed_init_method,
),
nprocs=nprocs,
)
run_torch_spawn(async_tp_pass_on_test_model, num_processes)
def test_async_tp_pass_requires_full_graph_compilation():
vllm_config = VllmConfig()
vllm_config.compilation_config.use_inductor_graph_partition = False
vllm_config.compilation_config.splitting_ops = [
"vllm::unified_attention_with_output"
]
async_tp_pass = object.__new__(AsyncTPPass)
async_tp_pass.compilation_config = vllm_config.compilation_config
with pytest.raises(
AssertionError, match="AsyncTPPass requires full-graph compilation"
):
async_tp_pass.is_applicable_for_range(Range(start=8, end=8))
def async_tp_pass_on_test_model(
local_rank: int,
world_size: int,
test_model_cls: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
dynamic: bool,
distributed_init_method: str,
check_numerics: bool = False,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
}
)
# initialize distributed
init_distributed_environment(
world_size=world_size,
rank=local_rank,
distributed_init_method=distributed_init_method,
local_rank=local_rank,
)
# configure vllm config for SequenceParallelismPass
vllm_config = VllmConfig()
vllm_config.compilation_config = CompilationConfig(
pass_config=PassConfig(
fuse_gemm_comms=True,
),
)
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
# this is a fake model name to construct the model config
# in the vllm_config, it's not really used.
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
vllm_config.model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
async_tp_pass = AsyncTPPass(vllm_config)
backend = TestBackend(async_tp_pass)
assert (
async_tp_pass.compilation_config.splitting_ops
== vllm_config.compilation_config.splitting_ops
)
assert (
async_tp_pass.compilation_config.use_inductor_graph_partition
== vllm_config.compilation_config.use_inductor_graph_partition
)
model = test_model_cls(hidden_size, dtype) # Pass dtype to model constructor
hidden_states = torch.randn(
(batch_size * seq_len, hidden_size), dtype=dtype, requires_grad=False
)
if check_numerics:
hidden_states += local_rank * 0.125
if dynamic:
torch._dynamo.mark_dynamic(hidden_states, 0)
compiled_model = torch.compile(model, backend=backend)
actual = compiled_model(hidden_states)
if check_numerics:
# These are new checks, so no existing tolerance is relaxed.
# Allow BF16 rounding vs. eager, but keep the lifetime check exact.
expected = model(hidden_states)
torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2)
snapshot = actual.clone()
changed = compiled_model(-hidden_states)
torch.testing.assert_close(changed, -expected, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(actual, snapshot, rtol=0, atol=0)
assert async_tp_pass.matched_count == 1
# In pre-nodes, all gather or reduce scatter should exist,
# fused_matmul_reduce_scatter or fused_all_gather_matmul should not
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
# In post-nodes, fused_matmul_reduce_scatter or \
# fused_all_gather_matmul should exist
backend.check_after_ops(model.ops_in_model_after())
@@ -1,343 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.envs as envs
from tests.compile.backend import TestBackend
from tests.utils import TestFP8Layer, multi_gpu_test
from vllm.compilation.passes.fusion.rms_quant_fusion import RMSNormQuantFusionPass
from vllm.compilation.passes.fusion.sequence_parallelism import SequenceParallelismPass
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.vllm_inductor_pass import VllmInductorPass
from vllm.config import (
CompilationConfig,
CUDAGraphMode,
DeviceConfig,
ModelConfig,
PassConfig,
VllmConfig,
get_current_vllm_config,
set_current_vllm_config,
)
from vllm.config.utils import Range
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.utils.system_utils import update_environment_variables
from vllm.utils.torch_utils import set_random_seed
DEVICE_TYPE = current_platform.device_type
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
FP8_DTYPE = current_platform.fp8_dtype()
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
class TestAllReduceRMSNormModel(torch.nn.Module):
def __init__(self, hidden_size=16, eps=1e-6):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
def forward(self, x):
z = torch.relu(x)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = torch.mm(y, self.w[0])
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = torch.mm(y2, self.w[1])
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid)
z4 = torch.mm(y3, self.w[2])
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid)
return y4
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
return [
torch.ops.vllm.all_gather.default,
torch.ops.vllm.reduce_scatter.default,
]
def ops_in_model(self):
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm,
]
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(self, hidden_size=16, eps=1e-6):
super().__init__()
self.vllm_config = get_current_vllm_config()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.fp8_linear_layers = [
TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=self.vllm_config.model_config.dtype,
)
for i in range(3)
]
def forward(self, hidden_states):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(hidden_states)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = self.fp8_linear_layers[0](y)
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = self.fp8_linear_layers[1](y2)
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid) # use resid here
z4 = self.fp8_linear_layers[2](y3)
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_after(self):
return [
torch.ops.vllm.all_gather.default,
torch.ops.vllm.reduce_scatter.default,
]
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
]
def ops_in_model(self):
if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
else:
quant_ops = (
[torch.ops._C.static_scaled_fp8_quant.default]
if any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers)
else [torch.ops.aten.reciprocal]
)
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm,
*quant_ops,
]
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"test_model_cls, custom_ops",
[
(TestAllReduceRMSNormModel, "+rms_norm"),
(TestAllReduceRMSNormModel, "-rms_norm"),
(TestAllReduceRMSNormStaticQuantFP8Model, "+rms_norm,+quant_fp8"),
(TestAllReduceRMSNormStaticQuantFP8Model, "+rms_norm,-quant_fp8"),
(TestAllReduceRMSNormStaticQuantFP8Model, "-rms_norm,+quant_fp8"),
(TestAllReduceRMSNormStaticQuantFP8Model, "-rms_norm,-quant_fp8"),
],
)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [16])
@pytest.mark.parametrize("hidden_size", [16])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("fuse_norm_quant", [True, False])
@pytest.mark.parametrize("dynamic", [False, True])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_sequence_parallelism_pass(
test_model_cls: type[torch.nn.Module],
custom_ops: str,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
fuse_norm_quant: bool,
dynamic: bool,
):
num_processes = 2
def run_torch_spawn(fn, nprocs):
# need to use torch.mp.spawn otherwise will have problems with
# torch.distributed and cuda
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
test_model_cls,
custom_ops,
batch_size,
seq_len,
hidden_size,
dtype,
fuse_norm_quant,
dynamic,
),
nprocs=nprocs,
)
run_torch_spawn(sequence_parallelism_pass_on_test_model, num_processes)
def test_sequence_parallelism_pass_requires_full_graph_compilation():
vllm_config = VllmConfig()
vllm_config.compilation_config.use_inductor_graph_partition = False
vllm_config.compilation_config.splitting_ops = [
"vllm::unified_attention_with_output"
]
sequence_parallelism_pass = object.__new__(SequenceParallelismPass)
sequence_parallelism_pass.compilation_config = vllm_config.compilation_config
sequence_parallelism_pass.min_token_num = 1
with pytest.raises(
AssertionError,
match="SequenceParallelismPass requires full-graph compilation",
):
sequence_parallelism_pass.is_applicable_for_range(Range(start=8, end=8))
def sequence_parallelism_pass_on_test_model(
local_rank: int,
world_size: int,
test_model_cls: type[torch.nn.Module],
custom_ops: str,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
fuse_norm_quant: bool,
dynamic: bool,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
}
)
# initialize distributed
init_distributed_environment()
# configure vllm config for SequenceParallelismPass
custom_ops_list = custom_ops.split(",") if custom_ops else []
compilation_config = CompilationConfig(
splitting_ops=[], # avoid automatic rms_norm enablement
cudagraph_mode=CUDAGraphMode.NONE, # avoid piecewise warnings
custom_ops=custom_ops_list,
pass_config=PassConfig(
enable_sp=True,
fuse_norm_quant=fuse_norm_quant,
eliminate_noops=True,
),
) # NoOp needed for fusion
device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
# this is a fake model name to construct the model config
# in the vllm_config, it's not really used.
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
vllm_config = VllmConfig(
model_config=model_config,
device_config=device_config,
compilation_config=compilation_config,
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
noop_pass = NoOpEliminationPass(vllm_config)
sequence_parallelism_pass = SequenceParallelismPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
assert (
sequence_parallelism_pass.compilation_config.splitting_ops
== vllm_config.compilation_config.splitting_ops
)
assert (
sequence_parallelism_pass.compilation_config.use_inductor_graph_partition
== vllm_config.compilation_config.use_inductor_graph_partition
)
passes_for_backend: list[VllmInductorPass] = [
noop_pass,
sequence_parallelism_pass,
]
if fuse_norm_quant:
fusion_pass = RMSNormQuantFusionPass(vllm_config)
passes_for_backend.append(fusion_pass)
passes_for_backend.append(cleanup_pass)
backend = TestBackend(*passes_for_backend)
model = test_model_cls(hidden_size)
hidden_states = torch.randn((batch_size * seq_len, hidden_size), dtype=dtype)
if dynamic:
torch._dynamo.mark_dynamic(hidden_states, 0)
compiled_model = torch.compile(model, backend=backend)
compiled_model(hidden_states)
assert sequence_parallelism_pass.matched_count == 4
# In pre-nodes, all reduce should be there,
# reduce scatter and all gather should not
for op in model.ops_in_model_before():
assert backend.op_count(op, before=True) == 4
# In post-nodes, reduce scatter and all gather should be there,
# all reduce should not
for op in model.ops_in_model_after():
assert backend.op_count(op, before=False) == 4
for op in model.ops_in_model():
assert backend.op_count(op, before=False) > 0
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-75
View File
@@ -1,75 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm AsyncTP numerical coverage kept outside the shared distributed suite."""
import pytest
import torch
from tests.compile.passes.distributed.test_async_tp import (
TestAGMMModel,
TestMMRSModel,
async_tp_pass_on_test_model,
)
from tests.utils import multi_gpu_test
from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_file_store_init_method
class TestRocmMMRSModel(TestMMRSModel):
def forward(self, hidden_states):
view = hidden_states.reshape(-1, self.hidden_size)
mm = torch.ops.vllm.rocm_unquantized_gemm(view, self.gate_proj, None)
return tensor_model_parallel_reduce_scatter(mm, dim=0)
def ops_in_model_before(self):
return super().ops_in_model_before() + [
torch.ops.vllm.rocm_unquantized_gemm.default
]
class TestRocmAGMMModel(TestAGMMModel):
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
super().__init__(hidden_size, dtype)
self.weight = torch.nn.Parameter(
torch.empty((hidden_size * 2, hidden_size)), requires_grad=False
)
torch.nn.init.normal_(self.weight, std=0.02)
def forward(self, hidden_states):
view = hidden_states.reshape(-1, self.hidden_size)
gathered = tensor_model_parallel_all_gather(view, dim=0)
return torch.ops.vllm.rocm_unquantized_gemm(gathered, self.weight, None)
def ops_in_model_before(self):
return super().ops_in_model_before() + [
torch.ops.vllm.rocm_unquantized_gemm.default
]
@multi_gpu_test(num_gpus=2)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend contract")
@pytest.mark.parametrize(
"test_model", [TestMMRSModel, TestAGMMModel, TestRocmMMRSModel, TestRocmAGMMModel]
)
@pytest.mark.parametrize("dynamic", [False, True])
def test_rocm_async_tp_bf16_rewrite_and_correctness(test_model, dynamic):
"""Real symmetric-memory AG/GEMM and GEMM/RS must replace the collectives."""
torch.multiprocessing.spawn(
async_tp_pass_on_test_model,
args=(
2,
test_model,
8,
16,
128,
torch.bfloat16,
dynamic,
get_file_store_init_method(),
True,
),
nprocs=2,
)
+12 -172
View File
@@ -464,49 +464,41 @@ def test_should_split():
(
"cudagraph_capture_sizes",
"max_cudagraph_capture_size",
"tp_size",
"enable_sp",
"max_num_batched_tokens",
"cudagraph_mode",
"expected_max_size",
),
[
(None, None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([1, 2, 4], 4, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
(None, None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([1, 2, 4], 4, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
(
[1, 2, 4],
8,
1,
False,
2048,
CUDAGraphMode.FULL_AND_PIECEWISE,
ValidationError,
),
([1, 256], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([], None, 1, False, 2048, CUDAGraphMode.NONE, 0),
(None, 0, 1, False, 2048, CUDAGraphMode.NONE, 0),
([1, 256], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
([], None, 2048, CUDAGraphMode.NONE, 0),
(None, 0, 2048, CUDAGraphMode.NONE, 0),
# truncated to nearest multiple of 8 or 16
(None, 257, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
(None, 257, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
# max_num_batched_tokens <= max_cudagraph_capture_size should always be
# captured even if not landing on a 16-stride step
(None, 2048, 1, False, 257, CUDAGraphMode.FULL_AND_PIECEWISE, 257),
(None, 2048, 257, CUDAGraphMode.FULL_AND_PIECEWISE, 257),
# max from list
([1, 2, 4, 15], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
# SP forces full-graph compilation, sizes are filtered by TP
([1, 2, 4, 15], None, 2, True, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
([1, 2, 4, 15], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
# limited by the max_tokens
([1, 2, 4, 15], None, 1, False, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
([1, 2, 4, 15], None, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
# the list should contain at least 1 element when use cudagraph
([], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
([], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
# the max capturing size should be >= 1 when use cudagraph
(None, 0, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
(None, 0, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
],
)
def test_cudagraph_sizes_post_init(
cudagraph_capture_sizes,
max_cudagraph_capture_size,
tp_size,
enable_sp,
max_num_batched_tokens,
cudagraph_mode,
expected_max_size,
@@ -515,10 +507,7 @@ def test_cudagraph_sizes_post_init(
if expected_max_size == ValidationError:
ctx = pytest.raises(expected_max_size)
with (
ctx,
patch.object(current_platform, "device_count", return_value=tp_size),
):
with ctx:
kwargs = {}
if cudagraph_capture_sizes is not None:
kwargs["cudagraph_capture_sizes"] = cudagraph_capture_sizes
@@ -526,18 +515,15 @@ def test_cudagraph_sizes_post_init(
kwargs["max_cudagraph_capture_size"] = max_cudagraph_capture_size
compilation_config = CompilationConfig(
pass_config=PassConfig(
enable_sp=enable_sp,
fuse_norm_quant=True,
fuse_act_quant=True,
eliminate_noops=True,
sp_min_token_num=512 if enable_sp else None,
),
cudagraph_mode=cudagraph_mode,
**kwargs,
)
engine_args = EngineArgs(
model="facebook/opt-125m",
tensor_parallel_size=tp_size,
max_num_seqs=min(max_num_batched_tokens, 128),
max_num_batched_tokens=max_num_batched_tokens,
compilation_config=compilation_config,
@@ -728,35 +714,6 @@ def test_default_cudagraph_capture_sizes_keep_all_sizes_bounded():
)
def test_cudagraph_capture_sizes_respect_sequence_parallelism():
"""Sequence-parallel capture sizes stay divisible by tensor parallel size."""
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
)
compilation_config.pass_config.enable_sp = True
config = _mock_config_for_cudagraph_sizes(
max_num_seqs=32,
num_speculative_tokens=16,
max_num_batched_tokens=32768,
compilation_config=compilation_config,
)
config.parallel_config = SimpleNamespace(tensor_parallel_size=2)
config.update_sizes_for_sequence_parallelism = lambda sizes: (
VllmConfig.update_sizes_for_sequence_parallelism(config, sizes)
)
with patch.object(
current_platform,
"is_device_capability_family",
return_value=False,
):
VllmConfig._set_cudagraph_sizes(config)
assert all(size % 2 == 0 for size in compilation_config.cudagraph_capture_sizes)
assert 544 not in compilation_config.cudagraph_capture_sizes
assert compilation_config.max_cudagraph_capture_size == 512
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
def test_real_vllm_config_caps_widest_ngram_decode_batch():
"""Real config post-init preserves the default capture ceiling."""
@@ -1046,123 +1003,6 @@ def test_blackwell_cudagraph_default(is_blackwell, expected_max_size):
)
@pytest.mark.skipif(
not current_platform.support_static_graph_mode(),
reason="Skip if not cudagraph mode supported",
)
@pytest.mark.parametrize(
(
"cudagraph_mode",
"use_inductor_graph_partition",
"expected_enable_sp",
"expected_cudagraph_mode",
"expected_piecewise_compile",
"expected_capture_sizes",
"expected_max_size",
),
[
(CUDAGraphMode.PIECEWISE, False, True, CUDAGraphMode.FULL, False, [2, 4], 4),
(
CUDAGraphMode.FULL_DECODE_ONLY,
False,
True,
CUDAGraphMode.FULL_DECODE_ONLY,
False,
[2, 4],
4,
),
(
CUDAGraphMode.FULL_AND_PIECEWISE,
False,
True,
CUDAGraphMode.FULL,
False,
[2, 4],
4,
),
(
CUDAGraphMode.FULL_AND_PIECEWISE,
True,
True,
CUDAGraphMode.FULL_AND_PIECEWISE,
True,
[2, 4],
4,
),
],
)
def test_sequence_parallelism_requires_full_graph_compilation(
cudagraph_mode: CUDAGraphMode,
use_inductor_graph_partition: bool,
expected_enable_sp: bool,
expected_cudagraph_mode: CUDAGraphMode,
expected_piecewise_compile: bool,
expected_capture_sizes: list[int],
expected_max_size: int,
):
with patch.object(current_platform, "device_count", return_value=2):
vllm_config = VllmConfig(
parallel_config=ParallelConfig(tensor_parallel_size=2),
scheduler_config=SchedulerConfig(
max_num_seqs=128,
max_num_batched_tokens=2048,
max_model_len=2048,
is_encoder_decoder=False,
),
)
vllm_config.model_config = MagicMock(
dtype=torch.float16,
enforce_eager=False,
is_moe=False,
disable_cascade_attn=False,
get_hidden_size=MagicMock(return_value=4096),
)
vllm_config.compilation_config = CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_capture_sizes=[1, 2, 4, 15],
max_cudagraph_capture_size=None,
compile_sizes=["cudagraph_capture_sizes"],
use_inductor_graph_partition=use_inductor_graph_partition,
pass_config=PassConfig(
enable_sp=True,
fuse_gemm_comms=True,
fuse_norm_quant=True,
fuse_act_quant=True,
eliminate_noops=True,
sp_min_token_num=512,
),
cudagraph_mode=cudagraph_mode,
)
vllm_config.compilation_config.set_splitting_ops_for_v1(
all2all_backend=vllm_config.parallel_config.all2all_backend,
data_parallel_size=1,
)
vllm_config._set_compile_ranges()
vllm_config._set_cudagraph_sizes()
assert (
vllm_config.compilation_config.use_inductor_graph_partition
== use_inductor_graph_partition
)
assert (
bool(vllm_config.compilation_config.splitting_ops) == expected_piecewise_compile
)
assert vllm_config.compilation_config.pass_config.enable_sp == expected_enable_sp
assert (
vllm_config.compilation_config.pass_config.fuse_gemm_comms == expected_enable_sp
)
assert vllm_config.compilation_config.cudagraph_mode == expected_cudagraph_mode
assert (
vllm_config.compilation_config.cudagraph_capture_sizes == expected_capture_sizes
)
assert (
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
)
assert (
511 in vllm_config.compilation_config.compile_ranges_endpoints
) == expected_enable_sp
def test_cached_compilation_config(default_vllm_config):
import torch
from torch._inductor.utils import run_and_get_code
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.compilation.passes.fusion.sequence_parallelism import (
SP_MIN_HIDDEN_SIZE,
SP_MIN_PER_GPU_SIZE_MB,
get_sequence_parallelism_threshold,
)
class TestGetSequenceParallelismThreshold:
"""Tests for get_sequence_parallelism_threshold function."""
def test_non_cuda_returns_none(self, mock_cuda_platform):
"""Non-CUDA platforms should return None."""
with mock_cuda_platform(is_cuda=False):
result = get_sequence_parallelism_threshold(
hidden_size=8192, tp_size=2, element_size=2
)
assert result is None
def test_unsupported_device_capability_returns_none(self, mock_cuda_platform):
"""Unsupported device capabilities (e.g., sm80) should return None."""
with mock_cuda_platform(capability=(8, 0)):
result = get_sequence_parallelism_threshold(
hidden_size=8192, tp_size=2, element_size=2
)
assert result is None
def test_small_hidden_size_returns_none(self, mock_cuda_platform):
"""H100 with hidden_size below threshold should return None."""
with mock_cuda_platform(capability=(9, 0)):
result = get_sequence_parallelism_threshold(
hidden_size=4096,
tp_size=2,
element_size=2, # 4096 < 8192
)
assert result is None
def test_h100_large_model_returns_threshold(self, mock_cuda_platform):
"""H100 with large enough hidden_size should return calculated threshold."""
with mock_cuda_platform(capability=(9, 0)):
hidden_size = 8192
tp_size = 2
element_size = 2 # float16/bfloat16
result = get_sequence_parallelism_threshold(
hidden_size=hidden_size,
tp_size=tp_size,
element_size=element_size,
)
# Verify calculation: (8 * 2 * 1024 * 1024) // (8192 * 2) = 1024
MiB = 1024 * 1024
expected = int(
(SP_MIN_PER_GPU_SIZE_MB[90] * tp_size * MiB)
// (hidden_size * element_size)
)
assert result == expected
assert result == 1024
@pytest.mark.parametrize(
"hidden_size,tp_size,element_size,expected",
[
# Boundary: exactly at min hidden size threshold, tp_size=1
# (8 * 1 * 1024 * 1024) // (8192 * 2) = 512
(8192, 1, 2, 512),
# Larger hidden size reduces token threshold
# (8 * 1 * 1024 * 1024) // (16384 * 2) = 256
(16384, 1, 2, 256),
# Larger tp_size increases token threshold
# (8 * 4 * 1024 * 1024) // (8192 * 2) = 2048
(8192, 4, 2, 2048),
# Larger element_size (fp32) reduces token threshold
# (8 * 2 * 1024 * 1024) // (8192 * 4) = 512
(8192, 2, 4, 512),
],
)
def test_threshold_calculation_variations(
self, mock_cuda_platform, hidden_size, tp_size, element_size, expected
):
"""Test threshold calculation with various parameter combinations."""
with mock_cuda_platform(capability=(9, 0)):
result = get_sequence_parallelism_threshold(
hidden_size=hidden_size,
tp_size=tp_size,
element_size=element_size,
)
assert result == expected
def test_hidden_size_boundary(self, mock_cuda_platform):
"""Test behavior at the exact hidden_size boundary."""
with mock_cuda_platform(capability=(9, 0)):
# Just below threshold
result = get_sequence_parallelism_threshold(
hidden_size=SP_MIN_HIDDEN_SIZE[90] - 1,
tp_size=2,
element_size=2,
)
assert result is None
# Exactly at threshold
result = get_sequence_parallelism_threshold(
hidden_size=SP_MIN_HIDDEN_SIZE[90],
tp_size=2,
element_size=2,
)
assert result is not None
# XPU-specific constants (must match sequence_parallelism.py values)
_XPU_MIN_HIDDEN_SIZE = 4096
_XPU_MIN_PER_GPU_SIZE_MB = 8.0
class TestGetSequenceParallelismThresholdXPU:
"""Tests for get_sequence_parallelism_threshold on XPU platform."""
def test_xpu_small_hidden_size_returns_none(self, mock_xpu_platform):
"""XPU with hidden_size below threshold should return None."""
with mock_xpu_platform():
result = get_sequence_parallelism_threshold(
hidden_size=_XPU_MIN_HIDDEN_SIZE - 1,
tp_size=2,
element_size=2,
)
assert result is None
def test_xpu_large_model_returns_threshold(self, mock_xpu_platform):
"""XPU with hidden_size >= threshold should return calculated value."""
with mock_xpu_platform():
hidden_size = _XPU_MIN_HIDDEN_SIZE
tp_size = 2
element_size = 2
result = get_sequence_parallelism_threshold(
hidden_size=hidden_size,
tp_size=tp_size,
element_size=element_size,
)
# (8 * 2 * 1024 * 1024) // (4096 * 2) = 2048
MiB = 1024 * 1024
expected = int(
(_XPU_MIN_PER_GPU_SIZE_MB * tp_size * MiB) // (hidden_size * element_size)
)
assert result == expected
assert result == 2048
@pytest.mark.parametrize(
"hidden_size,tp_size,element_size,expected",
[
# (8 * 1 * 1024 * 1024) // (4096 * 2) = 1024
(4096, 1, 2, 1024),
# (8 * 4 * 1024 * 1024) // (4096 * 2) = 4096
(4096, 4, 2, 4096),
# (8 * 2 * 1024 * 1024) // (8192 * 2) = 1024
(8192, 2, 2, 1024),
# (8 * 2 * 1024 * 1024) // (4096 * 4) = 1024
(4096, 2, 4, 1024),
],
)
def test_xpu_threshold_calculation_variations(
self, mock_xpu_platform, hidden_size, tp_size, element_size, expected
):
"""Test XPU threshold calculation with various parameter combinations."""
with mock_xpu_platform():
result = get_sequence_parallelism_threshold(
hidden_size=hidden_size,
tp_size=tp_size,
element_size=element_size,
)
assert result == expected
def test_xpu_hidden_size_boundary(self, mock_xpu_platform):
"""Test behavior at the exact XPU hidden_size boundary."""
with mock_xpu_platform():
# Just below threshold
result = get_sequence_parallelism_threshold(
hidden_size=_XPU_MIN_HIDDEN_SIZE - 1,
tp_size=2,
element_size=2,
)
assert result is None
# Exactly at threshold
result = get_sequence_parallelism_threshold(
hidden_size=_XPU_MIN_HIDDEN_SIZE,
tp_size=2,
element_size=2,
)
assert result is not None
@@ -582,10 +582,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch):
use_ubatching=False,
enable_elastic_ep=False,
),
compilation_config=SimpleNamespace(
mode=CompilationMode.NONE,
pass_config=SimpleNamespace(enable_sp=False),
),
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
cache_config=SimpleNamespace(
kv_sharing_fast_prefill=False,
mamba_cache_mode="none",
+1 -11
View File
@@ -843,7 +843,6 @@ def test_breakable_cudagraph_platform_default(
("bi-off", 2048, (16, 8), 6144, 1, False),
("opt-out", 2048, (16, 8), 6144, 1, False),
("eager", 2048, (16, 8), 6144, 1, False),
("sp", 2048, (16, 8), 6144, 1, False),
("fp16", 2048, (16, 8), 6144, 1, False),
("quantized", 2048, (16, 8), 6144, 1, False),
("untuned", 4096, (32, 32), 11008, 1, False),
@@ -886,11 +885,7 @@ def test_batch_invariant_breakable_cudagraph(
get_num_kv_heads=lambda pc: heads[1],
)
config.parallel_config = SimpleNamespace(tensor_parallel_size=tp)
config.compilation_config = (
CompilationConfig(pass_config=PassConfig(enable_sp=True))
if case == "sp"
else CompilationConfig()
)
config.compilation_config = CompilationConfig()
try:
assert config._maybe_enable_breakable_cudagraph() is expected
if expected:
@@ -1021,7 +1016,6 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
"FakeAttentionBackend",
uniform_decode_query_len=4,
use_v2_model_runner=use_v2_model_runner,
tensor_parallel_size=1,
)
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
@@ -1104,7 +1098,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
)
@@ -1117,7 +1110,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
is_profiling=True,
@@ -3359,8 +3351,6 @@ def test_vllm_config_explicit_overrides():
take precedence over callable defaults, across different models and
optimization levels.
"""
from vllm.config.compilation import PassConfig
quantized_model = ModelConfig("RedHatAI/Llama-3.2-1B-FP8")
moe_model = ModelConfig("deepseek-ai/DeepSeek-V2-Lite")
regular_model = ModelConfig("Qwen/Qwen1.5-7B")
-3
View File
@@ -378,9 +378,6 @@ def test_execute_model_waits_previous_pp_send_before_forward(
worker = SimpleNamespace(
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(
pass_config=SimpleNamespace(enable_sp=False)
),
parallel_config=SimpleNamespace(
pipeline_parallel_size=2, distributed_executor_backend="mp"
),