[Compile] Remove torch.compile-based sequence parallelism and AsyncTP (#60517)
This commit is contained in:
@@ -1,270 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import logging
|
||||
from contextlib import nullcontext
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.compile.fusions_e2e.common import FUSION_LOG_PATTERNS
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.utils import (
|
||||
compare_two_settings,
|
||||
create_new_process_for_each_test,
|
||||
multi_gpu_test,
|
||||
)
|
||||
from vllm.config import (
|
||||
CompilationMode,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
||||
NVFP4_HF_OVERRIDES = {
|
||||
"num_hidden_layers": 4,
|
||||
"hidden_size": 512,
|
||||
"intermediate_size": 800,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 1,
|
||||
}
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
|
||||
def test_rocm_async_tp_bf16_output_correctness(caplog_mp_spawn):
|
||||
"""Trained-model outputs must agree with unfused TP, including padded decode."""
|
||||
# A snapshot path pins both servers and the comparison helper's tokenizer.
|
||||
model = hf_api().snapshot_download(
|
||||
"Qwen/Qwen3-0.6B", revision="c1899de289a04d12100db370d81485cdf75e47ca"
|
||||
)
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--load-format",
|
||||
"auto",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--max-num-batched-tokens",
|
||||
"512",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(64 * 1024**2),
|
||||
"--gpu-memory-utilization",
|
||||
"0.01",
|
||||
"--attention-backend",
|
||||
"TRITON_ATTN",
|
||||
"--disable-custom-all-reduce",
|
||||
]
|
||||
|
||||
def settings(enabled):
|
||||
return [
|
||||
*common_args,
|
||||
"--compilation-config",
|
||||
json.dumps(
|
||||
{
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"cudagraph_mode": "NONE",
|
||||
"splitting_ops": [],
|
||||
"inductor_compile_config": {"force_disable_caches": True},
|
||||
"pass_config": {
|
||||
"enable_sp": enabled,
|
||||
"fuse_gemm_comms": enabled,
|
||||
"fuse_allreduce_rms": False,
|
||||
"sp_min_token_num": 1,
|
||||
},
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
env = {"VLLM_ROCM_USE_AITER": "0", "VLLM_DISABLE_COMPILE_CACHE": "1"}
|
||||
with caplog_mp_spawn(logging.DEBUG) as logs:
|
||||
compare_two_settings(
|
||||
model,
|
||||
settings(False),
|
||||
settings(True),
|
||||
env1=env,
|
||||
env2=env,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
matches = [
|
||||
int(count) for count in FUSION_LOG_PATTERNS["async_tp"].findall(logs.text)
|
||||
]
|
||||
print(f"AsyncTP rewrite counts: {matches}")
|
||||
assert len(matches) >= 2 and all(count > 0 for count in matches), (
|
||||
f"Expected positive AsyncTP rewrites on both TP workers, found {matches}"
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
["meta-llama/Llama-3.2-1B-Instruct", "RedHatAI/Llama-3.2-1B-Instruct-FP8"],
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("async_tp_enabled", [True])
|
||||
@pytest.mark.parametrize("distributed_backend", ["mp"])
|
||||
def test_async_tp_pass_correctness(
|
||||
model_id: str,
|
||||
tp_size: int,
|
||||
async_tp_enabled: bool,
|
||||
distributed_backend: str,
|
||||
num_gpus_available: int,
|
||||
monkeypatch,
|
||||
):
|
||||
# Disable FlashInfer FP8 scaled_mm kernel as it is incompatible with
|
||||
# async TP patterns. No-op on H100 (kernel requires CC >= 100).
|
||||
monkeypatch.setenv("VLLM_DISABLED_KERNELS", "FlashInferFP8ScaledMMLinearKernel")
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
|
||||
pp_size = 1
|
||||
if num_gpus_available < tp_size:
|
||||
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
|
||||
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
]
|
||||
|
||||
compilation_config = {
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_sizes": [2, 4, 8],
|
||||
"splitting_ops": [],
|
||||
"pass_config": {"fuse_gemm_comms": async_tp_enabled},
|
||||
}
|
||||
|
||||
async_tp_args = [
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--distributed-executor-backend",
|
||||
distributed_backend,
|
||||
"--compilation_config",
|
||||
json.dumps(compilation_config),
|
||||
]
|
||||
|
||||
tp_args = [
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
]
|
||||
|
||||
compare_two_settings(
|
||||
model_id,
|
||||
async_tp_args,
|
||||
tp_args,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, caplog_mp_spawn):
|
||||
is_blackwell = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
)
|
||||
is_rocm_cdna = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
is_rocm_cdna = on_cdna()
|
||||
if not (is_blackwell or is_rocm_cdna):
|
||||
pytest.skip("NVFP4 AsyncTP requires Blackwell or ROCm CDNA")
|
||||
if is_blackwell and not has_flashinfer():
|
||||
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
|
||||
|
||||
tp_size = 2
|
||||
if num_gpus_available < tp_size:
|
||||
pytest.skip(f"Need at least {tp_size} GPUs")
|
||||
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
*(["--linear-backend", "flashinfer_cutlass"] if is_blackwell else []),
|
||||
"--hf-overrides",
|
||||
json.dumps(NVFP4_HF_OVERRIDES),
|
||||
]
|
||||
|
||||
compilation_config = {
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_sizes": [2, 4, 8],
|
||||
"splitting_ops": [],
|
||||
"pass_config": {
|
||||
"enable_sp": True,
|
||||
"fuse_gemm_comms": True,
|
||||
"fuse_allreduce_rms": False,
|
||||
"sp_min_token_num": 1,
|
||||
},
|
||||
}
|
||||
if is_rocm_cdna:
|
||||
compilation_config["inductor_compile_config"] = {"force_disable_caches": True}
|
||||
|
||||
async_tp_args = [
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
"--compilation_config",
|
||||
json.dumps(compilation_config),
|
||||
]
|
||||
|
||||
tp_args = [
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
]
|
||||
|
||||
env = {"VLLM_DISABLE_COMPILE_CACHE": "1"} if is_rocm_cdna else None
|
||||
log_context = caplog_mp_spawn(logging.DEBUG) if is_rocm_cdna else nullcontext()
|
||||
with log_context as logs:
|
||||
compare_two_settings(
|
||||
NVFP4_MODEL_ID,
|
||||
async_tp_args,
|
||||
tp_args,
|
||||
env1=env,
|
||||
env2=env,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
if is_rocm_cdna:
|
||||
assert logs is not None
|
||||
matches_by_rank = {
|
||||
rank: [
|
||||
int(count)
|
||||
for line in logs.text.splitlines()
|
||||
if line.startswith(f"(Worker_TP{rank} pid=")
|
||||
for count in FUSION_LOG_PATTERNS["async_tp"].findall(line)
|
||||
]
|
||||
for rank in range(tp_size)
|
||||
}
|
||||
assert all(
|
||||
any(count > 0 for count in matches) for matches in matches_by_rank.values()
|
||||
), (
|
||||
"Expected positive NVFP4 AsyncTP rewrites on both TP workers, "
|
||||
f"found {matches_by_rank}"
|
||||
)
|
||||
@@ -1,414 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, NamedTuple
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from ...models.registry import HF_EXAMPLE_MODELS, _HfExamplesInfo
|
||||
from ...utils import compare_all_settings, create_new_process_for_each_test
|
||||
|
||||
logger = init_logger("test_sequence_parallel")
|
||||
|
||||
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
|
||||
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
||||
NVFP4_MODEL_INFO = _HfExamplesInfo(NVFP4_MODEL_ID)
|
||||
|
||||
|
||||
class ParallelSetup(NamedTuple):
|
||||
tp_size: int
|
||||
pp_size: int
|
||||
fuse_norm_quant: bool
|
||||
fuse_act_quant: bool
|
||||
eager_mode: bool
|
||||
|
||||
|
||||
class SPTestOptions(NamedTuple):
|
||||
multi_node_only: bool
|
||||
load_format: str | None = None
|
||||
model_info: _HfExamplesInfo | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SPTestSettings:
|
||||
parallel_setups: list[ParallelSetup]
|
||||
distributed_backends: list[str]
|
||||
runner: RunnerOption
|
||||
test_options: SPTestOptions
|
||||
|
||||
@staticmethod
|
||||
def fast(
|
||||
*,
|
||||
tp_base: int = 2,
|
||||
pp_base: int = 1,
|
||||
runner: RunnerOption = "auto",
|
||||
multi_node_only: bool = False,
|
||||
load_format: str | None = None,
|
||||
):
|
||||
parallel_setups = [
|
||||
ParallelSetup(
|
||||
tp_size=tp_base,
|
||||
pp_size=pp_multiplier * pp_base,
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=False,
|
||||
eager_mode=False,
|
||||
)
|
||||
for pp_multiplier in [1, 2]
|
||||
]
|
||||
return SPTestSettings(
|
||||
parallel_setups=parallel_setups,
|
||||
distributed_backends=["mp", "ray"],
|
||||
runner=runner,
|
||||
test_options=SPTestOptions(
|
||||
multi_node_only=multi_node_only, load_format=load_format
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fp8_quant(
|
||||
*,
|
||||
tp_base: int = 2,
|
||||
pp_base: int = 1,
|
||||
runner: RunnerOption = "auto",
|
||||
multi_node_only: bool = False,
|
||||
load_format: str | None = None,
|
||||
):
|
||||
parallel_setups = []
|
||||
for fusion_val in [False, True]:
|
||||
parallel_setups.append(
|
||||
ParallelSetup(
|
||||
tp_size=tp_base,
|
||||
pp_size=pp_base,
|
||||
fuse_norm_quant=fusion_val,
|
||||
fuse_act_quant=fusion_val,
|
||||
eager_mode=True,
|
||||
)
|
||||
)
|
||||
return SPTestSettings(
|
||||
parallel_setups=parallel_setups,
|
||||
distributed_backends=["mp", "ray"],
|
||||
runner=runner,
|
||||
test_options=SPTestOptions(
|
||||
multi_node_only=multi_node_only, load_format=load_format
|
||||
),
|
||||
)
|
||||
|
||||
def iter_params(self, model_id: str):
|
||||
opts = self.test_options
|
||||
|
||||
for parallel_setup in self.parallel_setups:
|
||||
for backend in self.distributed_backends:
|
||||
yield (
|
||||
model_id,
|
||||
parallel_setup,
|
||||
backend,
|
||||
self.runner,
|
||||
opts,
|
||||
)
|
||||
|
||||
|
||||
def _build_sp_args(
|
||||
model_id: str,
|
||||
parallel_setup: ParallelSetup,
|
||||
distributed_backend: str,
|
||||
runner: RunnerOption,
|
||||
test_options: SPTestOptions,
|
||||
num_gpus_available: int,
|
||||
use_inductor_graph_partition: bool,
|
||||
fuse_gemm_comms: bool,
|
||||
enable_prompt_embeds: bool,
|
||||
*,
|
||||
is_multimodal: bool,
|
||||
) -> tuple[list[str], list[str]] | None:
|
||||
(
|
||||
tp_size,
|
||||
pp_size,
|
||||
fuse_norm_quant,
|
||||
fuse_act_quant,
|
||||
eager_mode,
|
||||
) = parallel_setup
|
||||
|
||||
multi_node_only = test_options.multi_node_only
|
||||
load_format = test_options.load_format
|
||||
|
||||
model_info = test_options.model_info or HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
trust_remote_code = model_info.trust_remote_code
|
||||
tokenizer_mode = model_info.tokenizer_mode
|
||||
hf_overrides = dict(model_info.hf_overrides)
|
||||
require_embed_inputs = model_info.require_embed_inputs
|
||||
|
||||
if load_format == "dummy":
|
||||
# Avoid OOM
|
||||
text_overrides = {
|
||||
"num_hidden_layers": 4,
|
||||
"hidden_size": 512,
|
||||
"intermediate_size": 800,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 1,
|
||||
}
|
||||
|
||||
if is_multimodal:
|
||||
hf_overrides.update({"text_config": text_overrides})
|
||||
else:
|
||||
hf_overrides.update(text_overrides)
|
||||
else:
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
|
||||
if num_gpus_available < tp_size * pp_size:
|
||||
return None
|
||||
if VLLM_MULTI_NODE and distributed_backend == "mp":
|
||||
return None
|
||||
if multi_node_only and not VLLM_MULTI_NODE:
|
||||
return None
|
||||
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
]
|
||||
if eager_mode:
|
||||
common_args.append("-cc.cudagraph_mode=none")
|
||||
if runner != "auto":
|
||||
common_args.extend(["--runner", runner])
|
||||
if trust_remote_code:
|
||||
common_args.append("--trust-remote-code")
|
||||
if tokenizer_mode:
|
||||
common_args.extend(["--tokenizer-mode", tokenizer_mode])
|
||||
if load_format:
|
||||
common_args.extend(["--load-format", load_format])
|
||||
if hf_overrides:
|
||||
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
|
||||
if require_embed_inputs:
|
||||
common_args.extend(
|
||||
[
|
||||
"--skip-tokenizer-init",
|
||||
"--enable-prompt-embeds",
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
)
|
||||
elif enable_prompt_embeds:
|
||||
common_args.append("--enable-prompt-embeds")
|
||||
|
||||
compilation_config = {
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_sizes": [4, 8],
|
||||
"pass_config": {
|
||||
"enable_sp": True,
|
||||
"fuse_gemm_comms": fuse_gemm_comms,
|
||||
"fuse_norm_quant": fuse_norm_quant,
|
||||
"fuse_act_quant": fuse_act_quant,
|
||||
"fuse_allreduce_rms": False,
|
||||
"eliminate_noops": True,
|
||||
"sp_min_token_num": 0,
|
||||
},
|
||||
"use_inductor_graph_partition": use_inductor_graph_partition,
|
||||
}
|
||||
if not use_inductor_graph_partition:
|
||||
compilation_config["splitting_ops"] = []
|
||||
|
||||
tp_sp_args = [
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--pipeline-parallel-size",
|
||||
str(pp_size),
|
||||
"--distributed-executor-backend",
|
||||
distributed_backend,
|
||||
"--compilation_config",
|
||||
json.dumps(compilation_config),
|
||||
]
|
||||
|
||||
tp_args = [
|
||||
*common_args,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
]
|
||||
|
||||
return tp_args, tp_sp_args
|
||||
|
||||
|
||||
def _compare_sp_settings(
|
||||
model_id: str,
|
||||
settings: list[tuple[list[str], list[str]]],
|
||||
*,
|
||||
method: Literal["generate", "encode"],
|
||||
) -> None:
|
||||
if not settings:
|
||||
pytest.skip("No supported sequence-parallel configurations")
|
||||
|
||||
settings_by_tp_args: dict[tuple[str, ...], list[list[str]]] = {}
|
||||
for tp_args, tp_sp_args in settings:
|
||||
settings_by_tp_args.setdefault(tuple(tp_args), []).append(tp_sp_args)
|
||||
|
||||
for tp_args_key, grouped_tp_sp_args in settings_by_tp_args.items():
|
||||
all_args = [list(tp_args_key), *grouped_tp_sp_args]
|
||||
compare_all_settings(
|
||||
model_id,
|
||||
all_args,
|
||||
[None] * len(all_args),
|
||||
method=method,
|
||||
force_v1_runner=True,
|
||||
)
|
||||
|
||||
|
||||
SP_TEXT_GENERATION_MODELS = {
|
||||
# [Decoder-only]
|
||||
"hmellor/tiny-random-LlamaForCausalLM": SPTestSettings.fast(),
|
||||
"RedHatAI/Llama-3.2-1B-Instruct-FP8": SPTestSettings.fp8_quant(),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "settings"),
|
||||
[
|
||||
pytest.param(model_id, settings, id=model_id)
|
||||
for model_id, settings in SP_TEXT_GENERATION_MODELS.items()
|
||||
],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_tp_sp_generation(
|
||||
model_id: str,
|
||||
settings: SPTestSettings,
|
||||
num_gpus_available,
|
||||
):
|
||||
# Skip FP8 SP-only test on sm89 (compute capability 8.9).
|
||||
# An unknown capability (None) is left to fail in the test body rather than
|
||||
# being silently skipped here.
|
||||
capability = current_platform.get_device_capability()
|
||||
if "fp8" in model_id.lower() and capability is not None and capability < (9, 0):
|
||||
pytest.skip("FP8 reduction support begins with sm90 capable devices.")
|
||||
|
||||
graph_partition_options = [False]
|
||||
if is_torch_equal_or_newer("2.9.0.dev"):
|
||||
graph_partition_options.append(True)
|
||||
|
||||
comparisons = []
|
||||
for _, parallel_setup, backend, runner, test_options in settings.iter_params(
|
||||
model_id
|
||||
):
|
||||
for use_inductor_graph_partition in graph_partition_options:
|
||||
comparison = _build_sp_args(
|
||||
model_id,
|
||||
parallel_setup,
|
||||
backend,
|
||||
runner,
|
||||
test_options,
|
||||
num_gpus_available,
|
||||
use_inductor_graph_partition,
|
||||
fuse_gemm_comms=False, # TODO: enable async TP
|
||||
enable_prompt_embeds=False,
|
||||
is_multimodal=False,
|
||||
)
|
||||
if comparison is not None:
|
||||
comparisons.append(comparison)
|
||||
|
||||
_compare_sp_settings(model_id, comparisons, method="generate")
|
||||
|
||||
|
||||
# Focused regression test for the SP + prompt_embeds graph-rewrite path.
|
||||
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
|
||||
# we don't double the matrix of `test_tp_sp_generation` above.
|
||||
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
|
||||
ParallelSetup(
|
||||
tp_size=2,
|
||||
pp_size=pp_size,
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=False,
|
||||
eager_mode=False,
|
||||
)
|
||||
for pp_size in [1, 2]
|
||||
]
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_tp_sp_generation_prompt_embeds(
|
||||
num_gpus_available,
|
||||
):
|
||||
model_id = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
graph_partition_options = [False]
|
||||
if is_torch_equal_or_newer("2.9.0.dev"):
|
||||
graph_partition_options.append(True)
|
||||
|
||||
comparisons = []
|
||||
for parallel_setup in SP_PROMPT_EMBEDS_PARALLEL_SETUPS:
|
||||
for use_inductor_graph_partition in graph_partition_options:
|
||||
comparison = _build_sp_args(
|
||||
model_id,
|
||||
parallel_setup,
|
||||
distributed_backend="mp",
|
||||
runner="auto",
|
||||
test_options=SPTestOptions(multi_node_only=False, load_format=None),
|
||||
num_gpus_available=num_gpus_available,
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
fuse_gemm_comms=False,
|
||||
enable_prompt_embeds=True,
|
||||
is_multimodal=False,
|
||||
)
|
||||
if comparison is not None:
|
||||
comparisons.append(comparison)
|
||||
|
||||
_compare_sp_settings(model_id, comparisons, method="generate")
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_tp_sp_nvfp4_generation(num_gpus_available: int):
|
||||
is_blackwell = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
)
|
||||
is_rocm_cdna = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
is_rocm_cdna = on_cdna()
|
||||
if not (is_blackwell or is_rocm_cdna):
|
||||
pytest.skip("NVFP4 sequence parallelism requires Blackwell or ROCm CDNA")
|
||||
|
||||
comparison = _build_sp_args(
|
||||
NVFP4_MODEL_ID,
|
||||
ParallelSetup(
|
||||
tp_size=2,
|
||||
pp_size=1,
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=True,
|
||||
eager_mode=True,
|
||||
),
|
||||
"mp",
|
||||
"auto",
|
||||
SPTestOptions(
|
||||
multi_node_only=False,
|
||||
load_format="dummy",
|
||||
model_info=NVFP4_MODEL_INFO,
|
||||
),
|
||||
num_gpus_available,
|
||||
use_inductor_graph_partition=False,
|
||||
fuse_gemm_comms=False,
|
||||
enable_prompt_embeds=False,
|
||||
is_multimodal=False,
|
||||
)
|
||||
_compare_sp_settings(
|
||||
NVFP4_MODEL_ID,
|
||||
[] if comparison is None else [comparison],
|
||||
method="generate",
|
||||
)
|
||||
@@ -21,8 +21,6 @@ class Matches(NamedTuple):
|
||||
# distributed
|
||||
ar_rms_fusion: int = 0
|
||||
aiter_ar_rms_fusion: int = 0
|
||||
sequence_parallel: int = 0
|
||||
async_tp: int = 0
|
||||
|
||||
|
||||
class ModelFusionInfo(NamedTuple):
|
||||
@@ -143,8 +141,4 @@ FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
"aiter_ar_rms_fusion": re.compile(
|
||||
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
"sequence_parallel": re.compile(
|
||||
r"sequence_parallelism.py:\d+] Replaced (\d+) patterns"
|
||||
),
|
||||
"async_tp": re.compile(r"collective_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
}
|
||||
|
||||
@@ -170,26 +170,8 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
|
||||
for match_name in matches_check:
|
||||
log_matches = list(int(ms) for ms in log_matches_dict[match_name])
|
||||
|
||||
# AR+RMS skips the largest range; SP skips the smallest.
|
||||
# When both are enabled, AR+RMS activation count is
|
||||
# model-dependent (hidden_size affects threshold), so derive
|
||||
# from log data.
|
||||
if (
|
||||
match_name == "ar_rms_fusion"
|
||||
and "sequence_parallel" in matches_check
|
||||
and num_compile_ranges >= 2
|
||||
):
|
||||
assert (
|
||||
len(log_matches) >= tp_size and len(log_matches) % tp_size == 0
|
||||
), (
|
||||
f"Expected multiple of {tp_size} ar_rms log entries, "
|
||||
f"found {len(log_matches)}"
|
||||
)
|
||||
num_ranges_activated = len(log_matches) // tp_size
|
||||
elif (
|
||||
match_name in ("ar_rms_fusion", "sequence_parallel")
|
||||
and num_compile_ranges >= 2
|
||||
):
|
||||
# AR+RMS skips the largest range.
|
||||
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
|
||||
num_ranges_activated = num_compile_ranges - 1
|
||||
else:
|
||||
num_ranges_activated = num_compile_ranges
|
||||
@@ -233,44 +215,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
|
||||
f"Expecting at least {expected_matches - matches.ar_rms_fusion} "
|
||||
f"where ar+rms+quant was activated"
|
||||
)
|
||||
elif (
|
||||
match_name == "async_tp"
|
||||
and "sequence_parallel" in matches_check
|
||||
and num_compile_ranges >= 2
|
||||
):
|
||||
# AsyncTP only finds patterns on ranges where SP ran.
|
||||
n_sp_ranges = num_compile_ranges - 1
|
||||
assert (
|
||||
sum(m == expected_matches for m in log_matches)
|
||||
== tp_size * n_sp_ranges
|
||||
), (
|
||||
f"Expecting {expected_matches} async_tp on "
|
||||
f"{tp_size * n_sp_ranges} SP-range entries, "
|
||||
f"found: {log_matches}"
|
||||
)
|
||||
assert sum(m == 0 for m in log_matches) == tp_size, (
|
||||
f"Expecting 0 async_tp on {tp_size} small-range entries "
|
||||
f"(no SP), found: {log_matches}"
|
||||
)
|
||||
elif (
|
||||
match_name == "ar_rms_fusion"
|
||||
and "sequence_parallel" in matches_check
|
||||
and num_compile_ranges >= 2
|
||||
):
|
||||
# SP consumes allreduce patterns first, so AR+RMS finds
|
||||
# full matches only on the smallest range (no SP).
|
||||
assert sum(m == expected_matches for m in log_matches) == tp_size, (
|
||||
f"Expecting {expected_matches} ar_rms on "
|
||||
f"{tp_size} small-range entries, found: {log_matches}"
|
||||
)
|
||||
assert sum(m == 0 for m in log_matches) == tp_size * (
|
||||
num_ranges_activated - 1
|
||||
), (
|
||||
f"Expecting 0 ar_rms on "
|
||||
f"{tp_size * (num_ranges_activated - 1)} large-range "
|
||||
f"entries (SP took precedence), found: {log_matches}"
|
||||
)
|
||||
|
||||
elif match_name == "act_quant_fusion":
|
||||
actual_match = match_table.get("activation_quant_fusion_pass", 0)
|
||||
assert actual_match == expected_matches * n_expected, (
|
||||
@@ -305,17 +249,4 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
)
|
||||
|
||||
if match_name == "sequence_parallel" and num_compile_ranges >= 2:
|
||||
log_matches = re.findall(
|
||||
r"pass_manager.py:\d+] Skipping "
|
||||
r".*SequenceParallelismPass.* with compile range",
|
||||
log_holder.text,
|
||||
)
|
||||
|
||||
n_expected = tp_size * (num_compile_ranges - num_ranges_activated)
|
||||
assert len(log_matches) == n_expected, (
|
||||
f'Could not find {n_expected} "Skipping SequenceParallelismPass" '
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
)
|
||||
|
||||
return run
|
||||
|
||||
@@ -73,8 +73,6 @@ llama3_8b = ModelFusionInfo(
|
||||
matches=lambda n_layers: Matches(
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
aiter_ar_rms_fusion=n_layers * 2 + 1,
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 4,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -85,8 +83,6 @@ llama3_8b_fp8 = ModelFusionInfo(
|
||||
act_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 4,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -96,8 +92,6 @@ llama3_8b_fp4 = ModelFusionInfo(
|
||||
act_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 4,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -127,8 +121,6 @@ llama4_scout_fp8 = ModelFusionInfo(
|
||||
rms_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2,
|
||||
sequence_parallel=n_layers * 2,
|
||||
async_tp=n_layers * 2 - 1,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -138,8 +130,6 @@ llama4_scout_fp4 = ModelFusionInfo(
|
||||
matches=lambda n_layers: Matches(
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2,
|
||||
sequence_parallel=n_layers * 2,
|
||||
async_tp=n_layers * 2 - 1,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -149,8 +139,6 @@ qwen3_a3b = ModelFusionInfo(
|
||||
norm_rope_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
aiter_ar_rms_fusion=n_layers * 2 + 1,
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 2,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -161,8 +149,6 @@ qwen3_a3b_fp8 = ModelFusionInfo(
|
||||
norm_rope_fusion=n_layers,
|
||||
attn_quant_fusion=0, # attn + group quant not supported
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 2,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -192,9 +178,6 @@ deepseek_v3_fp8 = ModelFusionInfo(
|
||||
# MLA attn + per-group FP8 quant
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
# TODO
|
||||
# sequence_parallel= n_layers * 2 + 1,
|
||||
# async_tp=n_layers * 2,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -213,8 +196,6 @@ gpt_oss_20b = ModelFusionInfo(
|
||||
matches=lambda n_layers: Matches(
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
aiter_ar_rms_fusion=n_layers + 1,
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 2,
|
||||
),
|
||||
model_kwargs=(
|
||||
{"quantization_config": {"moe": {"activation": "mxfp8"}}}
|
||||
|
||||
@@ -1,339 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .common import (
|
||||
INDUCTOR_GRAPH_PARTITION,
|
||||
AttentionBackendCase,
|
||||
Matches,
|
||||
custom_ops_combos,
|
||||
is_blackwell,
|
||||
nvfp4_kernel_exposes_input_quant_key,
|
||||
)
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
TRITON_ATTN,
|
||||
llama3_8b,
|
||||
llama3_8b_fp4,
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp8,
|
||||
qwen3_a3b,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp8, llama4_scout_fp8],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
def test_tp2_async_tp_fp8_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=False,
|
||||
# Override threshold for testing (models have small hidden_size)
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"rms_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
"attn_quant_fusion",
|
||||
"sequence_parallel",
|
||||
"async_tp",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp4],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_tp2_async_tp_nvfp4_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
if nvfp4_kernel_exposes_input_quant_key():
|
||||
pytest.skip(
|
||||
"NVFP4 kernel exposes input_quant_key; manual fusion fires "
|
||||
"instead of compiler pass-based fusion"
|
||||
)
|
||||
|
||||
# NVFP4 currently wires the all-gather + GEMM path only.
|
||||
matches = matches_fn(n_layers)._replace(async_tp=n_layers * 2)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=False,
|
||||
# Override threshold for testing (models have small hidden_size)
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"attn_quant_fusion",
|
||||
"sequence_parallel",
|
||||
"async_tp",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b, qwen3_a3b],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
def test_tp2_async_tp_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=False,
|
||||
# Override threshold for testing (models have small hidden_size)
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"norm_rope_fusion",
|
||||
"sequence_parallel",
|
||||
"async_tp",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp8, llama4_scout_fp8],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
def test_tp2_sp_ar_rms_fp8_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=True,
|
||||
# Override threshold for testing (models have small hidden_size)
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"rms_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
"attn_quant_fusion",
|
||||
"ar_rms_fusion",
|
||||
"sequence_parallel",
|
||||
"async_tp",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b, qwen3_a3b],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
def test_tp2_sp_ar_rms_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=True,
|
||||
# Override threshold for testing (models have small hidden_size)
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"norm_rope_fusion",
|
||||
"ar_rms_fusion",
|
||||
"sequence_parallel",
|
||||
"async_tp",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Bounded ROCm model coverage for actual sequence-parallel AsyncTP rewrites."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .common import INDUCTOR_GRAPH_PARTITION, custom_ops_combos
|
||||
from .models import TRITON_ATTN, llama3_8b
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.parametrize("custom_ops", tuple(custom_ops_combos("rms_norm")))
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
|
||||
def test_tp2_rocm_async_tp_bf16_fusions(
|
||||
inductor_graph_partition, custom_ops, attn_backend, run_e2e_fusion_test
|
||||
):
|
||||
n_layers = 4
|
||||
model_kwargs = dict(
|
||||
hf_overrides=dict(
|
||||
num_hidden_layers=n_layers,
|
||||
hidden_size=512,
|
||||
intermediate_size=1024,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=2,
|
||||
head_dim=64,
|
||||
),
|
||||
load_format="dummy",
|
||||
max_model_len=1024,
|
||||
kv_cache_memory_bytes=64 * 1024**2,
|
||||
gpu_memory_utilization=0.01,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_seqs=32,
|
||||
)
|
||||
run_e2e_fusion_test(
|
||||
llama3_8b.model_name,
|
||||
llama3_8b.matches(n_layers),
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=False,
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
),
|
||||
["norm_rope_fusion", "sequence_parallel", "async_tp"],
|
||||
tp_size=2,
|
||||
)
|
||||
@@ -1,60 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm MoE coverage for the TP2 GEMM/collective fusion contract."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .common import INDUCTOR_GRAPH_PARTITION
|
||||
from .models import TRITON_ATTN, qwen3_a3b
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION[:1])
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
|
||||
def test_tp2_rocm_moe_async_tp_bf16_fusions(
|
||||
inductor_graph_partition, attn_backend, run_e2e_fusion_test
|
||||
):
|
||||
"""MoE dense projections retain the Qwen SP/AsyncTP rewrite counts."""
|
||||
n_layers = 4
|
||||
run_e2e_fusion_test(
|
||||
qwen3_a3b.model_name,
|
||||
qwen3_a3b.matches(n_layers),
|
||||
dict(
|
||||
hf_overrides=dict(
|
||||
num_hidden_layers=n_layers,
|
||||
hidden_size=512,
|
||||
intermediate_size=1024,
|
||||
moe_intermediate_size=128,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=2,
|
||||
head_dim=64,
|
||||
num_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
),
|
||||
load_format="dummy",
|
||||
max_model_len=1024,
|
||||
kv_cache_memory_bytes=64 * 1024**2,
|
||||
gpu_memory_utilization=0.01,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_seqs=32,
|
||||
),
|
||||
attn_backend,
|
||||
dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=["-rms_norm"],
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_allreduce_rms=False,
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
),
|
||||
["norm_rope_fusion", "sequence_parallel", "async_tp"],
|
||||
tp_size=2,
|
||||
)
|
||||
@@ -1,394 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.utils import (
|
||||
multi_gpu_test,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.collective_fusion import AsyncTPPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
DeviceConfig,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
class TestMMRSModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.dtype = dtype
|
||||
self.gate_proj = torch.nn.Parameter(
|
||||
torch.empty((self.hidden_size * 2, hidden_size)), requires_grad=False
|
||||
)
|
||||
# Initialize weights
|
||||
torch.nn.init.normal_(self.gate_proj, std=0.02)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
"""Forward pass implementing the mm + reduce scatter in the FX graph."""
|
||||
# Reshape input
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
|
||||
# matrix multiplication
|
||||
permute = self.gate_proj.permute(1, 0)
|
||||
mm = torch.mm(view, permute)
|
||||
reduce_scatter = tensor_model_parallel_reduce_scatter(mm, dim=0)
|
||||
return reduce_scatter
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.reduce_scatter.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.symm_mem.fused_matmul_reduce_scatter.default]
|
||||
|
||||
|
||||
class TestAGMMModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.dtype = dtype
|
||||
self.weight = torch.nn.Parameter(
|
||||
torch.empty((hidden_size, hidden_size)), requires_grad=False
|
||||
)
|
||||
# Initialize weights
|
||||
torch.nn.init.normal_(self.weight, std=0.02)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
"""Forward pass implementing the mm + all gather in the FX graph."""
|
||||
# Reshape input
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
all_gather = tensor_model_parallel_all_gather(view, dim=0)
|
||||
permute = self.weight.permute(1, 0)
|
||||
mm = torch.mm(all_gather, permute)
|
||||
return mm
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_gather.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.symm_mem.fused_all_gather_matmul.default]
|
||||
|
||||
|
||||
class _BaseScaledMMModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.dtype = dtype
|
||||
self.weight = (
|
||||
torch.empty([hidden_size, hidden_size], dtype=FP8_DTYPE)
|
||||
.contiguous()
|
||||
.transpose(0, 1)
|
||||
)
|
||||
|
||||
# Initialize scale_b for _scaled_mm.
|
||||
self.scale_b = torch.ones(1, self.hidden_size, dtype=torch.float32)
|
||||
|
||||
|
||||
class TestScaledMMRSModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""Forward pass implementing the scaled_mm + reduce scatter in the FX graph."""
|
||||
fp8_input = input.to(FP8_DTYPE)
|
||||
scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32)
|
||||
scaled_mm = torch._scaled_mm(
|
||||
fp8_input,
|
||||
self.weight,
|
||||
scale_a=scale_a,
|
||||
scale_b=self.scale_b,
|
||||
out_dtype=self.dtype,
|
||||
)
|
||||
reduce_scatter = tensor_model_parallel_reduce_scatter(scaled_mm, dim=0)
|
||||
return reduce_scatter
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.reduce_scatter.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.patched_fused_scaled_matmul_reduce_scatter.default]
|
||||
|
||||
|
||||
class TestAGScaledMMModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""Forward pass implementing the all gather + scaled_mm in the FX graph."""
|
||||
# Reshape input
|
||||
fp8_input = input.to(FP8_DTYPE)
|
||||
all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0)
|
||||
|
||||
scale_a = torch.ones(all_gather.shape[0], 1, dtype=torch.float32)
|
||||
scaled_mm = torch._scaled_mm(
|
||||
all_gather,
|
||||
self.weight,
|
||||
scale_a=scale_a,
|
||||
scale_b=self.scale_b,
|
||||
out_dtype=self.dtype,
|
||||
)
|
||||
return scaled_mm
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_gather.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.symm_mem.fused_all_gather_scaled_matmul.default]
|
||||
|
||||
|
||||
class TestCutlassScaledMMRSModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""Forward pass implementing the cutlass_scaled_mm + reduce scatter
|
||||
in the FX graph
|
||||
|
||||
"""
|
||||
fp8_input = input.to(FP8_DTYPE)
|
||||
scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32)
|
||||
mm_out = torch.empty(
|
||||
(fp8_input.shape[0], self.weight.shape[1]),
|
||||
dtype=self.dtype,
|
||||
device=input.device,
|
||||
)
|
||||
torch.ops._C.cutlass_scaled_mm(
|
||||
mm_out, fp8_input, self.weight, scale_a, self.scale_b, None
|
||||
)
|
||||
reduce_scatter = tensor_model_parallel_reduce_scatter(mm_out, dim=0)
|
||||
return reduce_scatter
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.reduce_scatter.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.patched_fused_scaled_matmul_reduce_scatter.default]
|
||||
|
||||
|
||||
class TestAGCutlassScaledMMModel(_BaseScaledMMModel):
|
||||
def forward(self, input: torch.Tensor):
|
||||
"""Forward pass implementing the all gather + cutlass_scaled_mm
|
||||
in the FX graph
|
||||
"""
|
||||
# Reshape input
|
||||
fp8_input = input.to(FP8_DTYPE)
|
||||
all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0)
|
||||
|
||||
scale_a = torch.ones(all_gather.shape[0], 1, dtype=torch.float32)
|
||||
|
||||
mm_out = torch.empty(
|
||||
(all_gather.shape[0], self.weight.shape[1]),
|
||||
dtype=self.dtype,
|
||||
device=all_gather.device,
|
||||
)
|
||||
torch.ops._C.cutlass_scaled_mm(
|
||||
mm_out, all_gather, self.weight, scale_a, self.scale_b, None
|
||||
)
|
||||
return mm_out
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_gather.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.symm_mem.fused_all_gather_scaled_matmul.default]
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"test_model",
|
||||
[
|
||||
TestMMRSModel,
|
||||
TestAGMMModel,
|
||||
TestScaledMMRSModel,
|
||||
TestAGScaledMMModel,
|
||||
pytest.param(
|
||||
TestCutlassScaledMMRSModel,
|
||||
marks=pytest.mark.skipif(
|
||||
not hasattr(torch.ops._C, "cutlass_scaled_mm"),
|
||||
reason="Requires cutlass_scaled_mm",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAGCutlassScaledMMModel,
|
||||
marks=pytest.mark.skipif(
|
||||
not hasattr(torch.ops._C, "cutlass_scaled_mm"),
|
||||
reason="Requires cutlass_scaled_mm",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [16])
|
||||
@pytest.mark.parametrize("hidden_size", [16])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("dynamic", [True, False])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_async_tp_pass_replace(
|
||||
test_model: str,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
dynamic: bool,
|
||||
):
|
||||
num_processes = 2
|
||||
distributed_init_method = get_file_store_init_method()
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
# need to use torch.mp.spawn otherwise will have problems with
|
||||
# torch.distributed and cuda
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
test_model,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
dtype,
|
||||
dynamic,
|
||||
distributed_init_method,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(async_tp_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def test_async_tp_pass_requires_full_graph_compilation():
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.compilation_config.use_inductor_graph_partition = False
|
||||
vllm_config.compilation_config.splitting_ops = [
|
||||
"vllm::unified_attention_with_output"
|
||||
]
|
||||
|
||||
async_tp_pass = object.__new__(AsyncTPPass)
|
||||
async_tp_pass.compilation_config = vllm_config.compilation_config
|
||||
|
||||
with pytest.raises(
|
||||
AssertionError, match="AsyncTPPass requires full-graph compilation"
|
||||
):
|
||||
async_tp_pass.is_applicable_for_range(Range(start=8, end=8))
|
||||
|
||||
|
||||
def async_tp_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
test_model_cls: torch.nn.Module,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
dynamic: bool,
|
||||
distributed_init_method: str,
|
||||
check_numerics: bool = False,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
}
|
||||
)
|
||||
|
||||
# initialize distributed
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=local_rank,
|
||||
distributed_init_method=distributed_init_method,
|
||||
local_rank=local_rank,
|
||||
)
|
||||
|
||||
# configure vllm config for SequenceParallelismPass
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.compilation_config = CompilationConfig(
|
||||
pass_config=PassConfig(
|
||||
fuse_gemm_comms=True,
|
||||
),
|
||||
)
|
||||
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
|
||||
|
||||
# this is a fake model name to construct the model config
|
||||
# in the vllm_config, it's not really used.
|
||||
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
|
||||
vllm_config.model_config = ModelConfig(
|
||||
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
async_tp_pass = AsyncTPPass(vllm_config)
|
||||
backend = TestBackend(async_tp_pass)
|
||||
|
||||
assert (
|
||||
async_tp_pass.compilation_config.splitting_ops
|
||||
== vllm_config.compilation_config.splitting_ops
|
||||
)
|
||||
assert (
|
||||
async_tp_pass.compilation_config.use_inductor_graph_partition
|
||||
== vllm_config.compilation_config.use_inductor_graph_partition
|
||||
)
|
||||
|
||||
model = test_model_cls(hidden_size, dtype) # Pass dtype to model constructor
|
||||
|
||||
hidden_states = torch.randn(
|
||||
(batch_size * seq_len, hidden_size), dtype=dtype, requires_grad=False
|
||||
)
|
||||
if check_numerics:
|
||||
hidden_states += local_rank * 0.125
|
||||
|
||||
if dynamic:
|
||||
torch._dynamo.mark_dynamic(hidden_states, 0)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend)
|
||||
actual = compiled_model(hidden_states)
|
||||
|
||||
if check_numerics:
|
||||
# These are new checks, so no existing tolerance is relaxed.
|
||||
# Allow BF16 rounding vs. eager, but keep the lifetime check exact.
|
||||
expected = model(hidden_states)
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2)
|
||||
snapshot = actual.clone()
|
||||
changed = compiled_model(-hidden_states)
|
||||
torch.testing.assert_close(changed, -expected, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(actual, snapshot, rtol=0, atol=0)
|
||||
|
||||
assert async_tp_pass.matched_count == 1
|
||||
|
||||
# In pre-nodes, all gather or reduce scatter should exist,
|
||||
# fused_matmul_reduce_scatter or fused_all_gather_matmul should not
|
||||
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
|
||||
|
||||
# In post-nodes, fused_matmul_reduce_scatter or \
|
||||
# fused_all_gather_matmul should exist
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
@@ -1,343 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.utils import TestFP8Layer, multi_gpu_test
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||
from vllm.compilation.passes.fusion.sequence_parallelism import SequenceParallelismPass
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.compilation.passes.vllm_inductor_pass import VllmInductorPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CUDAGraphMode,
|
||||
DeviceConfig,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceRMSNormModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size=16, eps=1e-6):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
|
||||
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
|
||||
|
||||
def forward(self, x):
|
||||
z = torch.relu(x)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
y = self.norm[0](x)
|
||||
|
||||
z2 = torch.mm(y, self.w[0])
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
z3 = torch.mm(y2, self.w[1])
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
|
||||
y3, resid = self.norm[2](x3, resid)
|
||||
|
||||
z4 = torch.mm(y3, self.w[2])
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
|
||||
y4, resid = self.norm[3](x4, resid)
|
||||
return y4
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_reduce.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [
|
||||
torch.ops.vllm.all_gather.default,
|
||||
torch.ops.vllm.reduce_scatter.default,
|
||||
]
|
||||
|
||||
def ops_in_model(self):
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
quant_key = kFp8StaticTensorSym
|
||||
|
||||
def __init__(self, hidden_size=16, eps=1e-6):
|
||||
super().__init__()
|
||||
self.vllm_config = get_current_vllm_config()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
|
||||
self.fp8_linear_layers = [
|
||||
TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
input_dtype=self.vllm_config.model_config.dtype,
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
def forward(self, hidden_states):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
z = torch.relu(hidden_states)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
y = self.norm[0](x)
|
||||
|
||||
z2 = self.fp8_linear_layers[0](y)
|
||||
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
z3 = self.fp8_linear_layers[1](y2)
|
||||
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
y3, resid = self.norm[2](x3, resid) # use resid here
|
||||
|
||||
z4 = self.fp8_linear_layers[2](y3)
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
y4, resid = self.norm[3](x4, resid) # use resid here
|
||||
return y4
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [
|
||||
torch.ops.vllm.all_gather.default,
|
||||
torch.ops.vllm.reduce_scatter.default,
|
||||
]
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.all_reduce.default,
|
||||
]
|
||||
|
||||
def ops_in_model(self):
|
||||
if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
|
||||
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
|
||||
else:
|
||||
quant_ops = (
|
||||
[torch.ops._C.static_scaled_fp8_quant.default]
|
||||
if any(layer.is_quant_fp8_enabled() for layer in self.fp8_linear_layers)
|
||||
else [torch.ops.aten.reciprocal]
|
||||
)
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm,
|
||||
*quant_ops,
|
||||
]
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"test_model_cls, custom_ops",
|
||||
[
|
||||
(TestAllReduceRMSNormModel, "+rms_norm"),
|
||||
(TestAllReduceRMSNormModel, "-rms_norm"),
|
||||
(TestAllReduceRMSNormStaticQuantFP8Model, "+rms_norm,+quant_fp8"),
|
||||
(TestAllReduceRMSNormStaticQuantFP8Model, "+rms_norm,-quant_fp8"),
|
||||
(TestAllReduceRMSNormStaticQuantFP8Model, "-rms_norm,+quant_fp8"),
|
||||
(TestAllReduceRMSNormStaticQuantFP8Model, "-rms_norm,-quant_fp8"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [16])
|
||||
@pytest.mark.parametrize("hidden_size", [16])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("fuse_norm_quant", [True, False])
|
||||
@pytest.mark.parametrize("dynamic", [False, True])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_sequence_parallelism_pass(
|
||||
test_model_cls: type[torch.nn.Module],
|
||||
custom_ops: str,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
fuse_norm_quant: bool,
|
||||
dynamic: bool,
|
||||
):
|
||||
num_processes = 2
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
# need to use torch.mp.spawn otherwise will have problems with
|
||||
# torch.distributed and cuda
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
test_model_cls,
|
||||
custom_ops,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
dtype,
|
||||
fuse_norm_quant,
|
||||
dynamic,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(sequence_parallelism_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def test_sequence_parallelism_pass_requires_full_graph_compilation():
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.compilation_config.use_inductor_graph_partition = False
|
||||
vllm_config.compilation_config.splitting_ops = [
|
||||
"vllm::unified_attention_with_output"
|
||||
]
|
||||
|
||||
sequence_parallelism_pass = object.__new__(SequenceParallelismPass)
|
||||
sequence_parallelism_pass.compilation_config = vllm_config.compilation_config
|
||||
sequence_parallelism_pass.min_token_num = 1
|
||||
|
||||
with pytest.raises(
|
||||
AssertionError,
|
||||
match="SequenceParallelismPass requires full-graph compilation",
|
||||
):
|
||||
sequence_parallelism_pass.is_applicable_for_range(Range(start=8, end=8))
|
||||
|
||||
|
||||
def sequence_parallelism_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
test_model_cls: type[torch.nn.Module],
|
||||
custom_ops: str,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
fuse_norm_quant: bool,
|
||||
dynamic: bool,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12345",
|
||||
}
|
||||
)
|
||||
|
||||
# initialize distributed
|
||||
init_distributed_environment()
|
||||
|
||||
# configure vllm config for SequenceParallelismPass
|
||||
custom_ops_list = custom_ops.split(",") if custom_ops else []
|
||||
compilation_config = CompilationConfig(
|
||||
splitting_ops=[], # avoid automatic rms_norm enablement
|
||||
cudagraph_mode=CUDAGraphMode.NONE, # avoid piecewise warnings
|
||||
custom_ops=custom_ops_list,
|
||||
pass_config=PassConfig(
|
||||
enable_sp=True,
|
||||
fuse_norm_quant=fuse_norm_quant,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
) # NoOp needed for fusion
|
||||
device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
|
||||
|
||||
# this is a fake model name to construct the model config
|
||||
# in the vllm_config, it's not really used.
|
||||
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
|
||||
model_config = ModelConfig(
|
||||
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
device_config=device_config,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
sequence_parallelism_pass = SequenceParallelismPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
assert (
|
||||
sequence_parallelism_pass.compilation_config.splitting_ops
|
||||
== vllm_config.compilation_config.splitting_ops
|
||||
)
|
||||
assert (
|
||||
sequence_parallelism_pass.compilation_config.use_inductor_graph_partition
|
||||
== vllm_config.compilation_config.use_inductor_graph_partition
|
||||
)
|
||||
passes_for_backend: list[VllmInductorPass] = [
|
||||
noop_pass,
|
||||
sequence_parallelism_pass,
|
||||
]
|
||||
|
||||
if fuse_norm_quant:
|
||||
fusion_pass = RMSNormQuantFusionPass(vllm_config)
|
||||
passes_for_backend.append(fusion_pass)
|
||||
|
||||
passes_for_backend.append(cleanup_pass)
|
||||
|
||||
backend = TestBackend(*passes_for_backend)
|
||||
|
||||
model = test_model_cls(hidden_size)
|
||||
|
||||
hidden_states = torch.randn((batch_size * seq_len, hidden_size), dtype=dtype)
|
||||
|
||||
if dynamic:
|
||||
torch._dynamo.mark_dynamic(hidden_states, 0)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend)
|
||||
compiled_model(hidden_states)
|
||||
|
||||
assert sequence_parallelism_pass.matched_count == 4
|
||||
|
||||
# In pre-nodes, all reduce should be there,
|
||||
# reduce scatter and all gather should not
|
||||
for op in model.ops_in_model_before():
|
||||
assert backend.op_count(op, before=True) == 4
|
||||
|
||||
# In post-nodes, reduce scatter and all gather should be there,
|
||||
# all reduce should not
|
||||
for op in model.ops_in_model_after():
|
||||
assert backend.op_count(op, before=False) == 4
|
||||
|
||||
for op in model.ops_in_model():
|
||||
assert backend.op_count(op, before=False) > 0
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,75 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm AsyncTP numerical coverage kept outside the shared distributed suite."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.compile.passes.distributed.test_async_tp import (
|
||||
TestAGMMModel,
|
||||
TestMMRSModel,
|
||||
async_tp_pass_on_test_model,
|
||||
)
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
|
||||
|
||||
class TestRocmMMRSModel(TestMMRSModel):
|
||||
def forward(self, hidden_states):
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
mm = torch.ops.vllm.rocm_unquantized_gemm(view, self.gate_proj, None)
|
||||
return tensor_model_parallel_reduce_scatter(mm, dim=0)
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return super().ops_in_model_before() + [
|
||||
torch.ops.vllm.rocm_unquantized_gemm.default
|
||||
]
|
||||
|
||||
|
||||
class TestRocmAGMMModel(TestAGMMModel):
|
||||
def __init__(self, hidden_size=16, dtype=torch.bfloat16):
|
||||
super().__init__(hidden_size, dtype)
|
||||
self.weight = torch.nn.Parameter(
|
||||
torch.empty((hidden_size * 2, hidden_size)), requires_grad=False
|
||||
)
|
||||
torch.nn.init.normal_(self.weight, std=0.02)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
gathered = tensor_model_parallel_all_gather(view, dim=0)
|
||||
return torch.ops.vllm.rocm_unquantized_gemm(gathered, self.weight, None)
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return super().ops_in_model_before() + [
|
||||
torch.ops.vllm.rocm_unquantized_gemm.default
|
||||
]
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend contract")
|
||||
@pytest.mark.parametrize(
|
||||
"test_model", [TestMMRSModel, TestAGMMModel, TestRocmMMRSModel, TestRocmAGMMModel]
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic", [False, True])
|
||||
def test_rocm_async_tp_bf16_rewrite_and_correctness(test_model, dynamic):
|
||||
"""Real symmetric-memory AG/GEMM and GEMM/RS must replace the collectives."""
|
||||
torch.multiprocessing.spawn(
|
||||
async_tp_pass_on_test_model,
|
||||
args=(
|
||||
2,
|
||||
test_model,
|
||||
8,
|
||||
16,
|
||||
128,
|
||||
torch.bfloat16,
|
||||
dynamic,
|
||||
get_file_store_init_method(),
|
||||
True,
|
||||
),
|
||||
nprocs=2,
|
||||
)
|
||||
+12
-172
@@ -464,49 +464,41 @@ def test_should_split():
|
||||
(
|
||||
"cudagraph_capture_sizes",
|
||||
"max_cudagraph_capture_size",
|
||||
"tp_size",
|
||||
"enable_sp",
|
||||
"max_num_batched_tokens",
|
||||
"cudagraph_mode",
|
||||
"expected_max_size",
|
||||
),
|
||||
[
|
||||
(None, None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
([1, 2, 4], 4, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
(None, None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
([1, 2, 4], 4, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
(
|
||||
[1, 2, 4],
|
||||
8,
|
||||
1,
|
||||
False,
|
||||
2048,
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
ValidationError,
|
||||
),
|
||||
([1, 256], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
([], None, 1, False, 2048, CUDAGraphMode.NONE, 0),
|
||||
(None, 0, 1, False, 2048, CUDAGraphMode.NONE, 0),
|
||||
([1, 256], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
([], None, 2048, CUDAGraphMode.NONE, 0),
|
||||
(None, 0, 2048, CUDAGraphMode.NONE, 0),
|
||||
# truncated to nearest multiple of 8 or 16
|
||||
(None, 257, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
(None, 257, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),
|
||||
# max_num_batched_tokens <= max_cudagraph_capture_size should always be
|
||||
# captured even if not landing on a 16-stride step
|
||||
(None, 2048, 1, False, 257, CUDAGraphMode.FULL_AND_PIECEWISE, 257),
|
||||
(None, 2048, 257, CUDAGraphMode.FULL_AND_PIECEWISE, 257),
|
||||
# max from list
|
||||
([1, 2, 4, 15], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
|
||||
# SP forces full-graph compilation, sizes are filtered by TP
|
||||
([1, 2, 4, 15], None, 2, True, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
([1, 2, 4, 15], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),
|
||||
# limited by the max_tokens
|
||||
([1, 2, 4, 15], None, 1, False, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
([1, 2, 4, 15], None, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),
|
||||
# the list should contain at least 1 element when use cudagraph
|
||||
([], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
|
||||
([], None, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
|
||||
# the max capturing size should be >= 1 when use cudagraph
|
||||
(None, 0, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
|
||||
(None, 0, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),
|
||||
],
|
||||
)
|
||||
def test_cudagraph_sizes_post_init(
|
||||
cudagraph_capture_sizes,
|
||||
max_cudagraph_capture_size,
|
||||
tp_size,
|
||||
enable_sp,
|
||||
max_num_batched_tokens,
|
||||
cudagraph_mode,
|
||||
expected_max_size,
|
||||
@@ -515,10 +507,7 @@ def test_cudagraph_sizes_post_init(
|
||||
if expected_max_size == ValidationError:
|
||||
ctx = pytest.raises(expected_max_size)
|
||||
|
||||
with (
|
||||
ctx,
|
||||
patch.object(current_platform, "device_count", return_value=tp_size),
|
||||
):
|
||||
with ctx:
|
||||
kwargs = {}
|
||||
if cudagraph_capture_sizes is not None:
|
||||
kwargs["cudagraph_capture_sizes"] = cudagraph_capture_sizes
|
||||
@@ -526,18 +515,15 @@ def test_cudagraph_sizes_post_init(
|
||||
kwargs["max_cudagraph_capture_size"] = max_cudagraph_capture_size
|
||||
compilation_config = CompilationConfig(
|
||||
pass_config=PassConfig(
|
||||
enable_sp=enable_sp,
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=True,
|
||||
eliminate_noops=True,
|
||||
sp_min_token_num=512 if enable_sp else None,
|
||||
),
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
**kwargs,
|
||||
)
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
tensor_parallel_size=tp_size,
|
||||
max_num_seqs=min(max_num_batched_tokens, 128),
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
compilation_config=compilation_config,
|
||||
@@ -728,35 +714,6 @@ def test_default_cudagraph_capture_sizes_keep_all_sizes_bounded():
|
||||
)
|
||||
|
||||
|
||||
def test_cudagraph_capture_sizes_respect_sequence_parallelism():
|
||||
"""Sequence-parallel capture sizes stay divisible by tensor parallel size."""
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
compilation_config.pass_config.enable_sp = True
|
||||
config = _mock_config_for_cudagraph_sizes(
|
||||
max_num_seqs=32,
|
||||
num_speculative_tokens=16,
|
||||
max_num_batched_tokens=32768,
|
||||
compilation_config=compilation_config,
|
||||
)
|
||||
config.parallel_config = SimpleNamespace(tensor_parallel_size=2)
|
||||
config.update_sizes_for_sequence_parallelism = lambda sizes: (
|
||||
VllmConfig.update_sizes_for_sequence_parallelism(config, sizes)
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
return_value=False,
|
||||
):
|
||||
VllmConfig._set_cudagraph_sizes(config)
|
||||
|
||||
assert all(size % 2 == 0 for size in compilation_config.cudagraph_capture_sizes)
|
||||
assert 544 not in compilation_config.cudagraph_capture_sizes
|
||||
assert compilation_config.max_cudagraph_capture_size == 512
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_real_vllm_config_caps_widest_ngram_decode_batch():
|
||||
"""Real config post-init preserves the default capture ceiling."""
|
||||
@@ -1046,123 +1003,6 @@ def test_blackwell_cudagraph_default(is_blackwell, expected_max_size):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.support_static_graph_mode(),
|
||||
reason="Skip if not cudagraph mode supported",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"cudagraph_mode",
|
||||
"use_inductor_graph_partition",
|
||||
"expected_enable_sp",
|
||||
"expected_cudagraph_mode",
|
||||
"expected_piecewise_compile",
|
||||
"expected_capture_sizes",
|
||||
"expected_max_size",
|
||||
),
|
||||
[
|
||||
(CUDAGraphMode.PIECEWISE, False, True, CUDAGraphMode.FULL, False, [2, 4], 4),
|
||||
(
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
False,
|
||||
True,
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
False,
|
||||
[2, 4],
|
||||
4,
|
||||
),
|
||||
(
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
False,
|
||||
True,
|
||||
CUDAGraphMode.FULL,
|
||||
False,
|
||||
[2, 4],
|
||||
4,
|
||||
),
|
||||
(
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
True,
|
||||
True,
|
||||
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
True,
|
||||
[2, 4],
|
||||
4,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sequence_parallelism_requires_full_graph_compilation(
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
use_inductor_graph_partition: bool,
|
||||
expected_enable_sp: bool,
|
||||
expected_cudagraph_mode: CUDAGraphMode,
|
||||
expected_piecewise_compile: bool,
|
||||
expected_capture_sizes: list[int],
|
||||
expected_max_size: int,
|
||||
):
|
||||
with patch.object(current_platform, "device_count", return_value=2):
|
||||
vllm_config = VllmConfig(
|
||||
parallel_config=ParallelConfig(tensor_parallel_size=2),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_num_seqs=128,
|
||||
max_num_batched_tokens=2048,
|
||||
max_model_len=2048,
|
||||
is_encoder_decoder=False,
|
||||
),
|
||||
)
|
||||
vllm_config.model_config = MagicMock(
|
||||
dtype=torch.float16,
|
||||
enforce_eager=False,
|
||||
is_moe=False,
|
||||
disable_cascade_attn=False,
|
||||
get_hidden_size=MagicMock(return_value=4096),
|
||||
)
|
||||
vllm_config.compilation_config = CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_capture_sizes=[1, 2, 4, 15],
|
||||
max_cudagraph_capture_size=None,
|
||||
compile_sizes=["cudagraph_capture_sizes"],
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
pass_config=PassConfig(
|
||||
enable_sp=True,
|
||||
fuse_gemm_comms=True,
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=True,
|
||||
eliminate_noops=True,
|
||||
sp_min_token_num=512,
|
||||
),
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
)
|
||||
vllm_config.compilation_config.set_splitting_ops_for_v1(
|
||||
all2all_backend=vllm_config.parallel_config.all2all_backend,
|
||||
data_parallel_size=1,
|
||||
)
|
||||
vllm_config._set_compile_ranges()
|
||||
vllm_config._set_cudagraph_sizes()
|
||||
|
||||
assert (
|
||||
vllm_config.compilation_config.use_inductor_graph_partition
|
||||
== use_inductor_graph_partition
|
||||
)
|
||||
assert (
|
||||
bool(vllm_config.compilation_config.splitting_ops) == expected_piecewise_compile
|
||||
)
|
||||
assert vllm_config.compilation_config.pass_config.enable_sp == expected_enable_sp
|
||||
assert (
|
||||
vllm_config.compilation_config.pass_config.fuse_gemm_comms == expected_enable_sp
|
||||
)
|
||||
assert vllm_config.compilation_config.cudagraph_mode == expected_cudagraph_mode
|
||||
assert (
|
||||
vllm_config.compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||
)
|
||||
assert (
|
||||
vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size
|
||||
)
|
||||
assert (
|
||||
511 in vllm_config.compilation_config.compile_ranges_endpoints
|
||||
) == expected_enable_sp
|
||||
|
||||
|
||||
def test_cached_compilation_config(default_vllm_config):
|
||||
import torch
|
||||
from torch._inductor.utils import run_and_get_code
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.compilation.passes.fusion.sequence_parallelism import (
|
||||
SP_MIN_HIDDEN_SIZE,
|
||||
SP_MIN_PER_GPU_SIZE_MB,
|
||||
get_sequence_parallelism_threshold,
|
||||
)
|
||||
|
||||
|
||||
class TestGetSequenceParallelismThreshold:
|
||||
"""Tests for get_sequence_parallelism_threshold function."""
|
||||
|
||||
def test_non_cuda_returns_none(self, mock_cuda_platform):
|
||||
"""Non-CUDA platforms should return None."""
|
||||
with mock_cuda_platform(is_cuda=False):
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=8192, tp_size=2, element_size=2
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_unsupported_device_capability_returns_none(self, mock_cuda_platform):
|
||||
"""Unsupported device capabilities (e.g., sm80) should return None."""
|
||||
with mock_cuda_platform(capability=(8, 0)):
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=8192, tp_size=2, element_size=2
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_small_hidden_size_returns_none(self, mock_cuda_platform):
|
||||
"""H100 with hidden_size below threshold should return None."""
|
||||
with mock_cuda_platform(capability=(9, 0)):
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=4096,
|
||||
tp_size=2,
|
||||
element_size=2, # 4096 < 8192
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_h100_large_model_returns_threshold(self, mock_cuda_platform):
|
||||
"""H100 with large enough hidden_size should return calculated threshold."""
|
||||
with mock_cuda_platform(capability=(9, 0)):
|
||||
hidden_size = 8192
|
||||
tp_size = 2
|
||||
element_size = 2 # float16/bfloat16
|
||||
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=hidden_size,
|
||||
tp_size=tp_size,
|
||||
element_size=element_size,
|
||||
)
|
||||
|
||||
# Verify calculation: (8 * 2 * 1024 * 1024) // (8192 * 2) = 1024
|
||||
MiB = 1024 * 1024
|
||||
expected = int(
|
||||
(SP_MIN_PER_GPU_SIZE_MB[90] * tp_size * MiB)
|
||||
// (hidden_size * element_size)
|
||||
)
|
||||
assert result == expected
|
||||
assert result == 1024
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,tp_size,element_size,expected",
|
||||
[
|
||||
# Boundary: exactly at min hidden size threshold, tp_size=1
|
||||
# (8 * 1 * 1024 * 1024) // (8192 * 2) = 512
|
||||
(8192, 1, 2, 512),
|
||||
# Larger hidden size reduces token threshold
|
||||
# (8 * 1 * 1024 * 1024) // (16384 * 2) = 256
|
||||
(16384, 1, 2, 256),
|
||||
# Larger tp_size increases token threshold
|
||||
# (8 * 4 * 1024 * 1024) // (8192 * 2) = 2048
|
||||
(8192, 4, 2, 2048),
|
||||
# Larger element_size (fp32) reduces token threshold
|
||||
# (8 * 2 * 1024 * 1024) // (8192 * 4) = 512
|
||||
(8192, 2, 4, 512),
|
||||
],
|
||||
)
|
||||
def test_threshold_calculation_variations(
|
||||
self, mock_cuda_platform, hidden_size, tp_size, element_size, expected
|
||||
):
|
||||
"""Test threshold calculation with various parameter combinations."""
|
||||
with mock_cuda_platform(capability=(9, 0)):
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=hidden_size,
|
||||
tp_size=tp_size,
|
||||
element_size=element_size,
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
def test_hidden_size_boundary(self, mock_cuda_platform):
|
||||
"""Test behavior at the exact hidden_size boundary."""
|
||||
with mock_cuda_platform(capability=(9, 0)):
|
||||
# Just below threshold
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=SP_MIN_HIDDEN_SIZE[90] - 1,
|
||||
tp_size=2,
|
||||
element_size=2,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
# Exactly at threshold
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=SP_MIN_HIDDEN_SIZE[90],
|
||||
tp_size=2,
|
||||
element_size=2,
|
||||
)
|
||||
assert result is not None
|
||||
|
||||
|
||||
# XPU-specific constants (must match sequence_parallelism.py values)
|
||||
_XPU_MIN_HIDDEN_SIZE = 4096
|
||||
_XPU_MIN_PER_GPU_SIZE_MB = 8.0
|
||||
|
||||
|
||||
class TestGetSequenceParallelismThresholdXPU:
|
||||
"""Tests for get_sequence_parallelism_threshold on XPU platform."""
|
||||
|
||||
def test_xpu_small_hidden_size_returns_none(self, mock_xpu_platform):
|
||||
"""XPU with hidden_size below threshold should return None."""
|
||||
with mock_xpu_platform():
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=_XPU_MIN_HIDDEN_SIZE - 1,
|
||||
tp_size=2,
|
||||
element_size=2,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_xpu_large_model_returns_threshold(self, mock_xpu_platform):
|
||||
"""XPU with hidden_size >= threshold should return calculated value."""
|
||||
with mock_xpu_platform():
|
||||
hidden_size = _XPU_MIN_HIDDEN_SIZE
|
||||
tp_size = 2
|
||||
element_size = 2
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=hidden_size,
|
||||
tp_size=tp_size,
|
||||
element_size=element_size,
|
||||
)
|
||||
# (8 * 2 * 1024 * 1024) // (4096 * 2) = 2048
|
||||
MiB = 1024 * 1024
|
||||
expected = int(
|
||||
(_XPU_MIN_PER_GPU_SIZE_MB * tp_size * MiB) // (hidden_size * element_size)
|
||||
)
|
||||
assert result == expected
|
||||
assert result == 2048
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,tp_size,element_size,expected",
|
||||
[
|
||||
# (8 * 1 * 1024 * 1024) // (4096 * 2) = 1024
|
||||
(4096, 1, 2, 1024),
|
||||
# (8 * 4 * 1024 * 1024) // (4096 * 2) = 4096
|
||||
(4096, 4, 2, 4096),
|
||||
# (8 * 2 * 1024 * 1024) // (8192 * 2) = 1024
|
||||
(8192, 2, 2, 1024),
|
||||
# (8 * 2 * 1024 * 1024) // (4096 * 4) = 1024
|
||||
(4096, 2, 4, 1024),
|
||||
],
|
||||
)
|
||||
def test_xpu_threshold_calculation_variations(
|
||||
self, mock_xpu_platform, hidden_size, tp_size, element_size, expected
|
||||
):
|
||||
"""Test XPU threshold calculation with various parameter combinations."""
|
||||
with mock_xpu_platform():
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=hidden_size,
|
||||
tp_size=tp_size,
|
||||
element_size=element_size,
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
def test_xpu_hidden_size_boundary(self, mock_xpu_platform):
|
||||
"""Test behavior at the exact XPU hidden_size boundary."""
|
||||
with mock_xpu_platform():
|
||||
# Just below threshold
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=_XPU_MIN_HIDDEN_SIZE - 1,
|
||||
tp_size=2,
|
||||
element_size=2,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
# Exactly at threshold
|
||||
result = get_sequence_parallelism_threshold(
|
||||
hidden_size=_XPU_MIN_HIDDEN_SIZE,
|
||||
tp_size=2,
|
||||
element_size=2,
|
||||
)
|
||||
assert result is not None
|
||||
@@ -582,10 +582,7 @@ def test_v2_model_runner_accepts_routed_experts(monkeypatch):
|
||||
use_ubatching=False,
|
||||
enable_elastic_ep=False,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
mode=CompilationMode.NONE,
|
||||
pass_config=SimpleNamespace(enable_sp=False),
|
||||
),
|
||||
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
|
||||
cache_config=SimpleNamespace(
|
||||
kv_sharing_fast_prefill=False,
|
||||
mamba_cache_mode="none",
|
||||
|
||||
+1
-11
@@ -843,7 +843,6 @@ def test_breakable_cudagraph_platform_default(
|
||||
("bi-off", 2048, (16, 8), 6144, 1, False),
|
||||
("opt-out", 2048, (16, 8), 6144, 1, False),
|
||||
("eager", 2048, (16, 8), 6144, 1, False),
|
||||
("sp", 2048, (16, 8), 6144, 1, False),
|
||||
("fp16", 2048, (16, 8), 6144, 1, False),
|
||||
("quantized", 2048, (16, 8), 6144, 1, False),
|
||||
("untuned", 4096, (32, 32), 11008, 1, False),
|
||||
@@ -886,11 +885,7 @@ def test_batch_invariant_breakable_cudagraph(
|
||||
get_num_kv_heads=lambda pc: heads[1],
|
||||
)
|
||||
config.parallel_config = SimpleNamespace(tensor_parallel_size=tp)
|
||||
config.compilation_config = (
|
||||
CompilationConfig(pass_config=PassConfig(enable_sp=True))
|
||||
if case == "sp"
|
||||
else CompilationConfig()
|
||||
)
|
||||
config.compilation_config = CompilationConfig()
|
||||
try:
|
||||
assert config._maybe_enable_breakable_cudagraph() is expected
|
||||
if expected:
|
||||
@@ -1021,7 +1016,6 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=4,
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
@@ -1104,7 +1098,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=1,
|
||||
use_v2_model_runner=True,
|
||||
tensor_parallel_size=1,
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_num_reqs=256,
|
||||
)
|
||||
@@ -1117,7 +1110,6 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
"FakeAttentionBackend",
|
||||
uniform_decode_query_len=1,
|
||||
use_v2_model_runner=True,
|
||||
tensor_parallel_size=1,
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_num_reqs=256,
|
||||
is_profiling=True,
|
||||
@@ -3359,8 +3351,6 @@ def test_vllm_config_explicit_overrides():
|
||||
take precedence over callable defaults, across different models and
|
||||
optimization levels.
|
||||
"""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
quantized_model = ModelConfig("RedHatAI/Llama-3.2-1B-FP8")
|
||||
moe_model = ModelConfig("deepseek-ai/DeepSeek-V2-Lite")
|
||||
regular_model = ModelConfig("Qwen/Qwen1.5-7B")
|
||||
|
||||
@@ -378,9 +378,6 @@ def test_execute_model_waits_previous_pp_send_before_forward(
|
||||
|
||||
worker = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(
|
||||
pass_config=SimpleNamespace(enable_sp=False)
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
pipeline_parallel_size=2, distributed_executor_backend="mp"
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user