Add NVFP4 all-gather GEMM fusion for AsyncTP (#41882)

Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
baonudesifeizhai
2026-05-10 01:13:22 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 006af4b956
commit bc5fdc1e6a
7 changed files with 605 additions and 6 deletions
@@ -13,6 +13,17 @@ from tests.utils import (
from vllm.config import (
CompilationMode,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
NVFP4_HF_OVERRIDES = {
"num_hidden_layers": 4,
"hidden_size": 512,
"intermediate_size": 800,
"num_attention_heads": 4,
"num_key_value_heads": 1,
}
@create_new_process_for_each_test()
@@ -82,3 +93,65 @@ def test_async_tp_pass_correctness(
]
compare_two_settings(model_id, async_tp_args, tp_args, method="generate")
@create_new_process_for_each_test()
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
if (
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
):
pytest.skip("NVFP4 requires Blackwell")
if not has_flashinfer():
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass")
tp_size = 2
if num_gpus_available < tp_size:
pytest.skip(f"Need at least {tp_size} GPUs")
common_args = [
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--max-num-seqs",
"8",
"--load-format",
"dummy",
"--hf-overrides",
json.dumps(NVFP4_HF_OVERRIDES),
]
compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
"compile_sizes": [2, 4, 8],
"splitting_ops": [],
"pass_config": {
"enable_sp": True,
"fuse_gemm_comms": True,
"fuse_allreduce_rms": False,
"sp_min_token_num": 1,
},
}
async_tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
"mp",
"--compilation_config",
json.dumps(compilation_config),
]
tp_args = [
*common_args,
"--tensor-parallel-size",
str(tp_size),
"--distributed-executor-backend",
"mp",
]
compare_two_settings(NVFP4_MODEL_ID, async_tp_args, tp_args, method="generate")
@@ -21,12 +21,14 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from ...models.registry import HF_EXAMPLE_MODELS
from ...models.registry import HF_EXAMPLE_MODELS, _HfExamplesInfo
from ...utils import compare_two_settings, create_new_process_for_each_test
logger = init_logger("test_sequence_parallel")
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
NVFP4_MODEL_INFO = _HfExamplesInfo(NVFP4_MODEL_ID)
class ParallelSetup(NamedTuple):
@@ -41,6 +43,7 @@ class ParallelSetup(NamedTuple):
class SPTestOptions(NamedTuple):
multi_node_only: bool
load_format: str | None = None
model_info: _HfExamplesInfo | None = None
@dataclass
@@ -170,6 +173,7 @@ def _compare_sp(
*,
method: Literal["generate", "encode"],
is_multimodal: bool,
dtype: str = "float16",
):
(
tp_size,
@@ -180,14 +184,15 @@ def _compare_sp(
chunked_prefill,
) = parallel_setup
multi_node_only, load_format = test_options
multi_node_only = test_options.multi_node_only
load_format = test_options.load_format
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info = test_options.model_info or HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_transformers_version(on_fail="skip")
trust_remote_code = model_info.trust_remote_code
tokenizer_mode = model_info.tokenizer_mode
hf_overrides = model_info.hf_overrides
hf_overrides = dict(model_info.hf_overrides)
require_embed_inputs = model_info.require_embed_inputs
if load_format == "dummy":
@@ -220,7 +225,7 @@ def _compare_sp(
common_args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"float16",
dtype,
"--max-model-len",
"2048",
"--max-num-seqs",
@@ -352,3 +357,37 @@ def test_tp_sp_generation(
method="generate",
is_multimodal=False,
)
@create_new_process_for_each_test()
def test_tp_sp_nvfp4_generation(num_gpus_available: int):
if (
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
):
pytest.skip("NVFP4 requires Blackwell")
_compare_sp(
NVFP4_MODEL_ID,
ParallelSetup(
tp_size=2,
pp_size=1,
fuse_norm_quant=True,
fuse_act_quant=True,
eager_mode=True,
chunked_prefill=False,
),
"mp",
"auto",
SPTestOptions(
multi_node_only=False,
load_format="dummy",
model_info=NVFP4_MODEL_INFO,
),
num_gpus_available,
use_inductor_graph_partition=False,
fuse_gemm_comms=False,
method="generate",
is_multimodal=False,
dtype="bfloat16",
)