[ROCm][Compile] Support BF16 AsyncTP fusion (#58098)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -2,18 +2,22 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.compile.fusions_e2e.common import FUSION_LOG_PATTERNS
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.utils import (
|
||||
compare_two_settings,
|
||||
create_new_process_for_each_test,
|
||||
multi_gpu_test,
|
||||
)
|
||||
from vllm.config import (
|
||||
CompilationMode,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
NVFP4_MODEL_ID = "nvidia/Llama-3.1-8B-Instruct-NVFP4"
|
||||
@@ -26,6 +30,78 @@ NVFP4_HF_OVERRIDES = {
|
||||
}
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract")
|
||||
def test_rocm_async_tp_bf16_output_correctness(caplog_mp_spawn):
|
||||
"""Trained-model outputs must agree with unfused TP, including padded decode."""
|
||||
# A snapshot path pins both servers and the comparison helper's tokenizer.
|
||||
model = hf_api().snapshot_download(
|
||||
"Qwen/Qwen3-0.6B", revision="c1899de289a04d12100db370d81485cdf75e47ca"
|
||||
)
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--load-format",
|
||||
"auto",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--max-num-batched-tokens",
|
||||
"512",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(64 * 1024**2),
|
||||
"--gpu-memory-utilization",
|
||||
"0.01",
|
||||
"--attention-backend",
|
||||
"TRITON_ATTN",
|
||||
"--disable-custom-all-reduce",
|
||||
]
|
||||
|
||||
def settings(enabled):
|
||||
return [
|
||||
*common_args,
|
||||
"--compilation-config",
|
||||
json.dumps(
|
||||
{
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"cudagraph_mode": "NONE",
|
||||
"splitting_ops": [],
|
||||
"inductor_compile_config": {"force_disable_caches": True},
|
||||
"pass_config": {
|
||||
"enable_sp": enabled,
|
||||
"fuse_gemm_comms": enabled,
|
||||
"fuse_allreduce_rms": False,
|
||||
"sp_min_token_num": 1,
|
||||
},
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
env = {"VLLM_ROCM_USE_AITER": "0", "VLLM_DISABLE_COMPILE_CACHE": "1"}
|
||||
with caplog_mp_spawn(logging.DEBUG) as logs:
|
||||
compare_two_settings(
|
||||
model,
|
||||
settings(False),
|
||||
settings(True),
|
||||
env1=env,
|
||||
env2=env,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
matches = [
|
||||
int(count) for count in FUSION_LOG_PATTERNS["async_tp"].findall(logs.text)
|
||||
]
|
||||
print(f"AsyncTP rewrite counts: {matches}")
|
||||
assert len(matches) >= 2 and all(count > 0 for count in matches), (
|
||||
f"Expected positive AsyncTP rewrites on both TP workers, found {matches}"
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
|
||||
Reference in New Issue
Block a user