[ROCm][CI] Validate Mooncake and NIXL prefill/decode accuracy (#58095)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-23 11:16:31 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent b715699df4
commit 8033bd08c1
7 changed files with 409 additions and 0 deletions
+43
View File
@@ -3562,6 +3562,49 @@ steps:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
timeout_in_minutes: 60
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
- label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
timeout_in_minutes: 30
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
num_gpus: 4
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- examples/disaggregated/mooncake_connector/
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
- label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy" # TBD
timeout_in_minutes: 40
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+23
View File
@@ -42,6 +42,29 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Preserve all six model/TP pairs with native AITER attention.
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy"
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
@@ -19,6 +19,30 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- examples/disaggregated/mooncake_connector/
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Same full accuracy/TP sweep, using same-host HIP GPU transport.
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
key: mooncake-ec-tcp-e2e-2-gpus
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake P/D accuracy with same-host HIP transport on ROCm."""
import pytest
from tests.utils import spawn_new_process_for_each_test
from tests.v1.kv_connector.rocm_pd_accuracy_utils import (
QWEN_MODEL,
run_rocm_pd_accuracy,
)
from vllm.platforms import current_platform
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"),
pytest.mark.distributed(num_gpus=4),
]
@pytest.mark.parametrize(
"model,prefill_tp,decode_tp",
[
pytest.param(QWEN_MODEL, 1, 1, id="mooncake-hip-qwen-p1-d1"),
pytest.param(QWEN_MODEL, 2, 2, id="mooncake-hip-qwen-p2-d2"),
],
)
@spawn_new_process_for_each_test
def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch):
run_rocm_pd_accuracy("mooncake", model, prefill_tp, decode_tp, monkeypatch)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL P/D accuracy with AITER unified attention on ROCm."""
import pytest
from tests.utils import spawn_new_process_for_each_test
from tests.v1.kv_connector.rocm_pd_accuracy_utils import (
DEEPSEEK_MODEL,
QWEN_MODEL,
run_rocm_pd_accuracy,
)
from vllm.platforms import current_platform
pytestmark = [
pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm P/D equivalents"),
pytest.mark.distributed(num_gpus=4),
]
@pytest.mark.parametrize(
"model,prefill_tp,decode_tp",
[
pytest.param(QWEN_MODEL, 2, 2, id="nixl-aiter-qwen-p2-d2"),
pytest.param(QWEN_MODEL, 1, 2, id="nixl-aiter-qwen-p1-d2"),
pytest.param(QWEN_MODEL, 2, 1, id="nixl-aiter-qwen-p2-d1"),
pytest.param(DEEPSEEK_MODEL, 1, 1, id="nixl-aiter-deepseek-p1-d1"),
pytest.param(DEEPSEEK_MODEL, 1, 2, id="nixl-aiter-deepseek-p1-d2"),
pytest.param(DEEPSEEK_MODEL, 2, 1, id="nixl-aiter-deepseek-p2-d1"),
],
)
@spawn_new_process_for_each_test
def test_rocm_pd_accuracy(model, prefill_tp, decode_tp, monkeypatch):
run_rocm_pd_accuracy("nixl", model, prefill_tp, decode_tp, monkeypatch)
@@ -0,0 +1,254 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared ROCm P/D accuracy harness for Mooncake and NIXL integration tests.
Keep the models, TP pairs, full GSM8K workload, and acceptance checks of the
Mooncake and FlashInfer/NIXL sweeps. Mooncake uses same-host HIP IPC; NIXL uses
AITER unified attention. These cases do not exercise RDMA or FlashInfer.
"""
import contextlib
import importlib
import json
import os
import signal
import subprocess
import sys
import time
from pathlib import Path
import requests
from prometheus_client.parser import text_string_to_metric_families
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
_ROOT = Path(__file__).resolve().parents[3]
QWEN_MODEL = "Qwen/Qwen3-0.6B"
DEEPSEEK_MODEL = "deepseek-ai/deepseek-vl2-tiny"
_REVISIONS = {
QWEN_MODEL: "c1899de289a04d12100db370d81485cdf75e47ca",
DEEPSEEK_MODEL: "66c54660eae7e90c9ba259bfdf92d07d6e3ce8aa",
}
def _metric_total(server, name: str, **labels: str) -> float:
response = requests.get(server.url_for("metrics"), timeout=10)
response.raise_for_status()
return sum(
sample.value
for family in text_string_to_metric_families(response.text)
for sample in family.samples
if sample.name == name
and all(sample.labels.get(key) == value for key, value in labels.items())
)
@contextlib.contextmanager
def _proxy(connector: str, prefill, decode, bootstrap_port: int):
port = get_open_port()
if connector == "mooncake":
script = (
_ROOT / "examples/disaggregated/mooncake_connector/"
"mooncake_connector_proxy.py"
)
args = [
"--prefill",
prefill.url_root,
str(bootstrap_port),
"--decode",
decode.url_root,
]
else:
script = Path(__file__).parent / "nixl_integration/toy_proxy_server.py"
args = [
"--prefiller-hosts",
"127.0.0.1",
"--prefiller-ports",
str(prefill.port),
"--decoder-hosts",
"127.0.0.1",
"--decoder-ports",
str(decode.port),
]
process = subprocess.Popen(
[sys.executable, str(script), "--port", str(port), *args],
start_new_session=True,
)
try:
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
assert process.poll() is None, "P/D proxy exited during startup"
try:
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
break
except requests.ConnectionError:
time.sleep(0.2)
else:
raise TimeoutError("P/D proxy did not start")
yield f"http://127.0.0.1:{port}/v1"
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=10)
def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch):
"""Transferred KV must feed decoding while full upstream accuracy holds."""
# Preserve the caller's allocation instead of querying host-wide SMI indices.
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
selector = next((key for key in selectors if key in os.environ), selectors[0])
visible = os.getenv(selector)
devices = (
visible.split(",")
if visible is not None
else [str(i) for i in range(current_platform.device_count())]
)
assert len(devices) >= prefill_tp + decode_tp, "Insufficient allocated GPUs"
for key in selectors:
if key != selector:
monkeypatch.delenv(key, raising=False)
if connector == "mooncake":
import mooncake.engine
assert mooncake.engine.SUPPORT_HIP, "Mooncake requires its ROCm/HIP build"
connector_name = "MooncakeConnector"
extra = {"mooncake_protocol": "hip", "device_name": ""}
else:
from vllm.distributed.nixl_utils import NixlWrapper
assert NixlWrapper is not None, "NIXL ROCm bindings must import"
connector_name = "NixlConnector"
extra = {}
common_args = [
"--revision",
_REVISIONS[model],
"--dtype",
"bfloat16",
"--enforce-eager",
"--distributed-executor-backend",
"mp",
"--block-size",
"128",
"--max-num-seqs",
"128",
"--max-num-batched-tokens",
"4096",
"--gpu-memory-utilization",
"0.2",
"--kv-cache-memory-bytes",
str(8 * 1024**3),
"--moe-backend",
"triton",
"--attention-backend",
"ROCM_AITER_UNIFIED_ATTN",
]
common_env = {
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
"VLLM_HOST_IP": "127.0.0.1",
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
"VLLM_USE_V2_MODEL_RUNNER": "1",
"VLLM_ROCM_USE_AITER": "1",
"VLLM_KV_CACHE_LAYOUT": "HND",
}
bootstrap_port = get_open_port()
with contextlib.ExitStack() as stack:
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
for role, tp, assigned in (
("kv_producer", prefill_tp, devices[:prefill_tp]),
("kv_consumer", decode_tp, devices[prefill_tp : prefill_tp + decode_tp]),
):
config = {
"kv_connector": connector_name,
"kv_role": role,
"kv_connector_extra_config": extra,
}
env = {
**common_env,
selector: ",".join(assigned),
"VLLM_PORT": str(get_open_port()),
"VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()),
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port),
}
servers.append(
RemoteOpenAIServer(
model,
[
*common_args,
"--tensor-parallel-size",
str(tp),
"--kv-transfer-config",
json.dumps(config),
],
env_dict=env,
max_wait_seconds=1200,
)
)
prefill, decode = servers
proxy_url = stack.enter_context(
_proxy(connector, prefill, decode, bootstrap_port)
)
# Reuse the full upstream evaluation and its model-specific assertions.
accuracy = importlib.import_module(
f"tests.v1.kv_connector.{connector}_integration.test_accuracy"
)
monkeypatch.setattr(accuracy, "MODEL_NAME", model)
monkeypatch.setattr(accuracy, "BASE_URL", proxy_url)
assert model in accuracy.EXPECTED_VALUES
def transfer_totals():
totals = {
"external_tokens": _metric_total(
decode,
"vllm:prompt_tokens_by_source_total",
source="external_kv_transfer",
)
}
if connector == "nixl":
totals["nixl_bytes"] = _metric_total(
decode, "vllm:nixl_bytes_transferred_sum"
)
return totals
simple_evaluate = accuracy.lm_eval.simple_evaluate
def evaluate_with_transfer_checks(*args, **kwargs):
# The upstream helper runs a smoke prompt before GSM8K. Wait until
# its counters appear, then require new transfers during evaluation.
deadline = time.monotonic() + 15
before = transfer_totals()
while not all(value > 0 for value in before.values()):
assert time.monotonic() < deadline, (
f"Smoke prompt did not transfer KV: {before}"
)
time.sleep(1)
before = transfer_totals()
# The API adapter loads a local tokenizer because these proxies do
# not expose /tokenize. Match it to the server's pinned checkpoint.
kwargs["model_args"] += f",revision={_REVISIONS[model]}"
results = simple_evaluate(*args, **kwargs)
deadline = time.monotonic() + 15
after = transfer_totals()
while not all(after[key] > value for key, value in before.items()):
assert time.monotonic() < deadline, (
f"GSM8K did not transfer KV: before={before}, after={after}"
)
time.sleep(1)
after = transfer_totals()
print(f"GSM8K transfer counters: before={before}, after={after}")
return results
monkeypatch.setattr(
accuracy.lm_eval, "simple_evaluate", evaluate_with_transfer_checks
)
accuracy.test_accuracy()