[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import itertools
|
||||
from collections.abc import Callable, Iterable
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
class Matches(NamedTuple):
|
||||
# simple pointwise
|
||||
aiter_rms_quant_fusion: int = 0
|
||||
rms_quant_fusion: int = 0
|
||||
act_quant_fusion: int = 0
|
||||
norm_rope_fusion: int = 0
|
||||
attn_quant_fusion: int = 0
|
||||
# distributed
|
||||
ar_rms_fusion: int = 0
|
||||
aiter_ar_rms_fusion: int = 0
|
||||
|
||||
|
||||
class ModelFusionInfo(NamedTuple):
|
||||
model_name: str
|
||||
matches: Callable[[int], Matches]
|
||||
"""Given number of hidden layers, produces the matches object"""
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
|
||||
|
||||
|
||||
class AttentionBackendCase(NamedTuple):
|
||||
backend: AttentionBackendEnum
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
"""Additional args required for attn+quant fusion"""
|
||||
|
||||
|
||||
is_blackwell = lambda: current_platform.is_device_capability_family(100)
|
||||
"""Are we running on Blackwell, a lot of tests depend on it"""
|
||||
|
||||
|
||||
def nvfp4_kernel_exposes_input_quant_key() -> bool:
|
||||
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
|
||||
|
||||
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
|
||||
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
|
||||
output format and the cutedsl backend's expected input format.
|
||||
|
||||
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
|
||||
manual fusion.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fallback: assume no manual fusion support
|
||||
return False
|
||||
|
||||
|
||||
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
|
||||
"""Generate all combinations of custom ops for parametrization."""
|
||||
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
|
||||
for op_list in itertools.product(*custom_ops_lists):
|
||||
yield ",".join(op_list)
|
||||
|
||||
|
||||
# Quick inline validation
|
||||
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
|
||||
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
|
||||
"-quant_fp8,-rms_norm",
|
||||
"-quant_fp8,+rms_norm",
|
||||
"+quant_fp8,-rms_norm",
|
||||
"+quant_fp8,+rms_norm",
|
||||
]
|
||||
|
||||
|
||||
def has_cuda_graph_wrapper_metadata() -> bool:
|
||||
from importlib import import_module
|
||||
|
||||
try:
|
||||
module = import_module("torch._inductor.utils")
|
||||
module.CUDAGraphWrapperMetadata # noqa B018
|
||||
except AttributeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
INDUCTOR_GRAPH_PARTITION = [
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not has_cuda_graph_wrapper_metadata(),
|
||||
reason="torch version does not support Inductor partition",
|
||||
),
|
||||
id="inductor_partition",
|
||||
),
|
||||
pytest.param(False, id="dynamo_partition"),
|
||||
]
|
||||
|
||||
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
"aiter_rms_quant_fusion": re.compile(
|
||||
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
# The combined pass can consume these sites before the standalone pass.
|
||||
"norm_rope_fusion": re.compile(
|
||||
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
|
||||
r"qk_norm_rope_kvcache_fusion.py:\d+] "
|
||||
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
|
||||
),
|
||||
"attn_quant_fusion": re.compile(
|
||||
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
|
||||
),
|
||||
"ar_rms_fusion": re.compile(
|
||||
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
|
||||
),
|
||||
"aiter_ar_rms_fusion": re.compile(
|
||||
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
|
||||
|
||||
from .common import FUSION_LOG_PATTERNS, AttentionBackendCase, Matches
|
||||
|
||||
|
||||
def run_model(
|
||||
vllm_runner,
|
||||
compile_config: CompilationMode | CompilationConfig,
|
||||
model: str,
|
||||
**model_kwargs,
|
||||
):
|
||||
"""Run a model with the given compilation config for E2E fusion tests."""
|
||||
compilation_config = (
|
||||
compile_config
|
||||
if isinstance(compile_config, CompilationConfig)
|
||||
else CompilationConfig(mode=compile_config)
|
||||
)
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
# Allow override from model_kwargs
|
||||
model_kwargs = {"tensor_parallel_size": 1, **model_kwargs}
|
||||
model_kwargs = {"disable_custom_all_reduce": True, **model_kwargs}
|
||||
|
||||
# No cudagraphs by default
|
||||
if compilation_config.cudagraph_mode is None:
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# trust_remote_code/enable_chunked_prefill/block_size/kernel_config are
|
||||
# pinned to LLM()'s own defaults since VllmRunner overrides them for
|
||||
# other test needs (e.g. it forces block_size=16, which some attention
|
||||
# backends like FLASHINFER_MLA reject, and disables JIT warmup).
|
||||
model_kwargs.setdefault("kernel_config", {"enable_jit_warmup": True})
|
||||
with vllm_runner(
|
||||
model,
|
||||
compilation_config=compilation_config,
|
||||
trust_remote_code=False,
|
||||
enable_chunked_prefill=None,
|
||||
block_size=None,
|
||||
**model_kwargs,
|
||||
) as vllm_model:
|
||||
llm = vllm_model.llm
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
# Print the outputs.
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
|
||||
# Get the compile ranges endpoints after vllm config post init
|
||||
# in order to compute compile ranges correctly
|
||||
compilation_config.compile_ranges_endpoints = (
|
||||
llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
|
||||
)
|
||||
|
||||
# Fetch match table from each worker via RPC and sum across workers.
|
||||
worker_tables: list[dict[str, int]] = llm.llm_engine.engine_core.collective_rpc(
|
||||
"get_compilation_match_table"
|
||||
)
|
||||
combined: defaultdict[str, int] = defaultdict(int)
|
||||
for table in worker_tables:
|
||||
for k, v in table.items():
|
||||
combined[k] += v
|
||||
return dict(combined)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn, vllm_runner):
|
||||
def run(
|
||||
model_name: str,
|
||||
matches: Matches,
|
||||
model_kwargs: dict,
|
||||
attn_backend: AttentionBackendCase,
|
||||
compilation_config: dict,
|
||||
matches_check: list[str],
|
||||
use_deepgemm: bool = False,
|
||||
use_aiter: bool = False,
|
||||
tp_size: int = 1,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1" if use_aiter else "0")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
# Filter here to reduce code duplication
|
||||
backend_name = attn_backend.backend.name.lower()
|
||||
requires_mla = "deepseek" in model_name.lower()
|
||||
is_mla = "mla" in backend_name
|
||||
|
||||
if requires_mla != is_mla:
|
||||
pytest.skip(
|
||||
f"Incompatible model '{model_name}' and "
|
||||
f"attention backend '{attn_backend.backend.name}'"
|
||||
)
|
||||
|
||||
if backend_name == "rocm_attn" and model_name == "openai/gpt-oss-20b":
|
||||
pytest.skip(
|
||||
"ROCM_ATTN does not support attention sinks (required by gpt-oss-20b)"
|
||||
)
|
||||
|
||||
if attn_backend.backend.name == "FLASHINFER":
|
||||
from vllm.utils.flashinfer import supports_trtllm_attention
|
||||
|
||||
if not supports_trtllm_attention():
|
||||
matches = matches._replace(attn_quant_fusion=0)
|
||||
|
||||
# Disable, compile cache to make sure custom passes run.
|
||||
# Otherwise, we can't verify fusion happened through the logs.
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
# To capture subprocess logs, we need to know whether spawn or fork is used.
|
||||
# Force spawn as it is more general.
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
model_kwargs = {**attn_backend.model_kwargs, **model_kwargs}
|
||||
model_kwargs["attention_config"] = {"backend": attn_backend.backend.name}
|
||||
model_kwargs["tensor_parallel_size"] = tp_size
|
||||
|
||||
# Cap warmup memory: tests use small max_model_len (1024) but the
|
||||
# engine default max_num_batched_tokens is 16384. Warming up large
|
||||
# models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM.
|
||||
model_kwargs.setdefault("max_num_batched_tokens", 8192)
|
||||
|
||||
# Always compile the full graph instead of piecewise
|
||||
if not compilation_config["use_inductor_graph_partition"]:
|
||||
compilation_config["splitting_ops"] = []
|
||||
|
||||
full_compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
inductor_compile_config={"force_disable_caches": True},
|
||||
**compilation_config,
|
||||
)
|
||||
|
||||
with caplog_mp_spawn(logging.DEBUG) as log_holder:
|
||||
match_table = run_model(
|
||||
vllm_runner, full_compilation_config, model_name, **model_kwargs
|
||||
)
|
||||
|
||||
num_compile_ranges = len(full_compilation_config.get_compile_ranges())
|
||||
assert num_compile_ranges in [1, 2, 3]
|
||||
|
||||
print(f"Compile ranges: {full_compilation_config.get_compile_ranges()}")
|
||||
print("Fusion results:")
|
||||
|
||||
# Iterate through all so printing happens before asserting
|
||||
log_matches_dict = {}
|
||||
for match_name, pattern in FUSION_LOG_PATTERNS.items():
|
||||
log_matches_dict[match_name] = list(pattern.findall(log_holder.text))
|
||||
print(f"- {match_name}={','.join(log_matches_dict[match_name])}")
|
||||
|
||||
# Now check the matches
|
||||
for match_name in matches_check:
|
||||
log_matches = list(int(ms) for ms in log_matches_dict[match_name])
|
||||
|
||||
# AR+RMS skips the largest range.
|
||||
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
|
||||
num_ranges_activated = num_compile_ranges - 1
|
||||
else:
|
||||
num_ranges_activated = num_compile_ranges
|
||||
|
||||
# TODO: Remove log counting in unit tests
|
||||
# once all matchers implement VllmFusionPatternMatcherPass
|
||||
n_expected = tp_size * num_ranges_activated
|
||||
if match_name not in (
|
||||
"attn_quant_fusion",
|
||||
"act_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
):
|
||||
assert len(log_matches) == n_expected, (
|
||||
f"Could not find {n_expected} {match_name} "
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
)
|
||||
|
||||
expected_matches = getattr(matches, match_name)
|
||||
|
||||
if match_name == "norm_rope_fusion":
|
||||
# Opaque LayerName lets the combined pass consume short-range
|
||||
# sites before the standalone pass; count either owner.
|
||||
assert len(log_matches) >= n_expected
|
||||
assert all(m in (0, expected_matches) for m in log_matches)
|
||||
assert sum(log_matches) == expected_matches * n_expected, (
|
||||
f"{match_name} expected {expected_matches * n_expected} "
|
||||
f"sites across both passes, found: {log_matches}"
|
||||
)
|
||||
elif match_name == "rms_quant_fusion" and "ar_rms_fusion" in matches_check:
|
||||
# AR+rms+quant takes precedence over rms+quant if activated.
|
||||
# That means we get full matching where ar+rms+quant was not
|
||||
# activated, and less where it was (only the smallest range).
|
||||
assert sum(m == expected_matches for m in log_matches) == tp_size * (
|
||||
num_ranges_activated - 1
|
||||
), "Expecting full rms+quant fusion where ar+rms+quant not activated"
|
||||
|
||||
assert all(
|
||||
expected_matches - matches.ar_rms_fusion <= m <= expected_matches
|
||||
for m in log_matches
|
||||
), (
|
||||
f"Expecting at least {expected_matches - matches.ar_rms_fusion} "
|
||||
f"where ar+rms+quant was activated"
|
||||
)
|
||||
elif match_name == "act_quant_fusion":
|
||||
actual_match = match_table.get("activation_quant_fusion_pass", 0)
|
||||
assert actual_match == expected_matches * n_expected, (
|
||||
f"Could not find {expected_matches * n_expected} "
|
||||
f"{match_name} (found {actual_match})."
|
||||
)
|
||||
elif match_name == "attn_quant_fusion":
|
||||
actual_match = match_table.get(
|
||||
"attn_quant_fusion", 0
|
||||
) + match_table.get("mla_attn_quant_fusion", 0)
|
||||
assert actual_match == expected_matches * n_expected, (
|
||||
f"Could not find {expected_matches * n_expected} "
|
||||
f"{match_name} (found {actual_match})."
|
||||
)
|
||||
else:
|
||||
expected_matches_list = [expected_matches] * n_expected
|
||||
assert sorted(log_matches) == expected_matches_list, (
|
||||
f"{match_name} expected: {expected_matches_list}, "
|
||||
f"found: {sorted(log_matches)}"
|
||||
)
|
||||
|
||||
if match_name == "ar_rms_fusion" and num_compile_ranges >= 2:
|
||||
log_matches = re.findall(
|
||||
r"pass_manager.py:\d+] Skipping "
|
||||
r".*AllReduceFusionPass.* with compile range",
|
||||
log_holder.text,
|
||||
)
|
||||
|
||||
n_expected = tp_size * (num_compile_ranges - num_ranges_activated)
|
||||
assert len(log_matches) == n_expected, (
|
||||
f'Could not find {n_expected} "Skipping AllReduceFusionPass" '
|
||||
f"(found {len(log_matches)}) in:\n {log_holder.text}"
|
||||
)
|
||||
|
||||
return run
|
||||
Reference in New Issue
Block a user