[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,19 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils.torch_utils import (
|
||||
create_kv_caches_with_random,
|
||||
create_kv_caches_with_random_flash,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kv_cache_factory():
|
||||
return create_kv_caches_with_random
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kv_cache_factory_flashinfer():
|
||||
return create_kv_caches_with_random_flash
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
|
||||
from .common import Config
|
||||
from .mk_objects import (
|
||||
MK_ALL_PREPARE_FINALIZE_TYPES,
|
||||
MK_FUSED_EXPERT_TYPES,
|
||||
MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES,
|
||||
)
|
||||
|
||||
|
||||
def make_config_arg_parser(description: str):
|
||||
def to_pf_class_type(s: str) -> mk.FusedMoEPrepareAndFinalizeModular:
|
||||
for pf in MK_ALL_PREPARE_FINALIZE_TYPES:
|
||||
if pf.__name__ == s:
|
||||
return pf
|
||||
raise ValueError(f"Cannot find a PrepareFinalize type that matches {s}")
|
||||
|
||||
def to_experts_class_type(s: str) -> mk.FusedMoEExpertsModular:
|
||||
for fe in MK_FUSED_EXPERT_TYPES:
|
||||
if fe.__name__ == s:
|
||||
return fe
|
||||
raise ValueError(f"Cannot find a FusedExperts type that matches {s}")
|
||||
|
||||
def to_quant_torch_dtype(s: str) -> torch.dtype:
|
||||
if s == "torch.float8_e4m3fn":
|
||||
return torch.float8_e4m3fn
|
||||
raise ValueError(f"Unsupported quant type {s}")
|
||||
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
|
||||
parser.add_argument(
|
||||
"--world-size",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of ranks that participate in all2all",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pf-type",
|
||||
type=to_pf_class_type,
|
||||
required=True,
|
||||
help=(
|
||||
"Choose a PrepareFinalize Type : "
|
||||
f"{[x.__name__ for x in MK_ALL_PREPARE_FINALIZE_TYPES]}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--experts-type",
|
||||
type=to_experts_class_type,
|
||||
required=True,
|
||||
help=(
|
||||
f"Choose a FusedExpert type : {[x.__name__ for x in MK_FUSED_EXPERT_TYPES]}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-m",
|
||||
nargs="+",
|
||||
type=int,
|
||||
default=[64],
|
||||
help="num tokens per rank",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-k",
|
||||
type=int,
|
||||
default=7168,
|
||||
help="hidden-size",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
type=int,
|
||||
default=1024,
|
||||
help="N dimension of the first fused-moe matmul",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-experts", type=int, default=32, help="Global num experts"
|
||||
)
|
||||
parser.add_argument("--topk", nargs="+", type=int, default=[4, 1], help="num topk")
|
||||
|
||||
# Quant args
|
||||
parser.add_argument(
|
||||
"--quant-dtype", type=to_quant_torch_dtype, help="Quant datatype"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--per-token-quantized-activations",
|
||||
action="store_true",
|
||||
help=("The input activations must be per-token quantized"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--per-channel-quantized-weights",
|
||||
action="store_true",
|
||||
help="The weights must be per-channel quantized.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--block-shape", nargs="+", type=int, help="Quantization block shape"
|
||||
)
|
||||
|
||||
# Torch trace profile generation args
|
||||
parser.add_argument(
|
||||
"--torch-trace-dir-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Get torch trace for single execution",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def _validate_args(args: argparse.Namespace):
|
||||
if args.quant_dtype is not None:
|
||||
assert args.quant_dtype == torch.float8_e4m3fn
|
||||
if args.block_shape is not None:
|
||||
assert len(args.block_shape) == 2, (
|
||||
f"block shape must have 2 elements. got {args.block_shape}"
|
||||
)
|
||||
|
||||
if args.experts_type in MK_SINGLE_GPU_PREPARE_FINALIZE_TYPES:
|
||||
assert args.world_size == 1, "Single GPU objects need world size set to 1"
|
||||
|
||||
if args.torch_trace_dir_path is not None:
|
||||
from pathlib import Path
|
||||
|
||||
assert Path(args.torch_trace_dir_path).is_dir(), (
|
||||
f"Please create {args.torch_trace_dir_path}"
|
||||
)
|
||||
|
||||
|
||||
def make_config(args: argparse.Namespace) -> Config:
|
||||
_validate_args(args)
|
||||
|
||||
quant_config = None
|
||||
if args.quant_dtype is not None:
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
quant_dtype=args.quant_dtype,
|
||||
per_act_token_quant=args.per_token_quantized_activations,
|
||||
per_out_ch_quant=args.per_channel_quantized_weights,
|
||||
block_shape=args.block_shape,
|
||||
)
|
||||
|
||||
return Config(
|
||||
Ms=args.m,
|
||||
K=args.k,
|
||||
N=args.n,
|
||||
E=args.num_experts,
|
||||
topks=args.topk,
|
||||
dtype=torch.bfloat16, # hard-code
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=args.pf_type,
|
||||
fused_experts_type=args.experts_type,
|
||||
world_size=args.world_size,
|
||||
torch_trace_dir_path=args.torch_trace_dir_path,
|
||||
)
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
from enum import Enum
|
||||
from itertools import product
|
||||
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.config import FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
from .common import (
|
||||
Config,
|
||||
RankTensors,
|
||||
WeightTensors,
|
||||
reference_moe_impl,
|
||||
run_modular_kernel,
|
||||
)
|
||||
from .mk_objects import (
|
||||
MK_FUSED_EXPERT_TYPES,
|
||||
MK_MULTI_GPU_PREPARE_FINALIZE_TYPES,
|
||||
MK_QUANT_CONFIGS,
|
||||
)
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch_with_config
|
||||
|
||||
|
||||
class Result(Enum):
|
||||
PASS = 1
|
||||
FAIL = 2
|
||||
SKIP = 3
|
||||
|
||||
|
||||
def rank_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
):
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
# get weights to this device
|
||||
weights.to_current_device()
|
||||
|
||||
Ms = config.Ms
|
||||
assert isinstance(Ms, list)
|
||||
TOPKs = config.topks
|
||||
assert isinstance(TOPKs, list)
|
||||
|
||||
for m, topk in product(Ms, TOPKs):
|
||||
print(f"Running m={m}, topk={topk} ...")
|
||||
# override m and topk
|
||||
cfgx = copy.deepcopy(config)
|
||||
cfgx.Ms = m
|
||||
cfgx.topks = topk
|
||||
|
||||
# inputs for rank
|
||||
rank_tensors = RankTensors.make(cfgx, pgi)
|
||||
|
||||
# modular kernel out
|
||||
mk_out = run_modular_kernel(pgi, vllm_config, cfgx, weights, rank_tensors)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = reference_moe_impl(cfgx, weights, rank_tensors)
|
||||
|
||||
torch.testing.assert_close(ref_out, mk_out, atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def make_feature_matrix(csv_file_path: str):
|
||||
from dataclasses import asdict
|
||||
|
||||
import pandas as pd
|
||||
|
||||
def add_to_results(
|
||||
config: Config, success: Result, results_df: pd.DataFrame | None = None
|
||||
):
|
||||
config_dict = asdict(config)
|
||||
config_dict["prepare_finalize_type"] = config_dict[
|
||||
"prepare_finalize_type"
|
||||
].__name__
|
||||
config_dict["fused_experts_type"] = config_dict["fused_experts_type"].__name__
|
||||
config_dict["per_tensor_act_quant"] = config.is_per_tensor_act_quant
|
||||
quant_config_dict = config_dict["quant_config"]
|
||||
del config_dict["quant_config"]
|
||||
if quant_config_dict is None:
|
||||
quant_config = FUSED_MOE_UNQUANTIZED_CONFIG
|
||||
quant_config_dict = asdict(quant_config)
|
||||
|
||||
config_dict |= quant_config_dict
|
||||
result_dict = config_dict | {"success": success.name}
|
||||
|
||||
result_df = pd.DataFrame([result_dict])
|
||||
if results_df is None:
|
||||
results_df = result_df
|
||||
else:
|
||||
results_df = pd.concat([results_df, result_df], ignore_index=True)
|
||||
|
||||
return results_df
|
||||
|
||||
Ms = [64]
|
||||
Ks = [7168] # hidden sizes
|
||||
Ns = [2048]
|
||||
TOPKs = [[4, 1]]
|
||||
Es = [32]
|
||||
DTYPEs = [torch.bfloat16]
|
||||
PF_TYPES = MK_MULTI_GPU_PREPARE_FINALIZE_TYPES
|
||||
FE_TYPES = MK_FUSED_EXPERT_TYPES
|
||||
Q_TYPES = MK_QUANT_CONFIGS
|
||||
|
||||
combinations = list(
|
||||
product(Ms, Ks, Ns, Es, TOPKs, DTYPEs, PF_TYPES, FE_TYPES, Q_TYPES)
|
||||
)
|
||||
|
||||
results_df: pd.DataFrame | None = None
|
||||
for m, k, n, e, topks, dtype, pf_type, experts_type, quant_config in tqdm(
|
||||
combinations
|
||||
):
|
||||
config = Config(
|
||||
Ms=[m],
|
||||
K=k,
|
||||
N=n,
|
||||
E=e,
|
||||
topks=topks,
|
||||
dtype=dtype,
|
||||
prepare_finalize_type=pf_type,
|
||||
fused_experts_type=experts_type,
|
||||
quant_config=quant_config,
|
||||
world_size=2,
|
||||
)
|
||||
|
||||
success = None
|
||||
if config.is_valid()[0]:
|
||||
print(f"Running config : {config.describe()} ...")
|
||||
try:
|
||||
weights: WeightTensors = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
rank_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
)
|
||||
success = Result.PASS
|
||||
except Exception as _:
|
||||
success = Result.FAIL
|
||||
else:
|
||||
success = Result.SKIP
|
||||
|
||||
results_df = add_to_results(config, success, results_df)
|
||||
|
||||
if results_df is not None:
|
||||
results_df.to_csv(f"{csv_file_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Make ModularKernel feature matrix \n"
|
||||
"Example : python3 -m tests.kernels.moe.modular_kernel_tools.make_feature_matrix " # noqa: E501
|
||||
"-f ./feature_matrices/feature_matrix.csv"
|
||||
)
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--feature-matrix-csv-file-path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="File name to Generate a .csv file",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
csv_path = args.feature_matrix_csv_file_path
|
||||
assert csv_path.endswith("csv"), (
|
||||
f"Need a file path ending with .csv, got {csv_path}"
|
||||
)
|
||||
assert Path(csv_path).parent.is_dir(), (
|
||||
f"Cannot find parent directory for {Path(csv_path).parent}"
|
||||
)
|
||||
|
||||
make_feature_matrix(args.feature_matrix_csv_file_path)
|
||||
@@ -1,182 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from .common import (
|
||||
Config,
|
||||
RankTensors,
|
||||
WeightTensors,
|
||||
_make_gscale,
|
||||
make_modular_kernel,
|
||||
)
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch_with_config
|
||||
|
||||
|
||||
def do_profile(
|
||||
fn: Callable,
|
||||
fn_kwargs: dict[Any, Any],
|
||||
pgi: ProcessGroupInfo,
|
||||
config: Config,
|
||||
num_warmups: int = 5,
|
||||
):
|
||||
for _ in range(num_warmups):
|
||||
fn(**fn_kwargs)
|
||||
|
||||
with torch.profiler.profile(
|
||||
activities=[
|
||||
torch.profiler.ProfilerActivity.CPU,
|
||||
torch.profiler.ProfilerActivity.CUDA,
|
||||
],
|
||||
with_stack=True,
|
||||
record_shapes=True,
|
||||
) as tprof:
|
||||
fn(**fn_kwargs)
|
||||
device = torch.accelerator.current_device_index()
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
# TODO (varun): Add a descriptive trace file name
|
||||
tprof.export_chrome_trace(
|
||||
f"{config.torch_trace_dir_path}/m{config.M}_{pgi.rank}_trace.json"
|
||||
)
|
||||
|
||||
|
||||
def profile_modular_kernel(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
rank_tensors: RankTensors,
|
||||
) -> None:
|
||||
assert isinstance(config.Ms, int)
|
||||
assert isinstance(config.topks, int)
|
||||
|
||||
# weights for rank
|
||||
rank_weights = weights.slice_weights(pgi.rank, config.num_local_experts)
|
||||
|
||||
if config.quant_dtype == "nvfp4":
|
||||
gscale = _make_gscale(config.num_local_experts)
|
||||
else:
|
||||
gscale = None
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
config.quant_dtype,
|
||||
w1_scale=rank_weights.w1_scale,
|
||||
w2_scale=rank_weights.w2_scale,
|
||||
a1_scale=rank_tensors.hidden_states_scale,
|
||||
g1_alphas=(1 / rank_weights.w1_gs) if rank_weights.w1_gs is not None else None,
|
||||
g2_alphas=(1 / rank_weights.w2_gs) if rank_weights.w2_gs is not None else None,
|
||||
a1_gscale=gscale,
|
||||
a2_gscale=gscale,
|
||||
block_shape=config.quant_block_shape,
|
||||
per_act_token_quant=config.is_per_act_token_quant,
|
||||
per_out_ch_quant=config.is_per_out_ch_quant,
|
||||
)
|
||||
|
||||
# make modular kernel
|
||||
mk = make_modular_kernel(config, vllm_config, quant_config)
|
||||
|
||||
topk_ids = rank_tensors.topk_ids.to(
|
||||
mk.prepare_finalize.topk_indices_dtype() or rank_tensors.topk_ids.dtype
|
||||
)
|
||||
|
||||
# impls might update the tensor in place
|
||||
hidden_states = rank_tensors.hidden_states.clone()
|
||||
|
||||
mk_kwargs = {
|
||||
"hidden_states": hidden_states,
|
||||
"w1": rank_weights.w1,
|
||||
"w2": rank_weights.w2,
|
||||
"topk_weights": rank_tensors.topk_weights,
|
||||
"topk_ids": topk_ids,
|
||||
"activation": MoEActivation.SILU,
|
||||
"expert_map": rank_tensors.expert_map,
|
||||
"global_num_experts": config.E,
|
||||
"apply_router_weight_on_input": config.topk == 1
|
||||
and config.supports_apply_weight_on_input(),
|
||||
}
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
num_tokens_across_dp = torch.tensor(
|
||||
[num_tokens] * config.world_size, device="cpu", dtype=torch.int
|
||||
)
|
||||
|
||||
with set_forward_context(
|
||||
None,
|
||||
vllm_config,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
):
|
||||
do_profile(mk.apply, mk_kwargs, pgi, config)
|
||||
|
||||
|
||||
def rank_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
):
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
# workspace manager is normally initialized by GPUModelRunner; we initialize
|
||||
# it here for the standalone benchmark process.
|
||||
init_workspace_manager(torch.device(f"cuda:{pgi.local_rank}"))
|
||||
|
||||
# get weights to this device
|
||||
weights.to_current_device()
|
||||
|
||||
Ms = config.Ms
|
||||
assert isinstance(Ms, list)
|
||||
TOPKs = config.topks
|
||||
assert isinstance(TOPKs, list)
|
||||
|
||||
for m, topk in product(Ms, TOPKs):
|
||||
print(f"Running m={m}, topk={topk} ...")
|
||||
# override m and topk
|
||||
cfgx = copy.deepcopy(config)
|
||||
cfgx.Ms = m
|
||||
cfgx.topks = topk
|
||||
|
||||
# inputs for rank
|
||||
rank_tensors = RankTensors.make(cfgx, pgi)
|
||||
profile_modular_kernel(pgi, vllm_config, cfgx, weights, rank_tensors)
|
||||
|
||||
|
||||
def run(config: Config):
|
||||
weights: WeightTensors = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
parallel_launch_with_config(
|
||||
config.world_size, rank_worker, vllm_config, env_dict, config, weights
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from .cli_args import make_config, make_config_arg_parser
|
||||
|
||||
parser = make_config_arg_parser(
|
||||
description=(
|
||||
"Run single prepare-finalize & fused-experts combination test"
|
||||
"Example : python3 -m tests.kernels.moe.modular_kernel_tools.profile_modular_kernel " # noqa: E501
|
||||
"--pf-type DeepEPLLPrepareAndFinalize --experts-type BatchedTritonExperts"
|
||||
)
|
||||
)
|
||||
args = parser.parse_args()
|
||||
assert args.torch_trace_dir_path is not None, (
|
||||
"Please pass in a directory to store torch traces"
|
||||
)
|
||||
config = make_config(args)
|
||||
|
||||
run(config)
|
||||
@@ -1,259 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Platform-agnostic eligibility tests for GateLinear's ROCm router GEMMs.
|
||||
|
||||
These assert the dispatch flags directly, so they run device-free by mocking
|
||||
the platform predicates. ``allow_cublas_router_gemm`` selects the
|
||||
bf16xbf16->fp32 ``torch.mm`` epilogue, while ``allow_fp32_router_gemm`` selects
|
||||
the gfx950 low-M kernel with fp32 weights and output.
|
||||
|
||||
The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
|
||||
term; a biased gate must fall back so the bias is not silently dropped.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.router.gate_linear as gate_linear_mod
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
|
||||
def _make_gate(
|
||||
monkeypatch,
|
||||
*,
|
||||
is_rocm: bool,
|
||||
is_cuda: bool = False,
|
||||
bias: bool = False,
|
||||
params_dtype: torch.dtype = torch.bfloat16,
|
||||
out_dtype: torch.dtype | None = torch.float32,
|
||||
input_size: int = 2048,
|
||||
output_size: int = 64,
|
||||
on_gfx950: bool = False,
|
||||
parallel_world_size: int = 1,
|
||||
) -> GateLinear:
|
||||
"""Build a GateLinear with platform predicates mocked, no GPU needed."""
|
||||
for target in (
|
||||
"vllm.model_executor.layers.linear",
|
||||
"vllm.model_executor.parameter",
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
f"{target}.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
f"{target}.get_tensor_model_parallel_world_size",
|
||||
lambda: parallel_world_size,
|
||||
)
|
||||
|
||||
platform = gate_linear_mod.current_platform
|
||||
monkeypatch.setattr(platform, "is_cuda", lambda: is_cuda)
|
||||
monkeypatch.setattr(platform, "is_rocm", lambda: is_rocm)
|
||||
monkeypatch.setattr(platform, "is_device_capability", lambda *a, **k: False)
|
||||
monkeypatch.setattr(platform, "is_device_capability_family", lambda *a, **k: False)
|
||||
if is_rocm:
|
||||
import vllm.platforms.rocm as rocm_platform
|
||||
|
||||
monkeypatch.setattr(rocm_platform, "on_gfx950", lambda: on_gfx950)
|
||||
|
||||
return GateLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=bias,
|
||||
out_dtype=out_dtype,
|
||||
params_dtype=params_dtype,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_no_bias_bf16_fp32_enables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=False)
|
||||
assert not gate.allow_specialized_router_gemm
|
||||
assert gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bias_disables_fused_gemm(monkeypatch):
|
||||
# torch.mm cannot add a bias, so a biased gate must not take the fused path.
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=True)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_fp32_weight_disables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, params_dtype=torch.float32)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_non_fp32_out_dtype_disables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_non_rocm_non_cuda_disables_fused_gemm(monkeypatch):
|
||||
# Neither the CUDA specialized path nor the ROCm branch applies.
|
||||
gate = _make_gate(monkeypatch, is_rocm=False, is_cuda=False)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_set_out_dtype_enables_fused_gemm(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=False, out_dtype=None)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
|
||||
gate = _make_gate(monkeypatch, is_rocm=True, bias=True, out_dtype=None)
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size"),
|
||||
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
|
||||
)
|
||||
def test_rocm_gfx950_enables_fp32_router_gemm(
|
||||
monkeypatch, input_size: int, output_size: int
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=torch.float32,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
on_gfx950=True,
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert not gate.allow_cublas_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_size", [2, 4, 8])
|
||||
def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
|
||||
monkeypatch, ep_size: int
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=torch.float32,
|
||||
input_size=6144,
|
||||
output_size=128,
|
||||
on_gfx950=True,
|
||||
parallel_world_size=ep_size,
|
||||
)
|
||||
assert gate.weight.shape == (128, 6144)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
|
||||
[
|
||||
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
|
||||
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
|
||||
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
|
||||
],
|
||||
)
|
||||
def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
|
||||
monkeypatch,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
bias: bool,
|
||||
on_gfx950: bool,
|
||||
) -> None:
|
||||
gate = _make_gate(
|
||||
monkeypatch,
|
||||
is_rocm=True,
|
||||
params_dtype=params_dtype,
|
||||
bias=bias,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
on_gfx950=on_gfx950,
|
||||
)
|
||||
assert not gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ROCm BF16x3 tier (fp32 router weights)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
|
||||
"""Build a gate with the ROCm bf16x3 platform check mocked."""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
|
||||
kwargs.setdefault("params_dtype", torch.float32)
|
||||
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
|
||||
|
||||
|
||||
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
# Built by process_weights_after_loading, not lazily in forward().
|
||||
# Registered up front but still None, so named_buffers() skips it.
|
||||
assert gate._bf16x3_weight is None
|
||||
assert "_bf16x3_weight" in gate._buffers
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
|
||||
# torch.mm has no bias term.
|
||||
gate = _make_bf16x3_gate(monkeypatch, bias=True)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
|
||||
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
|
||||
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
|
||||
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
|
||||
gate = _make_bf16x3_gate(monkeypatch, supported=False)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
assert not gate._rocm_bf16x3_weight_eligible
|
||||
|
||||
|
||||
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
|
||||
"""An ineligible out_dtype must not allocate a split that is never read;
|
||||
it is 1.5x the router weight on every layer."""
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
assert gate._bf16x3_weight is None
|
||||
|
||||
|
||||
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_dispatch_is_a_custom_op():
|
||||
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
|
||||
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
|
||||
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
|
||||
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
|
||||
assert out.shape == (8192, 128)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
|
||||
"""The gfx950 low-M fp32 tier claims these shapes too and returns
|
||||
unconditionally, so bf16x3 is only reachable if checked first."""
|
||||
gate = _make_bf16x3_gate(
|
||||
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
source = inspect.getsource(type(gate).forward)
|
||||
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
|
||||
"allow_fp32_router_gemm"
|
||||
)
|
||||
Reference in New Issue
Block a user