[Benchmark] Add vllm bench forward: exact-shape forward timing (#61339)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2c74fd1873
commit
9d3dfa2886
@@ -0,0 +1,227 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Balanced MoE routing for benchmarks with dummy weights.
|
||||
|
||||
Dummy gate weights give degenerate routing, which skews MoE and all-to-all
|
||||
cost. The routing still runs; its result is replaced by the fixture at
|
||||
whichever boundary the layer exposes:
|
||||
|
||||
- Gate output: fixed logits whose top-k is the fixture. Covers every backend
|
||||
that routes from logits, including monolithic kernels that run top-k
|
||||
internally (e.g. FlashInfer TRTLLM).
|
||||
- Experts input: models that compute top-k themselves and pass
|
||||
`topk_ids` to their experts (e.g. MegaMoE with a fused gate) get the
|
||||
fixture's IDs. The routing weights are kept; they don't affect cost.
|
||||
|
||||
Hash-routed layers ignore logits and keep their tables unless their experts
|
||||
take `topk_ids`.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Fixture logits: chosen experts score far above the rest under any monotonic
|
||||
# scoring function, so top-k picks exactly them.
|
||||
_CHOSEN_LOGIT = 4.0
|
||||
_OTHER_LOGIT = -4.0
|
||||
|
||||
|
||||
def balanced_topk_ids(
|
||||
num_tokens: int, num_experts: int, top_k: int, ep_size: int, rng
|
||||
) -> np.ndarray:
|
||||
"""Expert IDs [num_tokens, top_k] giving every EP rank equal load.
|
||||
|
||||
Each token's slots go round-robin over EP ranks, to distinct experts.
|
||||
"""
|
||||
assert num_experts % ep_size == 0
|
||||
local = num_experts // ep_size
|
||||
slots = np.arange(num_tokens)[:, None] * top_k + np.arange(top_k)
|
||||
ranks = slots % ep_size
|
||||
# Slots of one token that land on the same rank take consecutive experts.
|
||||
repeat = np.zeros_like(ranks)
|
||||
for j in range(1, top_k):
|
||||
repeat[:, j] = (ranks[:, :j] == ranks[:, j : j + 1]).sum(axis=1)
|
||||
assert repeat.max() < local
|
||||
base = rng.integers(local, size=(num_tokens, ep_size))
|
||||
offset = np.take_along_axis(base, ranks, axis=1)
|
||||
return ranks * local + (offset + repeat) % local
|
||||
|
||||
|
||||
def _moe_layers(model: torch.nn.Module) -> list[tuple[torch.nn.Module, GateLinear]]:
|
||||
"""(MoE module, gate) per layer, in model order."""
|
||||
layers: dict[GateLinear, torch.nn.Module] = {}
|
||||
for module in model.modules():
|
||||
gate = getattr(module, "gate", None)
|
||||
if not isinstance(gate, GateLinear) or gate in layers:
|
||||
continue
|
||||
if _top_k(module) is not None:
|
||||
layers[gate] = module
|
||||
return [(module, gate) for gate, module in layers.items()]
|
||||
|
||||
|
||||
def _top_k(module: torch.nn.Module) -> int | None:
|
||||
"""The MoE layer's top-k, from the module or its experts submodules."""
|
||||
for sub in module.modules():
|
||||
for name in ("top_k", "n_activated_experts"):
|
||||
top_k = getattr(sub, name, None)
|
||||
if isinstance(top_k, int):
|
||||
return top_k
|
||||
return None
|
||||
|
||||
|
||||
class BalancedRouting:
|
||||
"""Installs balanced routing for one case; `remove()` restores the model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
num_tokens: int,
|
||||
dp_rank: int,
|
||||
dp_size: int,
|
||||
ep_size: int,
|
||||
seed: int,
|
||||
device: torch.device,
|
||||
):
|
||||
layers = _moe_layers(model)
|
||||
if not layers:
|
||||
raise ValueError("No MoE gates found; use --routing real.")
|
||||
self._hooks = []
|
||||
self._biases: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
self._fired = [0] * len(layers)
|
||||
self._layers = layers
|
||||
# Per-rank rows, with headroom for padding; every rank generates the
|
||||
# same global fixture and takes its slice.
|
||||
rows = -(-num_tokens // 256) * 256 + 256
|
||||
for layer, (module, gate) in enumerate(layers):
|
||||
if (getattr(module, "num_expert_group", None) or 1) > 1:
|
||||
raise NotImplementedError("Balanced routing with grouped top-k.")
|
||||
num_experts = gate.weight.shape[0]
|
||||
top_k = _top_k(module)
|
||||
assert top_k is not None
|
||||
rng = np.random.default_rng([seed, layer])
|
||||
ids = balanced_topk_ids(rows * dp_size, num_experts, top_k, ep_size, rng)[
|
||||
dp_rank * rows : (dp_rank + 1) * rows
|
||||
]
|
||||
logits = torch.full((rows, num_experts), _OTHER_LOGIT, dtype=torch.float32)
|
||||
logits.scatter_(1, torch.from_numpy(ids), _CHOSEN_LOGIT)
|
||||
# Distinct values: no ties for any top-k implementation.
|
||||
logits += torch.from_numpy(rng.uniform(0, 0.5, logits.shape)).float()
|
||||
self._hooks.append(
|
||||
gate.register_forward_hook(
|
||||
self._substitute(logits.to(device), self._fired, layer)
|
||||
)
|
||||
)
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is None:
|
||||
bias = getattr(module, "e_score_correction_bias", None)
|
||||
if isinstance(bias, torch.Tensor):
|
||||
self._biases.append((bias, bias.detach().clone()))
|
||||
bias.data.zero_()
|
||||
experts = getattr(module, "experts", None)
|
||||
if isinstance(experts, torch.nn.Module):
|
||||
params = list(inspect.signature(experts.forward).parameters)
|
||||
if "topk_ids" in params:
|
||||
self._hooks.append(
|
||||
experts.register_forward_pre_hook(
|
||||
self._substitute_ids(
|
||||
torch.from_numpy(ids).to(device),
|
||||
params.index("topk_ids"),
|
||||
self._fired,
|
||||
layer,
|
||||
),
|
||||
with_kwargs=True,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _substitute(fixture: torch.Tensor, fired: list[int], layer: int):
|
||||
cast = [fixture]
|
||||
|
||||
def hook(module, inputs, output):
|
||||
fired[layer] += 1
|
||||
logits = output[0] if isinstance(output, tuple) else output
|
||||
if cast[0].dtype != logits.dtype:
|
||||
cast[0] = fixture.to(logits.dtype) # Once, at warmup.
|
||||
replaced = cast[0][: logits.shape[0]]
|
||||
return (replaced, *output[1:]) if isinstance(output, tuple) else replaced
|
||||
|
||||
return hook
|
||||
|
||||
@staticmethod
|
||||
def _substitute_ids(fixture: torch.Tensor, index: int, fired, layer: int):
|
||||
cast = [fixture]
|
||||
|
||||
def hook(module, args, kwargs):
|
||||
fired[layer] += 1
|
||||
ids = kwargs["topk_ids"] if "topk_ids" in kwargs else args[index]
|
||||
if cast[0].dtype != ids.dtype:
|
||||
cast[0] = fixture.to(ids.dtype) # Once, at warmup.
|
||||
replaced = cast[0][: ids.shape[0]]
|
||||
if "topk_ids" in kwargs:
|
||||
return args, {**kwargs, "topk_ids": replaced}
|
||||
return (*args[:index], replaced, *args[index + 1 :]), kwargs
|
||||
|
||||
return hook
|
||||
|
||||
def check_applied(self) -> None:
|
||||
"""Fail if a layer bypassed both the gate and the experts hooks."""
|
||||
missed = [
|
||||
type(module).__name__
|
||||
for (module, _), fired in zip(self._layers, self._fired)
|
||||
if not fired
|
||||
]
|
||||
if missed:
|
||||
raise RuntimeError(
|
||||
f"Balanced routing didn't reach {len(missed)} MoE layers "
|
||||
f"({missed[0]}): their gate is fused away. Use --routing real."
|
||||
)
|
||||
|
||||
def remove(self) -> None:
|
||||
for hook in self._hooks:
|
||||
hook.remove()
|
||||
for bias, saved in self._biases:
|
||||
bias.data.copy_(saved)
|
||||
|
||||
|
||||
def install_routing(
|
||||
runner, requested: str, dummy_weights: bool, num_tokens: int, seed: int
|
||||
) -> tuple[str, BalancedRouting | None]:
|
||||
"""Resolve `--routing` and install balanced routing if selected.
|
||||
|
||||
`auto` is balanced with dummy weights and real otherwise; it falls back to
|
||||
real routing when the model doesn't support the fixture.
|
||||
"""
|
||||
mode = requested
|
||||
if mode == "auto":
|
||||
mode = "balanced" if dummy_weights else "real"
|
||||
if mode == "real":
|
||||
return mode, None
|
||||
parallel = runner.parallel_config
|
||||
ep_size = (
|
||||
parallel.data_parallel_size * parallel.tensor_parallel_size
|
||||
if parallel.enable_expert_parallel
|
||||
else 1
|
||||
)
|
||||
try:
|
||||
routing = BalancedRouting(
|
||||
runner.model,
|
||||
num_tokens,
|
||||
runner.dp_rank,
|
||||
runner.dp_size,
|
||||
ep_size,
|
||||
seed,
|
||||
runner.device,
|
||||
)
|
||||
except (ValueError, NotImplementedError) as e:
|
||||
if requested == "balanced":
|
||||
raise
|
||||
logger.warning("Using real routing: %s", e)
|
||||
return "real", None
|
||||
return mode, routing
|
||||
Reference in New Issue
Block a user