[Benchmark] Remove the temporary bench forward dummy loader (#61676)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7c3f7e5500
commit
2e8a5930f6
@@ -51,9 +51,8 @@ Read `median_us` (each replay is already the max over ranks).
|
||||
## Caveats
|
||||
|
||||
- **Dummy weights lie for data-dependent kernels.** Dummy gates route every
|
||||
token to the same experts (hence balanced routing). Dummy norm weights make
|
||||
DSV4 indexer logits degenerate, so `topKPerRowPrefill` runs ~2.6x slow
|
||||
until #61323 lands. Cross-check suspicious kernels with real weights.
|
||||
token to the same experts (hence balanced routing). Cross-check suspicious
|
||||
kernels with real weights.
|
||||
- **torch.compile models (e.g. Qwen3/3.5) look slow**: no compile here. Use
|
||||
them for correctness only; trust perf for models under `vllm/models/`.
|
||||
- `sample spread > 0.5%` warns of unstable timing. DSV4.1 DEP prefill is
|
||||
|
||||
@@ -85,6 +85,7 @@ def test_jitter_kv_lens():
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
# TODO(woosuk): Support ROCm and re-enable this test there.
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="Hangs on ROCm; the bench is CUDA-only."
|
||||
)
|
||||
|
||||
@@ -151,6 +151,8 @@ class ForwardBench:
|
||||
# restore: neither rank skew, the restore nor host launch gaps reach
|
||||
# the timed interval.
|
||||
flag = torch.zeros(1, device=self.device)
|
||||
# TODO(woosuk): Support ROCm. The bench hangs there, likely on these
|
||||
# in-graph events.
|
||||
self._events = (
|
||||
torch.cuda.Event(enable_timing=True, external=True),
|
||||
torch.cuda.Event(enable_timing=True, external=True),
|
||||
|
||||
@@ -97,36 +97,10 @@ def _setup_env() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _register_dummy_loader() -> str:
|
||||
"""Dummy loader that also runs load-time model finalization.
|
||||
|
||||
TODO(woosuk): Remove once #60096 lands. DeepSeek-V4.1 fuses its MegaMoE
|
||||
shared experts at the end of `load_weights`, before the generic linear
|
||||
post-processing repacks their scales. Dummy loading never calls
|
||||
`load_weights`, so the fusion would fail and the bench would time unfused
|
||||
shared experts.
|
||||
"""
|
||||
from vllm.model_executor.model_loader import register_model_loader
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
|
||||
name = "forward_bench_dummy"
|
||||
|
||||
@register_model_loader(name)
|
||||
class ForwardBenchDummyLoader(DummyModelLoader):
|
||||
def load_weights(self, model, model_config) -> None:
|
||||
super().load_weights(model, model_config)
|
||||
if any(hasattr(m, "finalize_mega_moe_weights") for m in model.modules()):
|
||||
model.process_weights_after_loading()
|
||||
|
||||
return name
|
||||
|
||||
|
||||
def _make_engine_args(args: argparse.Namespace, cases: list[BenchCase]):
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
if engine_args.load_format == "dummy":
|
||||
engine_args.load_format = _register_dummy_loader()
|
||||
engine_args.distributed_executor_backend = "external_launcher"
|
||||
engine_args.enable_prefix_caching = False
|
||||
engine_args.compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
Reference in New Issue
Block a user