Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
@@ -4,7 +4,7 @@
import json
import sys
from collections import defaultdict
from contextlib import nullcontext
from contextlib import contextmanager, nullcontext
from inspect import signature
from pathlib import Path
from types import ModuleType, SimpleNamespace
@@ -12,8 +12,10 @@ from typing import Any
from unittest.mock import Mock, call, patch
import pytest
import torch
from vllm.model_executor.warmup import kernel_warmup as warmup
import vllm.model_executor.warmup.kernel_warmup as warmup
import vllm.utils.flashinfer as fi_utils
from vllm.model_executor.warmup.kernel_warmup import (
_flashinfer_autotune_token_counts,
_run_flashinfer_autotune_dummy_runs,
@@ -45,7 +47,12 @@ def _make_runner(modules, *, max_tokens: int = 8192):
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens),
vllm_config=SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=None),
parallel_config=SimpleNamespace(data_parallel_rank=0),
parallel_config=SimpleNamespace(
enable_elastic_ep=False,
nnodes=1,
pipeline_parallel_size=1,
data_parallel_rank=0,
),
),
get_model=Mock(
return_value=SimpleNamespace(modules=Mock(return_value=modules))
@@ -133,6 +140,258 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
]
@pytest.mark.parametrize(
"has_v2,elastic_ep,node_count",
[(False, False, 1), (True, False, 1), (True, True, 1), (True, False, 2)],
ids=["v1", "v2", "elastic-v1", "multi-node-v1"],
)
@pytest.mark.parametrize(
"hisparse,world_size,rank,skip_ops,failure",
[
(False, 1, 0, [], None),
(False, 2, 0, [], None),
(False, 2, 1, [], None),
(True, 2, 0, ["fp4_gemm", "bmm_fp8"], None),
(False, 2, 1, ["fp4_gemm"], "dummy"),
(False, 2, 1, [], "replayssm"),
(True, 2, 0, [], "kimi"),
],
)
def test_flashinfer_autotune_lifecycle(
monkeypatch,
tmp_path,
has_v2,
elastic_ep,
node_count,
hisparse,
world_size,
rank,
skip_ops,
failure,
):
use_v2 = has_v2 and not elastic_ep and node_count == 1 and world_size == 1
runner = _make_runner([])
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
runner.vllm_config.attention_config.hisparse_config = object() if hisparse else None
model = runner.get_model()
cache_path = tmp_path / "autotune.json"
rank_cache_path = cache_path.with_name(f"autotune_dp0_rank{rank}.json")
rank_cache_path.write_text(json.dumps({"_metadata": {}, "op": 1}))
events = Mock()
events.tuner.load_configs.return_value = True
events.tuner._dirty = True
world = SimpleNamespace(
rank_in_group=rank,
world_size=world_size,
cpu_group=object(),
broadcast_object=events.broadcast,
barrier=events.barrier,
)
@contextmanager
def autotune_context(**kwargs):
assert torch.is_inference_mode_enabled()
events.enter()
try:
yield
finally:
events.exit()
events.autotune.side_effect = autotune_context
if failure is not None:
getattr(events, failure).side_effect = RuntimeError("warmup failed")
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
SimpleNamespace(
AutoTuner=SimpleNamespace(get=lambda: events.tuner),
set_autotune_process_group=events.set_group,
),
)
monkeypatch.setitem(
sys.modules,
"flashinfer",
SimpleNamespace(autotune_v2=events.autotune, autotune_v2_reload=events.reload),
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_world_group", lambda: world
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_pp_group",
lambda: SimpleNamespace(world_size=1),
)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: has_v2)
monkeypatch.setattr(fi_utils, "autotune", events.autotune)
monkeypatch.setattr(warmup.envs, "VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS", skip_ops)
resolve_file = Mock(return_value=cache_path)
resolve_root = Mock(return_value=str(tmp_path))
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_file", resolve_file)
monkeypatch.setattr(warmup, "resolve_flashinfer_autotune_v2_root", resolve_root)
def gather(out, value, *, group):
out[:] = [value] * world_size
monkeypatch.setattr(torch.distributed, "all_gather_object", gather)
monkeypatch.setattr(warmup, "_run_flashinfer_autotune_dummy_runs", events.dummy)
monkeypatch.setattr(warmup, "replayssm_autotune_warmup", events.replayssm)
monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", events.kimi)
monkeypatch.setattr(
warmup, "autotune_hisparse_flashinfer_attention", events.hisparse
)
if failure is None:
warmup.flashinfer_autotune(runner)
else:
with pytest.raises(RuntimeError, match="warmup failed"):
warmup.flashinfer_autotune(runner)
kwargs = {"skip_ops": set(skip_ops)} if skip_ops else {}
expected = []
if use_v2:
resolve_file.assert_not_called()
resolve_root.assert_called_once_with()
expected.append(call.autotune(mode="tune", cache_root=str(tmp_path), **kwargs))
else:
resolve_root.assert_not_called()
resolve_file.assert_called_once_with(runner)
expected.extend(
[
call.tuner.load_configs(str(rank_cache_path)),
call.autotune(tune_mode=True, **kwargs),
]
)
expected.extend(
[call.set_group(world.cpu_group if world_size > 1 else None), call.enter()]
)
if hisparse:
expected.append(call.hisparse(runner))
for name, arg, kwargs in [
("dummy", runner, {"skip_attn": hisparse}),
("replayssm", runner, {}),
("kimi", model, {}),
]:
expected.append(getattr(call, name)(arg, **kwargs))
if failure == name:
break
expected.append(call.exit())
expected.append(call.set_group(None))
if failure is None:
if world_size > 1:
expected.append(call.barrier())
if use_v2 and world_size > 1:
expected.extend([call.reload(), call.barrier()])
elif not use_v2:
expected.append(call.tuner.save_configs(str(rank_cache_path)))
assert events.mock_calls == expected
@pytest.mark.parametrize(
"node_count,elastic_ep,pp_size",
[(1, False, 1), (2, False, 1), (1, True, 1), (1, False, 2)],
)
@pytest.mark.parametrize("world_size,rank", [(1, 0), (2, 0), (2, 1)])
@pytest.mark.parametrize("model_runner_v2,max_reqs", [(False, 2), (True, 1), (True, 2)])
def test_sparse_autotune_respects_cache_selection(
monkeypatch,
tmp_path,
node_count,
elastic_ep,
pp_size,
world_size,
rank,
model_runner_v2,
max_reqs,
):
import vllm.model_executor.warmup.flashinfer_sparse_mla_warmup as sparse
runner = _make_runner([])
runner.max_num_reqs = max_reqs
runner.vllm_config.use_v2_model_runner = model_runner_v2
runner.vllm_config.kernel_config = SimpleNamespace(enable_flashinfer_autotune=True)
runner.vllm_config.parallel_config.enable_elastic_ep = elastic_ep
runner.vllm_config.parallel_config.pipeline_parallel_size = pp_size
worker = SimpleNamespace(
model_runner=runner,
vllm_config=runner.vllm_config,
execute_model=Mock(),
sample_tokens=Mock(),
)
world = SimpleNamespace(
rank_in_group=rank,
world_size=world_size,
broadcast_object=Mock(return_value=b"cached"),
barrier=Mock(),
)
managed = Mock(return_value=nullcontext())
legacy = Mock(return_value=nullcontext())
reload = Mock()
tuner = Mock()
monkeypatch.setitem(
sys.modules,
"flashinfer",
SimpleNamespace(autotune_v2=managed, autotune_v2_reload=reload),
)
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
SimpleNamespace(AutoTuner=SimpleNamespace(get=lambda: tuner)),
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_world_group", lambda: world
)
monkeypatch.setattr(
"vllm.distributed.parallel_state.get_node_count", lambda: node_count
)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: True)
monkeypatch.setattr(sparse, "has_flashinfer", lambda: True)
monkeypatch.setattr(
sparse.current_platform, "is_device_capability_family", lambda _: True
)
monkeypatch.setattr(
sparse, "_flashinfer_sparse_mla_decode_label", lambda *_: "DSv3.2"
)
monkeypatch.setattr(sparse, "resolve_flashinfer_autotune_v2_root", lambda: tmp_path)
cache_path = tmp_path / "autotune.json"
cache_path.write_bytes(b"cached")
monkeypatch.setattr(
sparse, "resolve_flashinfer_autotune_file", lambda _: cache_path
)
monkeypatch.setattr(sparse, "flashinfer_autotune", legacy)
def mixed_warmup(*args, mixed_step_context=None, **kwargs):
context = (
mixed_step_context if mixed_step_context is not None else nullcontext()
)
with context:
assert torch.is_inference_mode_enabled()
return True
mixed = Mock(side_effect=mixed_warmup)
monkeypatch.setattr(sparse, "run_mixed_prefill_decode_warmup", mixed)
assert sparse._run_flashinfer_sparse_mla_decode_autotune(worker, 16, frozenset())
if node_count == 1 and not elastic_ep and pp_size == 1 and world_size == 1:
managed.assert_called_once_with(mode="tune", cache_root=tmp_path)
legacy.assert_not_called()
world.broadcast_object.assert_not_called()
tuner.load_configs.assert_not_called()
reload.assert_not_called()
world.barrier.assert_not_called()
else:
managed.assert_not_called()
reload.assert_not_called()
assert legacy.call_count == (1 if rank == 0 else 0)
world.broadcast_object.assert_called_once_with(
b"cached" if rank == 0 else None, src=0
)
tuner.load_configs.assert_called_once_with(str(cache_path))
assert mixed.call_count == (1 if model_runner_v2 and max_reqs >= 2 else 0)
assert runner._dummy_run.call_count == (0 if mixed.called else 1)
def test_flashinfer_autotune_buckets_cover_drafter_tokens():
"""The drafter's M can exceed the pass size; buckets must include it."""
runner = _make_runner([])
@@ -266,7 +525,9 @@ class _AutotuneRun:
for rank in range(self.pp * self.tp):
self.rank = rank
self.tuners[rank] = _AutotuneTuner(self)
flashinfer_autotune(_make_runner([]))
runner = _make_runner([])
runner.vllm_config.parallel_config.pipeline_parallel_size = self.pp
flashinfer_autotune(runner)
return self
def assert_collectives_match(self):
@@ -283,6 +544,8 @@ def autotune_run(monkeypatch, tmp_path):
def make_run(*, pp=2, tp=4):
run = _AutotuneRun(pp, tp, tmp_path)
monkeypatch.setattr(fi_utils, "has_flashinfer_autotune_v2", lambda: pp > 1)
monkeypatch.setattr(parallel_state, "get_node_count", lambda: 1)
autotuner = ModuleType("flashinfer.autotuner")
monkeypatch.setattr(
autotuner,
@@ -168,23 +168,28 @@ def _warmup_store_kernel(out, VALUE: tl.constexpr):
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
def test_registered_warmup_overlaps_kernels_and_runtime_stays_serial(
monkeypatch,
) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2))
def make_store(value):
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=value)
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler.
# Initialize Triton's binder before wrapping its compiler.
store.get_warmup_keys()
return store
stores = [make_store(value) for value in (1, 2)]
caller = threading.get_ident()
barrier = threading.Barrier(2, timeout=10)
compiled_on = []
@@ -200,14 +205,15 @@ def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
registry = JitWarmupRegistry(None)
with registry.activate():
store.register_warmup()
for store in stores:
store.register_warmup()
registry.warmup()
assert len(compiled_on) == 2
assert all(thread != caller for thread in compiled_on)
assert active_mode.get() is None
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
for value in (1, 2):
for store, value in zip(stores, (1, 2)):
store(out, value)
assert out.item() == value
assert len(compiled_on) == 2
@@ -220,3 +220,44 @@ def test_real_triton_kernel_override_end_to_end() -> None:
"out_ptr",
"BLOCK",
)
def test_cpu_slot_mapping_override_forwards_kernel_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm.utils import cpu_triton_utils
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
calls: list[tuple[Any, ...]] = []
fake_torch = types.SimpleNamespace(
ops=types.SimpleNamespace(
_C=types.SimpleNamespace(
compute_slot_mapping_kernel_impl=lambda *args: calls.append(args)
)
)
)
monkeypatch.setattr(cpu_triton_utils, "torch", fake_torch)
override = KernelOverride(
ComputeSlotMappingKernel.kernel,
cpu_triton_utils._compute_slot_mapping_kernel_impl,
)
override[(2,)](
num_tokens=2,
max_num_tokens=4,
query_start_loc_ptr="query_start_loc",
positions_ptr="positions",
block_table_ptr="block_table",
block_table_stride=128,
slot_mapping_ptr="slot_mapping",
KV_CACHE_BLOCK_SIZE=16,
TOTAL_CP_WORLD_SIZE=1,
TOTAL_CP_RANK=0,
CP_KV_CACHE_INTERLEAVE_SIZE=1,
PAD_ID=-1,
BLOCK_SIZE=1024,
)
assert calls == [
("query_start_loc", "positions", "block_table", "slot_mapping", 16)
]