[6/N][warmup][DSv4] Migrate sampling, and DFlash JIT kernels (#56323)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 17:24:24 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent e19a3e172e
commit aed894c190
27 changed files with 2288 additions and 558 deletions
+32
View File
@@ -171,6 +171,19 @@ def test_warmup_range_validates_custom_advancement() -> None:
)
def test_warmup_cases_support_lambda_advancement() -> None:
def warmup_inputs() -> dict[str, Any]:
value: Any = WarmupIntRange(1, 9, advance=lambda value: value * 2)
return dict(value=value)
assert list(ToyKernel()._expand_warmup_cases(warmup_inputs)) == [
{"value": 1},
{"value": 2},
{"value": 4},
{"value": 8},
]
def test_compile_key_uses_defaults_locals_attributes_and_expressions() -> None:
cfg = _config(bias=3, disabled=True, name="cfg", vectorized=True)
@@ -577,6 +590,25 @@ def test_registry_records_only_inside_model_setup_context() -> None:
assert kernel.compiled == []
def test_registry_capture() -> None:
class Owner:
@JitWarmupRegistry.capture
def __init__(
self,
vllm_config: Any,
kernel: RecordingToyKernel,
) -> None:
kernel.register_warmup(3, vllm_config)
kernel = RecordingToyKernel()
config = _config()
owner = Owner(config, kernel)
assert len(owner.jit_warmup_registry) == 1 # type: ignore[attr-defined]
kernel.register_warmup(5, config)
assert len(owner.jit_warmup_registry) == 1 # type: ignore[attr-defined]
def test_registry_expands_requests_and_deduplicates_owner_keys() -> None:
registry = JitWarmupRegistry(_config())
kernel = RecordingToyKernel()
@@ -4,22 +4,68 @@
from dataclasses import dataclass
from typing import Any
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
import pytest
import torch
from vllm.model_executor.warmup import jit_warmup_triton_helper
from vllm.model_executor.warmup.jit_warmup import (
WarmupChoices,
WarmupIntRange,
kernel_launcher,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonJitKey,
TritonWarmupTensor,
VllmTritonJitKernel,
triton_kernel_dispatcher_with_warmup,
triton_warmup_inputs,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _binder_test_kernel(x, value, BLOCK: tl.constexpr):
pass
@triton.jit
def _second_binder_test_kernel(x, value, BLOCK: tl.constexpr):
pass
@triton.jit
def _pointer_group_test_kernel(x_ptr, value, BLOCK: tl.constexpr):
pass
class _FakeTritonKernel:
arg_names = ("first", "second", "CONST")
arg_names: tuple[str, ...] = ("first", "second", "CONST")
def __init__(self) -> None:
self.warmup_calls: list[dict[str, Any]] = []
self.runtime_calls: list[tuple[Any, tuple[Any, ...], dict[str, Any]]] = []
def warmup(self, **kwargs: Any) -> None:
self.warmup_calls.append(kwargs)
def __getitem__(self, grid: Any) -> Any:
def launch(*args: Any, **kwargs: Any) -> None:
self.runtime_calls.append((grid, args, kwargs))
return launch
def _patch_key_deriver(
monkeypatch: pytest.MonkeyPatch,
derive: Any,
) -> None:
monkeypatch.setattr(
jit_warmup_triton_helper,
"_triton_key_deriver",
lambda kernel: lambda kwargs: derive(kernel, kwargs),
)
class _TestTritonKernel(VllmTritonJitKernel["_TestTritonKernel.CompileKey"]):
kernel = _FakeTritonKernel()
@@ -91,6 +137,291 @@ def test_triton_launcher_supports_cpu_function_wrappers() -> None:
assert calls == [("runtime", 2, 7)]
def test_triton_kernel_decorator_returns_launcher(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
kernel.arg_names = (
"FIRST_PTR",
"FIRST_STRIDE0",
"aliased_ptr",
"aliased_stride_0",
"SECOND",
"CONST",
)
def warmup_inputs() -> dict[str, Any]:
return dict(
first=TritonWarmupTensor(torch.float32, shape=(2, 3), strides=(5, 1)),
second=WarmupChoices(1, 2),
)
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def launch(first: str, second: int = 2, config: int = 7) -> LaunchSpec:
return (2,), dict(aliased_ptr=first, CONST=config)
def fake_keys(kernel: Any, kwargs: Any) -> set[TritonJitKey]:
return {TritonJitKey(id(kernel), "fake", 0, kwargs["SECOND"])}
_patch_key_deriver(monkeypatch, fake_keys)
keys = launch.get_warmup_keys()
keys_with_config = launch.get_warmup_keys(vllm_config=object())
assert keys_with_config == keys
assert [dict(key.inputs)["second"] for key in keys] == [1, 2]
launch.compile(keys[0])
assert kernel.warmup_calls == [
{
"grid": (1,),
"FIRST_PTR": TritonWarmupTensor(
torch.float32, shape=(2, 3), strides=(5, 1)
),
"FIRST_STRIDE0": 5,
"aliased_ptr": TritonWarmupTensor(
torch.float32, shape=(2, 3), strides=(5, 1)
),
"aliased_stride_0": 5,
"SECOND": 1,
"CONST": 7,
}
]
first = torch.empty_strided((2, 3), (5, 1))
launch(first)
assert kernel.runtime_calls == [
(
(2,),
(),
{
"FIRST_PTR": first,
"FIRST_STRIDE0": 5,
"aliased_ptr": first,
"aliased_stride_0": 5,
"SECOND": 2,
"CONST": 7,
},
)
]
assert launch.__name__ == "launch"
with pytest.raises(TypeError, match="unexpected keyword"):
launch(first, 1, 7, stale_constexpr=True)
def test_triton_kernel_decorator_exhausts_large_ranges_before_deduplication(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
dispatched: list[int] = []
def warmup_inputs() -> dict[str, Any]:
tokens: Any = WarmupIntRange(1, 8193)
return dict(first="warmup", second=tokens)
def specialization(second: int) -> int:
return 7 if second <= 97 else 8
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def dispatch(first: str, second: int) -> LaunchSpec:
dispatched.append(second)
return (second,), dict(CONST=specialization(second))
def fake_keys(kernel: Any, kwargs: Any) -> set[TritonJitKey]:
return {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])}
_patch_key_deriver(monkeypatch, fake_keys)
keys = dispatch.get_warmup_keys()
assert dispatched == list(range(1, 8193))
assert len(keys) == 2
assert {specialization(dict(key.inputs)["second"]) for key in keys} == {7, 8}
def test_triton_kernel_decorator_propagates_dispatch_assertions(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
def warmup_inputs() -> dict[str, Any]:
return dict(first="warmup", second=WarmupChoices(1, 2))
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def dispatch(first: str, second: int) -> LaunchSpec:
assert second != 2, "broken dispatch"
return (1,), dict(CONST=second)
_patch_key_deriver(
monkeypatch,
lambda kernel, kwargs: {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])},
)
with pytest.raises(AssertionError, match="broken dispatch"):
dispatch.get_warmup_keys()
def test_triton_kernel_dispatch_uses_cuda_fake_tensors(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
def warmup_inputs() -> dict[str, Any]:
return dict(
first=TritonWarmupTensor(torch.float32, shape=(2, 3)),
second=1,
)
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def dispatch(first: torch.Tensor, second: int) -> LaunchSpec:
assert isinstance(first, torch.Tensor)
assert first.is_cuda
assert first[0].is_contiguous()
return (first.shape[0],), dict(CONST=first[0].numel())
_patch_key_deriver(
monkeypatch,
lambda kernel, kwargs: {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])},
)
keys = dispatch.get_warmup_keys()
assert len(keys) == 1
assert dict(keys[0].inputs)["first"].device.type == "cuda"
def test_triton_kernel_decorates_native_launchers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
def warmup_inputs(kernel: Any = kernel) -> dict[str, Any]:
return triton_warmup_inputs(
kernel,
"warmup",
WarmupChoices(1, 2),
grid=(2,),
CONST=7,
)
launcher = triton_kernel_dispatcher_with_warmup(warmup_inputs=warmup_inputs)(kernel)
launcher[(3,)]("runtime", 4, CONST=8)
assert kernel.runtime_calls == [((3,), ("runtime", 4), {"CONST": 8})]
_patch_key_deriver(
monkeypatch,
lambda kernel, kwargs: {
TritonJitKey(id(kernel), "fake", 0, (kwargs["second"], kwargs["CONST"]))
},
)
keys = launcher.get_warmup_keys()
assert len(keys) == 2
launcher.compile(keys[0])
assert kernel.warmup_calls == [
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
]
def test_triton_warmup_inputs_expands_explicit_pointer_dtypes() -> None:
inputs = triton_warmup_inputs(
_pointer_group_test_kernel,
grid=(1,),
pointer_dtypes={torch.float32: ("x_ptr",)},
value=2,
BLOCK=16,
)
assert inputs["x_ptr"] == TritonWarmupTensor(torch.float32)
assert inputs["value"] == 2
with pytest.raises(ValueError, match="not a pointer: value"):
triton_warmup_inputs(
_pointer_group_test_kernel,
grid=(1,),
pointer_dtypes={torch.float32: ("value",)},
x_ptr=TritonWarmupTensor(torch.float32),
BLOCK=16,
)
with pytest.raises(ValueError, match="Missing Triton pointer inputs: x_ptr"):
triton_warmup_inputs(
_pointer_group_test_kernel,
grid=(1,),
pointer_dtypes={},
value=2,
BLOCK=16,
)
def test_triton_key_derivation_applies_wrappers_and_runtime_options(
monkeypatch: pytest.MonkeyPatch,
) -> None:
knobs = triton.knobs
driver = triton.runtime.driver
device = "test-device"
monkeypatch.setattr(driver.active, "get_current_device", lambda: device)
calls: list[dict[str, Any]] = []
def binder(x: Any, value: int, BLOCK: int, **options: Any) -> Any:
calls.append(dict(BLOCK=BLOCK, **options))
specialization = [("pointer", x.dtype), ("i32", value), ("constexpr", BLOCK)]
return {}, specialization, options
cache: dict[Any, Any] = {}
_binder_test_kernel.device_caches[device] = ({}, cache, None, None, binder)
heuristic_kernel = triton.heuristics(
{"BLOCK": lambda args: 16 if args["value"] <= 16 else 32}
)(_binder_test_kernel)
keys = jit_warmup_triton_helper._triton_key_deriver(heuristic_kernel)(
{"x": TritonWarmupTensor(tl.float32), "value": 2}
)
assert len(keys) == 1
expected_options = {
"debug": knobs.runtime.debug,
"instrumentation_mode": knobs.compilation.instrumentation_mode,
}
if hasattr(knobs.compilation, "fpsan_homomorphic_casts"):
expected_options["fpsan_homomorphic_casts"] = (
knobs.compilation.fpsan_homomorphic_casts
)
assert calls == [{"BLOCK": 16, **expected_options}]
def test_triton_key_derivation_covers_autotune_configs_and_jit_identity(
monkeypatch: pytest.MonkeyPatch,
) -> None:
driver = triton.runtime.driver
device = "test-device"
monkeypatch.setattr(driver.active, "get_current_device", lambda: device)
def binder(x: Any, value: int, BLOCK: int, **options: Any) -> Any:
specialization = [("pointer", x.dtype), ("i32", value), ("constexpr", BLOCK)]
return {}, specialization, options
for kernel in (_binder_test_kernel, _second_binder_test_kernel):
kernel.device_caches[device] = ({}, {}, None, None, binder)
autotuned = triton.autotune(
configs=[
triton.Config({"BLOCK": 16}, num_warps=2),
triton.Config({"BLOCK": 32}, num_warps=4),
],
key=["value"],
)(_binder_test_kernel)
inputs = {"x": TritonWarmupTensor(tl.float32), "value": 2}
autotune_keys = jit_warmup_triton_helper._triton_key_deriver(autotuned)(inputs)
other_kernel_keys = jit_warmup_triton_helper._triton_key_deriver(
_second_binder_test_kernel
)(inputs | {"BLOCK": 16})
assert len(autotune_keys) == 2
assert {key.jit_function_key for key in autotune_keys} == {
_binder_test_kernel.cache_key
}
assert {key.jit_function_id for key in autotune_keys} == {id(_binder_test_kernel)}
assert autotune_keys.isdisjoint(other_kernel_keys)
def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
@@ -1,121 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
from vllm.model_executor.warmup.spec_decode_rejection_warmup import (
spec_decode_rejection_warmup,
)
MASTER_KEY = 20250910
def _worker(
watermark_config: WatermarkConfig | None,
draft_sample_method: str = "greedy",
) -> SimpleNamespace:
return SimpleNamespace(
device=torch.device("cpu"),
vllm_config=SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=2,
rejection_sample_method="standard",
draft_sample_method=draft_sample_method,
),
model_config=SimpleNamespace(
get_vocab_size=lambda: 128, dtype=torch.bfloat16
),
watermark_config=watermark_config,
),
)
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
calls: list[dict] = []
def fake_rejection_sample(**kwargs):
calls.append(kwargs)
return None, None
monkeypatch.setattr(
"vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils.rejection_sample",
fake_rejection_sample,
)
return calls
def test_warmup_token_id_dtypes_match_the_runtime_buffers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None))
assert calls
for kwargs in calls:
assert kwargs["draft_sampled"].dtype == torch.int32
assert kwargs["pos"].dtype == torch.int64
def test_warmup_index_mapping_dtypes_match_the_runtime_buffers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None))
assert calls
for kwargs in calls:
assert kwargs["idx_mapping"].dtype == torch.int64
assert kwargs["expanded_idx_mapping"].dtype == torch.int64
assert kwargs["expanded_local_pos"].dtype == torch.int32
def test_warmup_watermark_args_match_the_runtime_kernel_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
watermark_config = WatermarkConfig(
key=MASTER_KEY, algorithm="dual_key_gumbel", context_width=3
)
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(
_worker(watermark_config, draft_sample_method="probabilistic")
)
assert calls
key_b = derive_watermark_key(MASTER_KEY, b"key_b")
for kwargs in calls:
contexts = kwargs["contexts"]
assert contexts.dtype == torch.int32
assert contexts.shape == (3, 3) and contexts.stride() == (3, 1)
assert kwargs["watermarking"].dtype == torch.bool
assert kwargs["watermark_key"] == key_b
assert key_b & 0xFFFFFFFF >= 2**31 and key_b >> 32 >= 2**31
def test_warmup_is_skipped_when_the_recovery_key_cannot_be_derived(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
monkeypatch.setattr(
"vllm.v1.watermarking.spec_decode.speculative_target_watermark_key",
lambda _: (_ for _ in ()).throw(NotImplementedError("no philox")),
)
spec_decode_rejection_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert calls == []
def test_warmup_draft_logits_match_the_runtime_draft_sample_method(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None))
assert calls
assert all(kwargs["draft_logits"] is None for kwargs in calls)
calls = _capture_calls(monkeypatch)
spec_decode_rejection_warmup(_worker(None, draft_sample_method="probabilistic"))
assert calls
assert all(kwargs["draft_logits"] is not None for kwargs in calls)