[2/N][Feat][Perf] Add new warmup infrastructure for JITs. Add predicate filtering for JIT warmup, and migrate Inkling FA4 (#49315)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-08-11 07:46:07 -04:00
committed by GitHub
co-authored by Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d8f840071e
commit 6c95a641e9
12 changed files with 739 additions and 386 deletions
+2 -22
View File
@@ -35,11 +35,10 @@ from vllm.v1.kv_cache_interface import (
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .ops.fa4_rel_attention import (
INKLING_FA4_REL_ATTENTION_KERNEL,
bucket_max_seqlen_q,
inkling_fa4_num_splits,
inkling_fa4_rel_attention,
)
from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup
from .ops.qkvr_prep import fused_qkvr_prep
from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata
from .short_conv import InklingShortConv
@@ -175,25 +174,6 @@ class InklingAttention(nn.Module, AttentionLayerBase):
compilation_config.static_forward_context[prefix] = self
self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache
register_fa4_warmup(
InklingFA4WarmupConfig(
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
rel_extent=self.rel_extent,
window_size=self.window_size,
is_local=self.is_local,
max_kv_len=self._max_kv_len,
dtype=vllm_config.model_config.dtype,
kv_dtype=self.kv_cache_torch_dtype,
block_size=vllm_config.cache_config.block_size,
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
max_num_batched_tokens=(
vllm_config.scheduler_config.max_num_batched_tokens
),
)
)
def get_attn_backend(self) -> type[AttentionBackend]:
return FlashAttentionBackend
@@ -309,7 +289,7 @@ class InklingAttention(nn.Module, AttentionLayerBase):
num_kv_heads=self.num_kv_heads,
max_kv_len=self._max_kv_len,
)
inkling_fa4_rel_attention(
INKLING_FA4_REL_ATTENTION_KERNEL(
q[:nt],
key_cache,
value_cache,
+4 -2
View File
@@ -15,11 +15,13 @@ from .sconv import fused_sconv
_LAZY_EXPORTS = {
"silu_and_mul_triton": "silu_and_mul",
"sink_silu_mul_epilogue": "silu_and_mul",
"inkling_fa4_rel_attention": "fa4_rel_attention",
"INKLING_FA4_REL_ATTENTION_KERNEL": "fa4_rel_attention",
}
if TYPE_CHECKING:
from .fa4_rel_attention import inkling_fa4_rel_attention # noqa: F401
from .fa4_rel_attention import ( # noqa: F401
INKLING_FA4_REL_ATTENTION_KERNEL,
)
from .silu_and_mul import ( # noqa: F401
silu_and_mul_triton,
sink_silu_mul_epilogue,
@@ -3,12 +3,23 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from functools import cache
from typing import Any
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
WarmupIntRange,
zip_inputs,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import kv_cache_dtype_str_to_dtype
if TYPE_CHECKING:
from vllm.config import VllmConfig
def bucket_max_seqlen_q(max_seqlen_q: int) -> int:
@@ -98,73 +109,328 @@ def inkling_fa4_num_splits(
)
def inkling_fa4_rel_attention(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
block_table: torch.Tensor,
cache_seqlens: torch.Tensor,
cu_seqlens_q: torch.Tensor,
max_seqlen_q: int,
softmax_scale: float,
causal: bool,
window_size: tuple[int, int],
rel_extent: int,
rel_logits: torch.Tensor,
num_splits: int = 32,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Paged varlen FA4 over the bound K/V cache with the Inkling relative bias.
def _num_warps_bucket(num_reqs: int) -> int:
num_warps = min((num_reqs + 30) // 31, 32)
return 1 << (num_warps - 1).bit_length()
``q`` is ``(num_tokens, num_heads, head_dim)``; ``key_cache`` / ``value_cache``
are the paged caches ``(num_blocks, block_size, num_kv_heads, head_dim)``;
``block_table`` is the per-request page table and ``cache_seqlens`` the
per-request KV lengths (``seqused_k``). ``rel_logits`` is
``(num_tokens, num_heads, rel_extent)``.
Hopper uses standard FA4's score-mod gather. Blackwell uses tml-fa4's
sheared relative-bias layout.
"""
# cute uses (None, None) to mean "no window".
cute_window = (None, None) if window_size == (-1, -1) else window_size
class InklingFA4RelAttentionKernel(
VllmJitKernel["InklingFA4RelAttentionKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
is_local: bool
num_heads: int
num_kv_heads: int
head_dim: int
rel_extent: int
dtype: torch.dtype
kv_dtype: torch.dtype
block_size: int
window_size: tuple[int, int]
max_seqlen_q: int
num_splits: int
num_warps_bucket: int | None
large_num_reqs: bool
rel_logits = rel_logits.contiguous()
flash_attn_varlen_func: Callable[..., Any]
if _use_sheared_bias():
from vllm.third_party.tml_fa4 import (
flash_attn_varlen_func as tml_flash_attn_varlen_func,
@staticmethod
def kernel(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
block_table: torch.Tensor,
cache_seqlens: torch.Tensor,
cu_seqlens_q: torch.Tensor,
max_seqlen_q: int,
softmax_scale: float,
causal: bool,
window_size: tuple[int, int],
rel_extent: int,
rel_logits: torch.Tensor,
num_splits: int = 32,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Paged varlen FA4 over the bound K/V cache with the Inkling relative bias.
``q`` is ``(num_tokens, num_heads, head_dim)``; ``key_cache`` / ``value_cache``
are the paged caches ``(num_blocks, block_size, num_kv_heads, head_dim)``;
``block_table`` is the per-request page table and ``cache_seqlens`` the
per-request KV lengths (``seqused_k``). ``rel_logits`` is
``(num_tokens, num_heads, rel_extent)``.
Hopper uses standard FA4's score-mod gather. Blackwell uses tml-fa4's
sheared relative-bias layout.
"""
# cute uses (None, None) to mean "no window".
cute_window = (None, None) if window_size == (-1, -1) else window_size
rel_logits = rel_logits.contiguous()
flash_attn_varlen_func: Callable[..., Any]
if _use_sheared_bias():
from vllm.third_party.tml_fa4 import (
flash_attn_varlen_func as tml_flash_attn_varlen_func,
)
flash_attn_varlen_func = tml_flash_attn_varlen_func
bias_kwargs: dict[str, Any] = {"rel_bias": rel_logits}
else:
from vllm.vllm_flash_attn.cute import (
flash_attn_varlen_func as cute_flash_attn_varlen_func,
)
flash_attn_varlen_func = cute_flash_attn_varlen_func
bias_kwargs = {
"score_mod": _get_score_mod(rel_extent),
"aux_tensors": [rel_logits],
}
ret = flash_attn_varlen_func(
q=q,
k=key_cache,
v=value_cache,
cu_seqlens_q=cu_seqlens_q,
seqused_k=cache_seqlens,
max_seqlen_q=max_seqlen_q,
page_table=block_table,
softmax_scale=softmax_scale,
causal=causal,
window_size=cute_window,
num_splits=num_splits,
return_lse=False,
out=out,
**bias_kwargs,
)
if isinstance(ret, tuple):
return ret[0]
return ret
def dispatch( # type: ignore[override]
self,
*,
is_local: bool,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rel_extent: int,
dtype: torch.dtype,
kv_dtype: torch.dtype,
block_size: int,
window_size: tuple[int, int],
max_kv_len: int,
query_len: int,
num_reqs: int,
) -> CompileKey:
max_seqlen_q = bucket_max_seqlen_q(query_len)
num_splits = inkling_fa4_num_splits(
is_local=is_local,
batch_size=num_reqs,
max_query_len=max_seqlen_q,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
max_kv_len=max_kv_len,
)
return self.CompileKey(
is_local=is_local,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
rel_extent=rel_extent,
dtype=dtype,
kv_dtype=kv_dtype,
block_size=block_size,
window_size=window_size,
max_seqlen_q=max_seqlen_q,
num_splits=num_splits,
num_warps_bucket=(_num_warps_bucket(num_reqs) if num_splits > 1 else None),
large_num_reqs=num_reqs > 1024,
)
flash_attn_varlen_func = tml_flash_attn_varlen_func
bias_kwargs: dict[str, Any] = {"rel_bias": rel_logits}
else:
from vllm.vllm_flash_attn.cute import (
flash_attn_varlen_func as cute_flash_attn_varlen_func,
def _is_valid_warmup_dispatch(
self,
*,
query_len: int,
num_reqs: int,
max_num_batched_tokens: int,
) -> bool:
return query_len + num_reqs <= max_num_batched_tokens + 1
def get_warmup_keys(self, vllm_config: VllmConfig) -> list[CompileKey]:
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
if max_num_reqs <= 0 or max_num_batched_tokens <= 0:
return []
hf_config = vllm_config.model_config.hf_config
get_text_config = getattr(hf_config, "get_text_config", None)
config = get_text_config() if callable(get_text_config) else hf_config
tp_size = get_tensor_model_parallel_world_size()
dtype = vllm_config.model_config.dtype
kv_dtype = kv_cache_dtype_str_to_dtype(
vllm_config.cache_config.cache_dtype,
vllm_config.model_config,
)
block_size = vllm_config.cache_config.block_size
local_extent = config.sliding_window_size
global_num_kv_heads = config.num_key_value_heads
local_num_kv_heads = config.swa_num_key_value_heads
assert config.num_attention_heads % tp_size == 0
assert config.swa_num_attention_heads % tp_size == 0
if global_num_kv_heads >= tp_size:
assert global_num_kv_heads % tp_size == 0
else:
assert tp_size % global_num_kv_heads == 0
if local_num_kv_heads >= tp_size:
assert local_num_kv_heads % tp_size == 0
else:
assert tp_size % local_num_kv_heads == 0
return self._trace_dispatch(self.dispatch)(
zip_inputs(
dict(
is_local=False,
num_heads=config.num_attention_heads // tp_size,
num_kv_heads=max(1, global_num_kv_heads // tp_size),
head_dim=config.head_dim,
rel_extent=config.rel_extent,
dtype=dtype,
kv_dtype=kv_dtype,
block_size=block_size,
window_size=(-1, -1),
max_kv_len=vllm_config.model_config.max_model_len,
),
dict(
is_local=True,
num_heads=config.swa_num_attention_heads // tp_size,
num_kv_heads=max(1, local_num_kv_heads // tp_size),
head_dim=config.swa_head_dim,
rel_extent=local_extent,
dtype=dtype,
kv_dtype=kv_dtype,
block_size=block_size,
window_size=(local_extent - 1, 0),
max_kv_len=local_extent,
),
),
query_len=WarmupIntRange(
1,
max_num_batched_tokens + 1,
advance=lambda value: bucket_max_seqlen_q(value) + 1,
),
num_reqs=WarmupIntRange(1, max_num_reqs + 1),
max_num_batched_tokens=max_num_batched_tokens,
_when=self._is_valid_warmup_dispatch,
)
flash_attn_varlen_func = cute_flash_attn_varlen_func
bias_kwargs = {
"score_mod": _get_score_mod(rel_extent),
"aux_tensors": [rel_logits],
}
def compile(self, compile_key: CompileKey) -> None:
from torch._subclasses.fake_tensor import FakeTensorMode
ret = flash_attn_varlen_func(
q=q,
k=key_cache,
v=value_cache,
cu_seqlens_q=cu_seqlens_q,
seqused_k=cache_seqlens,
max_seqlen_q=max_seqlen_q,
page_table=block_table,
softmax_scale=softmax_scale,
causal=causal,
window_size=cute_window,
num_splits=num_splits,
return_lse=False,
out=out,
**bias_kwargs,
)
if isinstance(ret, tuple):
return ret[0]
return ret
if compile_key.num_splits == 1 or compile_key.num_warps_bucket is None:
num_reqs = 1025 if compile_key.large_num_reqs else 1
else:
min_num_warps = (
1
if compile_key.num_warps_bucket == 1
else compile_key.num_warps_bucket // 2 + 1
)
min_num_reqs = (min_num_warps - 1) * 31 + 1
num_reqs = max(
min_num_reqs,
1025 if compile_key.large_num_reqs else 1,
)
with FakeTensorMode():
device = torch.accelerator.current_accelerator()
total_q = compile_key.max_seqlen_q + num_reqs - 1
q = torch.empty(
total_q,
compile_key.num_heads,
compile_key.head_dim,
dtype=compile_key.dtype,
device=device,
)
kv = torch.empty(
1,
2,
compile_key.block_size,
compile_key.num_kv_heads,
compile_key.head_dim,
dtype=compile_key.kv_dtype,
device=device,
)
key_cache, value_cache = kv.unbind(1)
self.kernel(
q,
key_cache,
value_cache,
block_table=torch.empty(
num_reqs,
1,
dtype=torch.int32,
device=device,
),
cache_seqlens=torch.empty(
num_reqs,
dtype=torch.int32,
device=device,
),
cu_seqlens_q=torch.empty(
num_reqs + 1,
dtype=torch.int32,
device=device,
),
max_seqlen_q=compile_key.max_seqlen_q,
softmax_scale=compile_key.head_dim**-1,
causal=True,
window_size=compile_key.window_size,
rel_extent=compile_key.rel_extent,
rel_logits=torch.empty(
total_q,
compile_key.num_heads,
compile_key.rel_extent,
dtype=compile_key.dtype,
device=device,
),
num_splits=compile_key.num_splits,
out=torch.empty_like(q),
)
def __call__(
self,
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
block_table: torch.Tensor,
cache_seqlens: torch.Tensor,
cu_seqlens_q: torch.Tensor,
max_seqlen_q: int,
softmax_scale: float,
causal: bool,
window_size: tuple[int, int],
rel_extent: int,
rel_logits: torch.Tensor,
num_splits: int = 32,
out: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=max_seqlen_q,
softmax_scale=softmax_scale,
causal=causal,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
num_splits=num_splits,
out=out,
)
INKLING_FA4_REL_ATTENTION_KERNEL = InklingFA4RelAttentionKernel()
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Startup compilation of the FA4 kernels used by Inkling."""
from __future__ import annotations
from collections.abc import Iterator
from dataclasses import dataclass
from functools import partial
import torch
from vllm.model_executor.warmup.cutedsl_warmup import (
CuTeDSLCompileUnit,
register_cutedsl_warmup_provider,
)
from .fa4_rel_attention import (
bucket_max_seqlen_q,
inkling_fa4_num_splits,
inkling_fa4_rel_attention,
)
@dataclass(frozen=True)
class InklingFA4WarmupConfig:
num_heads: int
num_kv_heads: int
head_dim: int
rel_extent: int
window_size: tuple[int, int]
is_local: bool
max_kv_len: int
dtype: torch.dtype
kv_dtype: torch.dtype
block_size: int
max_num_reqs: int
max_num_batched_tokens: int
def _num_warps_bucket(num_reqs: int) -> int:
num_warps = min((num_reqs + 30) // 31, 32)
return 1 << (num_warps - 1).bit_length()
def _compile(config: InklingFA4WarmupConfig, max_seqlen_q: int, num_reqs: int) -> None:
from torch._subclasses.fake_tensor import FakeTensorMode
num_splits = inkling_fa4_num_splits(
is_local=config.is_local,
batch_size=num_reqs,
max_query_len=max_seqlen_q,
num_heads=config.num_heads,
num_kv_heads=config.num_kv_heads,
max_kv_len=config.max_kv_len,
)
with FakeTensorMode():
device = torch.accelerator.current_accelerator()
total_q = max_seqlen_q + num_reqs - 1
q = torch.empty(
total_q,
config.num_heads,
config.head_dim,
dtype=config.dtype,
device=device,
)
kv = torch.empty(
1,
2,
config.block_size,
config.num_kv_heads,
config.head_dim,
dtype=config.kv_dtype,
device=device,
)
key_cache, value_cache = kv.unbind(1)
inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=torch.empty(num_reqs, 1, dtype=torch.int32, device=device),
cache_seqlens=torch.empty(num_reqs, dtype=torch.int32, device=device),
cu_seqlens_q=torch.empty(num_reqs + 1, dtype=torch.int32, device=device),
max_seqlen_q=max_seqlen_q,
softmax_scale=1.0 / config.head_dim,
causal=True,
window_size=config.window_size,
rel_extent=config.rel_extent,
rel_logits=torch.empty(
total_q,
config.num_heads,
config.rel_extent,
dtype=config.dtype,
device=device,
),
num_splits=num_splits,
out=torch.empty_like(q),
)
def _iter_compile_units(
config: InklingFA4WarmupConfig,
) -> Iterator[CuTeDSLCompileUnit]:
max_bucket = bucket_max_seqlen_q(config.max_num_batched_tokens)
max_seqlen_q = 1
while max_seqlen_q <= max_bucket:
min_query_len = max_seqlen_q // 2 + 1
max_num_reqs = min(
config.max_num_reqs,
config.max_num_batched_tokens - min_query_len + 1,
)
seen: set[tuple[int, int, int | None, bool]] = set()
for num_reqs in range(1, max_num_reqs + 1):
num_splits = inkling_fa4_num_splits(
is_local=config.is_local,
batch_size=num_reqs,
max_query_len=max_seqlen_q,
num_heads=config.num_heads,
num_kv_heads=config.num_kv_heads,
max_kv_len=config.max_kv_len,
)
key = (
max_seqlen_q,
num_splits,
_num_warps_bucket(num_reqs) if num_splits > 1 else None,
num_reqs > 1024,
)
if key in seen:
continue
seen.add(key)
yield CuTeDSLCompileUnit(
name="inkling_fa4",
key=("inkling_fa4", config, key),
compile=partial(_compile, config, max_seqlen_q, num_reqs),
)
max_seqlen_q *= 2
class _WarmupProvider:
def __init__(self) -> None:
self.configs: set[InklingFA4WarmupConfig] = set()
def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
return tuple(
unit for config in self.configs for unit in _iter_compile_units(config)
)
_PROVIDER = _WarmupProvider()
def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None:
_PROVIDER.configs.add(config)
register_cutedsl_warmup_provider(_PROVIDER)