[2/N][Feat][Perf] Add new warmup infrastructure for JITs. Add predicate filtering for JIT warmup, and migrate Inkling FA4 (#49315)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d8f840071e
commit
6c95a641e9
@@ -35,11 +35,10 @@ from vllm.v1.kv_cache_interface import (
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .ops.fa4_rel_attention import (
|
||||
INKLING_FA4_REL_ATTENTION_KERNEL,
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_num_splits,
|
||||
inkling_fa4_rel_attention,
|
||||
)
|
||||
from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup
|
||||
from .ops.qkvr_prep import fused_qkvr_prep
|
||||
from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata
|
||||
from .short_conv import InklingShortConv
|
||||
@@ -175,25 +174,6 @@ class InklingAttention(nn.Module, AttentionLayerBase):
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache
|
||||
|
||||
register_fa4_warmup(
|
||||
InklingFA4WarmupConfig(
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
rel_extent=self.rel_extent,
|
||||
window_size=self.window_size,
|
||||
is_local=self.is_local,
|
||||
max_kv_len=self._max_kv_len,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
kv_dtype=self.kv_cache_torch_dtype,
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return FlashAttentionBackend
|
||||
|
||||
@@ -309,7 +289,7 @@ class InklingAttention(nn.Module, AttentionLayerBase):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
max_kv_len=self._max_kv_len,
|
||||
)
|
||||
inkling_fa4_rel_attention(
|
||||
INKLING_FA4_REL_ATTENTION_KERNEL(
|
||||
q[:nt],
|
||||
key_cache,
|
||||
value_cache,
|
||||
|
||||
@@ -15,11 +15,13 @@ from .sconv import fused_sconv
|
||||
_LAZY_EXPORTS = {
|
||||
"silu_and_mul_triton": "silu_and_mul",
|
||||
"sink_silu_mul_epilogue": "silu_and_mul",
|
||||
"inkling_fa4_rel_attention": "fa4_rel_attention",
|
||||
"INKLING_FA4_REL_ATTENTION_KERNEL": "fa4_rel_attention",
|
||||
}
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .fa4_rel_attention import inkling_fa4_rel_attention # noqa: F401
|
||||
from .fa4_rel_attention import ( # noqa: F401
|
||||
INKLING_FA4_REL_ATTENTION_KERNEL,
|
||||
)
|
||||
from .silu_and_mul import ( # noqa: F401
|
||||
silu_and_mul_triton,
|
||||
sink_silu_mul_epilogue,
|
||||
|
||||
@@ -3,12 +3,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
VllmJitKernel,
|
||||
WarmupIntRange,
|
||||
zip_inputs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import kv_cache_dtype_str_to_dtype
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
def bucket_max_seqlen_q(max_seqlen_q: int) -> int:
|
||||
@@ -98,73 +109,328 @@ def inkling_fa4_num_splits(
|
||||
)
|
||||
|
||||
|
||||
def inkling_fa4_rel_attention(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
block_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
max_seqlen_q: int,
|
||||
softmax_scale: float,
|
||||
causal: bool,
|
||||
window_size: tuple[int, int],
|
||||
rel_extent: int,
|
||||
rel_logits: torch.Tensor,
|
||||
num_splits: int = 32,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Paged varlen FA4 over the bound K/V cache with the Inkling relative bias.
|
||||
def _num_warps_bucket(num_reqs: int) -> int:
|
||||
num_warps = min((num_reqs + 30) // 31, 32)
|
||||
return 1 << (num_warps - 1).bit_length()
|
||||
|
||||
``q`` is ``(num_tokens, num_heads, head_dim)``; ``key_cache`` / ``value_cache``
|
||||
are the paged caches ``(num_blocks, block_size, num_kv_heads, head_dim)``;
|
||||
``block_table`` is the per-request page table and ``cache_seqlens`` the
|
||||
per-request KV lengths (``seqused_k``). ``rel_logits`` is
|
||||
``(num_tokens, num_heads, rel_extent)``.
|
||||
|
||||
Hopper uses standard FA4's score-mod gather. Blackwell uses tml-fa4's
|
||||
sheared relative-bias layout.
|
||||
"""
|
||||
# cute uses (None, None) to mean "no window".
|
||||
cute_window = (None, None) if window_size == (-1, -1) else window_size
|
||||
class InklingFA4RelAttentionKernel(
|
||||
VllmJitKernel["InklingFA4RelAttentionKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
is_local: bool
|
||||
num_heads: int
|
||||
num_kv_heads: int
|
||||
head_dim: int
|
||||
rel_extent: int
|
||||
dtype: torch.dtype
|
||||
kv_dtype: torch.dtype
|
||||
block_size: int
|
||||
window_size: tuple[int, int]
|
||||
max_seqlen_q: int
|
||||
num_splits: int
|
||||
num_warps_bucket: int | None
|
||||
large_num_reqs: bool
|
||||
|
||||
rel_logits = rel_logits.contiguous()
|
||||
flash_attn_varlen_func: Callable[..., Any]
|
||||
if _use_sheared_bias():
|
||||
from vllm.third_party.tml_fa4 import (
|
||||
flash_attn_varlen_func as tml_flash_attn_varlen_func,
|
||||
@staticmethod
|
||||
def kernel(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
block_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
max_seqlen_q: int,
|
||||
softmax_scale: float,
|
||||
causal: bool,
|
||||
window_size: tuple[int, int],
|
||||
rel_extent: int,
|
||||
rel_logits: torch.Tensor,
|
||||
num_splits: int = 32,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Paged varlen FA4 over the bound K/V cache with the Inkling relative bias.
|
||||
|
||||
``q`` is ``(num_tokens, num_heads, head_dim)``; ``key_cache`` / ``value_cache``
|
||||
are the paged caches ``(num_blocks, block_size, num_kv_heads, head_dim)``;
|
||||
``block_table`` is the per-request page table and ``cache_seqlens`` the
|
||||
per-request KV lengths (``seqused_k``). ``rel_logits`` is
|
||||
``(num_tokens, num_heads, rel_extent)``.
|
||||
|
||||
Hopper uses standard FA4's score-mod gather. Blackwell uses tml-fa4's
|
||||
sheared relative-bias layout.
|
||||
"""
|
||||
# cute uses (None, None) to mean "no window".
|
||||
cute_window = (None, None) if window_size == (-1, -1) else window_size
|
||||
|
||||
rel_logits = rel_logits.contiguous()
|
||||
flash_attn_varlen_func: Callable[..., Any]
|
||||
if _use_sheared_bias():
|
||||
from vllm.third_party.tml_fa4 import (
|
||||
flash_attn_varlen_func as tml_flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = tml_flash_attn_varlen_func
|
||||
bias_kwargs: dict[str, Any] = {"rel_bias": rel_logits}
|
||||
else:
|
||||
from vllm.vllm_flash_attn.cute import (
|
||||
flash_attn_varlen_func as cute_flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = cute_flash_attn_varlen_func
|
||||
bias_kwargs = {
|
||||
"score_mod": _get_score_mod(rel_extent),
|
||||
"aux_tensors": [rel_logits],
|
||||
}
|
||||
|
||||
ret = flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
seqused_k=cache_seqlens,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
page_table=block_table,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
window_size=cute_window,
|
||||
num_splits=num_splits,
|
||||
return_lse=False,
|
||||
out=out,
|
||||
**bias_kwargs,
|
||||
)
|
||||
if isinstance(ret, tuple):
|
||||
return ret[0]
|
||||
return ret
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
is_local: bool,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rel_extent: int,
|
||||
dtype: torch.dtype,
|
||||
kv_dtype: torch.dtype,
|
||||
block_size: int,
|
||||
window_size: tuple[int, int],
|
||||
max_kv_len: int,
|
||||
query_len: int,
|
||||
num_reqs: int,
|
||||
) -> CompileKey:
|
||||
max_seqlen_q = bucket_max_seqlen_q(query_len)
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=is_local,
|
||||
batch_size=num_reqs,
|
||||
max_query_len=max_seqlen_q,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
return self.CompileKey(
|
||||
is_local=is_local,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
rel_extent=rel_extent,
|
||||
dtype=dtype,
|
||||
kv_dtype=kv_dtype,
|
||||
block_size=block_size,
|
||||
window_size=window_size,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
num_splits=num_splits,
|
||||
num_warps_bucket=(_num_warps_bucket(num_reqs) if num_splits > 1 else None),
|
||||
large_num_reqs=num_reqs > 1024,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = tml_flash_attn_varlen_func
|
||||
bias_kwargs: dict[str, Any] = {"rel_bias": rel_logits}
|
||||
else:
|
||||
from vllm.vllm_flash_attn.cute import (
|
||||
flash_attn_varlen_func as cute_flash_attn_varlen_func,
|
||||
def _is_valid_warmup_dispatch(
|
||||
self,
|
||||
*,
|
||||
query_len: int,
|
||||
num_reqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
) -> bool:
|
||||
return query_len + num_reqs <= max_num_batched_tokens + 1
|
||||
|
||||
def get_warmup_keys(self, vllm_config: VllmConfig) -> list[CompileKey]:
|
||||
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
if max_num_reqs <= 0 or max_num_batched_tokens <= 0:
|
||||
return []
|
||||
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
get_text_config = getattr(hf_config, "get_text_config", None)
|
||||
config = get_text_config() if callable(get_text_config) else hf_config
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
dtype = vllm_config.model_config.dtype
|
||||
kv_dtype = kv_cache_dtype_str_to_dtype(
|
||||
vllm_config.cache_config.cache_dtype,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
local_extent = config.sliding_window_size
|
||||
|
||||
global_num_kv_heads = config.num_key_value_heads
|
||||
local_num_kv_heads = config.swa_num_key_value_heads
|
||||
assert config.num_attention_heads % tp_size == 0
|
||||
assert config.swa_num_attention_heads % tp_size == 0
|
||||
if global_num_kv_heads >= tp_size:
|
||||
assert global_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % global_num_kv_heads == 0
|
||||
if local_num_kv_heads >= tp_size:
|
||||
assert local_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % local_num_kv_heads == 0
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(
|
||||
dict(
|
||||
is_local=False,
|
||||
num_heads=config.num_attention_heads // tp_size,
|
||||
num_kv_heads=max(1, global_num_kv_heads // tp_size),
|
||||
head_dim=config.head_dim,
|
||||
rel_extent=config.rel_extent,
|
||||
dtype=dtype,
|
||||
kv_dtype=kv_dtype,
|
||||
block_size=block_size,
|
||||
window_size=(-1, -1),
|
||||
max_kv_len=vllm_config.model_config.max_model_len,
|
||||
),
|
||||
dict(
|
||||
is_local=True,
|
||||
num_heads=config.swa_num_attention_heads // tp_size,
|
||||
num_kv_heads=max(1, local_num_kv_heads // tp_size),
|
||||
head_dim=config.swa_head_dim,
|
||||
rel_extent=local_extent,
|
||||
dtype=dtype,
|
||||
kv_dtype=kv_dtype,
|
||||
block_size=block_size,
|
||||
window_size=(local_extent - 1, 0),
|
||||
max_kv_len=local_extent,
|
||||
),
|
||||
),
|
||||
query_len=WarmupIntRange(
|
||||
1,
|
||||
max_num_batched_tokens + 1,
|
||||
advance=lambda value: bucket_max_seqlen_q(value) + 1,
|
||||
),
|
||||
num_reqs=WarmupIntRange(1, max_num_reqs + 1),
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
_when=self._is_valid_warmup_dispatch,
|
||||
)
|
||||
|
||||
flash_attn_varlen_func = cute_flash_attn_varlen_func
|
||||
bias_kwargs = {
|
||||
"score_mod": _get_score_mod(rel_extent),
|
||||
"aux_tensors": [rel_logits],
|
||||
}
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
ret = flash_attn_varlen_func(
|
||||
q=q,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
seqused_k=cache_seqlens,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
page_table=block_table,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
window_size=cute_window,
|
||||
num_splits=num_splits,
|
||||
return_lse=False,
|
||||
out=out,
|
||||
**bias_kwargs,
|
||||
)
|
||||
if isinstance(ret, tuple):
|
||||
return ret[0]
|
||||
return ret
|
||||
if compile_key.num_splits == 1 or compile_key.num_warps_bucket is None:
|
||||
num_reqs = 1025 if compile_key.large_num_reqs else 1
|
||||
else:
|
||||
min_num_warps = (
|
||||
1
|
||||
if compile_key.num_warps_bucket == 1
|
||||
else compile_key.num_warps_bucket // 2 + 1
|
||||
)
|
||||
min_num_reqs = (min_num_warps - 1) * 31 + 1
|
||||
num_reqs = max(
|
||||
min_num_reqs,
|
||||
1025 if compile_key.large_num_reqs else 1,
|
||||
)
|
||||
|
||||
with FakeTensorMode():
|
||||
device = torch.accelerator.current_accelerator()
|
||||
total_q = compile_key.max_seqlen_q + num_reqs - 1
|
||||
q = torch.empty(
|
||||
total_q,
|
||||
compile_key.num_heads,
|
||||
compile_key.head_dim,
|
||||
dtype=compile_key.dtype,
|
||||
device=device,
|
||||
)
|
||||
kv = torch.empty(
|
||||
1,
|
||||
2,
|
||||
compile_key.block_size,
|
||||
compile_key.num_kv_heads,
|
||||
compile_key.head_dim,
|
||||
dtype=compile_key.kv_dtype,
|
||||
device=device,
|
||||
)
|
||||
key_cache, value_cache = kv.unbind(1)
|
||||
self.kernel(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=torch.empty(
|
||||
num_reqs,
|
||||
1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
cache_seqlens=torch.empty(
|
||||
num_reqs,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
cu_seqlens_q=torch.empty(
|
||||
num_reqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
max_seqlen_q=compile_key.max_seqlen_q,
|
||||
softmax_scale=compile_key.head_dim**-1,
|
||||
causal=True,
|
||||
window_size=compile_key.window_size,
|
||||
rel_extent=compile_key.rel_extent,
|
||||
rel_logits=torch.empty(
|
||||
total_q,
|
||||
compile_key.num_heads,
|
||||
compile_key.rel_extent,
|
||||
dtype=compile_key.dtype,
|
||||
device=device,
|
||||
),
|
||||
num_splits=compile_key.num_splits,
|
||||
out=torch.empty_like(q),
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
block_table: torch.Tensor,
|
||||
cache_seqlens: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
max_seqlen_q: int,
|
||||
softmax_scale: float,
|
||||
causal: bool,
|
||||
window_size: tuple[int, int],
|
||||
rel_extent: int,
|
||||
rel_logits: torch.Tensor,
|
||||
num_splits: int = 32,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
num_splits=num_splits,
|
||||
out=out,
|
||||
)
|
||||
|
||||
|
||||
INKLING_FA4_REL_ATTENTION_KERNEL = InklingFA4RelAttentionKernel()
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Startup compilation of the FA4 kernels used by Inkling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import (
|
||||
CuTeDSLCompileUnit,
|
||||
register_cutedsl_warmup_provider,
|
||||
)
|
||||
|
||||
from .fa4_rel_attention import (
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_num_splits,
|
||||
inkling_fa4_rel_attention,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InklingFA4WarmupConfig:
|
||||
num_heads: int
|
||||
num_kv_heads: int
|
||||
head_dim: int
|
||||
rel_extent: int
|
||||
window_size: tuple[int, int]
|
||||
is_local: bool
|
||||
max_kv_len: int
|
||||
dtype: torch.dtype
|
||||
kv_dtype: torch.dtype
|
||||
block_size: int
|
||||
max_num_reqs: int
|
||||
max_num_batched_tokens: int
|
||||
|
||||
|
||||
def _num_warps_bucket(num_reqs: int) -> int:
|
||||
num_warps = min((num_reqs + 30) // 31, 32)
|
||||
return 1 << (num_warps - 1).bit_length()
|
||||
|
||||
|
||||
def _compile(config: InklingFA4WarmupConfig, max_seqlen_q: int, num_reqs: int) -> None:
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=config.is_local,
|
||||
batch_size=num_reqs,
|
||||
max_query_len=max_seqlen_q,
|
||||
num_heads=config.num_heads,
|
||||
num_kv_heads=config.num_kv_heads,
|
||||
max_kv_len=config.max_kv_len,
|
||||
)
|
||||
with FakeTensorMode():
|
||||
device = torch.accelerator.current_accelerator()
|
||||
total_q = max_seqlen_q + num_reqs - 1
|
||||
q = torch.empty(
|
||||
total_q,
|
||||
config.num_heads,
|
||||
config.head_dim,
|
||||
dtype=config.dtype,
|
||||
device=device,
|
||||
)
|
||||
kv = torch.empty(
|
||||
1,
|
||||
2,
|
||||
config.block_size,
|
||||
config.num_kv_heads,
|
||||
config.head_dim,
|
||||
dtype=config.kv_dtype,
|
||||
device=device,
|
||||
)
|
||||
key_cache, value_cache = kv.unbind(1)
|
||||
inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=torch.empty(num_reqs, 1, dtype=torch.int32, device=device),
|
||||
cache_seqlens=torch.empty(num_reqs, dtype=torch.int32, device=device),
|
||||
cu_seqlens_q=torch.empty(num_reqs + 1, dtype=torch.int32, device=device),
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
softmax_scale=1.0 / config.head_dim,
|
||||
causal=True,
|
||||
window_size=config.window_size,
|
||||
rel_extent=config.rel_extent,
|
||||
rel_logits=torch.empty(
|
||||
total_q,
|
||||
config.num_heads,
|
||||
config.rel_extent,
|
||||
dtype=config.dtype,
|
||||
device=device,
|
||||
),
|
||||
num_splits=num_splits,
|
||||
out=torch.empty_like(q),
|
||||
)
|
||||
|
||||
|
||||
def _iter_compile_units(
|
||||
config: InklingFA4WarmupConfig,
|
||||
) -> Iterator[CuTeDSLCompileUnit]:
|
||||
max_bucket = bucket_max_seqlen_q(config.max_num_batched_tokens)
|
||||
max_seqlen_q = 1
|
||||
while max_seqlen_q <= max_bucket:
|
||||
min_query_len = max_seqlen_q // 2 + 1
|
||||
max_num_reqs = min(
|
||||
config.max_num_reqs,
|
||||
config.max_num_batched_tokens - min_query_len + 1,
|
||||
)
|
||||
seen: set[tuple[int, int, int | None, bool]] = set()
|
||||
for num_reqs in range(1, max_num_reqs + 1):
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=config.is_local,
|
||||
batch_size=num_reqs,
|
||||
max_query_len=max_seqlen_q,
|
||||
num_heads=config.num_heads,
|
||||
num_kv_heads=config.num_kv_heads,
|
||||
max_kv_len=config.max_kv_len,
|
||||
)
|
||||
key = (
|
||||
max_seqlen_q,
|
||||
num_splits,
|
||||
_num_warps_bucket(num_reqs) if num_splits > 1 else None,
|
||||
num_reqs > 1024,
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield CuTeDSLCompileUnit(
|
||||
name="inkling_fa4",
|
||||
key=("inkling_fa4", config, key),
|
||||
compile=partial(_compile, config, max_seqlen_q, num_reqs),
|
||||
)
|
||||
max_seqlen_q *= 2
|
||||
|
||||
|
||||
class _WarmupProvider:
|
||||
def __init__(self) -> None:
|
||||
self.configs: set[InklingFA4WarmupConfig] = set()
|
||||
|
||||
def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
|
||||
return tuple(
|
||||
unit for config in self.configs for unit in _iter_compile_units(config)
|
||||
)
|
||||
|
||||
|
||||
_PROVIDER = _WarmupProvider()
|
||||
|
||||
|
||||
def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None:
|
||||
_PROVIDER.configs.add(config)
|
||||
register_cutedsl_warmup_provider(_PROVIDER)
|
||||
Reference in New Issue
Block a user