[2/N][Feat][Perf] Add new warmup infrastructure for JITs. Add predicate filtering for JIT warmup, and migrate Inkling FA4 (#49315)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d8f840071e
commit
6c95a641e9
@@ -35,11 +35,10 @@ from vllm.v1.kv_cache_interface import (
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .ops.fa4_rel_attention import (
|
||||
INKLING_FA4_REL_ATTENTION_KERNEL,
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_num_splits,
|
||||
inkling_fa4_rel_attention,
|
||||
)
|
||||
from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup
|
||||
from .ops.qkvr_prep import fused_qkvr_prep
|
||||
from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata
|
||||
from .short_conv import InklingShortConv
|
||||
@@ -175,25 +174,6 @@ class InklingAttention(nn.Module, AttentionLayerBase):
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache
|
||||
|
||||
register_fa4_warmup(
|
||||
InklingFA4WarmupConfig(
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
rel_extent=self.rel_extent,
|
||||
window_size=self.window_size,
|
||||
is_local=self.is_local,
|
||||
max_kv_len=self._max_kv_len,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
kv_dtype=self.kv_cache_torch_dtype,
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return FlashAttentionBackend
|
||||
|
||||
@@ -309,7 +289,7 @@ class InklingAttention(nn.Module, AttentionLayerBase):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
max_kv_len=self._max_kv_len,
|
||||
)
|
||||
inkling_fa4_rel_attention(
|
||||
INKLING_FA4_REL_ATTENTION_KERNEL(
|
||||
q[:nt],
|
||||
key_cache,
|
||||
value_cache,
|
||||
|
||||
Reference in New Issue
Block a user