[Perf][GDN] Internal prefill checkpoints for the FlashInfer GDN prefill backend (#60050)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jiangyun Zhu
2026-10-14 14:49:46 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent a2fb482732
commit badfcdbf91
12 changed files with 794 additions and 151 deletions
+27 -2
View File
@@ -2,11 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from collections.abc import Iterable
from math import prod
from math import lcm, prod
import torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backend import AttentionBackend
@@ -62,6 +62,13 @@ class MambaBase(AttentionLayerBase):
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
def get_prefill_checkpoint_alignment(self) -> int | None:
"""Token alignment of the checkpoint offsets this layer can export from
inside a prefill, if any; in align mode, the platform makes Mamba blocks
multiples of it.
"""
return None
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
mamba_block_size = vllm_config.cache_config.mamba_block_size
assert mamba_block_size is not None
@@ -86,3 +93,21 @@ class MambaBase(AttentionLayerBase):
def get_attn_backend(self) -> type[AttentionBackend]:
"""Get the attention backend class for this Mamba layer."""
return get_mamba_attn_backend(self.mamba_type)
def get_mamba_prefill_checkpoint_alignment(vllm_config: VllmConfig) -> int | None:
"""LCM of the Mamba layers' prefill checkpoint alignments.
None unless every Mamba layer declares one: the scheduler takes internal
checkpoints only when every Mamba group can export them.
"""
alignment = 1
for layer in get_layers_from_vllm_config(
vllm_config,
MambaBase, # type: ignore[type-abstract]
).values():
layer_alignment = layer.get_prefill_checkpoint_alignment()
if layer_alignment is None:
return None
alignment = lcm(alignment, layer_alignment)
return alignment
+126 -10
View File
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass, replace
import torch
from vllm.config import VllmConfig
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
@@ -71,9 +71,11 @@ def _gather_checkpoint_state_indices(
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Host copy of ``checkpoint_offsets``, for backends that lay out the
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
# Host copies of ``checkpoint_offsets`` and of each row's query length, for
# backends that lay out the checkpoint before any tensor exists (Mamba2's
# SSD chunk split, FlashInfer GDN's checkpoint plan).
offsets: list[int] | None = None
query_lens: list[int] | None = None
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
@@ -150,20 +152,134 @@ class MambaPrefillCheckpointBuilder:
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
offsets=checkpoint_offsets,
query_lens=query_lens,
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
class MambaPrefillCheckpointExporter(ABC):
"""Export a mid-prefill checkpoint into backend-specific paged states."""
@dataclass(frozen=True)
class MambaPrefillCheckpointExporter:
"""Store each checkpoint's pre-convolution window and recurrent state.
The recurrent states are rows of a buffer the prefill kernel emitted;
``recurrent_rows`` picks each checkpoint row's state out of it, and by
default row ``i`` holds checkpoint ``i``'s.
"""
state_len: int | None = None
@abstractmethod
def export(
self,
checkpoint: MambaPrefillCheckpointMetadata,
*args,
**kwargs,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
recurrent_checkpoint: torch.Tensor,
recurrent_state: torch.Tensor,
cu_seqlens: torch.Tensor,
recurrent_rows: torch.Tensor | None = None,
) -> None:
"""Write checkpoint state into the paged cache."""
raise NotImplementedError
state_len = (
self.state_len if self.state_len is not None else conv_state.shape[-1]
)
width = raw_qkv.shape[-1]
recurrent_row_size = recurrent_checkpoint[0].numel()
block_size = 256
store_cache_checkpoints_kernel[
(
checkpoint.checkpoint_offsets.numel(),
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
)
](
raw_qkv,
conv_state,
recurrent_checkpoint,
recurrent_state,
cu_seqlens,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
recurrent_rows if recurrent_rows is not None else checkpoint.state_indices,
raw_qkv.stride(0),
raw_qkv.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
recurrent_checkpoint.stride(0),
recurrent_state.stride(0),
checkpoint.checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
recurrent_rows is not None,
block_size,
)
@triton.jit
def store_cache_checkpoints_kernel(
x_ptr,
conv_state_ptr,
recurrent_checkpoint_ptr,
recurrent_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
checkpoint_state_indices_ptr,
recurrent_rows_ptr,
x_stride_0: tl.constexpr,
x_stride_1: tl.constexpr,
state_stride_0: tl.constexpr,
state_stride_1: tl.constexpr,
state_stride_2: tl.constexpr,
checkpoint_stride_0: tl.constexpr,
recurrent_state_stride_0: tl.constexpr,
checkpoint_offset_stride: tl.constexpr,
STATE_LEN: tl.constexpr,
WIDTH: tl.constexpr,
RECURRENT_ROW_SIZE: tl.constexpr,
NULL_STATE_IDX: tl.constexpr,
HAS_RECURRENT_ROWS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
seq_idx = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
checkpoint_offset = tl.load(
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
)
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
valid_conv = (
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
)
width_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
token_idx = checkpoint_end - STATE_LEN + history_idx
values = tl.load(
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
mask=valid_conv,
)
tl.store(
conv_state_ptr
+ state_idx * state_stride_0
+ width_idx * state_stride_1
+ history_idx * state_stride_2,
values,
mask=valid_conv,
)
if HAS_RECURRENT_ROWS:
recurrent_row = tl.load(recurrent_rows_ptr + seq_idx).to(tl.int64)
else:
recurrent_row = seq_idx.to(tl.int64)
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
recurrent = tl.load(
recurrent_checkpoint_ptr + recurrent_row * checkpoint_stride_0 + cols,
mask=valid_recurrent,
)
tl.store(
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
recurrent,
mask=valid_recurrent,
)
@@ -2,7 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen3-Next/Qwen3.5 model."""
import functools
import itertools
import math
import os
from dataclasses import replace
from typing import Literal
import torch
@@ -15,6 +19,7 @@ from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
CacheConfig,
VllmConfig,
get_current_vllm_config,
)
@@ -30,6 +35,12 @@ from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import (
get_mamba_prefill_checkpoint_alignment,
)
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader
from vllm.model_executor.layers.mamba.mamba_utils import (
@@ -59,13 +70,19 @@ from vllm.third_party.flash_linear_attention.ops import (
)
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
FLASHINFER_GDN_CHECKPOINT_ALIGNMENT,
GDNAttentionMetadata,
)
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
from vllm.v1.worker.workspace import current_workspace_manager
# Optional ROCm AITER Triton kernels for the GDN decode path.
# Availability is checked centrally via rocm_aiter_ops; the actual function
@@ -241,6 +258,10 @@ def fi_chunk_gated_delta_rule(
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
state_checkpoints: torch.Tensor | None = None,
checkpoint_cu_starts: torch.Tensor | None = None,
checkpoint_every_n_tokens: int = 0,
use_cp: Literal["auto"] | bool = "auto",
):
from flashinfer.gdn_prefill import (
chunk_gated_delta_rule as chunk_gated_delta_rule_fi,
@@ -271,6 +292,10 @@ def fi_chunk_gated_delta_rule(
initial_state=fi_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=checkpoint_cu_starts,
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
use_cp=use_cp,
backend="flashinfer",
)
# FlashInfer returns (output, state) when output_final_state=True,
@@ -283,6 +308,20 @@ def fi_chunk_gated_delta_rule(
return result.unsqueeze(0), None
def _checkpoint_grid(cache_config: CacheConfig) -> int:
"""Token grid that FlashInfer GDN checkpoint offsets are kept on.
Prefix-cache boundaries, made multiples of FlashInfer's checkpoint
alignment; the platform already keeps blocks on it.
"""
if cache_config.prefix_match_unit is not None:
return math.lcm(
cache_config.prefix_match_unit, FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
)
assert cache_config.mamba_block_size is not None
return cache_config.mamba_block_size
@CustomOp.register("chunk_gated_delta_rule")
class ChunkGatedDeltaRule(CustomOp):
def __init__(self) -> None:
@@ -591,6 +630,21 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
self.gdn_prefill_backend = self.chunk_gated_delta_rule.gdn_prefill_backend
self._prefill_kernels_warmed_up = False
# FlashInfer's prefill emits mid-prefill states, so an align-mode
# checkpoint needs no split prefill.
self._checkpoint_alignment: int | None = None
if (
vllm_config.cache_config.mamba_cache_mode == "align"
and self.gdn_prefill_backend == "flashinfer"
):
self._checkpoint_alignment = FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
self._vllm_config = vllm_config
self._max_num_batched_tokens = (
vllm_config.scheduler_config.max_num_batched_tokens
)
self._checkpoint_exporter = MambaPrefillCheckpointExporter(
state_len=self.conv_kernel_size - 1
)
self.enable_packed_recurrent_decode = (
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
)
@@ -624,6 +678,46 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_prefill_checkpoint_alignment(self) -> int | None:
return self._checkpoint_alignment
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if not self._exports_checkpoints or not isinstance(spec, MambaSpec):
return spec
# FlashInfer emits states at one stride for the whole batch, the gcd of
# the offsets; keeping every offset on the prefix-cache grid keeps that
# stride a multiple of the grid, which bounds the states emitted.
return replace(
spec,
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=_checkpoint_grid(vllm_config.cache_config),
)
@functools.cached_property
def _exports_checkpoints(self) -> bool:
# Known only once every layer is built: a Mamba layer that cannot
# checkpoint (e.g. Qwen4Exp's PLE short conv) keeps the scheduler
# splitting the prefill instead.
return (
self._checkpoint_alignment is not None
and get_mamba_prefill_checkpoint_alignment(self._vllm_config) is not None
)
@functools.cached_property
def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]:
# Offsets on the grid emit at most one state per grid step of a batch.
# Read lazily: the block size is final only after the model is built.
return (
(
cdiv(self._max_num_batched_tokens, _checkpoint_grid(self.cache_config)),
self.num_v_heads // self.tp_size,
self.head_v_dim,
self.head_k_dim,
),
torch.float32,
)
def _fused_gdn_decode_unsupported_reason(
self, vllm_config: VllmConfig
) -> str | None:
@@ -1167,6 +1261,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
if self._prefill_kernels_warmed_up:
return
self._prefill_kernels_warmed_up = True
if self._exports_checkpoints:
# Size the workspace while profiling; it is locked afterwards.
current_workspace_manager().get_simultaneous(self._checkpoint_buffer_spec)
device = qkv_or_qkvz.device
dtype = qkv_or_qkvz.dtype
@@ -1242,6 +1339,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
if self.gdn_prefill_backend == "flashinfer":
self._warmup_flashinfer_prefill_variants(q, k, v, g, beta, state)
except Exception:
logger.warning(
"GDN prefill kernel warmup (T=%d) failed for "
@@ -1276,6 +1375,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
torch.accelerator.empty_cache()
def _warmup_flashinfer_prefill_variants(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
state: torch.Tensor,
) -> None:
"""Compile every FlashInfer GDN prefill kernel a batch can dispatch to.
FlashInfer builds separate kernels for its context-parallel route (few
sequences) and its regular route, each with and without checkpoint
output. A single sequence under ``use_cp="auto"`` takes the CP route
wherever the heuristic can pick it; ``use_cp=False`` forces the other.
"""
checkpoint_buffers: list[torch.Tensor | None] = [None]
if self._exports_checkpoints:
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
self._checkpoint_buffer_spec
)
checkpoint_buffers.append(state_checkpoints[:1])
cu_seqlens = torch.tensor([0, q.shape[1]], device=q.device, dtype=torch.int32)
checkpoint_cu_starts = torch.tensor([0, 1], device=q.device)
for use_cp, state_checkpoints in itertools.product(
("auto", False), checkpoint_buffers
):
has_checkpoint = state_checkpoints is not None
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=state,
output_final_state=True,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
use_cp=use_cp,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=checkpoint_cu_starts if has_checkpoint else None,
checkpoint_every_n_tokens=q.shape[1] if has_checkpoint else 0,
)
def _forward_core_rocm(
self,
qkvz: torch.Tensor,
@@ -1443,6 +1586,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
)
# 1.2: Process the remaining part
conv_input_non_spec = mixed_qkv_non_spec
if attn_metadata.num_prefills > 0:
assert mixed_qkv_non_spec is not None
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
@@ -1598,23 +1742,59 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
assert prefill_has_initial_state is not None
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
(
core_attn_out_non_spec,
last_recurrent_state,
) = self.chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
plan = attn_metadata.flashinfer_checkpoint_plan
if plan is None:
(
core_attn_out_non_spec,
last_recurrent_state,
) = self.chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
else:
# Only FlashInfer's prefill emits checkpoints.
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
self._checkpoint_buffer_spec
)
state_checkpoints = state_checkpoints[: plan.num_checkpoints]
(
core_attn_out_non_spec,
last_recurrent_state,
) = fi_chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=plan.cu_starts,
checkpoint_every_n_tokens=plan.every_n_tokens,
)
assert attn_metadata.checkpoint is not None
assert non_spec_query_start_loc is not None
self._checkpoint_exporter.export(
attn_metadata.checkpoint,
raw_qkv=conv_input_non_spec,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=non_spec_query_start_loc,
recurrent_rows=plan.rows,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
@@ -1,130 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
return 16 if backend == "flashkda" else None
@dataclass(frozen=True)
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
"""Store FlashKDA recurrent and convolution checkpoint states."""
state_len: int | None = None
def export(
self,
checkpoint: MambaPrefillCheckpointMetadata,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
recurrent_checkpoint: torch.Tensor,
recurrent_state: torch.Tensor,
cu_seqlens: torch.Tensor,
) -> None:
state_len = (
self.state_len if self.state_len is not None else conv_state.shape[-1]
)
width = raw_qkv.shape[-1]
recurrent_row_size = recurrent_checkpoint[0].numel()
block_size = 256
store_cache_checkpoints_kernel[
(
checkpoint.checkpoint_offsets.numel(),
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
)
](
raw_qkv,
conv_state,
recurrent_checkpoint,
recurrent_state,
cu_seqlens,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
raw_qkv.stride(0),
raw_qkv.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
recurrent_checkpoint.stride(0),
recurrent_state.stride(0),
checkpoint.checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
block_size,
)
@triton.jit
def store_cache_checkpoints_kernel(
x_ptr,
conv_state_ptr,
recurrent_checkpoint_ptr,
recurrent_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
checkpoint_state_indices_ptr,
x_stride_0: tl.constexpr,
x_stride_1: tl.constexpr,
state_stride_0: tl.constexpr,
state_stride_1: tl.constexpr,
state_stride_2: tl.constexpr,
checkpoint_stride_0: tl.constexpr,
recurrent_state_stride_0: tl.constexpr,
checkpoint_offset_stride: tl.constexpr,
STATE_LEN: tl.constexpr,
WIDTH: tl.constexpr,
RECURRENT_ROW_SIZE: tl.constexpr,
NULL_STATE_IDX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
seq_idx = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
checkpoint_offset = tl.load(
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
)
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
valid_conv = (
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
)
width_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
token_idx = checkpoint_end - STATE_LEN + history_idx
values = tl.load(
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
mask=valid_conv,
)
tl.store(
conv_state_ptr
+ state_idx * state_stride_0
+ width_idx * state_stride_1
+ history_idx * state_stride_2,
values,
mask=valid_conv,
)
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
recurrent = tl.load(
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
mask=valid_recurrent,
)
tl.store(
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
recurrent,
mask=valid_recurrent,
)