Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
@@ -0,0 +1,51 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,51 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"32": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"16": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"64": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"128": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"256": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"8192": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"16384": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"32768": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 8,
"num_warps": 8
},
"16": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"16": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"32": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"64": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"48": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"96": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"1536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"3072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"6144": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"12288": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"24576": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,39 @@
{
"triton_version": "3.8.0",
"48": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"384": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"768": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"1536": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"3072": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"6144": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
},
"12288": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"24576": {
"BLOCK_SIZE_M": 8,
"num_warps": 2
},
"49152": {
"BLOCK_SIZE_M": 4,
"num_warps": 2
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"32": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"256": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 8,
"num_warps": 1
}
}
@@ -0,0 +1,63 @@
{
"triton_version": "3.8.0",
"8": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"32": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"64": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"128": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"256": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"512": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"1024": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"2048": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"4096": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"8192": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"16384": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"32768": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"65536": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
},
"131072": {
"BLOCK_SIZE_M": 4,
"num_warps": 1
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"80": {
"BLOCK_SIZE_M": 16,
"num_warps": 4
},
"640": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"1280": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2560": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"5120": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"10240": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"20480": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"40960": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"81920": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"122880": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
},
"163840": {
"BLOCK_SIZE_M": 32,
"num_warps": 8
}
}
@@ -0,0 +1,47 @@
{
"triton_version": "3.8.0",
"80": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"640": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"1280": {
"BLOCK_SIZE_M": 64,
"num_warps": 8
},
"2560": {
"BLOCK_SIZE_M": 8,
"num_warps": 4
},
"5120": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"10240": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"20480": {
"BLOCK_SIZE_M": 16,
"num_warps": 8
},
"40960": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"81920": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"122880": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
},
"163840": {
"BLOCK_SIZE_M": 16,
"num_warps": 2
}
}
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _store_prefill_checkpoint_kernel(
x_ptr,
conv_state_ptr,
chunk_states_ptr,
ssm_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
state_indices_ptr,
chunk_idx_ptr,
stride_x_dim,
stride_x_token,
stride_conv_block,
stride_conv_dim,
stride_conv_token,
stride_chunk_states,
stride_ssm_block,
DIM: tl.constexpr,
STATE_LEN: tl.constexpr,
SSM_ROW_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
valid = checkpoint_offset > 0
dst = tl.load(state_indices_ptr + row).to(tl.int64)
conv_mask = valid & (cols < DIM * STATE_LEN)
dim_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
token_idx = (
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
).to(tl.int64)
conv = tl.load(
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
)
tl.store(
conv_state_ptr
+ dst * stride_conv_block
+ dim_idx * stride_conv_dim
+ history_idx * stride_conv_token,
conv,
mask=conv_mask,
)
ssm_mask = valid & (cols < SSM_ROW_SIZE)
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
def store_prefill_checkpoint(
checkpoint: MambaPrefillCheckpointMetadata,
chunk_idx: torch.Tensor,
x: torch.Tensor,
conv_state: torch.Tensor,
chunk_states: torch.Tensor,
ssm_state: torch.Tensor,
query_start_loc: torch.Tensor,
) -> None:
"""Write each prefill row's mid-prefill checkpoint into its paged block.
Rows without a checkpoint (offset 0) write nothing.
Args:
checkpoint: Per-row offsets and destination blocks.
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
chunk_states: SSD states at every chunk end.
ssm_state: Paged SSM state.
query_start_loc: Prefill query start locations.
"""
dim = x.size(0)
state_len = conv_state.size(-1)
ssm_row_size = chunk_states[0].numel()
block_size = 1024
grid = (
chunk_idx.numel(),
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
)
_store_prefill_checkpoint_kernel[grid](
x,
conv_state,
chunk_states,
ssm_state,
query_start_loc,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
chunk_idx,
x.stride(0),
x.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
chunk_states.stride(0),
ssm_state.stride(0),
DIM=dim,
STATE_LEN=state_len,
SSM_ROW_SIZE=ssm_row_size,
BLOCK_SIZE=block_size,
)