[Kernel] Optimize long-context MLA cache gathers (#51739)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
parent
65b7662d3f
commit
c76a425278
@@ -1032,83 +1032,121 @@ void convert_fp8(torch::stable::Tensor& dst_cache,
|
||||
|
||||
namespace vllm {
|
||||
|
||||
// grid is launched with dimensions (batch, num_splits)
|
||||
struct GatherPageTask {
|
||||
int32_t req_id;
|
||||
int32_t logical_block;
|
||||
int32_t page_token_begin;
|
||||
int32_t page_token_end;
|
||||
int32_t output_token_begin;
|
||||
};
|
||||
|
||||
template <bool has_terminal_start>
|
||||
__device__ __forceinline__ bool map_gather_page_task(
|
||||
int32_t task, const int32_t* __restrict__ output_starts, int32_t num_reqs,
|
||||
int32_t total_tokens, int32_t block_size,
|
||||
const int32_t* __restrict__ seq_starts, GatherPageTask& page) {
|
||||
int32_t relative_page = task;
|
||||
for (int32_t req_id = 0; req_id < num_reqs; ++req_id) {
|
||||
const int32_t output_begin = min(output_starts[req_id], total_tokens);
|
||||
int32_t output_end;
|
||||
if constexpr (has_terminal_start) {
|
||||
output_end = min(output_starts[req_id + 1], total_tokens);
|
||||
} else {
|
||||
output_end =
|
||||
min(req_id + 1 < num_reqs ? output_starts[req_id + 1] : total_tokens,
|
||||
total_tokens);
|
||||
}
|
||||
const int32_t seq_len = max(output_end - output_begin, 0);
|
||||
const int32_t source_begin = seq_starts == nullptr ? 0 : seq_starts[req_id];
|
||||
const int32_t first_block = source_begin / block_size;
|
||||
const int32_t num_pages =
|
||||
cuda_utils::ceil_div(source_begin + seq_len, block_size) - first_block;
|
||||
if (relative_page < num_pages) {
|
||||
page.req_id = req_id;
|
||||
page.logical_block = first_block + relative_page;
|
||||
const int32_t page_begin = page.logical_block * block_size;
|
||||
const int32_t copy_begin = max(source_begin, page_begin);
|
||||
const int32_t copy_end =
|
||||
min(source_begin + seq_len, page_begin + block_size);
|
||||
page.page_token_begin = copy_begin - page_begin;
|
||||
page.page_token_end = copy_end - page_begin;
|
||||
page.output_token_begin = output_begin + copy_begin - source_begin;
|
||||
return true;
|
||||
}
|
||||
relative_page -= num_pages;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt,
|
||||
int ENTRY_SIZE, int CTA_SIZE>
|
||||
__global__ void gather_and_maybe_dequant_cache(
|
||||
const cache_t* __restrict__ src_cache, // [NUM_BLOCKS, BLOCK_SIZE,
|
||||
// ENTRIES...]
|
||||
scalar_t* __restrict__ dst, // [TOT_TOKENS, ENTRIES...]
|
||||
const int32_t* __restrict__ block_table, // [BATCH, BLOCK_INDICES]
|
||||
const int32_t* __restrict__ cu_seq_lens, // [BATCH+1]
|
||||
const int32_t* __restrict__ token_to_seq, // [MAX_TOKEN_ACROSS_CHUNK]
|
||||
int ENTRY_SIZE>
|
||||
__global__ void gather_and_maybe_dequant_cache_page(
|
||||
const cache_t* __restrict__ src_cache, scalar_t* __restrict__ dst,
|
||||
const int32_t* __restrict__ block_table,
|
||||
const int32_t* __restrict__ cu_seq_lens, const int32_t num_reqs,
|
||||
const int32_t num_tokens, const int32_t block_size,
|
||||
const int64_t block_table_stride, const int64_t cache_block_stride,
|
||||
const int64_t cache_entry_stride, const int64_t dst_entry_stride,
|
||||
const float* __restrict__ scale,
|
||||
const int32_t* __restrict__ seq_starts) { // Optional: starting offsets per
|
||||
// batch
|
||||
constexpr int vec_size = sizeof(float4) / sizeof(scalar_t);
|
||||
const float* __restrict__ scale, const int32_t* __restrict__ seq_starts) {
|
||||
constexpr int32_t vec_size = sizeof(float4) / sizeof(scalar_t);
|
||||
constexpr int32_t vec_iter_cnt = ENTRY_SIZE / vec_size;
|
||||
static_assert(ENTRY_SIZE % vec_size == 0);
|
||||
using ltype = vllm::vec_n_t<cache_t, vec_size>;
|
||||
using stype = vllm::vec_n_t<scalar_t, vec_size>;
|
||||
// We are adding this for code readability which will be optimized out when
|
||||
// build in release.
|
||||
assert(CTA_SIZE == blockDim.x);
|
||||
|
||||
#pragma unroll
|
||||
for (int token_id = blockIdx.x; token_id < num_tokens;
|
||||
token_id += gridDim.x) {
|
||||
int64_t batch_id = token_to_seq[token_id];
|
||||
int64_t batch_start = cu_seq_lens[batch_id];
|
||||
int64_t batch_end = cu_seq_lens[batch_id + 1];
|
||||
int32_t batch_offset = token_id - batch_start;
|
||||
__shared__ GatherPageTask page;
|
||||
__shared__ int32_t physical_block;
|
||||
__shared__ bool has_task;
|
||||
__shared__ bool copy_task;
|
||||
__shared__ float scale_value;
|
||||
|
||||
if (token_id >= batch_end) return;
|
||||
int32_t offset = 0;
|
||||
if (seq_starts != nullptr) {
|
||||
offset = seq_starts[batch_id];
|
||||
if (threadIdx.x == 0) {
|
||||
if constexpr (kv_dt != Fp8KVCacheDataType::kAuto) {
|
||||
scale_value = *scale;
|
||||
}
|
||||
batch_offset += offset;
|
||||
int32_t block_table_id = batch_offset / block_size;
|
||||
int32_t slot_id = batch_offset % block_size;
|
||||
// seq_starts may push the block index past the end of the batch's block
|
||||
// table row.
|
||||
if (block_table_id >= block_table_stride) continue;
|
||||
int32_t block_table_offset = batch_id * block_table_stride + block_table_id;
|
||||
int32_t block_id = block_table[block_table_offset];
|
||||
int64_t cache_offset =
|
||||
block_id * cache_block_stride + slot_id * cache_entry_stride;
|
||||
constexpr int32_t vec_iter_cnt = ENTRY_SIZE / vec_size;
|
||||
scalar_t* dst_ = dst + token_id * dst_entry_stride;
|
||||
cache_t* src_ = const_cast<cache_t*>(src_cache) + cache_offset;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
has_task =
|
||||
map_gather_page_task<true>(blockIdx.x, cu_seq_lens, num_reqs,
|
||||
num_tokens, block_size, seq_starts, page);
|
||||
copy_task = has_task && page.logical_block < block_table_stride;
|
||||
if (copy_task) {
|
||||
physical_block =
|
||||
block_table[page.req_id * block_table_stride + page.logical_block];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (!has_task) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (copy_task) {
|
||||
const int32_t page_vectors =
|
||||
(page.page_token_end - page.page_token_begin) * vec_iter_cnt;
|
||||
for (int32_t flat_idx = threadIdx.x; flat_idx < page_vectors;
|
||||
flat_idx += blockDim.x) {
|
||||
const int32_t token_offset = flat_idx / vec_iter_cnt;
|
||||
const int32_t idx = flat_idx - token_offset * vec_iter_cnt;
|
||||
const int32_t page_token = page.page_token_begin + token_offset;
|
||||
const int32_t output_token = page.output_token_begin + token_offset;
|
||||
const cache_t* src = src_cache + physical_block * cache_block_stride +
|
||||
page_token * cache_entry_stride;
|
||||
scalar_t* output = dst + output_token * dst_entry_stride;
|
||||
|
||||
#pragma unroll
|
||||
for (int idx = threadIdx.x; idx < vec_iter_cnt; idx += CTA_SIZE) {
|
||||
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
|
||||
reinterpret_cast<stype*>(dst_)[idx] =
|
||||
static_cast<stype>(reinterpret_cast<ltype*>(src_)[idx]);
|
||||
reinterpret_cast<stype*>(output)[idx] =
|
||||
static_cast<stype>(reinterpret_cast<const ltype*>(src)[idx]);
|
||||
} else {
|
||||
ltype loaded_val = reinterpret_cast<ltype*>(src_)[idx];
|
||||
stype store_val;
|
||||
const ltype loaded = reinterpret_cast<const ltype*>(src)[idx];
|
||||
stype converted;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < vec_size; ++j) {
|
||||
store_val.val[j] = fp8::scaled_convert<scalar_t, cache_t, kv_dt>(
|
||||
loaded_val.val[j], *scale);
|
||||
for (int32_t j = 0; j < vec_size; ++j) {
|
||||
converted.val[j] = fp8::scaled_convert<scalar_t, cache_t, kv_dt>(
|
||||
loaded.val[j], scale_value);
|
||||
}
|
||||
reinterpret_cast<stype*>(dst_)[idx] = store_val;
|
||||
}
|
||||
}
|
||||
// process tail
|
||||
constexpr int32_t tail_cnt = ENTRY_SIZE % vec_size;
|
||||
dst_ = dst_ + ENTRY_SIZE - tail_cnt;
|
||||
src_ = src_ + ENTRY_SIZE - tail_cnt;
|
||||
#pragma unroll
|
||||
for (int idx = threadIdx.x; idx < tail_cnt; idx += CTA_SIZE) {
|
||||
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
|
||||
dst_[idx] = static_cast<scalar_t>(src_[idx]);
|
||||
} else {
|
||||
dst_[idx] =
|
||||
fp8::scaled_convert<scalar_t, cache_t, kv_dt>(src_[idx], *scale);
|
||||
reinterpret_cast<stype*>(output)[idx] = converted;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1120,18 +1158,17 @@ __global__ void gather_and_maybe_dequant_cache(
|
||||
// SCALAR_T is the data type of the destination tensor.
|
||||
// CACHE_T is the stored data type of kv-cache.
|
||||
// KV_DTYPE is the real data type of kv-cache.
|
||||
#define CALL_GATHER_CACHE(SCALAR_T, CACHE_T, KV_DTYPE, ENTRY_SZ) \
|
||||
vllm::gather_and_maybe_dequant_cache<SCALAR_T, CACHE_T, KV_DTYPE, ENTRY_SZ, \
|
||||
thread_block_size> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
reinterpret_cast<CACHE_T*>(src_cache.data_ptr()), \
|
||||
reinterpret_cast<SCALAR_T*>(dst.data_ptr()), \
|
||||
block_table.const_data_ptr<int32_t>(), \
|
||||
cu_seq_lens.const_data_ptr<int32_t>(), \
|
||||
token_to_seq.const_data_ptr<int32_t>(), num_tokens, block_size, \
|
||||
block_table_stride, cache_block_stride, cache_entry_stride, \
|
||||
dst_entry_stride, reinterpret_cast<const float*>(scale.data_ptr()), \
|
||||
seq_starts_ptr);
|
||||
#define CALL_GATHER_CACHE(SCALAR_T, CACHE_T, KV_DTYPE, ENTRY_SZ) \
|
||||
vllm::gather_and_maybe_dequant_cache_page<SCALAR_T, CACHE_T, KV_DTYPE, \
|
||||
ENTRY_SZ> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
reinterpret_cast<CACHE_T*>(src_cache.data_ptr()), \
|
||||
reinterpret_cast<SCALAR_T*>(dst.data_ptr()), \
|
||||
block_table.const_data_ptr<int32_t>(), \
|
||||
cu_seq_lens.const_data_ptr<int32_t>(), num_reqs, \
|
||||
static_cast<int32_t>(num_tokens), block_size, block_table_stride, \
|
||||
cache_block_stride, cache_entry_stride, dst_entry_stride, \
|
||||
reinterpret_cast<const float*>(scale.data_ptr()), seq_starts_ptr)
|
||||
|
||||
#define CALL_GATHER_CACHE_576(SCALAR_T, CACHE_T, KV_DTYPE) \
|
||||
CALL_GATHER_CACHE(SCALAR_T, CACHE_T, KV_DTYPE, 576)
|
||||
@@ -1142,7 +1179,6 @@ __global__ void gather_and_maybe_dequant_cache(
|
||||
// Gather sequences from the cache into the destination tensor.
|
||||
// - cu_seq_lens contains the cumulative sequence lengths for each batch
|
||||
// - block_table contains the cache block indices for each sequence
|
||||
// - token_to_seq contains the back mapping from token_id to batch_id
|
||||
// - Optionally, seq_starts (if provided) offsets the starting block index by
|
||||
// (seq_starts[bid] / page_size)
|
||||
void gather_and_maybe_dequant_cache(
|
||||
@@ -1158,6 +1194,7 @@ void gather_and_maybe_dequant_cache(
|
||||
torch::stable::accelerator::DeviceGuard device_guard(
|
||||
src_cache.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
(void)token_to_seq;
|
||||
|
||||
int32_t block_size = src_cache.size(1);
|
||||
int32_t head_dim = dst.size(-1);
|
||||
@@ -1189,14 +1226,21 @@ void gather_and_maybe_dequant_cache(
|
||||
"src_cache and seq_starts must be on the same device");
|
||||
}
|
||||
|
||||
int64_t block_table_stride = block_table.stride(0);
|
||||
int64_t cache_block_stride = src_cache.stride(0);
|
||||
int64_t cache_entry_stride = src_cache.stride(1);
|
||||
int64_t dst_entry_stride = dst.stride(0);
|
||||
if (num_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr int32_t thread_block_size = 64;
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(thread_block_size);
|
||||
const int32_t num_reqs = cu_seq_lens.size(0) - 1;
|
||||
const int64_t block_table_stride = block_table.stride(0);
|
||||
const int64_t cache_block_stride = src_cache.stride(0);
|
||||
const int64_t cache_entry_stride = src_cache.stride(1);
|
||||
const int64_t dst_entry_stride = dst.stride(0);
|
||||
const int32_t page_threads = num_tokens >= (1 << 20) ? 128 : 256;
|
||||
const int32_t required_blocks =
|
||||
cuda_utils::ceil_div(static_cast<int32_t>(num_tokens), block_size) +
|
||||
2 * num_reqs;
|
||||
const dim3 grid(required_blocks);
|
||||
const dim3 block(page_threads);
|
||||
|
||||
const int32_t* seq_starts_ptr =
|
||||
seq_starts.has_value() ? seq_starts.value().const_data_ptr<int32_t>()
|
||||
@@ -1213,151 +1257,161 @@ void gather_and_maybe_dequant_cache(
|
||||
|
||||
namespace vllm {
|
||||
|
||||
// Gather and upconvert FP8 KV cache tokens to BF16 workspace
|
||||
// Similar to cp_gather_cache but specifically for FP8->BF16 conversion
|
||||
__global__ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
const uint8_t* __restrict__ src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656]
|
||||
__nv_bfloat16* __restrict__ dst, // [total_tokens, 576]
|
||||
const int32_t* __restrict__ block_table, // [num_reqs, BLOCK_INDICES]
|
||||
const int32_t* __restrict__ workspace_starts, // [num_reqs]
|
||||
const int32_t num_reqs, const int32_t block_size,
|
||||
const int32_t total_tokens, const int64_t block_table_stride,
|
||||
const int64_t cache_block_stride, const int64_t cache_entry_stride,
|
||||
const int64_t dst_entry_stride,
|
||||
const int32_t* __restrict__ seq_starts) { // Optional source offsets
|
||||
const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) >> 5;
|
||||
if (flat_warp_id >= total_tokens) return;
|
||||
const int lane_id = threadIdx.x & 31;
|
||||
__device__ __forceinline__ void gather_and_upconvert_fp8_token(
|
||||
const uint8_t* __restrict__ token_ptr, __nv_bfloat16* __restrict__ dst_ptr,
|
||||
int32_t lane_id) {
|
||||
const uint2* fp8_src = reinterpret_cast<const uint2*>(token_ptr);
|
||||
const float* scales = reinterpret_cast<const float*>(token_ptr + 512);
|
||||
int4* nope_dst = reinterpret_cast<int4*>(dst_ptr);
|
||||
|
||||
// Binary search to find which request owns this output token
|
||||
int lo = 0, hi = num_reqs - 1;
|
||||
while (lo < hi) {
|
||||
int mid = (lo + hi + 1) >> 1;
|
||||
if (workspace_starts[mid] <= flat_warp_id)
|
||||
lo = mid;
|
||||
else
|
||||
hi = mid - 1;
|
||||
}
|
||||
const int req_id = lo;
|
||||
|
||||
// Compute physical token address via block table
|
||||
const int out_token_id = flat_warp_id;
|
||||
int token_offset = out_token_id - workspace_starts[req_id];
|
||||
if (seq_starts != nullptr) token_offset += seq_starts[req_id];
|
||||
const int cache_block_idx = token_offset / block_size;
|
||||
const int offset_in_block = token_offset % block_size;
|
||||
const int physical_block =
|
||||
block_table[req_id * block_table_stride + cache_block_idx];
|
||||
|
||||
const uint8_t* token_ptr = src_cache + physical_block * cache_block_stride +
|
||||
offset_in_block * cache_entry_stride;
|
||||
|
||||
const int4* nope_src = reinterpret_cast<const int4*>(token_ptr);
|
||||
const int4 fp8_data = nope_src[lane_id];
|
||||
|
||||
const float* scales_ptr = reinterpret_cast<const float*>(token_ptr + 512);
|
||||
const float scale = scales_ptr[lane_id >> 3];
|
||||
|
||||
const uint2 fp8_lo = make_uint2(fp8_data.x, fp8_data.y);
|
||||
const uint2 fp8_hi = make_uint2(fp8_data.z, fp8_data.w);
|
||||
#pragma unroll
|
||||
for (int32_t phase = 0; phase < 2; ++phase) {
|
||||
const int32_t chunk = phase * 32 + lane_id;
|
||||
const uint2 fp8_data = fp8_src[chunk];
|
||||
const float scale = scales[chunk >> 4];
|
||||
#ifdef USE_ROCM
|
||||
const bf16_8_t bf16_lo =
|
||||
fp8::scaled_vec_conversion<bf16_8_t, uint2>(fp8_lo, scale);
|
||||
const bf16_8_t bf16_hi =
|
||||
fp8::scaled_vec_conversion<bf16_8_t, uint2>(fp8_hi, scale);
|
||||
const bf16_8_t bf16_data =
|
||||
fp8::scaled_vec_conversion<bf16_8_t, uint2>(fp8_data, scale);
|
||||
#else
|
||||
const bf16_8_t bf16_lo =
|
||||
fp8::scaled_vec_conversion<bf16_8_t, uint2>(fp8_lo, scale, __NV_E4M3);
|
||||
const bf16_8_t bf16_hi =
|
||||
fp8::scaled_vec_conversion<bf16_8_t, uint2>(fp8_hi, scale, __NV_E4M3);
|
||||
const bf16_8_t bf16_data =
|
||||
fp8::scaled_vec_conversion<bf16_8_t, uint2>(fp8_data, scale, __NV_E4M3);
|
||||
#endif
|
||||
|
||||
__nv_bfloat16* dst_ptr = dst + out_token_id * dst_entry_stride;
|
||||
int4* nope_dst = reinterpret_cast<int4*>(dst_ptr) + lane_id * 2;
|
||||
nope_dst[0] = *reinterpret_cast<const int4*>(&bf16_lo);
|
||||
nope_dst[1] = *reinterpret_cast<const int4*>(&bf16_hi);
|
||||
nope_dst[chunk] = *reinterpret_cast<const int4*>(&bf16_data);
|
||||
}
|
||||
|
||||
const int* rope_src = reinterpret_cast<const int*>(token_ptr + 528);
|
||||
int* rope_dst = reinterpret_cast<int*>(dst_ptr + 512);
|
||||
rope_dst[lane_id] = rope_src[lane_id];
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
// Note(hc): The cp_gather_cache allows seq_starts to no longer be divisible by
|
||||
// block_size.
|
||||
__global__ void cp_gather_cache(
|
||||
const scalar_t* __restrict__ src_cache, // [NUM_BLOCKS, BLOCK_SIZE,
|
||||
// ENTRY_SIZE]
|
||||
scalar_t* __restrict__ dst, // [TOT_TOKENS, ENTRY_SIZE]
|
||||
const int32_t* __restrict__ block_table, // [BATCH, BLOCK_INDICES]
|
||||
const int32_t* __restrict__ cu_seq_lens, // [BATCH+1]
|
||||
const int32_t block_size, const int32_t entry_size,
|
||||
__global__ void cp_gather_and_upconvert_fp8_kv_cache_page(
|
||||
const uint8_t* __restrict__ src_cache, __nv_bfloat16* __restrict__ dst,
|
||||
const int32_t* __restrict__ block_table,
|
||||
const int32_t* __restrict__ workspace_starts, const int32_t num_reqs,
|
||||
const int32_t block_size, const int32_t total_tokens,
|
||||
const int64_t block_table_stride, const int64_t cache_block_stride,
|
||||
const int64_t cache_entry_stride, const int64_t dst_entry_stride,
|
||||
const int32_t* __restrict__ seq_starts // Optional: starting offsets per
|
||||
// batch
|
||||
) {
|
||||
const int64_t bid = blockIdx.x; // Batch ID
|
||||
const int32_t num_splits = gridDim.y;
|
||||
const int32_t split = blockIdx.y;
|
||||
const int32_t seq_start = cu_seq_lens[bid];
|
||||
const int32_t seq_end = cu_seq_lens[bid + 1];
|
||||
const int32_t seq_len = seq_end - seq_start;
|
||||
const int32_t tot_slots = seq_len;
|
||||
const int32_t split_slots = cuda_utils::ceil_div(tot_slots, num_splits);
|
||||
const int32_t* __restrict__ seq_starts) {
|
||||
constexpr int32_t warps_per_cta = 16;
|
||||
__shared__ GatherPageTask page;
|
||||
__shared__ int32_t physical_block;
|
||||
__shared__ bool has_task;
|
||||
|
||||
const int32_t split_start = split * split_slots;
|
||||
const int32_t split_end = min((split + 1) * split_slots, tot_slots);
|
||||
|
||||
const bool is_active_split = (split_start < tot_slots);
|
||||
|
||||
if (!is_active_split) return;
|
||||
|
||||
// Adjust the pointer for the block_table for this batch.
|
||||
// If seq_starts is provided, compute an offset based on it
|
||||
const int32_t batch_offset = bid * block_table_stride;
|
||||
int32_t offset = split_start;
|
||||
if (seq_starts != nullptr) {
|
||||
offset += seq_starts[bid];
|
||||
if (threadIdx.x == 0) {
|
||||
has_task =
|
||||
map_gather_page_task<false>(blockIdx.x, workspace_starts, num_reqs,
|
||||
total_tokens, block_size, seq_starts, page);
|
||||
if (has_task) {
|
||||
physical_block =
|
||||
block_table[page.req_id * block_table_stride + page.logical_block];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (!has_task) {
|
||||
return;
|
||||
}
|
||||
int32_t offset_div = offset / block_size;
|
||||
offset = offset % block_size;
|
||||
const int32_t* batch_block_table = block_table + batch_offset;
|
||||
|
||||
// Adjust dst pointer based on the cumulative sequence lengths.
|
||||
dst += seq_start * dst_entry_stride;
|
||||
const int32_t warp_id = threadIdx.x >> 5;
|
||||
const int32_t lane_id = threadIdx.x & 31;
|
||||
for (int32_t page_token = page.page_token_begin + warp_id;
|
||||
page_token < page.page_token_end; page_token += warps_per_cta) {
|
||||
const int32_t output_token =
|
||||
page.output_token_begin + page_token - page.page_token_begin;
|
||||
const uint8_t* token_ptr = src_cache + physical_block * cache_block_stride +
|
||||
page_token * cache_entry_stride;
|
||||
__nv_bfloat16* dst_ptr = dst + output_token * dst_entry_stride;
|
||||
gather_and_upconvert_fp8_token(token_ptr, dst_ptr, lane_id);
|
||||
}
|
||||
}
|
||||
|
||||
auto copy_entry = [&](const scalar_t* __restrict__ _src,
|
||||
scalar_t* __restrict__ _dst) {
|
||||
for (int i = threadIdx.x; i < entry_size; i += blockDim.x)
|
||||
_dst[i] = _src[i];
|
||||
};
|
||||
template <bool contiguous_entries, bool vectorized>
|
||||
__global__ void cp_gather_cache_page(
|
||||
const uint8_t* __restrict__ src_cache, // [NUM_BLOCKS, BLOCK_SIZE,
|
||||
// ENTRY_SIZE_BYTES]
|
||||
uint8_t* __restrict__ dst, // [TOT_TOKENS, ENTRY_SIZE_BYTES]
|
||||
const int32_t* __restrict__ block_table, // [BATCH, BLOCK_INDICES]
|
||||
const int32_t* __restrict__ cu_seq_lens, // [BATCH+1]
|
||||
const int32_t num_reqs, const int32_t block_size,
|
||||
const int32_t entry_size_bytes, const int32_t total_tokens,
|
||||
const int64_t block_table_stride, const int64_t cache_block_stride,
|
||||
const int64_t cache_entry_stride, const int64_t dst_entry_stride,
|
||||
const int32_t* __restrict__ seq_starts) {
|
||||
constexpr int32_t threads_per_token = 32;
|
||||
constexpr int32_t tokens_per_cta = 256 / threads_per_token;
|
||||
__shared__ GatherPageTask page;
|
||||
__shared__ int32_t physical_block;
|
||||
__shared__ bool has_task;
|
||||
|
||||
for (int pid = split_start; pid < split_end; ++pid) {
|
||||
auto block_id = batch_block_table[offset_div];
|
||||
auto block_start_ptr = src_cache + block_id * cache_block_stride;
|
||||
auto block_dst_ptr = dst + pid * dst_entry_stride;
|
||||
copy_entry(block_start_ptr + offset * cache_entry_stride, block_dst_ptr);
|
||||
offset += 1;
|
||||
// bump to next block
|
||||
if (offset == block_size) {
|
||||
offset_div += 1;
|
||||
offset = 0;
|
||||
if (threadIdx.x == 0) {
|
||||
has_task =
|
||||
map_gather_page_task<true>(blockIdx.x, cu_seq_lens, num_reqs,
|
||||
total_tokens, block_size, seq_starts, page);
|
||||
if (has_task) {
|
||||
physical_block =
|
||||
block_table[page.req_id * block_table_stride + page.logical_block];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (!has_task) {
|
||||
return;
|
||||
}
|
||||
|
||||
if constexpr (contiguous_entries) {
|
||||
const int64_t src_offset = physical_block * cache_block_stride +
|
||||
page.page_token_begin * entry_size_bytes;
|
||||
const int64_t dst_offset = page.output_token_begin * entry_size_bytes;
|
||||
const int32_t copy_bytes =
|
||||
(page.page_token_end - page.page_token_begin) * entry_size_bytes;
|
||||
if constexpr (vectorized) {
|
||||
const int4* src = reinterpret_cast<const int4*>(src_cache + src_offset);
|
||||
int4* output = reinterpret_cast<int4*>(dst + dst_offset);
|
||||
const int32_t num_vectors = copy_bytes / sizeof(int4);
|
||||
int32_t idx = threadIdx.x;
|
||||
for (; idx + blockDim.x < num_vectors; idx += 2 * blockDim.x) {
|
||||
const int4 first = src[idx];
|
||||
const int4 second = src[idx + blockDim.x];
|
||||
output[idx] = first;
|
||||
output[idx + blockDim.x] = second;
|
||||
}
|
||||
for (; idx < num_vectors; idx += blockDim.x) {
|
||||
output[idx] = src[idx];
|
||||
}
|
||||
} else {
|
||||
const uint8_t* src = src_cache + src_offset;
|
||||
uint8_t* output = dst + dst_offset;
|
||||
for (int32_t idx = threadIdx.x; idx < copy_bytes; idx += blockDim.x) {
|
||||
output[idx] = src[idx];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const int32_t token_group = threadIdx.x / threads_per_token;
|
||||
const int32_t lane_id = threadIdx.x % threads_per_token;
|
||||
for (int32_t page_token = page.page_token_begin + token_group;
|
||||
page_token < page.page_token_end; page_token += tokens_per_cta) {
|
||||
const int32_t output_token =
|
||||
page.output_token_begin + page_token - page.page_token_begin;
|
||||
const uint8_t* src = src_cache + physical_block * cache_block_stride +
|
||||
page_token * cache_entry_stride;
|
||||
uint8_t* output = dst + output_token * dst_entry_stride;
|
||||
if constexpr (vectorized) {
|
||||
const int4* src_vec = reinterpret_cast<const int4*>(src);
|
||||
int4* dst_vec = reinterpret_cast<int4*>(output);
|
||||
const int32_t num_vectors = entry_size_bytes / sizeof(int4);
|
||||
for (int32_t idx = lane_id; idx < num_vectors;
|
||||
idx += threads_per_token) {
|
||||
dst_vec[idx] = src_vec[idx];
|
||||
}
|
||||
} else {
|
||||
for (int32_t idx = lane_id; idx < entry_size_bytes;
|
||||
idx += threads_per_token) {
|
||||
output[idx] = src[idx];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace vllm
|
||||
|
||||
// Macro to dispatch the kernel based on the data type.
|
||||
#define CALL_CP_GATHER_CACHE(CPY_DTYPE) \
|
||||
vllm::cp_gather_cache<CPY_DTYPE><<<grid, block, 0, stream>>>( \
|
||||
reinterpret_cast<CPY_DTYPE*>(src_cache.data_ptr()), \
|
||||
reinterpret_cast<CPY_DTYPE*>(dst.data_ptr()), \
|
||||
block_table.const_data_ptr<int32_t>(), \
|
||||
cu_seq_lens.const_data_ptr<int32_t>(), block_size, entry_size, \
|
||||
block_table_stride, cache_block_stride, cache_entry_stride, \
|
||||
dst_entry_stride, seq_starts_ptr);
|
||||
|
||||
// Gather sequences from the cache into the destination tensor.
|
||||
// - cu_seq_lens contains the cumulative sequence lengths for each batch
|
||||
// - block_table contains the cache block indices for each sequence
|
||||
@@ -1401,33 +1455,70 @@ void cp_gather_cache(
|
||||
"src_cache and seq_starts must be on the same device");
|
||||
}
|
||||
|
||||
int64_t block_table_stride = block_table.stride(0);
|
||||
int64_t cache_block_stride = src_cache.stride(0);
|
||||
int64_t cache_entry_stride = src_cache.stride(1);
|
||||
int64_t dst_entry_stride = dst.stride(0);
|
||||
|
||||
// Decide on the number of splits based on the batch size.
|
||||
int num_splits = batch_size > 128 ? 2 : batch_size > 64 ? 4 : 16;
|
||||
dim3 grid(batch_size, num_splits);
|
||||
dim3 block(1024);
|
||||
|
||||
STD_TORCH_CHECK(src_cache.scalar_type() == dst.scalar_type(),
|
||||
"src_cache and dst must have the same dtype");
|
||||
const int32_t element_size = src_cache.element_size();
|
||||
STD_TORCH_CHECK(element_size == 1 || element_size == 2 || element_size == 4,
|
||||
"Unsupported data type width: ", element_size * 8);
|
||||
STD_TORCH_CHECK(batch_size == cu_seq_lens.size(0) - 1,
|
||||
"batch_size must match cu_seq_lens");
|
||||
|
||||
const int dtype_bits = src_cache.element_size() * 8;
|
||||
const int32_t total_tokens = dst.size(0);
|
||||
if (total_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int32_t entry_size_bytes = entry_size * element_size;
|
||||
const int64_t block_table_stride = block_table.stride(0);
|
||||
const int64_t cache_block_stride = src_cache.stride(0) * element_size;
|
||||
const int64_t cache_entry_stride = src_cache.stride(1) * element_size;
|
||||
const int64_t dst_entry_stride = dst.stride(0) * element_size;
|
||||
const int32_t* seq_starts_ptr =
|
||||
seq_starts.has_value() ? seq_starts.value().const_data_ptr<int32_t>()
|
||||
: nullptr;
|
||||
|
||||
if (dtype_bits == 32) {
|
||||
CALL_CP_GATHER_CACHE(uint32_t);
|
||||
} else if (dtype_bits == 16) {
|
||||
CALL_CP_GATHER_CACHE(uint16_t);
|
||||
} else if (dtype_bits == 8) {
|
||||
CALL_CP_GATHER_CACHE(uint8_t);
|
||||
constexpr std::uintptr_t vector_alignment = alignof(int4);
|
||||
const bool vectorized =
|
||||
reinterpret_cast<std::uintptr_t>(src_cache.data_ptr()) %
|
||||
vector_alignment ==
|
||||
0 &&
|
||||
reinterpret_cast<std::uintptr_t>(dst.data_ptr()) % vector_alignment ==
|
||||
0 &&
|
||||
entry_size_bytes % vector_alignment == 0 &&
|
||||
cache_block_stride % vector_alignment == 0 &&
|
||||
cache_entry_stride % vector_alignment == 0 &&
|
||||
dst_entry_stride % vector_alignment == 0;
|
||||
const bool contiguous_entries = cache_entry_stride == entry_size_bytes &&
|
||||
dst_entry_stride == entry_size_bytes;
|
||||
|
||||
const int32_t num_reqs = static_cast<int32_t>(batch_size);
|
||||
const int32_t required_blocks =
|
||||
cuda_utils::ceil_div(total_tokens, block_size) + 2 * num_reqs;
|
||||
const dim3 grid(required_blocks);
|
||||
const dim3 block(256);
|
||||
|
||||
#define CALL_CP_GATHER_CACHE(CONTIGUOUS, VECTORIZED) \
|
||||
vllm::cp_gather_cache_page<CONTIGUOUS, VECTORIZED> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
reinterpret_cast<const uint8_t*>(src_cache.data_ptr()), \
|
||||
reinterpret_cast<uint8_t*>(dst.data_ptr()), \
|
||||
block_table.const_data_ptr<int32_t>(), \
|
||||
cu_seq_lens.const_data_ptr<int32_t>(), num_reqs, block_size, \
|
||||
entry_size_bytes, total_tokens, block_table_stride, \
|
||||
cache_block_stride, cache_entry_stride, dst_entry_stride, \
|
||||
seq_starts_ptr)
|
||||
|
||||
if (contiguous_entries && vectorized) {
|
||||
CALL_CP_GATHER_CACHE(true, true);
|
||||
} else if (contiguous_entries) {
|
||||
CALL_CP_GATHER_CACHE(true, false);
|
||||
} else if (vectorized) {
|
||||
CALL_CP_GATHER_CACHE(false, true);
|
||||
} else {
|
||||
STD_TORCH_CHECK(false, "Unsupported data type width: ", dtype_bits);
|
||||
CALL_CP_GATHER_CACHE(false, false);
|
||||
}
|
||||
|
||||
#undef CALL_CP_GATHER_CACHE
|
||||
}
|
||||
|
||||
void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
@@ -1491,15 +1582,18 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
}
|
||||
|
||||
const int total_tokens = dst.size(0);
|
||||
constexpr int warps_per_block = 8;
|
||||
const int grid_size = (total_tokens + warps_per_block - 1) / warps_per_block;
|
||||
const int block_size_threads = warps_per_block * 32; // 256 threads
|
||||
if (total_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
constexpr int warps_per_block = 16;
|
||||
const int block_size_threads = warps_per_block * 32;
|
||||
const int32_t* seq_starts_ptr =
|
||||
seq_starts.has_value() ? seq_starts.value().const_data_ptr<int32_t>()
|
||||
: nullptr;
|
||||
|
||||
vllm::cp_gather_and_upconvert_fp8_kv_cache<<<grid_size, block_size_threads, 0,
|
||||
stream>>>(
|
||||
const int required_blocks = cuda_utils::ceil_div(total_tokens, block_size) +
|
||||
2 * static_cast<int32_t>(batch_size);
|
||||
vllm::cp_gather_and_upconvert_fp8_kv_cache_page<<<
|
||||
required_blocks, block_size_threads, 0, stream>>>(
|
||||
src_ptr, reinterpret_cast<__nv_bfloat16*>(dst.data_ptr()),
|
||||
block_table.const_data_ptr<int32_t>(),
|
||||
workspace_starts.const_data_ptr<int32_t>(),
|
||||
|
||||
Reference in New Issue
Block a user