[Kimi-K3] DCP support (#50484)

Signed-off-by: girasoley <girasoley@inferact.ai>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Summer Yang
2026-08-10 12:54:26 -07:00
committed by GitHub
co-authored by Claude Fable 5 云挚 foraxe Woosuk Kwon
parent 3f142bd85e
commit 63ac04a61e
19 changed files with 3529 additions and 82 deletions
+5
View File
@@ -474,6 +474,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
endif()
if(VLLM_GPU_LANG STREQUAL "CUDA")
list(APPEND VLLM_STABLE_EXT_SRC
"csrc/libtorch_stable/attention/dcp_utils/dcp_direct_a2a_lse_reduce.cu"
"csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu"
"csrc/libtorch_stable/attention/dcp_utils/dcp_direct_q_gather.cu")
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
# Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
@@ -0,0 +1,381 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// Direct symmetric-memory DCP A2A LSE reduction.
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <math_constants.h>
#include <torch/csrc/stable/library.h>
#include <torch/headeronly/core/ScalarType.h>
#include <cstdio>
#include <optional>
#include <type_traits>
#include "dcp_direct_common.cuh"
namespace {
using vllm::direct_dcp::check_cuda_launch;
using vllm::direct_dcp::get_peer_ptr;
using vllm::direct_dcp::increment_epoch_kernel;
using vllm::direct_dcp::store_release_system;
using vllm::direct_dcp::wait_for_epoch;
__device__ __forceinline__ int64_t find_sequence(const int32_t* query_start_loc,
int64_t token_idx,
int64_t num_seqs) {
int64_t left = 0;
int64_t right = num_seqs;
while (left < right) {
int64_t mid = (left + right) / 2;
if (query_start_loc[mid] <= token_idx) {
left = mid + 1;
} else {
right = mid;
}
}
return left - 1;
}
template <typename scalar_t>
__device__ __forceinline__ float to_float(scalar_t value) {
if constexpr (std::is_same_v<scalar_t, float>) {
return value;
} else if constexpr (std::is_same_v<scalar_t, __half>) {
return __half2float(value);
} else {
return __bfloat162float(value);
}
}
template <typename scalar_t>
__device__ __forceinline__ scalar_t from_float(float value) {
if constexpr (std::is_same_v<scalar_t, __half>) {
return __float2half_rn(value);
} else {
return __float2bfloat16_rn(value);
}
}
// Send each destination's head slice and LSE to its symmetric buffer. Mask
// undefined empty-shard rows at the source so peers give them zero weight.
template <typename lse_t>
__global__ void dispatch_output_lse_kernel(
const uint4* partial_output, const lse_t* partial_lse,
const int32_t* seq_lens, const int32_t* query_start_loc,
const int64_t* peer_output_ptrs, const int64_t* peer_lse_ptrs,
const int64_t* epoch_ptr, int64_t world_size, int64_t rank,
int64_t num_tokens, int64_t max_num_tokens, int64_t num_seqs,
int64_t heads_per_rank, int64_t head_dim, int64_t output_token_stride,
int64_t lse_token_stride) {
int64_t item = static_cast<int64_t>(blockIdx.x);
int64_t destination_rank = item / num_tokens;
int64_t token_idx = item - destination_rank * num_tokens;
__shared__ bool empty_kv;
if (threadIdx.x == 0) {
empty_kv = false;
if (seq_lens != nullptr) {
int64_t seq_idx = find_sequence(query_start_loc, token_idx, num_seqs);
empty_kv = seq_lens[seq_idx] == 0;
}
}
__syncthreads();
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t parity = static_cast<int64_t>(epoch & 1u);
int64_t destination_item =
((parity * world_size + rank) * max_num_tokens + token_idx) *
heads_per_rank;
int64_t source_head = destination_rank * heads_per_rank;
if (!empty_kv) {
uint4* peer_output =
get_peer_ptr<uint4>(peer_output_ptrs, destination_rank);
int64_t vectors_per_item = heads_per_rank * head_dim / 8;
int64_t source_vector =
(token_idx * output_token_stride + source_head * head_dim) / 8;
int64_t destination_vector = destination_item * head_dim / 8;
for (int64_t vector_idx = threadIdx.x; vector_idx < vectors_per_item;
vector_idx += blockDim.x) {
peer_output[destination_vector + vector_idx] =
partial_output[source_vector + vector_idx];
}
}
float* peer_lse = get_peer_ptr<float>(peer_lse_ptrs, destination_rank);
int64_t source_lse = token_idx * lse_token_stride + source_head;
for (int64_t head_idx = threadIdx.x; head_idx < heads_per_rank;
head_idx += blockDim.x) {
peer_lse[destination_item + head_idx] =
empty_kv ? -CUDART_INF_F : to_float(partial_lse[source_lse + head_idx]);
}
}
// Publish completion after the stream-ordered payload writes.
__global__ void signal_kernel(const int64_t* peer_signal_ptrs,
const int64_t* epoch_ptr, int64_t world_size,
int64_t rank) {
int64_t destination_rank =
static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
if (destination_rank >= world_size) {
return;
}
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t parity = static_cast<int64_t>(epoch & 1u);
uint32_t* peer_signal =
get_peer_ptr<uint32_t>(peer_signal_ptrs, destination_rank);
int64_t signal_item = parity * world_size + rank;
store_release_system(peer_signal + signal_item, epoch);
}
// Form stable base-2 LSE weights and combine in FP32.
template <typename scalar_t>
__global__ void wait_lse_combine_kernel(
const scalar_t* received_output, const float* received_lse,
const uint32_t* received_signal, const int64_t* epoch_ptr,
scalar_t* combined_output, int64_t world_size, int64_t num_tokens,
int64_t max_num_tokens, int64_t heads_per_rank, int64_t head_dim,
bool is_lse_base_on_e) {
extern __shared__ float weights[];
int64_t item = static_cast<int64_t>(blockIdx.x);
int64_t token_idx = item / heads_per_rank;
int64_t head_idx = item - token_idx * heads_per_rank;
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t parity = static_cast<int64_t>(epoch & 1u);
if (threadIdx.x == 0) {
float lse_max = -CUDART_INF_F;
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
int64_t source_item =
((parity * world_size + source_rank) * max_num_tokens + token_idx) *
heads_per_rank +
head_idx;
int64_t signal_item = parity * world_size + source_rank;
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
printf(
"direct DCP A2A timeout source=%lld token=%lld head=%lld "
"epoch=%u\n",
static_cast<long long>(source_rank),
static_cast<long long>(token_idx), static_cast<long long>(head_idx),
epoch);
asm volatile("trap;");
}
float value = received_lse[source_item];
if (isnan(value) || value == CUDART_INF_F) {
value = -CUDART_INF_F;
}
if (is_lse_base_on_e) {
value *= CUDART_L2E_F;
}
weights[source_rank] = value;
lse_max = fmaxf(lse_max, value);
}
if (lse_max == -CUDART_INF_F) {
lse_max = 0.0f;
}
float lse_sum = 0.0f;
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
lse_sum += exp2f(weights[source_rank] - lse_max);
}
float inverse_lse_sum = lse_sum > 0.0f ? 1.0f / lse_sum : 0.0f;
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
weights[source_rank] =
exp2f(weights[source_rank] - lse_max) * inverse_lse_sum;
}
}
__syncthreads();
for (int64_t dim_idx = threadIdx.x; dim_idx < head_dim;
dim_idx += blockDim.x) {
float accumulator = 0.0f;
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
// Empty shards leave stale payloads, so never read zero-weight sources.
float weight = weights[source_rank];
if (weight == 0.0f) {
continue;
}
int64_t source_item =
((parity * world_size + source_rank) * max_num_tokens + token_idx) *
heads_per_rank +
head_idx;
accumulator +=
to_float(received_output[source_item * head_dim + dim_idx]) * weight;
}
combined_output[item * head_dim + dim_idx] =
from_float<scalar_t>(accumulator);
}
}
void direct_dcp_a2a_lse_reduce(
const torch::stable::Tensor& partial_output,
const torch::stable::Tensor& partial_lse,
const std::optional<torch::stable::Tensor>& seq_lens,
const std::optional<torch::stable::Tensor>& query_start_loc,
const torch::stable::Tensor& peer_output_ptrs,
const torch::stable::Tensor& peer_lse_ptrs,
const torch::stable::Tensor& peer_signal_ptrs,
torch::stable::Tensor& received_output, torch::stable::Tensor& received_lse,
torch::stable::Tensor& received_signal, torch::stable::Tensor& epoch,
torch::stable::Tensor& combined_output, int64_t world_size, int64_t rank,
int64_t max_num_tokens, bool is_lse_base_on_e) {
STD_TORCH_CHECK(partial_output.is_cuda() && partial_lse.is_cuda(),
"partial output and LSE must be CUDA tensors");
auto output_dtype = partial_output.scalar_type();
STD_TORCH_CHECK(output_dtype == torch::headeronly::ScalarType::Half ||
output_dtype == torch::headeronly::ScalarType::BFloat16,
"direct DCP A2A only supports FP16 and BF16 output");
auto lse_dtype = partial_lse.scalar_type();
STD_TORCH_CHECK(lse_dtype == torch::headeronly::ScalarType::Float ||
lse_dtype == torch::headeronly::ScalarType::Half ||
lse_dtype == torch::headeronly::ScalarType::BFloat16,
"partial LSE must be FP32, FP16, or BF16");
STD_TORCH_CHECK(partial_output.dim() == 3 && partial_lse.dim() == 2,
"expected output [T,H,D] and LSE [T,H]");
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
int64_t num_tokens = partial_output.size(0);
int64_t total_heads = partial_output.size(1);
int64_t head_dim = partial_output.size(2);
int64_t output_token_stride = partial_output.stride(0);
int64_t lse_token_stride = partial_lse.stride(0);
STD_TORCH_CHECK(
partial_output.stride(2) == 1 && partial_output.stride(1) == head_dim &&
output_token_stride >= total_heads * head_dim &&
output_token_stride % 8 == 0,
"partial output must have packed heads and an aligned token stride");
STD_TORCH_CHECK(partial_lse.stride(1) == 1 && lse_token_stride >= total_heads,
"partial LSE must have packed heads");
STD_TORCH_CHECK(num_tokens > 0 && num_tokens <= max_num_tokens,
"token count exceeds symmetric buffer capacity");
STD_TORCH_CHECK(total_heads % world_size == 0,
"attention heads must divide evenly across DCP ranks");
STD_TORCH_CHECK(
partial_lse.size(0) == num_tokens && partial_lse.size(1) == total_heads,
"LSE shape must match attention output");
STD_TORCH_CHECK(head_dim % 8 == 0,
"head_dim must be divisible by 8 for 16-byte stores");
int64_t heads_per_rank = total_heads / world_size;
STD_TORCH_CHECK(combined_output.scalar_type() == output_dtype &&
combined_output.is_contiguous() &&
combined_output.is_cuda(),
"combined output must match the contiguous CUDA input");
STD_TORCH_CHECK(combined_output.size(0) == num_tokens &&
combined_output.size(1) == heads_per_rank &&
combined_output.size(2) == head_dim,
"combined output has the wrong shape");
STD_TORCH_CHECK(
received_output.scalar_type() == output_dtype &&
received_lse.scalar_type() == torch::headeronly::ScalarType::Float &&
received_signal.scalar_type() == torch::headeronly::ScalarType::Int,
"invalid symmetric staging dtypes");
STD_TORCH_CHECK(
peer_output_ptrs.scalar_type() == torch::headeronly::ScalarType::Long &&
peer_lse_ptrs.scalar_type() == torch::headeronly::ScalarType::Long &&
peer_signal_ptrs.scalar_type() == torch::headeronly::ScalarType::Long,
"peer pointer tables must be int64");
const int32_t* seq_lens_ptr = nullptr;
const int32_t* query_start_loc_ptr = nullptr;
int64_t num_seqs = 0;
STD_TORCH_CHECK(seq_lens.has_value() == query_start_loc.has_value(),
"seq_lens and query_start_loc must be provided together");
if (seq_lens.has_value() && query_start_loc.has_value()) {
STD_TORCH_CHECK(
seq_lens->is_cuda() && seq_lens->dim() == 1 &&
seq_lens->stride(0) == 1 &&
seq_lens->scalar_type() == torch::headeronly::ScalarType::Int,
"seq_lens must be a contiguous 1-D int32 CUDA tensor");
STD_TORCH_CHECK(
query_start_loc->is_cuda() && query_start_loc->dim() == 1 &&
query_start_loc->stride(0) == 1 &&
query_start_loc->scalar_type() ==
torch::headeronly::ScalarType::Int,
"query_start_loc must be a contiguous 1-D int32 CUDA tensor");
num_seqs = seq_lens->size(0);
STD_TORCH_CHECK(num_seqs > 0 && query_start_loc->size(0) == num_seqs + 1,
"query_start_loc must contain one boundary per sequence");
seq_lens_ptr = seq_lens->const_data_ptr<int32_t>();
query_start_loc_ptr = query_start_loc->const_data_ptr<int32_t>();
}
const torch::stable::accelerator::DeviceGuard device_guard(
partial_output.get_device_index());
cudaStream_t stream = get_current_cuda_stream();
constexpr int kExchangeThreads = 256;
constexpr int kCombineThreads = 128;
increment_epoch_kernel<<<1, 1, 0, stream>>>(
epoch.mutable_data_ptr<int64_t>());
check_cuda_launch("direct DCP A2A");
int64_t dispatch_blocks = world_size * num_tokens;
auto launch_dispatch = [&]<typename lse_t>() {
dispatch_output_lse_kernel<lse_t>
<<<dispatch_blocks, kExchangeThreads, 0, stream>>>(
reinterpret_cast<const uint4*>(partial_output.data_ptr()),
reinterpret_cast<const lse_t*>(partial_lse.data_ptr()),
seq_lens_ptr, query_start_loc_ptr,
peer_output_ptrs.const_data_ptr<int64_t>(),
peer_lse_ptrs.const_data_ptr<int64_t>(),
epoch.const_data_ptr<int64_t>(), world_size, rank, num_tokens,
max_num_tokens, num_seqs, heads_per_rank, head_dim,
output_token_stride, lse_token_stride);
};
if (lse_dtype == torch::headeronly::ScalarType::Float) {
launch_dispatch.operator()<float>();
} else if (lse_dtype == torch::headeronly::ScalarType::BFloat16) {
launch_dispatch.operator()<__nv_bfloat16>();
} else {
launch_dispatch.operator()<__half>();
}
check_cuda_launch("direct DCP A2A");
int64_t signal_items = world_size;
int64_t signal_blocks =
(signal_items + kExchangeThreads - 1) / kExchangeThreads;
signal_kernel<<<signal_blocks, kExchangeThreads, 0, stream>>>(
peer_signal_ptrs.const_data_ptr<int64_t>(),
epoch.const_data_ptr<int64_t>(), world_size, rank);
check_cuda_launch("direct DCP A2A");
int64_t combine_blocks = num_tokens * heads_per_rank;
size_t shared_memory_bytes = world_size * sizeof(float);
auto launch_combine = [&]<typename scalar_t>() {
wait_lse_combine_kernel<scalar_t>
<<<combine_blocks, kCombineThreads, shared_memory_bytes, stream>>>(
reinterpret_cast<const scalar_t*>(received_output.data_ptr()),
received_lse.const_data_ptr<float>(),
reinterpret_cast<const uint32_t*>(
received_signal.const_data_ptr<int32_t>()),
epoch.const_data_ptr<int64_t>(),
reinterpret_cast<scalar_t*>(combined_output.mutable_data_ptr()),
world_size, num_tokens, max_num_tokens, heads_per_rank, head_dim,
is_lse_base_on_e);
};
if (output_dtype == torch::headeronly::ScalarType::BFloat16) {
launch_combine.operator()<__nv_bfloat16>();
} else {
launch_combine.operator()<__half>();
}
check_cuda_launch("direct DCP A2A");
}
} // namespace
STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_a2a_ops) {
direct_dcp_a2a_ops.def(
"direct_dcp_a2a_lse_reduce("
"Tensor partial_output, Tensor partial_lse, Tensor? seq_lens, "
"Tensor? query_start_loc, Tensor peer_output_ptrs, "
"Tensor peer_lse_ptrs, Tensor peer_signal_ptrs, Tensor! received_output, "
"Tensor! received_lse, Tensor! received_signal, Tensor! epoch, "
"Tensor! combined_output, int world_size, int rank, "
"int max_num_tokens, bool is_lse_base_on_e) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_a2a_ops) {
direct_dcp_a2a_ops.impl("direct_dcp_a2a_lse_reduce",
TORCH_BOX(&direct_dcp_a2a_lse_reduce));
}
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#pragma once
#include <cuda_runtime.h>
#include <cstdint>
#include <string>
#include "../../torch_utils.h"
namespace vllm::direct_dcp {
constexpr uint64_t kSpinLimit = 100000000;
// Advance the invocation ID; its low bit selects one of two staging slots.
static __global__ void increment_epoch_kernel(int64_t* epoch) {
if (blockIdx.x == 0 && threadIdx.x == 0) {
epoch[0] += 1;
}
}
template <typename T>
__device__ __forceinline__ T* get_peer_ptr(const int64_t* peer_ptrs,
int64_t peer) {
return reinterpret_cast<T*>(static_cast<uintptr_t>(peer_ptrs[peer]));
}
// Replicate one 16-byte payload to every symmetric-buffer replica.
__device__ __forceinline__ void multimem_store_16(uint4* mc_ptr, uint4 value) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
asm volatile("multimem.st.relaxed.sys.global.v4.f32 [%0], {%1,%2,%3,%4};"
:
: "l"(mc_ptr), "r"(value.x), "r"(value.y), "r"(value.z),
"r"(value.w)
: "memory");
#else
asm volatile("trap;");
#endif
}
// Publish prior system-scope writes and signal every replica.
__device__ __forceinline__ void multimem_store_release_system(uint32_t* mc_ptr,
uint32_t value) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
asm volatile("multimem.st.release.sys.global.u32 [%0], %1;"
:
: "l"(mc_ptr), "r"(value)
: "memory");
#else
asm volatile("trap;");
#endif
}
__device__ __forceinline__ void store_release_system(uint32_t* ptr,
uint32_t value) {
uint64_t address = reinterpret_cast<uint64_t>(ptr);
asm volatile("st.global.release.sys.u32 [%0], %1;"
:
: "l"(address), "r"(value)
: "memory");
}
__device__ __forceinline__ uint32_t load_acquire_system(const uint32_t* ptr) {
uint32_t value;
uint64_t address = reinterpret_cast<uint64_t>(ptr);
asm volatile("ld.global.acquire.sys.u32 %0, [%1];"
: "=r"(value)
: "l"(address)
: "memory");
return value;
}
__device__ __forceinline__ bool wait_for_epoch(const uint32_t* ptr,
uint32_t epoch) {
for (uint64_t spins = 0; spins < kSpinLimit; ++spins) {
if (load_acquire_system(ptr) == epoch) {
return true;
}
}
return false;
}
inline void check_cuda_launch(const char* operation) {
cudaError_t error = cudaGetLastError();
STD_TORCH_CHECK(error == cudaSuccess,
std::string(operation) +
" kernel launch failed: " + cudaGetErrorString(error));
}
} // namespace vllm::direct_dcp
@@ -0,0 +1,209 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// Direct symmetric-memory DCP KV gather.
#include <torch/csrc/stable/library.h>
#include <torch/headeronly/core/ScalarType.h>
#include <cstdio>
#include "dcp_direct_common.cuh"
namespace {
using vllm::direct_dcp::check_cuda_launch;
using vllm::direct_dcp::increment_epoch_kernel;
using vllm::direct_dcp::multimem_store_16;
using vllm::direct_dcp::multimem_store_release_system;
using vllm::direct_dcp::wait_for_epoch;
constexpr int kThreads = 256;
// KV chunks need many blocks in flight to saturate the fabric.
constexpr int64_t kMaxMulticastBlocks = 128;
constexpr int64_t kMaxCopyBlocks = 1024;
// Multicast each rank's KV slice and completion epoch to every replica.
__global__ void direct_dcp_kv_gather_multimem_kernel(
const uint4* local_kv, uint4* mc_kv, uint32_t* mc_signal,
const uint32_t* received_signal, const int64_t* epoch_ptr,
uint32_t* completion, int64_t world_size, int64_t rank, int64_t slice_bytes,
int64_t slot_stride_bytes) {
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t buffer_slot = static_cast<int64_t>(epoch & 1u);
int64_t item_count = slice_bytes / sizeof(uint4);
mc_kv += buffer_slot * slot_stride_bytes / sizeof(uint4);
mc_kv += rank * item_count;
int64_t item_stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
for (int64_t item =
static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
item < item_count; item += item_stride) {
multimem_store_16(mc_kv + item, local_kv[item]);
}
// Publish all multicast writes before incrementing completion.
__threadfence_system();
__syncthreads();
if (threadIdx.x != 0) {
return;
}
uint32_t completed = atomicAdd(completion + buffer_slot, 1u);
if (completed + 1u != gridDim.x) {
return;
}
atomicExch(completion + buffer_slot, 0u);
multimem_store_release_system(mc_signal + buffer_slot * world_size + rank,
epoch);
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
int64_t signal_item = buffer_slot * world_size + source_rank;
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
printf("direct DCP kv-gather multimem timeout source=%lld epoch=%u\n",
static_cast<long long>(source_rank), epoch);
asm volatile("trap;");
}
}
}
// Materialize the acquired slot into the caller's workspace.
template <typename copy_t>
__global__ void materialize_kv_gather_kernel(const copy_t* received_kv,
const int64_t* epoch_ptr,
copy_t* gathered_kv,
int64_t gathered_item_count,
int64_t slot_stride_items) {
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t buffer_slot = static_cast<int64_t>(epoch & 1u);
received_kv += buffer_slot * slot_stride_items;
int64_t item = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
int64_t item_stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
for (; item < gathered_item_count; item += item_stride) {
gathered_kv[item] = received_kv[item];
}
}
void direct_dcp_kv_gather(
const torch::stable::Tensor& local_kv, torch::stable::Tensor& received_kv,
torch::stable::Tensor& received_signal, torch::stable::Tensor& completion,
torch::stable::Tensor& epoch, torch::stable::Tensor& gathered_kv,
int64_t world_size, int64_t rank, int64_t max_gathered_tokens,
int64_t kv_mc_ptr, int64_t signal_mc_ptr) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(local_kv.is_cuda(), "local kv must be a CUDA tensor");
ScalarType dtype = local_kv.scalar_type();
STD_TORCH_CHECK(dtype == ScalarType::Half || dtype == ScalarType::BFloat16 ||
dtype == ScalarType::Float8_e4m3fn,
"direct DCP kv-gather only supports FP16, BF16, and FP8");
STD_TORCH_CHECK(local_kv.dim() == 2 && local_kv.is_contiguous(),
"local kv must be a contiguous [T,D] tensor");
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
int64_t num_tokens = local_kv.size(0);
int64_t token_dim = local_kv.size(1);
int64_t element_size = local_kv.element_size();
STD_TORCH_CHECK(num_tokens > 0 && token_dim > 0,
"local kv dimensions must be positive");
STD_TORCH_CHECK(num_tokens * world_size <= max_gathered_tokens,
"gathered kv exceeds symmetric kv-gather buffer capacity");
STD_TORCH_CHECK(received_kv.is_cuda() && received_kv.scalar_type() == dtype &&
received_kv.is_contiguous() && received_kv.dim() == 3 &&
received_kv.size(0) == 2 &&
received_kv.size(1) == max_gathered_tokens &&
received_kv.size(2) == token_dim,
"received kv has the wrong symmetric buffer layout");
STD_TORCH_CHECK(
received_signal.is_cuda() && received_signal.is_contiguous() &&
received_signal.scalar_type() == ScalarType::Int &&
received_signal.dim() == 2 && received_signal.size(0) == 2 &&
received_signal.size(1) == world_size,
"received signal has the wrong symmetric buffer layout");
STD_TORCH_CHECK(completion.is_cuda() && completion.is_contiguous() &&
completion.scalar_type() == ScalarType::Int &&
completion.numel() == 2,
"completion counter must be a two-element CUDA int32 tensor");
STD_TORCH_CHECK(epoch.is_cuda() && epoch.is_contiguous() &&
epoch.scalar_type() == ScalarType::Long &&
epoch.numel() == 1,
"epoch must be a one-element CUDA int64 tensor");
STD_TORCH_CHECK(gathered_kv.is_cuda() && gathered_kv.scalar_type() == dtype &&
gathered_kv.is_contiguous() && gathered_kv.dim() == 2 &&
gathered_kv.size(0) == num_tokens * world_size &&
gathered_kv.size(1) == token_dim,
"gathered kv must be contiguous with shape [world_size*T,D]");
int64_t device_index = local_kv.get_device_index();
STD_TORCH_CHECK(
received_kv.get_device_index() == device_index &&
received_signal.get_device_index() == device_index &&
completion.get_device_index() == device_index &&
epoch.get_device_index() == device_index &&
gathered_kv.get_device_index() == device_index,
"direct DCP kv-gather tensors must be on the same CUDA device");
int64_t slice_bytes = num_tokens * token_dim * element_size;
int64_t slot_stride_bytes = max_gathered_tokens * token_dim * element_size;
bool vectorized =
reinterpret_cast<uintptr_t>(local_kv.data_ptr()) % alignof(uint4) == 0 &&
reinterpret_cast<uintptr_t>(received_kv.data_ptr()) % alignof(uint4) ==
0 &&
reinterpret_cast<uintptr_t>(gathered_kv.data_ptr()) % alignof(uint4) ==
0 &&
slice_bytes % sizeof(uint4) == 0 &&
slot_stride_bytes % sizeof(uint4) == 0;
STD_TORCH_CHECK(
vectorized,
"direct DCP kv-gather requires 16-byte-aligned pointers and strides");
STD_TORCH_CHECK(kv_mc_ptr != 0 && signal_mc_ptr != 0,
"direct DCP kv-gather requires multicast pointers");
const torch::stable::accelerator::DeviceGuard device_guard(device_index);
cudaStream_t stream = get_current_cuda_stream();
increment_epoch_kernel<<<1, 1, 0, stream>>>(
epoch.mutable_data_ptr<int64_t>());
check_cuda_launch("direct DCP kv-gather");
int64_t item_count = slice_bytes / sizeof(uint4);
int64_t blocks = (item_count + kThreads - 1) / kThreads;
blocks = blocks < kMaxMulticastBlocks ? blocks : kMaxMulticastBlocks;
direct_dcp_kv_gather_multimem_kernel<<<blocks, kThreads, 0, stream>>>(
reinterpret_cast<const uint4*>(local_kv.data_ptr()),
reinterpret_cast<uint4*>(static_cast<uintptr_t>(kv_mc_ptr)),
reinterpret_cast<uint32_t*>(static_cast<uintptr_t>(signal_mc_ptr)),
reinterpret_cast<const uint32_t*>(
received_signal.const_data_ptr<int32_t>()),
epoch.const_data_ptr<int64_t>(),
reinterpret_cast<uint32_t*>(completion.mutable_data_ptr<int32_t>()),
world_size, rank, slice_bytes, slot_stride_bytes);
check_cuda_launch("direct DCP kv-gather");
int64_t gathered_item_count = world_size * item_count;
int64_t copy_blocks = (gathered_item_count + kThreads - 1) / kThreads;
copy_blocks = copy_blocks < kMaxCopyBlocks ? copy_blocks : kMaxCopyBlocks;
materialize_kv_gather_kernel<uint4><<<copy_blocks, kThreads, 0, stream>>>(
reinterpret_cast<const uint4*>(received_kv.data_ptr()),
epoch.const_data_ptr<int64_t>(),
reinterpret_cast<uint4*>(gathered_kv.mutable_data_ptr()),
gathered_item_count, slot_stride_bytes / sizeof(uint4));
check_cuda_launch("direct DCP kv-gather");
}
} // namespace
STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_kv_gather_ops) {
direct_dcp_kv_gather_ops.def(
"direct_dcp_kv_gather("
"Tensor local_kv, Tensor! received_kv, Tensor! received_signal, "
"Tensor! completion, "
"Tensor! epoch, Tensor! gathered_kv, "
"int world_size, int rank, int max_gathered_tokens, "
"int kv_mc_ptr, int signal_mc_ptr) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_kv_gather_ops) {
direct_dcp_kv_gather_ops.impl("direct_dcp_kv_gather",
TORCH_BOX(&direct_dcp_kv_gather));
}
@@ -0,0 +1,197 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// Direct symmetric-memory DCP query gather into the consumer-final buffer.
#include <torch/csrc/stable/library.h>
#include <torch/headeronly/core/ScalarType.h>
#include <cstdio>
#include "dcp_direct_common.cuh"
namespace {
using vllm::direct_dcp::check_cuda_launch;
using vllm::direct_dcp::increment_epoch_kernel;
using vllm::direct_dcp::multimem_store_16;
using vllm::direct_dcp::multimem_store_release_system;
using vllm::direct_dcp::wait_for_epoch;
// Multicast each rank's head slice directly into every consumer's final query
// buffer. Reuse is ordered by the downstream DCP output synchronization.
__global__ void direct_dcp_q_gather_multimem_kernel(
const uint4* local_query, uint4* mc_final_query, uint32_t* mc_signal,
const uint32_t* received_signal, int64_t* epoch_ptr, uint32_t* completion,
int64_t world_size, int64_t rank, int64_t num_tokens,
int64_t bytes_per_token, int64_t query_token_stride_bytes,
int64_t destination_token_stride_bytes) {
// The common one-token decode case uses one block. Fold the epoch update
// into that publication kernel to avoid an otherwise separate tiny launch.
if (gridDim.x == 1 && threadIdx.x == 0) {
epoch_ptr[0] += 1;
}
__syncthreads();
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
int64_t signal_slot = static_cast<int64_t>(epoch & 1u);
int64_t items_per_token = bytes_per_token / sizeof(uint4);
int64_t source_token_stride = query_token_stride_bytes / sizeof(uint4);
int64_t destination_token_stride =
destination_token_stride_bytes / sizeof(uint4);
int64_t destination_head_offset = rank * items_per_token;
for (int64_t token_idx = blockIdx.x; token_idx < num_tokens;
token_idx += gridDim.x) {
for (int64_t token_item = threadIdx.x; token_item < items_per_token;
token_item += blockDim.x) {
multimem_store_16(
mc_final_query + token_idx * destination_token_stride +
destination_head_offset + token_item,
local_query[token_idx * source_token_stride + token_item]);
}
}
// Publish all multicast writes before incrementing completion.
__threadfence_system();
__syncthreads();
if (threadIdx.x != 0) {
return;
}
if (gridDim.x > 1) {
uint32_t completed = atomicAdd(completion, 1u);
if (completed + 1u != gridDim.x) {
return;
}
atomicExch(completion, 0u);
}
multimem_store_release_system(mc_signal + signal_slot * world_size + rank,
epoch);
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
int64_t signal_item = signal_slot * world_size + source_rank;
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
printf("direct DCP q-gather multimem timeout source=%lld epoch=%u\n",
static_cast<long long>(source_rank), epoch);
asm volatile("trap;");
}
}
}
void direct_dcp_q_gather(const torch::stable::Tensor& local_query,
torch::stable::Tensor& final_query,
torch::stable::Tensor& received_signal,
torch::stable::Tensor& completion,
torch::stable::Tensor& epoch, int64_t world_size,
int64_t rank, int64_t max_num_tokens,
int64_t padded_num_heads, int64_t query_mc_ptr,
int64_t signal_mc_ptr) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(local_query.is_cuda(), "local query must be a CUDA tensor");
ScalarType dtype = local_query.scalar_type();
STD_TORCH_CHECK(local_query.dim() == 3,
"local query must have shape [T,H,D]");
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
int64_t num_tokens = local_query.size(0);
int64_t heads_per_rank = local_query.size(1);
int64_t head_dim = local_query.size(2);
int64_t gathered_num_heads = world_size * heads_per_rank;
int64_t element_size = local_query.element_size();
STD_TORCH_CHECK(num_tokens > 0 && num_tokens <= max_num_tokens,
"token count exceeds symmetric q-gather buffer capacity");
STD_TORCH_CHECK(heads_per_rank > 0 && head_dim > 0,
"query head dimensions must be positive");
STD_TORCH_CHECK(padded_num_heads >= gathered_num_heads,
"padded query heads must cover all gathered heads");
STD_TORCH_CHECK(local_query.stride(2) == 1 &&
local_query.stride(1) == head_dim &&
local_query.stride(0) >= heads_per_rank * head_dim,
"local query must have packed heads");
STD_TORCH_CHECK(
final_query.is_cuda() && final_query.scalar_type() == dtype &&
final_query.is_contiguous() && final_query.dim() == 3 &&
final_query.size(0) == num_tokens &&
final_query.size(1) == gathered_num_heads &&
final_query.size(2) == head_dim,
"final query must be contiguous with shape [T,world_size*H,D]");
STD_TORCH_CHECK(
received_signal.is_cuda() && received_signal.is_contiguous() &&
received_signal.scalar_type() == ScalarType::Int &&
received_signal.dim() == 2 && received_signal.size(0) == 2 &&
received_signal.size(1) == world_size,
"received signal has the wrong symmetric buffer layout");
STD_TORCH_CHECK(completion.is_cuda() && completion.is_contiguous() &&
completion.scalar_type() == ScalarType::Int &&
completion.numel() == 1,
"completion counter must be one CUDA int32 tensor");
STD_TORCH_CHECK(epoch.is_cuda() && epoch.is_contiguous() &&
epoch.scalar_type() == ScalarType::Long &&
epoch.numel() == 1,
"epoch must be a one-element CUDA int64 tensor");
int64_t device_index = local_query.get_device_index();
STD_TORCH_CHECK(
final_query.get_device_index() == device_index &&
received_signal.get_device_index() == device_index &&
completion.get_device_index() == device_index &&
epoch.get_device_index() == device_index,
"direct DCP q-gather tensors must be on the same CUDA device");
int64_t query_token_stride_bytes = local_query.stride(0) * element_size;
int64_t bytes_per_token = heads_per_rank * head_dim * element_size;
int64_t gathered_token_stride_bytes =
gathered_num_heads * head_dim * element_size;
bool vectorized =
reinterpret_cast<uintptr_t>(local_query.data_ptr()) % alignof(uint4) ==
0 &&
reinterpret_cast<uintptr_t>(final_query.data_ptr()) % alignof(uint4) ==
0 &&
query_token_stride_bytes % sizeof(uint4) == 0 &&
bytes_per_token % sizeof(uint4) == 0;
STD_TORCH_CHECK(
vectorized,
"direct DCP q-gather requires 16-byte-aligned pointers and strides");
STD_TORCH_CHECK(query_mc_ptr != 0 && signal_mc_ptr != 0,
"direct DCP q-gather requires multicast pointers");
const torch::stable::accelerator::DeviceGuard device_guard(device_index);
cudaStream_t stream = get_current_cuda_stream();
constexpr int kThreads = 256;
int64_t blocks = num_tokens < world_size ? num_tokens : world_size;
if (blocks > 1) {
increment_epoch_kernel<<<1, 1, 0, stream>>>(
epoch.mutable_data_ptr<int64_t>());
check_cuda_launch("direct DCP q-gather epoch");
}
direct_dcp_q_gather_multimem_kernel<<<blocks, kThreads, 0, stream>>>(
reinterpret_cast<const uint4*>(local_query.data_ptr()),
reinterpret_cast<uint4*>(static_cast<uintptr_t>(query_mc_ptr)),
reinterpret_cast<uint32_t*>(static_cast<uintptr_t>(signal_mc_ptr)),
reinterpret_cast<const uint32_t*>(
received_signal.const_data_ptr<int32_t>()),
epoch.mutable_data_ptr<int64_t>(),
reinterpret_cast<uint32_t*>(completion.mutable_data_ptr<int32_t>()),
world_size, rank, num_tokens, bytes_per_token, query_token_stride_bytes,
gathered_token_stride_bytes);
check_cuda_launch("direct DCP q-gather");
}
} // namespace
STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_q_gather_ops) {
direct_dcp_q_gather_ops.def(
"direct_dcp_q_gather("
"Tensor local_query, Tensor! final_query, Tensor! received_signal, "
"Tensor! completion, Tensor! epoch, "
"int world_size, int rank, int max_num_tokens, "
"int padded_num_heads, int query_mc_ptr, int signal_mc_ptr) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_q_gather_ops) {
direct_dcp_q_gather_ops.impl("direct_dcp_q_gather",
TORCH_BOX(&direct_dcp_q_gather));
}
+106 -2
View File
@@ -222,6 +222,49 @@ class TestLSEWeightedCombine:
assert result.shape == (B, H, D)
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
def test_empty_shard_ignores_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
result = _lse_weighted_combine(outputs, lses)
torch.testing.assert_close(result, outputs[1])
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
import vllm.v1.attention.ops.common as common
class FakeGroup:
world_size = 2
rank_in_group = 0
def all_gather(self, tensor, dim):
assert dim == 0
return torch.cat((tensor, tensor), dim=dim)
monkeypatch.setattr(
common,
"correct_attn_out",
lambda output, lses, *args, **kwargs: (output, lses[0]),
)
output = torch.ones(7, 1, 1)
lse = torch.ones(7, 1)
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
_, masked_lse = common._cp_lse_common(
output,
lse,
FakeGroup(),
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
assert torch.isneginf(masked_lse[:1]).all()
torch.testing.assert_close(masked_lse[1:5], torch.ones_like(masked_lse[1:5]))
assert torch.isneginf(masked_lse[5:]).all()
def test_mathematically_correct(self):
"""Verify mathematical correctness of LSE combination."""
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
@@ -344,7 +387,7 @@ class TestPackedA2AKernels:
world_size, B, h_per_rank, D = 4, 7, 2, 32
H = world_size * h_per_rank
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
cp_attn_lse = torch.randn(B, H, device=device, dtype=dtype)
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
send_buffer = torch.empty(
(world_size, B, h_per_rank, D + lse_pack_dim),
@@ -375,6 +418,67 @@ class TestPackedA2AKernels:
else:
_assert_packed_a2a_close(actual, expected_out, dtype)
@pytest.mark.skipif(
torch.accelerator.device_count() < 1, reason="CUDA is required."
)
def test_empty_seq_lens_ignore_undefined_output(self):
from vllm.v1.attention.ops.dcp_alltoall import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
_dcp_a2a_unpack_combine,
)
device = torch.device("cuda")
world_size, num_tokens, h_per_rank, head_dim = 2, 5, 1, 32
num_heads = world_size * h_per_rank
output = torch.randn(
num_tokens,
num_heads,
head_dim,
device=device,
dtype=torch.bfloat16,
)
output[:1] = float("nan")
lse = torch.randn(num_tokens, num_heads, device=device, dtype=output.dtype)
seq_lens = torch.tensor([0, 2], device=device, dtype=torch.int32)
query_start_loc = torch.tensor([0, 1, 5], device=device, dtype=torch.int32)
lse_pack_dim = _dcp_a2a_lse_pack_dim(output.dtype)
send_buffer = torch.empty(
(
world_size,
num_tokens,
h_per_rank,
head_dim + lse_pack_dim,
),
device=device,
dtype=output.dtype,
)
_dcp_a2a_pack_send(
output,
lse,
send_buffer,
world_size,
h_per_rank,
head_dim,
lse_pack_dim,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
actual_output, actual_lse = _dcp_a2a_unpack_combine(
send_buffer,
head_dim,
lse_pack_dim,
return_lse=True,
is_lse_base_on_e=True,
)
torch.testing.assert_close(
actual_output[:1], torch.zeros_like(actual_output[:1])
)
assert torch.isneginf(actual_lse[:1]).all()
assert torch.isfinite(actual_output[1:]).all()
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
update_environment_variables(env)
@@ -417,7 +521,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
B,
H,
device=f"cuda:{local_rank}",
dtype=torch.float32,
dtype=dtype,
generator=generator,
)
actual = dcp_a2a_lse_reduce(
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
from tests.conftest import VllmRunner
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams, TokensPrompt
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
def _make_tiny_overrides() -> dict:
linear_attn_config = {
"full_attn_layers": [4],
"head_dim": 128,
"kda_layers": [1, 2, 3],
"num_heads": 8,
"short_conv_kernel_size": 4,
"use_full_rank_gate": True,
}
return {
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"vocab_size": 256,
"hidden_size": 256,
"head_dim": 32,
"intermediate_size": 512,
"num_hidden_layers": 4,
"num_attention_heads": 8,
"num_key_value_heads": 8,
"num_experts": None,
"num_experts_per_token": None,
"num_shared_experts": 0,
"q_lora_rank": 128,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"mla_use_nope": True,
"mla_use_output_gate": True,
"linear_attn_config": linear_attn_config,
"max_position_embeddings": 512,
"model_max_length": 512,
}
def _run_tiny_model(
vllm_runner: type[VllmRunner], dcp_size: int
) -> list[tuple[list[int], list[float]]]:
with vllm_runner(
model_name=MODEL,
skip_tokenizer_init=True,
load_format="dummy",
hf_overrides=_make_tiny_overrides(),
tensor_parallel_size=2,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
distributed_executor_backend="mp",
dtype="bfloat16",
seed=0,
enforce_eager=True,
max_model_len=512,
max_num_seqs=8,
max_num_batched_tokens=1024,
gpu_memory_utilization=0.25,
enable_chunked_prefill=True,
enable_prefix_caching=False,
) as runner:
lengths = [1, 32, 33, 129]
prompts = [
TokensPrompt(
prompt_token_ids=[
3 + ((request_idx + token_idx) % 251) for token_idx in range(length)
]
)
for request_idx, length in enumerate(lengths)
]
outputs = runner.llm.generate(
prompts,
SamplingParams(
temperature=0,
max_tokens=8,
seed=0,
ignore_eos=True,
logprobs=20,
),
use_tqdm=False,
)
results = []
for request_output in outputs:
completion = request_output.outputs[0]
token_ids = list(completion.token_ids)
assert completion.logprobs is not None
selected_logprobs = [
step_logprobs[token_id].logprob
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
]
results.append((token_ids, selected_logprobs))
return results
@create_new_process_for_each_test()
@pytest.mark.distributed(num_gpus=2)
def test_kimi_linear_dcp_tiny(
vllm_runner: type[VllmRunner],
num_gpus_available: int,
) -> None:
if num_gpus_available < 2:
pytest.skip("Need at least 2 GPUs")
baseline = _run_tiny_model(vllm_runner, dcp_size=1)
dcp = _run_tiny_model(vllm_runner, dcp_size=2)
assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline]
logprob_drifts = []
for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp):
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
assert math.isfinite(baseline_logprob)
assert math.isfinite(dcp_logprob)
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
assert max(logprob_drifts) <= 1e-2
@@ -0,0 +1,246 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm._custom_ops as ops
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
fused_mla_decode_q_concat_kv_cache_insert,
fused_mla_key_concat_ds_mla_insert,
fused_mla_key_concat_kv_cache_insert,
fused_mla_qkv_quant_kv_cache_fp8_insert,
)
from vllm.platforms import current_platform
NUM_TOKENS = 4
NUM_HEADS = 2
KV_LORA_RANK = 512
NOPE_HEAD_DIM = 128
ROPE_HEAD_DIM = 64
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
V_HEAD_DIM = 128
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
DS_MLA_CACHE_ENTRY = 656
BLOCK_SIZE = 8
OWNED_TOKEN_INDICES = [1, 3]
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
)
def _inputs() -> dict[str, torch.Tensor]:
torch.manual_seed(0)
def make(*shape: int) -> torch.Tensor:
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
return {
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
}
def _slot_mapping() -> torch.Tensor:
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
def _owned(tensor: torch.Tensor) -> torch.Tensor:
return tensor[OWNED_TOKEN_INDICES]
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
return torch.cat((k_nope, shared_pe), dim=-1)
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
return torch.cat((ql_nope, q_pe), dim=-1)
def _make_cache(cache_format: str) -> torch.Tensor:
if cache_format == "bf16":
return torch.full(
(1, BLOCK_SIZE, CACHE_ENTRY),
-7,
device="cuda",
dtype=torch.bfloat16,
)
if cache_format == "fp8":
return torch.full(
(1, BLOCK_SIZE, CACHE_ENTRY),
1.0,
device="cuda",
dtype=torch.float32,
).to(torch.float8_e4m3fn)
assert cache_format == "fp8_ds_mla"
return torch.full(
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
165,
device="cuda",
dtype=torch.uint8,
)
def _reference_cache(
cache_format: str,
inputs: dict[str, torch.Tensor],
initial_cache: torch.Tensor,
slots: torch.Tensor,
) -> torch.Tensor:
reference_cache = initial_cache.clone()
valid_slots = _owned(slots)
if cache_format == "fp8_ds_mla":
ops.concat_and_cache_mla(
_owned(inputs["kv_c"]),
_owned(inputs["k_pe"]),
reference_cache,
valid_slots,
cache_format,
torch.ones(1, device="cuda", dtype=torch.float32),
)
return reference_cache
latent = torch.cat(
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
dim=-1,
)
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
for row, slot in zip(latent, valid_slots.tolist()):
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
return reference_cache
def _assert_cache_matches_reference(
cache: torch.Tensor,
reference_cache: torch.Tensor,
initial_cache: torch.Tensor,
) -> None:
assert torch.equal(cache, reference_cache)
for slot in (0, 5):
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
for slot in (1, 2, 3, 4, 6, 7):
assert torch.equal(cache[0, slot], initial_cache[0, slot])
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache(cache_format)
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
op = (
fused_mla_key_concat_kv_cache_insert
if cache_format == "bf16"
else fused_mla_key_concat_ds_mla_insert
)
output = op(
inputs["q"],
inputs["k_nope"],
inputs["k_pe"],
inputs["kv_c"],
mixed_cache,
slots,
)
torch.testing.assert_close(
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache("fp8")
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
inputs["q"],
inputs["k_nope"],
inputs["k_pe"],
inputs["kv_c"],
inputs["v"],
mixed_cache,
slots,
scale_inv,
scale_inv,
scale_inv,
scale_inv,
)
expected = (
inputs["q"].to(torch.float8_e4m3fn),
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
inputs["v"].to(torch.float8_e4m3fn),
)
for actual, reference in zip((q, k, v), expected):
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
inputs = _inputs()
slots = _slot_mapping()
initial_cache = _make_cache(cache_format)
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
kwargs = {
"ds_mla": cache_format == "fp8_ds_mla",
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
}
output = fused_mla_decode_q_concat_kv_cache_insert(
inputs["ql_nope"],
inputs["q_pe"],
inputs["kv_c"],
inputs["k_pe"],
mixed_cache,
slots,
**kwargs,
)
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
if cache_format == "fp8":
expected = expected.to(torch.float8_e4m3fn)
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
else:
torch.testing.assert_close(output, expected, rtol=0, atol=0)
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
def test_ds_mla_cache_insert_bit_compatible_with_reference() -> None:
"""Fused ds_mla insertion must match the reference bit-for-bit."""
torch.manual_seed(0)
num_tokens, num_blocks = 33, 16
dt = torch.bfloat16
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
ref = torch.full(
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
165,
device="cuda",
dtype=torch.uint8,
)
got = ref.clone()
scale = torch.ones(1, device="cuda", dtype=torch.float32)
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
assert torch.equal(ref, got)
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
import torch
from vllm.platforms import current_platform
requires_flashinfer_mla = pytest.mark.skipif(
not current_platform.has_device_capability(100),
reason="FlashInfer MLA requires compute capability 10 or above.",
)
def test_mla_dcp_gathered_query_reserves_backend_head_storage():
from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage
query = torch.randn(3, 24, 576, dtype=torch.bfloat16)
padded = reserve_query_head_storage(query, 128)
assert padded.shape == query.shape
assert padded.stride() == query.stride()
assert padded.untyped_storage().nbytes() >= 3 * 128 * 576 * 2
assert torch.equal(padded, query)
@requires_flashinfer_mla
def test_flashinfer_mla_selects_backend_for_gathered_heads():
from vllm.v1.attention.backends.mla.flashinfer_mla import (
_select_mla_decode_backend,
)
assert _select_mla_decode_backend(6) is None
assert _select_mla_decode_backend(24) == "cute-dsl"
@requires_flashinfer_mla
def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
import vllm.v1.attention.backends.mla.flashinfer_mla as flashinfer_mla
impl = MagicMock()
impl.bmm1_scale = 1.0
impl.bmm2_scale = 1.0
impl.need_to_return_lse_for_decode = True
impl.num_heads = 6
impl.qk_nope_head_dim = 128
impl.kv_lora_rank = 512
impl.qk_rope_head_dim = 64
impl.kv_cache_dtype = "auto"
attn_metadata = MagicMock()
attn_metadata.causal = True
attn_metadata.num_decode_tokens = 2
attn_metadata.num_decodes = 2
attn_metadata.max_seq_len = 1
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
kernel = MagicMock(
return_value=(
torch.ones(2, 1, 24, 512, dtype=torch.bfloat16),
torch.ones(2, 24, dtype=torch.float32),
)
)
monkeypatch.setattr(flashinfer_mla, "_get_workspace_buffer", MagicMock())
monkeypatch.setattr(flashinfer_mla, "trtllm_batch_decode_with_kv_cache_mla", kernel)
layer = MagicMock()
output, lse = flashinfer_mla.FlashInferMLAImpl.forward_mqa(
impl, query, kv_cache, attn_metadata, layer
)
assert kernel.call_args.kwargs["backend"] == "cute-dsl"
assert output.shape == (2, 24, 512)
assert lse is not None and lse.shape == (2, 24)
+13
View File
@@ -190,6 +190,9 @@ if TYPE_CHECKING:
VLLM_USE_DEEP_GEMM_E8M0: bool = True
VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
VLLM_DCP_Q_REPLICATE: bool = False
VLLM_USE_DIRECT_DCP_A2A: bool | None = None
VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None
VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None
VLLM_DEEP_GEMM_WARMUP: Literal[
"skip",
"full",
@@ -2057,6 +2060,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool(
int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0"))
),
# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
os.getenv("VLLM_USE_DIRECT_DCP_A2A")
),
"VLLM_USE_DIRECT_DCP_Q_GATHER": lambda: maybe_convert_bool(
os.getenv("VLLM_USE_DIRECT_DCP_Q_GATHER")
),
"VLLM_USE_DIRECT_DCP_KV_GATHER": lambda: maybe_convert_bool(
os.getenv("VLLM_USE_DIRECT_DCP_KV_GATHER")
),
# Whether to enable dual cuda streams for LoRA computation
# (used by both BaseLinearLayerWithLoRA and FusedMoEWithLoRA to
# overlap the base layer compute with the LoRA fast path).
@@ -206,6 +206,7 @@ from abc import abstractmethod
from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from math import lcm
from typing import ClassVar, Generic, TypeVar, cast
import numpy as np
@@ -222,7 +223,6 @@ from vllm.config import (
ModelConfig,
VllmConfig,
get_current_vllm_config,
get_current_vllm_config_or_none,
)
from vllm.config.cache import CacheDType
from vllm.distributed.parallel_state import (
@@ -264,7 +264,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down
from vllm.utils.math_utils import cdiv, round_down, round_up
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -292,8 +292,7 @@ from vllm.v1.attention.backends.utils import (
get_num_attention_heads_from_layers,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import (
@@ -579,12 +578,27 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.use_sparse = use_sparse
_vllm_config = get_current_vllm_config_or_none()
self.dcp_a2a = (
_vllm_config is not None
and _vllm_config.parallel_config.decode_context_parallel_size > 1
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
self.dcp_manager: MLADCPManager | None = None
if self.impl.dcp_world_size > 1:
query_dtype = (
current_platform.fp8_dtype()
if is_quantized_kv_cache(self.kv_cache_dtype)
and self.kv_cache_dtype != "fp8_ds_mla"
and self.impl.supports_quant_query_input
else dtype
)
self.dcp_manager = MLADCPManager(
vllm_config=vllm_config,
device=next(kv_b_proj.parameters()).device,
num_heads=self.num_heads,
query_head_dim=self.kv_lora_rank + self.qk_rope_head_dim,
output_head_dim=self.kv_lora_rank,
query_dtype=query_dtype,
output_dtype=dtype,
padded_num_heads=self.q_pad_num_heads,
is_lse_base_on_e=self.impl.lse_base_on_e,
use_pcp=self.use_pcp,
)
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
@@ -753,9 +767,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
return quant_output.fill_(0)
return output.fill_(0)
if self.impl.dcp_world_size == -1:
self.impl.dcp_world_size = get_dcp_group().world_size
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
num_actual_toks = attn_metadata.num_actual_tokens
@@ -914,6 +925,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
mqa_q = (mqa_ql_nope, mqa_q_pe)
# concatenate nope + pe -> (B, N, L + P) (fp8 op above may have fused)
if self.impl.dcp_world_size > 1:
assert self.dcp_manager is not None
if self.use_pcp:
if self.impl.dcp_world_size > self.impl.pcp_world_size:
if isinstance(mqa_q, tuple):
@@ -924,8 +936,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
if not qrep_decode:
# mqa_q do allgather in head dim.
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
assert self.dcp_manager.query_gather is not None
mqa_q = self.dcp_manager.query_gather(mqa_q)
# call decode attn
if not self.impl.is_sparse:
@@ -935,27 +947,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# correct dcp attn_out with lse.
if self.impl.dcp_world_size > 1:
assert lse is not None
if self.dcp_a2a:
attn_out = dcp_a2a_lse_reduce(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
elif self.use_pcp:
attn_out = cp_lse_ag_out_ar(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
else:
attn_out = cp_lse_ag_out_rs(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
assert self.dcp_manager is not None
seq_lens = (
attn_metadata.decode.seq_lens
if attn_metadata.decode is not None
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
: attn_metadata.num_decodes
]
)
query_start_loc = attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
]
attn_out = self.dcp_manager.combine(
attn_out,
lse,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
if self.use_pcp:
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
@@ -1427,6 +1435,7 @@ class MLACommonPrefillMetadata:
chunks: "list[MLACommonPrefillMetadata.ContextChunk]"
context_lens_list: list[int]
empty_token_slices: list[slice]
dcp_manager: MLADCPManager | None = None
block_table: torch.Tensor
query_start_loc: torch.Tensor
@@ -1684,6 +1693,25 @@ def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
def align_mla_chunked_context_workspace_size(
vllm_config: VllmConfig,
workspace_size: int,
) -> int:
parallel_config = vllm_config.parallel_config
alignment = vllm_config.cache_config.block_size
if parallel_config.decode_context_parallel_size > 1:
alignment = lcm(
alignment,
parallel_config.decode_context_parallel_size
* parallel_config.cp_kv_cache_interleave_size,
)
workspace_size = max(
workspace_size,
vllm_config.scheduler_config.max_num_seqs * alignment,
)
return round_up(workspace_size, alignment)
def build_mla_chunked_context_metadata(
*,
context_lens_cpu: torch.Tensor,
@@ -1696,6 +1724,7 @@ def build_mla_chunked_context_metadata(
dcp_world_size: int,
dcp_local_block_size: int,
dcp_virtual_block_size: int,
dcp_manager: MLADCPManager | None = None,
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
"""Build chunked-context metadata for an MLA prefill.
@@ -1714,6 +1743,7 @@ def build_mla_chunked_context_metadata(
dcp_world_size: Decode-context-parallel world size (1 if disabled).
dcp_local_block_size: Per-rank interleave block size for DCP.
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
dcp_manager: Shared MLA DCP collective manager.
Returns:
The chunked-context metadata, or None when no prefill has any context.
@@ -1902,6 +1932,7 @@ def build_mla_chunked_context_metadata(
chunks=chunks,
context_lens_list=context_lens,
empty_token_slices=empty_token_slices,
dcp_manager=dcp_manager,
)
@@ -1954,7 +1985,10 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
64 * 1024,
)
return max(chunked_prefill_workspace_size, cache_config.block_size)
return align_mla_chunked_context_workspace_size(
vllm_config,
chunked_prefill_workspace_size,
)
@staticmethod
def determine_prefill_query_data_type(
@@ -2033,6 +2067,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.q_data_type = self.determine_prefill_query_data_type(
vllm_config, self.model_config.dtype
)
attention_layer = self.compilation_config.static_forward_context[layer_names[0]]
try:
self.dcp_world_size = get_dcp_group().world_size
@@ -2050,6 +2085,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
)
use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
self.dcp_manager: MLADCPManager | None = None
if self.dcp_world_size > 1:
# Note(hc): The local kvcache is incomplete when DCP is triggered,
# an additional kvcache allgather across the DCP group is therefore
@@ -2067,6 +2103,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
else self.model_config.dtype,
device=device,
)
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
assert isinstance(self.dcp_manager, MLADCPManager)
self.dcp_manager.init_kv_gather(
self.chunked_prefill_workspace,
self.chunked_prefill_workspace_size,
)
else:
self.chunked_prefill_workspace = torch.empty(
(
@@ -2080,9 +2122,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
# Metadata builders are created per ubatch when DBO is enabled. MLA
# prefill backends keep the prepared metadata on the backend object, so
# each builder needs its own backend instance to avoid cross-ubatch races.
self._prefill_backend = self.compilation_config.static_forward_context[
layer_names[0]
].prefill_backend.clone()
self._prefill_backend = attention_layer.prefill_backend.clone()
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
self._init_reorder_batch_threshold(
@@ -2223,6 +2263,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
dcp_world_size=self.dcp_world_size,
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
)
prefill_metadata = MLACommonPrefillMetadata(
@@ -2660,9 +2701,8 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
]
assert toks * dcp_world_size <= cur_allgather_workspace.shape[0]
cur_allgather_kvcache = cur_allgather_workspace[: toks * dcp_world_size]
cur_allgather_kvcache.copy_(
get_dcp_group().all_gather(local_gathered_kvcache, dim=0)
)
dcp_manager = cast(MLADCPManager, chunked_context.dcp_manager)
dcp_manager.kv_gather(cur_allgather_kvcache, local_gathered_kvcache)
assert (
cur_allgather_kvcache.shape[-1]
== self.kv_lora_rank + self.qk_rope_head_dim
@@ -2732,8 +2772,6 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
output_scale: torch.Tensor | None = None,
) -> None:
assert attn_metadata.prefill is not None
assert self.dcp_world_size != -1
prefill_metadata = attn_metadata.prefill
assert prefill_metadata.prefill_backend is not None
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
@@ -2877,10 +2915,11 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
and (self.qk_rope_head_dim == 64)
)
self.dcp_world_size: int = -1
parallel_config = get_current_vllm_config().parallel_config
# Avoid requiring an initialized DCP group in tests.
self.dcp_world_size: int = parallel_config.decode_context_parallel_size
self.cp_kv_cache_interleave_size: int = (
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
parallel_config.cp_kv_cache_interleave_size
)
@abstractmethod
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
MLACommonMetadata,
MLACommonPrefillMetadata,
accumulate_mla_context_chunk,
align_mla_chunked_context_workspace_size,
build_mla_chunked_context_metadata,
get_mla_dims,
init_mla_context_partial,
@@ -31,6 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
if TYPE_CHECKING:
@@ -162,9 +164,19 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
dtype=torch.int32,
device=device,
)
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
attention_layer = vllm_config.compilation_config.static_forward_context[
layer_names[0]
].prefill_backend
]
layer_prefill_backend = attention_layer.prefill_backend
self.dcp_manager: MLADCPManager | None = None
if self.dcp_world_size > 1:
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
assert isinstance(self.dcp_manager, MLADCPManager)
if layer_prefill_backend is not None:
self.dcp_manager.init_kv_gather(
self.chunked_prefill_workspace,
self.chunked_prefill_workspace_size,
)
self._prefill_backend = (
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
)
@@ -184,7 +196,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
64 * 1024,
scheduler_config.max_num_seqs * topk_tokens,
)
return max(workspace_size, cache_config.block_size)
workspace_size = max(
workspace_size,
scheduler_config.max_num_seqs * cache_config.block_size,
)
if vllm_config.parallel_config.decode_context_parallel_size > 1:
return align_mla_chunked_context_workspace_size(vllm_config, workspace_size)
return workspace_size
def _build_req_id_per_token(
self,
@@ -232,6 +250,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
dcp_world_size=self.dcp_world_size,
dcp_local_block_size=self.dcp_local_block_size,
dcp_virtual_block_size=self.dcp_virtual_block_size,
dcp_manager=self.dcp_manager,
)
def build(
+69 -11
View File
@@ -24,8 +24,8 @@ KV cache, and absorbs ``kv_b_proj`` into ``W_UK_T`` / ``W_UV`` -- mirroring the
K3 specifics: optional rotary embedding (disabled for the target model's NoPE
layers, enabled for DSpark) and an optional sigmoid output gate (``g_proj``).
Out of scope (extension points, not wired here): context parallelism (DCP/PCP),
sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths.
Out of scope (extension points, not wired here): prefill context parallelism
(PCP), sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths.
"""
import math
@@ -35,7 +35,11 @@ import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
from vllm.config import (
CacheConfig,
VllmConfig,
get_current_vllm_config,
)
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
@@ -78,6 +82,7 @@ from vllm.v1.attention.backend import (
MLAAttentionImpl,
)
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode
@@ -310,10 +315,36 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
assert (
parallel_config.decode_context_parallel_size <= 1
and parallel_config.prefill_context_parallel_size <= 1
), "Kimi-K3 MultiHeadLatentAttention does not support context parallelism."
assert parallel_config.prefill_context_parallel_size == 1, (
"Kimi-K3 MultiHeadLatentAttention does not support prefill context "
"parallelism."
)
self.dcp_world_size = parallel_config.decode_context_parallel_size
assert self.dcp_world_size <= 1 or self.rotary_emb is None, (
"Kimi-K3 MultiHeadLatentAttention does not support RoPE with decode "
"context parallelism because gathered queries require gathered "
"positions."
)
self.dcp_manager: MLADCPManager | None = None
if self.dcp_world_size > 1:
query_dtype = (
torch.float8_e4m3fn
if is_quantized_kv_cache(self.kv_cache_dtype)
and self.kv_cache_dtype != "fp8_ds_mla"
else dtype
)
self.dcp_manager = MLADCPManager(
vllm_config=vllm_config,
device=next(self.kv_b_proj.parameters()).device,
num_heads=self.num_local_heads,
query_head_dim=self.head_size,
output_head_dim=self.kv_lora_rank,
query_dtype=query_dtype,
output_dtype=dtype,
padded_num_heads=self.q_pad_num_heads,
is_lse_base_on_e=self.impl.lse_base_on_e,
use_pcp=False,
)
self.prefill_backend = get_mla_prefill_backend(vllm_config)(
num_heads=self.num_local_heads,
scale=self.scale,
@@ -584,9 +615,25 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
cos_sin_cache,
slot_mapping[:num_mqa_tokens],
)
latent_out, _lse = self.impl.forward_mqa( # type: ignore[attr-defined]
if self.dcp_world_size > 1:
assert self.dcp_manager is not None
assert self.dcp_manager.query_gather is not None
mqa_q = self.dcp_manager.query_gather(mqa_q)
latent_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
mqa_q, self._attn_read_kv_cache(), attn_metadata, self
)
if self.dcp_world_size > 1:
assert lse is not None
assert self.dcp_manager is not None
assert attn_metadata.decode is not None
latent_out = self.dcp_manager.combine(
latent_out,
lse,
seq_lens=attn_metadata.decode.seq_lens,
query_start_loc=attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
],
)
self._v_up_proj(latent_out, out=attn_out[:num_mqa_tokens])
def _decode_concat_cache(
@@ -752,9 +799,20 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
)
if has_context:
context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined]
q, self._attn_read_kv_cache(), attn_metadata, self._k_scale
)
if self.dcp_world_size > 1:
context_output, context_lse = (
self.impl._context_parallel_compute_prefill_context( # type: ignore[attr-defined]
q,
self._attn_read_kv_cache(),
attn_metadata,
k_scale=self._k_scale,
dcp_world_size=self.dcp_world_size,
)
)
else:
context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined]
q, self._attn_read_kv_cache(), attn_metadata, self._k_scale
)
suffix_output, suffix_lse = output_prefill
out = out.view(-1, self.num_local_heads, self.v_head_dim)
merge_attn_states(
@@ -309,9 +309,11 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
return_lse = self.need_to_return_lse_for_decode
workspace_buffer = _get_workspace_buffer(return_lse)
# Parallel gathers can change the runtime Q heads from TP-local num_heads.
runtime_num_heads = q.shape[-2]
# trtllm-gen rejects MLA head counts it can't tile (e.g. 96);
# fall back to cute-dsl for those.
decode_backend = _select_mla_decode_backend(self.num_heads)
decode_backend = _select_mla_decode_backend(runtime_num_heads)
extra_kwargs = {}
if decode_backend:
extra_kwargs["backend"] = decode_backend
@@ -326,7 +328,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
if self._mla_counter_bytes is None:
self._mla_counter_bytes = get_trtllm_gen_multi_ctas_kv_counter_bytes(
self._mla_counter_max_batch,
self.num_heads,
runtime_num_heads,
get_device_sm_count(q.device),
)
extra_kwargs["multi_ctas_kv_counter_buffer"] = (
@@ -289,8 +289,9 @@ class TokenspeedMLAImpl(MLACommonImpl[MLACommonMetadata]):
self.output_scale = layer._k_scale_float
if self._workspace_buffer is None:
# Parallelism can change the runtime query head count.
self._workspace_buffer = _get_workspace(
q.device, self.num_heads, self.kv_lora_rank
q.device, q.shape[-2], self.kv_lora_rank
)
# vLLM kv_c_and_k_pe_cache is already (num_blocks, block_size, head_size).
+51 -4
View File
@@ -6,6 +6,34 @@ from vllm.distributed.parallel_state import GroupCoordinator
from vllm.triton_utils import tl, triton
def mask_dcp_empty_shards_(
lse: torch.Tensor,
seq_lens: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
) -> None:
if seq_lens is None and query_start_loc is None:
return
if seq_lens is None or query_start_loc is None:
raise ValueError("seq_lens and query_start_loc must be provided together")
if (
seq_lens.ndim != 1
or query_start_loc.ndim != 1
or query_start_loc.shape[0] != seq_lens.shape[0] + 1
):
raise ValueError("query_start_loc must contain one boundary per sequence")
row_indices = torch.arange(
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
)
sequence_indices = torch.searchsorted(
query_start_loc[1:], row_indices, right=True
).clamp_max(seq_lens.shape[0] - 1)
empty_rows = (row_indices >= query_start_loc[-1]) | (
seq_lens[sequence_indices] == 0
)
lse.masked_fill_(empty_rows[:, None], float("-inf"))
@triton.jit
def _correct_attn_cp_out_kernel(
outputs_ptr,
@@ -51,7 +79,7 @@ def _correct_attn_cp_out_kernel(
)
# calc final lse
lse = tl.load(lses_ptr + lse_offsets)
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
lse_max = tl.max(lse, axis=0)
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
@@ -80,7 +108,7 @@ def _correct_attn_cp_out_kernel(
lse_offset = (
lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H
)
lse_tmp = tl.load(lses_ptr + lse_offset)
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
lse_finally = lse_tmp - lse
lse_finally = tl.where(
(lse_finally != lse_finally) | (lse_finally == float("inf")),
@@ -185,6 +213,8 @@ def _cp_lse_common(
cp_group: GroupCoordinator,
ctx: CPTritonContext | None = None,
is_lse_base_on_e=True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
):
"""
cp_attn_out: [ B, H, D ]
@@ -197,6 +227,7 @@ def _cp_lse_common(
ctx = CPTritonContext()
cp_attn_lse = cp_attn_lse.contiguous()
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape(
(cp_group.world_size,) + cp_attn_lse.shape
)
@@ -217,13 +248,21 @@ def cp_lse_ag_out_rs(
ctx: CPTritonContext | None = None,
return_lse: bool = False,
is_lse_base_on_e=True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
):
"""
cp_attn_out: [ B, H, D ]
cp_attn_lse: [ B, H ]
"""
out, lse = _cp_lse_common(
cp_attn_out, cp_attn_lse, cp_group, ctx=ctx, is_lse_base_on_e=is_lse_base_on_e
cp_attn_out,
cp_attn_lse,
cp_group,
ctx=ctx,
is_lse_base_on_e=is_lse_base_on_e,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
out = cp_group.reduce_scatter(out, dim=1)
@@ -242,13 +281,21 @@ def cp_lse_ag_out_ar(
ctx: CPTritonContext | None = None,
return_lse: bool = False,
is_lse_base_on_e=True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
):
"""
cp_attn_out: [ B, H, D ]
cp_attn_lse: [ B, H ]
"""
out, lse = _cp_lse_common(
cp_attn_out, cp_attn_lse, cp_group, ctx=ctx, is_lse_base_on_e=is_lse_base_on_e
cp_attn_out,
cp_attn_lse,
cp_group,
ctx=ctx,
is_lse_base_on_e=is_lse_base_on_e,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
out = cp_group.all_reduce(out)
+20 -11
View File
@@ -26,6 +26,7 @@ import torch
import torch.distributed as dist
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.common import mask_dcp_empty_shards_
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
@@ -87,7 +88,9 @@ def _lse_weighted_combine(
weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H]
# Weighted combination: sum over N dimension
result = (outputs * weights.unsqueeze(-1)).sum(dim=0) # [B, H, D]
weights = weights.unsqueeze(-1)
outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs)
result = (outputs * weights).sum(dim=0) # [B, H, D]
if return_lse:
if is_lse_base_on_e:
@@ -172,7 +175,7 @@ def _dcp_a2a_pack_send_kernel(
lse_val = tl.load(
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
)
).to(tl.float32)
if LSE_PACK_DIM == 1:
tl.store(
send_ptr + send_base + HEAD_DIM * send_stride_D,
@@ -301,10 +304,11 @@ def _dcp_a2a_unpack_combine_kernel(
else:
weight = tl.exp2(lse_val - global_lse)
weight = tl.where(weight != weight, 0.0, weight)
acc += (
tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(tl.float32)
* weight
partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(
tl.float32
)
partial = tl.where(weight == 0.0, 0.0, partial)
acc += partial * weight
final_offsets = (
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
@@ -324,7 +328,10 @@ def _dcp_a2a_pack_send(
h_per_rank: int,
head_dim: int,
lse_pack_dim: int,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> None:
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
grid = (cp_attn_out.shape[0], h_per_rank, 1)
_dcp_a2a_pack_send_kernel[grid](
cp_attn_out,
@@ -396,20 +403,24 @@ def dcp_a2a_lse_reduce(
ctx: CPTritonContext | None = None,
return_lse: bool = False,
is_lse_base_on_e: bool = True,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
Combine partial attention outputs across DCP ranks using All-to-All.
The output and fp32 LSE are packed into a single output-dtype buffer, sent
The output and LSE are packed into a single output-dtype buffer, sent
with one All-to-All, then unpacked and combined with exact LSE weighting.
Args:
cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim
cp_attn_lse: [B, H] log-sum-exp values (fp32)
cp_attn_lse: [B, H] floating-point log-sum-exp values
cp_group: GroupCoordinator for DCP communication
ctx: CPTritonContext (unused, for signature compatibility)
return_lse: If True, also return the combined global LSE
is_lse_base_on_e: If True, LSE is base e; if False, base 2
seq_lens: Local KV lengths. Empty shards contribute zero weight.
query_start_loc: Cumulative query-token offsets for each request.
Returns:
Combined output [B, H/N, D] (head-scattered)
@@ -426,10 +437,6 @@ def dcp_a2a_lse_reduce(
if H % world_size != 0:
raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.")
H_per_rank = H // world_size
# The pack kernel bit-casts the LSE as fp32; some MLA backends return it in
# the activation dtype (bf16/fp16), so enforce the documented fp32 contract.
if cp_attn_lse.dtype != torch.float32:
cp_attn_lse = cp_attn_lse.to(torch.float32)
lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype)
send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers(
@@ -446,6 +453,8 @@ def dcp_a2a_lse_reduce(
H_per_rank,
D,
lse_pack_dim,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
work = dist.all_to_all_single(
+740
View File
@@ -0,0 +1,740 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MLA DCP collective selection and direct symmetric-memory implementations."""
from __future__ import annotations
import functools
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Protocol
import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_dcp_group
from vllm.distributed.parallel_state import in_the_same_node_as
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
logger = init_logger(__name__)
if TYPE_CHECKING:
from torch.distributed import ProcessGroup
from vllm.distributed.parallel_state import GroupCoordinator
try:
import torch.distributed._symmetric_memory as symm_mem
symm_mem_available = True
except ImportError:
symm_mem = None # type: ignore[assignment]
symm_mem_available = False
@functools.cache
def _symm_mem_spans_group(group: GroupCoordinator) -> bool:
"""Probe whether the group has NVLS symmetric memory."""
if not symm_mem_available:
return False
try:
from torch._C._autograd import DeviceType
from torch._C._distributed_c10d import _SymmetricMemory
device = torch.device("cuda", torch.accelerator.current_device_index())
if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index):
return False
probe = symm_mem.empty(8, dtype=torch.uint8, device=device)
probe.zero_()
torch.accelerator.synchronize()
handle = symm_mem.rendezvous(probe, group.device_group.group_name)
spans = handle is not None and handle.multicast_ptr != 0
except Exception as error:
logger.debug("Direct DCP symmetric-memory probe failed: %s", error)
return False
logger.debug_once(
"Direct DCP symmetric memory across %d ranks: %s",
group.world_size,
"available" if spans else "unavailable",
)
return spans
def _direct_dcp_enabled(
group: GroupCoordinator,
dtype: torch.dtype,
use_direct: bool | None,
supported_dtypes: tuple[torch.dtype, ...] | None = None,
) -> bool:
if use_direct is not None:
return use_direct
return (
symm_mem_available
and current_platform.is_cuda()
and (supported_dtypes is None or dtype in supported_dtypes)
and (
all(in_the_same_node_as(group.cpu_group, source_rank=0))
or _symm_mem_spans_group(group)
)
)
def _direct_dcp_multicast_enabled(
group: GroupCoordinator,
dtype: torch.dtype,
use_direct: bool | None,
supported_dtypes: tuple[torch.dtype, ...] | None = None,
) -> bool:
return _direct_dcp_enabled(
group, dtype, use_direct, supported_dtypes
) and _symm_mem_spans_group(group)
def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int:
scheduler_config = vllm_config.scheduler_config
speculative_config = vllm_config.speculative_config
speculative_tokens = vllm_config.num_speculative_tokens
tokens_per_seq = (
1
+ (
2
if speculative_config is not None and speculative_config.parallel_drafting
else 1
)
* speculative_tokens
)
return min(
scheduler_config.max_num_batched_tokens,
max(
scheduler_config.max_num_seqs * tokens_per_seq,
vllm_config.compilation_config.max_cudagraph_capture_size or 0,
),
)
class _DirectDCPWorkspace:
def __init__(
self,
group: ProcessGroup,
device: torch.device,
num_ubatches: int,
) -> None:
self.group = group
self.world_size = group.size()
self.rank = group.rank()
self.device = torch.device(device)
self.num_ubatches = num_ubatches
self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device)
self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = []
def _allocate(
self, shape: tuple[int, ...], dtype: torch.dtype
) -> tuple[torch.Tensor, torch.Tensor]:
storage = symm_mem.empty(shape, device=self.device, dtype=dtype)
storage.zero_()
torch.accelerator.synchronize()
handle = symm_mem.rendezvous(storage, self.group.group_name)
assert handle is not None, "DCP symmetric memory rendezvous returned None"
handle.barrier()
views = [
handle.get_buffer(peer, list(shape), dtype, 0)
for peer in range(self.world_size)
]
self.device = storage.device
peer_ptrs = torch.tensor(
[
[view[ubatch].data_ptr() for view in views]
for ubatch in range(self.num_ubatches)
],
dtype=torch.int64,
device=self.device,
)
self._allocations.append((storage, handle, views))
return storage, peer_ptrs
def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]:
disabled = [0] * self.num_ubatches
for allocated, handle, _ in self._allocations:
if allocated is storage:
break
else:
return disabled
try:
from torch._C._autograd import DeviceType
from torch._C._distributed_c10d import _SymmetricMemory
if not _SymmetricMemory.has_multicast_support(
DeviceType.CUDA, storage.device.index
):
return disabled
multicast_base = handle.multicast_ptr
except Exception:
return disabled
if not multicast_base:
return disabled
storage_base = storage.data_ptr()
return [
multicast_base + (storage[ubatch].data_ptr() - storage_base)
for ubatch in range(self.num_ubatches)
]
def reserve_query_head_storage(
query: torch.Tensor, padded_num_heads: int
) -> torch.Tensor:
"""Reserve backing storage for fixed-head decode kernels."""
assert query.ndim == 3
assert query.shape[1] <= padded_num_heads
padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2]))
padded.resize_(query.shape)
padded.copy_(query)
return padded
_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
class DirectDCPA2AWorkspace(_DirectDCPWorkspace):
"""Persistent symmetric buffers for direct DCP output exchange."""
def __init__(
self,
group: ProcessGroup,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype = torch.bfloat16,
num_ubatches: int = 1,
) -> None:
if dtype not in _A2A_SUPPORTED_DTYPES:
raise ValueError(f"Direct DCP A2A does not support {dtype}")
if num_ubatches < 1:
raise ValueError(
f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}"
)
super().__init__(group, device, num_ubatches)
self.max_num_tokens = max_num_tokens
self.heads_per_rank = heads_per_rank
self.head_dim = head_dim
output_shape = (
num_ubatches,
2,
self.world_size,
max_num_tokens,
heads_per_rank,
head_dim,
)
lse_shape = (
num_ubatches,
2,
self.world_size,
max_num_tokens,
heads_per_rank,
)
signal_shape = (num_ubatches, 2, self.world_size)
self.received_output, self.peer_output_ptrs = self._allocate(
output_shape, dtype
)
self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32)
self.received_signal, self.peer_signal_ptrs = self._allocate(
signal_shape, torch.int32
)
def lse_reduce(
self,
partial_output: torch.Tensor,
partial_lse: torch.Tensor,
is_lse_base_on_e: bool,
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> torch.Tensor:
ubatch = dbo_current_ubatch_id()
num_tokens = partial_output.shape[0]
output = partial_output.new_empty(
(num_tokens, self.heads_per_rank, self.head_dim)
)
torch.ops._C.direct_dcp_a2a_lse_reduce(
partial_output,
partial_lse,
seq_lens,
query_start_loc,
self.peer_output_ptrs[ubatch],
self.peer_lse_ptrs[ubatch],
self.peer_signal_ptrs[ubatch],
self.received_output[ubatch],
self.received_lse[ubatch],
self.received_signal[ubatch],
self.epoch[ubatch : ubatch + 1],
output,
self.world_size,
self.rank,
self.max_num_tokens,
is_lse_base_on_e,
)
return output
@functools.cache
def get_direct_dcp_a2a_workspace(
group: GroupCoordinator,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype,
num_ubatches: int,
) -> DirectDCPA2AWorkspace | None:
if not _direct_dcp_enabled(
group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES
):
return None
return DirectDCPA2AWorkspace(
group.device_group,
device,
max_num_tokens,
heads_per_rank,
head_dim,
dtype,
num_ubatches,
)
def _q_gather_layout_supported(
world_size: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype,
padded_num_heads: int | None,
) -> bool:
element_size = torch.empty((), dtype=dtype).element_size()
gathered_num_heads = world_size * heads_per_rank
storage_num_heads = (
gathered_num_heads if padded_num_heads is None else padded_num_heads
)
return (
heads_per_rank * head_dim * element_size % 16 == 0
and storage_num_heads * head_dim * element_size % 16 == 0
)
class DirectDCPQGatherWorkspace(_DirectDCPWorkspace):
"""Publish query shards directly into the consumer-final symmetric buffer.
The final buffer is reusable after the downstream DCP output combine. That
combine orders all ranks after attention has consumed the gathered query.
"""
def __init__(
self,
group: ProcessGroup,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype = torch.bfloat16,
num_ubatches: int = 1,
padded_num_heads: int | None = None,
) -> None:
if num_ubatches < 1:
raise ValueError(
"Direct DCP q-gather requires at least one ubatch slot, "
f"got {num_ubatches}"
)
if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1:
raise ValueError(
"Direct DCP q-gather dimensions must be positive, got "
f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}"
)
gathered_num_heads = group.size() * heads_per_rank
if not _q_gather_layout_supported(
group.size(), heads_per_rank, head_dim, dtype, padded_num_heads
):
raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.")
super().__init__(group, device, num_ubatches)
if self.world_size <= 1:
raise ValueError("Direct DCP q-gather requires at least two ranks")
self.max_num_tokens = max_num_tokens
self.heads_per_rank = heads_per_rank
self.gathered_num_heads = gathered_num_heads
self.padded_num_heads = (
self.gathered_num_heads if padded_num_heads is None else padded_num_heads
)
if self.padded_num_heads < self.gathered_num_heads:
raise ValueError(
"Direct DCP q-gather padded heads must cover gathered heads: "
f"{self.padded_num_heads} < {self.gathered_num_heads}"
)
self.head_dim = head_dim
query_shape = (
num_ubatches,
max_num_tokens,
self.padded_num_heads,
head_dim,
)
signal_shape = (num_ubatches, 2, self.world_size)
self.final_query, _ = self._allocate(query_shape, dtype)
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
query_multicast_ptrs = self._multicast_ptrs(self.final_query)
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
self.multicast_ptrs = list(
zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True)
)
if not all(
query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs
):
raise RuntimeError(
"Direct DCP q-gather requires NVLS symmetric-memory multicast."
)
self.completion = self.received_signal.new_zeros((num_ubatches, 1))
torch.accelerator.synchronize()
def gather(self, local_query: torch.Tensor) -> torch.Tensor:
ubatch = dbo_current_ubatch_id()
if not 0 <= ubatch < self.num_ubatches:
raise ValueError(
f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
)
if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank:
raise ValueError(
f"DCP q-gather expected {self.heads_per_rank} local query heads, "
f"got {local_query.shape[1]}"
)
num_tokens = local_query.shape[0]
output = torch.as_strided(
self.final_query[ubatch],
size=(num_tokens, self.gathered_num_heads, self.head_dim),
stride=(
self.gathered_num_heads * self.head_dim,
self.head_dim,
1,
),
)
query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
torch.ops._C.direct_dcp_q_gather(
local_query,
output,
self.received_signal[ubatch],
self.completion[ubatch],
self.epoch[ubatch : ubatch + 1],
self.world_size,
self.rank,
self.max_num_tokens,
self.padded_num_heads,
query_multicast_ptr,
signal_multicast_ptr,
)
return output
@functools.cache
def get_direct_dcp_q_gather_workspace(
group: GroupCoordinator,
device: torch.device,
max_num_tokens: int,
heads_per_rank: int,
head_dim: int,
dtype: torch.dtype,
num_ubatches: int,
padded_num_heads: int | None = None,
) -> DirectDCPQGatherWorkspace | None:
if not _direct_dcp_multicast_enabled(
group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER
):
return None
if not _q_gather_layout_supported(
group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads
):
return None
return DirectDCPQGatherWorkspace(
group.device_group,
device,
max_num_tokens,
heads_per_rank,
head_dim,
dtype,
num_ubatches,
padded_num_heads,
)
_KV_GATHER_SUPPORTED_DTYPES = (
torch.float16,
torch.bfloat16,
torch.float8_e4m3fn,
)
def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool:
return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0
class DirectDCPKVGatherWorkspace(_DirectDCPWorkspace):
"""Persistent symmetric buffers for direct DCP KV gather."""
def __init__(
self,
group: ProcessGroup,
device: torch.device,
max_gathered_tokens: int,
token_dim: int,
dtype: torch.dtype = torch.bfloat16,
num_ubatches: int = 1,
) -> None:
if dtype not in _KV_GATHER_SUPPORTED_DTYPES:
raise ValueError(f"Direct DCP kv-gather does not support {dtype}")
if num_ubatches < 1:
raise ValueError(
"Direct DCP kv-gather requires at least one ubatch slot, "
f"got {num_ubatches}"
)
if max_gathered_tokens < 1 or token_dim < 1:
raise ValueError(
"Direct DCP kv-gather dimensions must be positive, got "
f"T={max_gathered_tokens}, D={token_dim}"
)
if not _kv_gather_layout_supported(token_dim, dtype):
raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.")
super().__init__(group, device, num_ubatches)
if self.world_size <= 1:
raise ValueError("Direct DCP kv-gather requires at least two ranks")
if max_gathered_tokens % self.world_size != 0:
raise ValueError(
"Direct DCP kv-gather capacity must divide evenly across "
f"ranks: {max_gathered_tokens} % {self.world_size} != 0"
)
self.max_gathered_tokens = max_gathered_tokens
kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim)
signal_shape = (num_ubatches, 2, self.world_size)
self.received_kv, _ = self._allocate(kv_shape, dtype)
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
kv_multicast_ptrs = self._multicast_ptrs(self.received_kv)
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
self.multicast_ptrs = list(
zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True)
)
if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs):
raise RuntimeError(
"Direct DCP kv-gather requires NVLS symmetric-memory multicast."
)
self.completion = self.received_signal.new_zeros((num_ubatches, 2))
torch.accelerator.synchronize()
def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None:
ubatch = dbo_current_ubatch_id()
if not 0 <= ubatch < self.num_ubatches:
raise ValueError(
f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
)
kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
torch.ops._C.direct_dcp_kv_gather(
local_kv,
self.received_kv[ubatch],
self.received_signal[ubatch],
self.completion[ubatch],
self.epoch[ubatch : ubatch + 1],
gathered_kv,
self.world_size,
self.rank,
self.max_gathered_tokens,
kv_multicast_ptr,
signal_multicast_ptr,
)
@functools.cache
def get_direct_dcp_kv_gather_workspace(
group: GroupCoordinator,
device: torch.device,
max_gathered_tokens: int,
token_dim: int,
dtype: torch.dtype,
num_ubatches: int,
) -> DirectDCPKVGatherWorkspace | None:
if not _direct_dcp_multicast_enabled(
group,
dtype,
envs.VLLM_USE_DIRECT_DCP_KV_GATHER,
_KV_GATHER_SUPPORTED_DTYPES,
):
return None
if not _kv_gather_layout_supported(token_dim, dtype):
return None
return DirectDCPKVGatherWorkspace(
group.device_group,
device,
max_gathered_tokens,
token_dim,
dtype,
num_ubatches,
)
class DCPCombine(Protocol):
def __call__(
self,
partial_output: torch.Tensor,
partial_lse: torch.Tensor,
*,
seq_lens: torch.Tensor,
query_start_loc: torch.Tensor,
) -> torch.Tensor: ...
class MLADCPManager:
"""Select and own layer-level collective implementations for MLA DCP."""
_kv_gather: Callable[[torch.Tensor, torch.Tensor], object]
def __init__(
self,
vllm_config: VllmConfig,
device: torch.device,
num_heads: int,
query_head_dim: int,
output_head_dim: int,
query_dtype: torch.dtype,
output_dtype: torch.dtype,
padded_num_heads: int | None,
is_lse_base_on_e: bool,
use_pcp: bool,
) -> None:
parallel_config = vllm_config.parallel_config
self.group = get_dcp_group()
self.device = torch.device(device)
self.num_ubatches = max(parallel_config.num_ubatches, 1)
self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config)
self.use_a2a = parallel_config.dcp_comm_backend == "a2a"
self.padded_num_heads = padded_num_heads
self.combine = self._init_combine(
num_heads,
output_head_dim,
output_dtype,
is_lse_base_on_e,
use_pcp,
)
self.query_gather = (
None
if use_pcp
else self._init_query_gather(
num_heads,
query_head_dim,
query_dtype,
)
)
def _init_combine(
self,
num_heads: int,
head_dim: int,
dtype: torch.dtype,
is_lse_base_on_e: bool,
use_pcp: bool,
) -> DCPCombine:
direct_workspace = None
if self.use_a2a:
direct_workspace = get_direct_dcp_a2a_workspace(
self.group,
self.device,
self.max_num_tokens,
num_heads,
head_dim,
dtype,
self.num_ubatches,
)
if direct_workspace is not None:
logger.info_once("Using direct symmetric-memory DCP A2A for MLA.")
return functools.partial(
direct_workspace.lse_reduce,
is_lse_base_on_e=is_lse_base_on_e,
)
combine_fn = (
dcp_a2a_lse_reduce
if self.use_a2a
else cp_lse_ag_out_ar
if use_pcp
else cp_lse_ag_out_rs
)
return functools.partial(
combine_fn,
cp_group=self.group,
is_lse_base_on_e=is_lse_base_on_e,
)
def _init_query_gather(
self,
num_heads: int,
head_dim: int,
dtype: torch.dtype,
) -> Callable[[torch.Tensor], torch.Tensor]:
direct_workspace = get_direct_dcp_q_gather_workspace(
self.group,
self.device,
self.max_num_tokens,
num_heads,
head_dim,
dtype,
self.num_ubatches,
self.padded_num_heads,
)
if direct_workspace is not None:
logger.info_once("Using direct symmetric-memory DCP query gather for MLA.")
return direct_workspace.gather
return self._gather_query
def _gather_query(self, query: torch.Tensor) -> torch.Tensor:
query = self.group.all_gather(query, dim=1)
if self.padded_num_heads is not None:
query = reserve_query_head_storage(query, self.padded_num_heads)
return query
def init_kv_gather(
self,
workspace: torch.Tensor,
max_gathered_tokens: int,
) -> None:
world_size = self.group.world_size
assert max_gathered_tokens > 0
assert max_gathered_tokens % world_size == 0
assert workspace.ndim == 2
assert workspace.is_contiguous()
assert workspace.shape[0] == (
max_gathered_tokens + max_gathered_tokens // world_size
)
assert workspace.shape[1] > 0
direct_workspace = get_direct_dcp_kv_gather_workspace(
self.group,
workspace.device,
max_gathered_tokens,
workspace.shape[1],
workspace.dtype,
self.num_ubatches,
)
if direct_workspace is not None:
logger.info_once(
"Using direct symmetric-memory DCP chunked-context KV gather for MLA."
)
self._kv_gather = direct_workspace.gather
else:
self._kv_gather = functools.partial(
torch.distributed.all_gather_into_tensor,
group=self.group.device_group,
)
def kv_gather(
self,
gathered_kv: torch.Tensor,
local_kv: torch.Tensor,
) -> object:
return self._kv_gather(gathered_kv, local_kv)