[Kimi-K3] DCP support (#50484)
Signed-off-by: girasoley <girasoley@inferact.ai> Signed-off-by: Summer Yang <girasoleyang@gmail.com> Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: 云挚 <ningyunxiao.nyx@antgroup.com> Co-authored-by: foraxe <73625538+foraxe@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5
云挚
foraxe
Woosuk Kwon
parent
3f142bd85e
commit
63ac04a61e
@@ -474,6 +474,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
endif()
|
||||
|
||||
if(VLLM_GPU_LANG STREQUAL "CUDA")
|
||||
list(APPEND VLLM_STABLE_EXT_SRC
|
||||
"csrc/libtorch_stable/attention/dcp_utils/dcp_direct_a2a_lse_reduce.cu"
|
||||
"csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu"
|
||||
"csrc/libtorch_stable/attention/dcp_utils/dcp_direct_q_gather.cu")
|
||||
|
||||
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
|
||||
|
||||
# Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
// Direct symmetric-memory DCP A2A LSE reduction.
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <math_constants.h>
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
|
||||
#include "dcp_direct_common.cuh"
|
||||
|
||||
namespace {
|
||||
|
||||
using vllm::direct_dcp::check_cuda_launch;
|
||||
using vllm::direct_dcp::get_peer_ptr;
|
||||
using vllm::direct_dcp::increment_epoch_kernel;
|
||||
using vllm::direct_dcp::store_release_system;
|
||||
using vllm::direct_dcp::wait_for_epoch;
|
||||
|
||||
__device__ __forceinline__ int64_t find_sequence(const int32_t* query_start_loc,
|
||||
int64_t token_idx,
|
||||
int64_t num_seqs) {
|
||||
int64_t left = 0;
|
||||
int64_t right = num_seqs;
|
||||
while (left < right) {
|
||||
int64_t mid = (left + right) / 2;
|
||||
if (query_start_loc[mid] <= token_idx) {
|
||||
left = mid + 1;
|
||||
} else {
|
||||
right = mid;
|
||||
}
|
||||
}
|
||||
return left - 1;
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ float to_float(scalar_t value) {
|
||||
if constexpr (std::is_same_v<scalar_t, float>) {
|
||||
return value;
|
||||
} else if constexpr (std::is_same_v<scalar_t, __half>) {
|
||||
return __half2float(value);
|
||||
} else {
|
||||
return __bfloat162float(value);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ scalar_t from_float(float value) {
|
||||
if constexpr (std::is_same_v<scalar_t, __half>) {
|
||||
return __float2half_rn(value);
|
||||
} else {
|
||||
return __float2bfloat16_rn(value);
|
||||
}
|
||||
}
|
||||
|
||||
// Send each destination's head slice and LSE to its symmetric buffer. Mask
|
||||
// undefined empty-shard rows at the source so peers give them zero weight.
|
||||
template <typename lse_t>
|
||||
__global__ void dispatch_output_lse_kernel(
|
||||
const uint4* partial_output, const lse_t* partial_lse,
|
||||
const int32_t* seq_lens, const int32_t* query_start_loc,
|
||||
const int64_t* peer_output_ptrs, const int64_t* peer_lse_ptrs,
|
||||
const int64_t* epoch_ptr, int64_t world_size, int64_t rank,
|
||||
int64_t num_tokens, int64_t max_num_tokens, int64_t num_seqs,
|
||||
int64_t heads_per_rank, int64_t head_dim, int64_t output_token_stride,
|
||||
int64_t lse_token_stride) {
|
||||
int64_t item = static_cast<int64_t>(blockIdx.x);
|
||||
int64_t destination_rank = item / num_tokens;
|
||||
int64_t token_idx = item - destination_rank * num_tokens;
|
||||
|
||||
__shared__ bool empty_kv;
|
||||
if (threadIdx.x == 0) {
|
||||
empty_kv = false;
|
||||
if (seq_lens != nullptr) {
|
||||
int64_t seq_idx = find_sequence(query_start_loc, token_idx, num_seqs);
|
||||
empty_kv = seq_lens[seq_idx] == 0;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
|
||||
int64_t parity = static_cast<int64_t>(epoch & 1u);
|
||||
int64_t destination_item =
|
||||
((parity * world_size + rank) * max_num_tokens + token_idx) *
|
||||
heads_per_rank;
|
||||
int64_t source_head = destination_rank * heads_per_rank;
|
||||
|
||||
if (!empty_kv) {
|
||||
uint4* peer_output =
|
||||
get_peer_ptr<uint4>(peer_output_ptrs, destination_rank);
|
||||
int64_t vectors_per_item = heads_per_rank * head_dim / 8;
|
||||
int64_t source_vector =
|
||||
(token_idx * output_token_stride + source_head * head_dim) / 8;
|
||||
int64_t destination_vector = destination_item * head_dim / 8;
|
||||
for (int64_t vector_idx = threadIdx.x; vector_idx < vectors_per_item;
|
||||
vector_idx += blockDim.x) {
|
||||
peer_output[destination_vector + vector_idx] =
|
||||
partial_output[source_vector + vector_idx];
|
||||
}
|
||||
}
|
||||
|
||||
float* peer_lse = get_peer_ptr<float>(peer_lse_ptrs, destination_rank);
|
||||
int64_t source_lse = token_idx * lse_token_stride + source_head;
|
||||
for (int64_t head_idx = threadIdx.x; head_idx < heads_per_rank;
|
||||
head_idx += blockDim.x) {
|
||||
peer_lse[destination_item + head_idx] =
|
||||
empty_kv ? -CUDART_INF_F : to_float(partial_lse[source_lse + head_idx]);
|
||||
}
|
||||
}
|
||||
|
||||
// Publish completion after the stream-ordered payload writes.
|
||||
__global__ void signal_kernel(const int64_t* peer_signal_ptrs,
|
||||
const int64_t* epoch_ptr, int64_t world_size,
|
||||
int64_t rank) {
|
||||
int64_t destination_rank =
|
||||
static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
if (destination_rank >= world_size) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
|
||||
int64_t parity = static_cast<int64_t>(epoch & 1u);
|
||||
uint32_t* peer_signal =
|
||||
get_peer_ptr<uint32_t>(peer_signal_ptrs, destination_rank);
|
||||
int64_t signal_item = parity * world_size + rank;
|
||||
store_release_system(peer_signal + signal_item, epoch);
|
||||
}
|
||||
|
||||
// Form stable base-2 LSE weights and combine in FP32.
|
||||
template <typename scalar_t>
|
||||
__global__ void wait_lse_combine_kernel(
|
||||
const scalar_t* received_output, const float* received_lse,
|
||||
const uint32_t* received_signal, const int64_t* epoch_ptr,
|
||||
scalar_t* combined_output, int64_t world_size, int64_t num_tokens,
|
||||
int64_t max_num_tokens, int64_t heads_per_rank, int64_t head_dim,
|
||||
bool is_lse_base_on_e) {
|
||||
extern __shared__ float weights[];
|
||||
|
||||
int64_t item = static_cast<int64_t>(blockIdx.x);
|
||||
int64_t token_idx = item / heads_per_rank;
|
||||
int64_t head_idx = item - token_idx * heads_per_rank;
|
||||
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
|
||||
int64_t parity = static_cast<int64_t>(epoch & 1u);
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
float lse_max = -CUDART_INF_F;
|
||||
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
|
||||
int64_t source_item =
|
||||
((parity * world_size + source_rank) * max_num_tokens + token_idx) *
|
||||
heads_per_rank +
|
||||
head_idx;
|
||||
int64_t signal_item = parity * world_size + source_rank;
|
||||
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
|
||||
printf(
|
||||
"direct DCP A2A timeout source=%lld token=%lld head=%lld "
|
||||
"epoch=%u\n",
|
||||
static_cast<long long>(source_rank),
|
||||
static_cast<long long>(token_idx), static_cast<long long>(head_idx),
|
||||
epoch);
|
||||
asm volatile("trap;");
|
||||
}
|
||||
float value = received_lse[source_item];
|
||||
if (isnan(value) || value == CUDART_INF_F) {
|
||||
value = -CUDART_INF_F;
|
||||
}
|
||||
if (is_lse_base_on_e) {
|
||||
value *= CUDART_L2E_F;
|
||||
}
|
||||
weights[source_rank] = value;
|
||||
lse_max = fmaxf(lse_max, value);
|
||||
}
|
||||
if (lse_max == -CUDART_INF_F) {
|
||||
lse_max = 0.0f;
|
||||
}
|
||||
|
||||
float lse_sum = 0.0f;
|
||||
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
|
||||
lse_sum += exp2f(weights[source_rank] - lse_max);
|
||||
}
|
||||
float inverse_lse_sum = lse_sum > 0.0f ? 1.0f / lse_sum : 0.0f;
|
||||
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
|
||||
weights[source_rank] =
|
||||
exp2f(weights[source_rank] - lse_max) * inverse_lse_sum;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int64_t dim_idx = threadIdx.x; dim_idx < head_dim;
|
||||
dim_idx += blockDim.x) {
|
||||
float accumulator = 0.0f;
|
||||
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
|
||||
// Empty shards leave stale payloads, so never read zero-weight sources.
|
||||
float weight = weights[source_rank];
|
||||
if (weight == 0.0f) {
|
||||
continue;
|
||||
}
|
||||
int64_t source_item =
|
||||
((parity * world_size + source_rank) * max_num_tokens + token_idx) *
|
||||
heads_per_rank +
|
||||
head_idx;
|
||||
accumulator +=
|
||||
to_float(received_output[source_item * head_dim + dim_idx]) * weight;
|
||||
}
|
||||
combined_output[item * head_dim + dim_idx] =
|
||||
from_float<scalar_t>(accumulator);
|
||||
}
|
||||
}
|
||||
|
||||
void direct_dcp_a2a_lse_reduce(
|
||||
const torch::stable::Tensor& partial_output,
|
||||
const torch::stable::Tensor& partial_lse,
|
||||
const std::optional<torch::stable::Tensor>& seq_lens,
|
||||
const std::optional<torch::stable::Tensor>& query_start_loc,
|
||||
const torch::stable::Tensor& peer_output_ptrs,
|
||||
const torch::stable::Tensor& peer_lse_ptrs,
|
||||
const torch::stable::Tensor& peer_signal_ptrs,
|
||||
torch::stable::Tensor& received_output, torch::stable::Tensor& received_lse,
|
||||
torch::stable::Tensor& received_signal, torch::stable::Tensor& epoch,
|
||||
torch::stable::Tensor& combined_output, int64_t world_size, int64_t rank,
|
||||
int64_t max_num_tokens, bool is_lse_base_on_e) {
|
||||
STD_TORCH_CHECK(partial_output.is_cuda() && partial_lse.is_cuda(),
|
||||
"partial output and LSE must be CUDA tensors");
|
||||
auto output_dtype = partial_output.scalar_type();
|
||||
STD_TORCH_CHECK(output_dtype == torch::headeronly::ScalarType::Half ||
|
||||
output_dtype == torch::headeronly::ScalarType::BFloat16,
|
||||
"direct DCP A2A only supports FP16 and BF16 output");
|
||||
auto lse_dtype = partial_lse.scalar_type();
|
||||
STD_TORCH_CHECK(lse_dtype == torch::headeronly::ScalarType::Float ||
|
||||
lse_dtype == torch::headeronly::ScalarType::Half ||
|
||||
lse_dtype == torch::headeronly::ScalarType::BFloat16,
|
||||
"partial LSE must be FP32, FP16, or BF16");
|
||||
STD_TORCH_CHECK(partial_output.dim() == 3 && partial_lse.dim() == 2,
|
||||
"expected output [T,H,D] and LSE [T,H]");
|
||||
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
|
||||
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
|
||||
|
||||
int64_t num_tokens = partial_output.size(0);
|
||||
int64_t total_heads = partial_output.size(1);
|
||||
int64_t head_dim = partial_output.size(2);
|
||||
int64_t output_token_stride = partial_output.stride(0);
|
||||
int64_t lse_token_stride = partial_lse.stride(0);
|
||||
STD_TORCH_CHECK(
|
||||
partial_output.stride(2) == 1 && partial_output.stride(1) == head_dim &&
|
||||
output_token_stride >= total_heads * head_dim &&
|
||||
output_token_stride % 8 == 0,
|
||||
"partial output must have packed heads and an aligned token stride");
|
||||
STD_TORCH_CHECK(partial_lse.stride(1) == 1 && lse_token_stride >= total_heads,
|
||||
"partial LSE must have packed heads");
|
||||
STD_TORCH_CHECK(num_tokens > 0 && num_tokens <= max_num_tokens,
|
||||
"token count exceeds symmetric buffer capacity");
|
||||
STD_TORCH_CHECK(total_heads % world_size == 0,
|
||||
"attention heads must divide evenly across DCP ranks");
|
||||
STD_TORCH_CHECK(
|
||||
partial_lse.size(0) == num_tokens && partial_lse.size(1) == total_heads,
|
||||
"LSE shape must match attention output");
|
||||
STD_TORCH_CHECK(head_dim % 8 == 0,
|
||||
"head_dim must be divisible by 8 for 16-byte stores");
|
||||
int64_t heads_per_rank = total_heads / world_size;
|
||||
STD_TORCH_CHECK(combined_output.scalar_type() == output_dtype &&
|
||||
combined_output.is_contiguous() &&
|
||||
combined_output.is_cuda(),
|
||||
"combined output must match the contiguous CUDA input");
|
||||
STD_TORCH_CHECK(combined_output.size(0) == num_tokens &&
|
||||
combined_output.size(1) == heads_per_rank &&
|
||||
combined_output.size(2) == head_dim,
|
||||
"combined output has the wrong shape");
|
||||
STD_TORCH_CHECK(
|
||||
received_output.scalar_type() == output_dtype &&
|
||||
received_lse.scalar_type() == torch::headeronly::ScalarType::Float &&
|
||||
received_signal.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"invalid symmetric staging dtypes");
|
||||
STD_TORCH_CHECK(
|
||||
peer_output_ptrs.scalar_type() == torch::headeronly::ScalarType::Long &&
|
||||
peer_lse_ptrs.scalar_type() == torch::headeronly::ScalarType::Long &&
|
||||
peer_signal_ptrs.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"peer pointer tables must be int64");
|
||||
const int32_t* seq_lens_ptr = nullptr;
|
||||
const int32_t* query_start_loc_ptr = nullptr;
|
||||
int64_t num_seqs = 0;
|
||||
STD_TORCH_CHECK(seq_lens.has_value() == query_start_loc.has_value(),
|
||||
"seq_lens and query_start_loc must be provided together");
|
||||
if (seq_lens.has_value() && query_start_loc.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
seq_lens->is_cuda() && seq_lens->dim() == 1 &&
|
||||
seq_lens->stride(0) == 1 &&
|
||||
seq_lens->scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"seq_lens must be a contiguous 1-D int32 CUDA tensor");
|
||||
STD_TORCH_CHECK(
|
||||
query_start_loc->is_cuda() && query_start_loc->dim() == 1 &&
|
||||
query_start_loc->stride(0) == 1 &&
|
||||
query_start_loc->scalar_type() ==
|
||||
torch::headeronly::ScalarType::Int,
|
||||
"query_start_loc must be a contiguous 1-D int32 CUDA tensor");
|
||||
num_seqs = seq_lens->size(0);
|
||||
STD_TORCH_CHECK(num_seqs > 0 && query_start_loc->size(0) == num_seqs + 1,
|
||||
"query_start_loc must contain one boundary per sequence");
|
||||
seq_lens_ptr = seq_lens->const_data_ptr<int32_t>();
|
||||
query_start_loc_ptr = query_start_loc->const_data_ptr<int32_t>();
|
||||
}
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
partial_output.get_device_index());
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
constexpr int kExchangeThreads = 256;
|
||||
constexpr int kCombineThreads = 128;
|
||||
|
||||
increment_epoch_kernel<<<1, 1, 0, stream>>>(
|
||||
epoch.mutable_data_ptr<int64_t>());
|
||||
check_cuda_launch("direct DCP A2A");
|
||||
int64_t dispatch_blocks = world_size * num_tokens;
|
||||
auto launch_dispatch = [&]<typename lse_t>() {
|
||||
dispatch_output_lse_kernel<lse_t>
|
||||
<<<dispatch_blocks, kExchangeThreads, 0, stream>>>(
|
||||
reinterpret_cast<const uint4*>(partial_output.data_ptr()),
|
||||
reinterpret_cast<const lse_t*>(partial_lse.data_ptr()),
|
||||
seq_lens_ptr, query_start_loc_ptr,
|
||||
peer_output_ptrs.const_data_ptr<int64_t>(),
|
||||
peer_lse_ptrs.const_data_ptr<int64_t>(),
|
||||
epoch.const_data_ptr<int64_t>(), world_size, rank, num_tokens,
|
||||
max_num_tokens, num_seqs, heads_per_rank, head_dim,
|
||||
output_token_stride, lse_token_stride);
|
||||
};
|
||||
if (lse_dtype == torch::headeronly::ScalarType::Float) {
|
||||
launch_dispatch.operator()<float>();
|
||||
} else if (lse_dtype == torch::headeronly::ScalarType::BFloat16) {
|
||||
launch_dispatch.operator()<__nv_bfloat16>();
|
||||
} else {
|
||||
launch_dispatch.operator()<__half>();
|
||||
}
|
||||
check_cuda_launch("direct DCP A2A");
|
||||
int64_t signal_items = world_size;
|
||||
int64_t signal_blocks =
|
||||
(signal_items + kExchangeThreads - 1) / kExchangeThreads;
|
||||
signal_kernel<<<signal_blocks, kExchangeThreads, 0, stream>>>(
|
||||
peer_signal_ptrs.const_data_ptr<int64_t>(),
|
||||
epoch.const_data_ptr<int64_t>(), world_size, rank);
|
||||
check_cuda_launch("direct DCP A2A");
|
||||
int64_t combine_blocks = num_tokens * heads_per_rank;
|
||||
size_t shared_memory_bytes = world_size * sizeof(float);
|
||||
auto launch_combine = [&]<typename scalar_t>() {
|
||||
wait_lse_combine_kernel<scalar_t>
|
||||
<<<combine_blocks, kCombineThreads, shared_memory_bytes, stream>>>(
|
||||
reinterpret_cast<const scalar_t*>(received_output.data_ptr()),
|
||||
received_lse.const_data_ptr<float>(),
|
||||
reinterpret_cast<const uint32_t*>(
|
||||
received_signal.const_data_ptr<int32_t>()),
|
||||
epoch.const_data_ptr<int64_t>(),
|
||||
reinterpret_cast<scalar_t*>(combined_output.mutable_data_ptr()),
|
||||
world_size, num_tokens, max_num_tokens, heads_per_rank, head_dim,
|
||||
is_lse_base_on_e);
|
||||
};
|
||||
if (output_dtype == torch::headeronly::ScalarType::BFloat16) {
|
||||
launch_combine.operator()<__nv_bfloat16>();
|
||||
} else {
|
||||
launch_combine.operator()<__half>();
|
||||
}
|
||||
check_cuda_launch("direct DCP A2A");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_a2a_ops) {
|
||||
direct_dcp_a2a_ops.def(
|
||||
"direct_dcp_a2a_lse_reduce("
|
||||
"Tensor partial_output, Tensor partial_lse, Tensor? seq_lens, "
|
||||
"Tensor? query_start_loc, Tensor peer_output_ptrs, "
|
||||
"Tensor peer_lse_ptrs, Tensor peer_signal_ptrs, Tensor! received_output, "
|
||||
"Tensor! received_lse, Tensor! received_signal, Tensor! epoch, "
|
||||
"Tensor! combined_output, int world_size, int rank, "
|
||||
"int max_num_tokens, bool is_lse_base_on_e) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_a2a_ops) {
|
||||
direct_dcp_a2a_ops.impl("direct_dcp_a2a_lse_reduce",
|
||||
TORCH_BOX(&direct_dcp_a2a_lse_reduce));
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "../../torch_utils.h"
|
||||
|
||||
namespace vllm::direct_dcp {
|
||||
|
||||
constexpr uint64_t kSpinLimit = 100000000;
|
||||
|
||||
// Advance the invocation ID; its low bit selects one of two staging slots.
|
||||
static __global__ void increment_epoch_kernel(int64_t* epoch) {
|
||||
if (blockIdx.x == 0 && threadIdx.x == 0) {
|
||||
epoch[0] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ __forceinline__ T* get_peer_ptr(const int64_t* peer_ptrs,
|
||||
int64_t peer) {
|
||||
return reinterpret_cast<T*>(static_cast<uintptr_t>(peer_ptrs[peer]));
|
||||
}
|
||||
|
||||
// Replicate one 16-byte payload to every symmetric-buffer replica.
|
||||
__device__ __forceinline__ void multimem_store_16(uint4* mc_ptr, uint4 value) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
asm volatile("multimem.st.relaxed.sys.global.v4.f32 [%0], {%1,%2,%3,%4};"
|
||||
:
|
||||
: "l"(mc_ptr), "r"(value.x), "r"(value.y), "r"(value.z),
|
||||
"r"(value.w)
|
||||
: "memory");
|
||||
#else
|
||||
asm volatile("trap;");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Publish prior system-scope writes and signal every replica.
|
||||
__device__ __forceinline__ void multimem_store_release_system(uint32_t* mc_ptr,
|
||||
uint32_t value) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
asm volatile("multimem.st.release.sys.global.u32 [%0], %1;"
|
||||
:
|
||||
: "l"(mc_ptr), "r"(value)
|
||||
: "memory");
|
||||
#else
|
||||
asm volatile("trap;");
|
||||
#endif
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void store_release_system(uint32_t* ptr,
|
||||
uint32_t value) {
|
||||
uint64_t address = reinterpret_cast<uint64_t>(ptr);
|
||||
asm volatile("st.global.release.sys.u32 [%0], %1;"
|
||||
:
|
||||
: "l"(address), "r"(value)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t load_acquire_system(const uint32_t* ptr) {
|
||||
uint32_t value;
|
||||
uint64_t address = reinterpret_cast<uint64_t>(ptr);
|
||||
asm volatile("ld.global.acquire.sys.u32 %0, [%1];"
|
||||
: "=r"(value)
|
||||
: "l"(address)
|
||||
: "memory");
|
||||
return value;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ bool wait_for_epoch(const uint32_t* ptr,
|
||||
uint32_t epoch) {
|
||||
for (uint64_t spins = 0; spins < kSpinLimit; ++spins) {
|
||||
if (load_acquire_system(ptr) == epoch) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
inline void check_cuda_launch(const char* operation) {
|
||||
cudaError_t error = cudaGetLastError();
|
||||
STD_TORCH_CHECK(error == cudaSuccess,
|
||||
std::string(operation) +
|
||||
" kernel launch failed: " + cudaGetErrorString(error));
|
||||
}
|
||||
|
||||
} // namespace vllm::direct_dcp
|
||||
@@ -0,0 +1,209 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
// Direct symmetric-memory DCP KV gather.
|
||||
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include "dcp_direct_common.cuh"
|
||||
|
||||
namespace {
|
||||
|
||||
using vllm::direct_dcp::check_cuda_launch;
|
||||
using vllm::direct_dcp::increment_epoch_kernel;
|
||||
using vllm::direct_dcp::multimem_store_16;
|
||||
using vllm::direct_dcp::multimem_store_release_system;
|
||||
using vllm::direct_dcp::wait_for_epoch;
|
||||
|
||||
constexpr int kThreads = 256;
|
||||
// KV chunks need many blocks in flight to saturate the fabric.
|
||||
constexpr int64_t kMaxMulticastBlocks = 128;
|
||||
constexpr int64_t kMaxCopyBlocks = 1024;
|
||||
|
||||
// Multicast each rank's KV slice and completion epoch to every replica.
|
||||
__global__ void direct_dcp_kv_gather_multimem_kernel(
|
||||
const uint4* local_kv, uint4* mc_kv, uint32_t* mc_signal,
|
||||
const uint32_t* received_signal, const int64_t* epoch_ptr,
|
||||
uint32_t* completion, int64_t world_size, int64_t rank, int64_t slice_bytes,
|
||||
int64_t slot_stride_bytes) {
|
||||
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
|
||||
int64_t buffer_slot = static_cast<int64_t>(epoch & 1u);
|
||||
int64_t item_count = slice_bytes / sizeof(uint4);
|
||||
mc_kv += buffer_slot * slot_stride_bytes / sizeof(uint4);
|
||||
mc_kv += rank * item_count;
|
||||
|
||||
int64_t item_stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
|
||||
for (int64_t item =
|
||||
static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
item < item_count; item += item_stride) {
|
||||
multimem_store_16(mc_kv + item, local_kv[item]);
|
||||
}
|
||||
|
||||
// Publish all multicast writes before incrementing completion.
|
||||
__threadfence_system();
|
||||
__syncthreads();
|
||||
if (threadIdx.x != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t completed = atomicAdd(completion + buffer_slot, 1u);
|
||||
if (completed + 1u != gridDim.x) {
|
||||
return;
|
||||
}
|
||||
atomicExch(completion + buffer_slot, 0u);
|
||||
|
||||
multimem_store_release_system(mc_signal + buffer_slot * world_size + rank,
|
||||
epoch);
|
||||
|
||||
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
|
||||
int64_t signal_item = buffer_slot * world_size + source_rank;
|
||||
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
|
||||
printf("direct DCP kv-gather multimem timeout source=%lld epoch=%u\n",
|
||||
static_cast<long long>(source_rank), epoch);
|
||||
asm volatile("trap;");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Materialize the acquired slot into the caller's workspace.
|
||||
template <typename copy_t>
|
||||
__global__ void materialize_kv_gather_kernel(const copy_t* received_kv,
|
||||
const int64_t* epoch_ptr,
|
||||
copy_t* gathered_kv,
|
||||
int64_t gathered_item_count,
|
||||
int64_t slot_stride_items) {
|
||||
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
|
||||
int64_t buffer_slot = static_cast<int64_t>(epoch & 1u);
|
||||
received_kv += buffer_slot * slot_stride_items;
|
||||
int64_t item = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
int64_t item_stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
|
||||
for (; item < gathered_item_count; item += item_stride) {
|
||||
gathered_kv[item] = received_kv[item];
|
||||
}
|
||||
}
|
||||
|
||||
void direct_dcp_kv_gather(
|
||||
const torch::stable::Tensor& local_kv, torch::stable::Tensor& received_kv,
|
||||
torch::stable::Tensor& received_signal, torch::stable::Tensor& completion,
|
||||
torch::stable::Tensor& epoch, torch::stable::Tensor& gathered_kv,
|
||||
int64_t world_size, int64_t rank, int64_t max_gathered_tokens,
|
||||
int64_t kv_mc_ptr, int64_t signal_mc_ptr) {
|
||||
using torch::headeronly::ScalarType;
|
||||
|
||||
STD_TORCH_CHECK(local_kv.is_cuda(), "local kv must be a CUDA tensor");
|
||||
ScalarType dtype = local_kv.scalar_type();
|
||||
STD_TORCH_CHECK(dtype == ScalarType::Half || dtype == ScalarType::BFloat16 ||
|
||||
dtype == ScalarType::Float8_e4m3fn,
|
||||
"direct DCP kv-gather only supports FP16, BF16, and FP8");
|
||||
STD_TORCH_CHECK(local_kv.dim() == 2 && local_kv.is_contiguous(),
|
||||
"local kv must be a contiguous [T,D] tensor");
|
||||
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
|
||||
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
|
||||
|
||||
int64_t num_tokens = local_kv.size(0);
|
||||
int64_t token_dim = local_kv.size(1);
|
||||
int64_t element_size = local_kv.element_size();
|
||||
STD_TORCH_CHECK(num_tokens > 0 && token_dim > 0,
|
||||
"local kv dimensions must be positive");
|
||||
STD_TORCH_CHECK(num_tokens * world_size <= max_gathered_tokens,
|
||||
"gathered kv exceeds symmetric kv-gather buffer capacity");
|
||||
|
||||
STD_TORCH_CHECK(received_kv.is_cuda() && received_kv.scalar_type() == dtype &&
|
||||
received_kv.is_contiguous() && received_kv.dim() == 3 &&
|
||||
received_kv.size(0) == 2 &&
|
||||
received_kv.size(1) == max_gathered_tokens &&
|
||||
received_kv.size(2) == token_dim,
|
||||
"received kv has the wrong symmetric buffer layout");
|
||||
STD_TORCH_CHECK(
|
||||
received_signal.is_cuda() && received_signal.is_contiguous() &&
|
||||
received_signal.scalar_type() == ScalarType::Int &&
|
||||
received_signal.dim() == 2 && received_signal.size(0) == 2 &&
|
||||
received_signal.size(1) == world_size,
|
||||
"received signal has the wrong symmetric buffer layout");
|
||||
STD_TORCH_CHECK(completion.is_cuda() && completion.is_contiguous() &&
|
||||
completion.scalar_type() == ScalarType::Int &&
|
||||
completion.numel() == 2,
|
||||
"completion counter must be a two-element CUDA int32 tensor");
|
||||
STD_TORCH_CHECK(epoch.is_cuda() && epoch.is_contiguous() &&
|
||||
epoch.scalar_type() == ScalarType::Long &&
|
||||
epoch.numel() == 1,
|
||||
"epoch must be a one-element CUDA int64 tensor");
|
||||
STD_TORCH_CHECK(gathered_kv.is_cuda() && gathered_kv.scalar_type() == dtype &&
|
||||
gathered_kv.is_contiguous() && gathered_kv.dim() == 2 &&
|
||||
gathered_kv.size(0) == num_tokens * world_size &&
|
||||
gathered_kv.size(1) == token_dim,
|
||||
"gathered kv must be contiguous with shape [world_size*T,D]");
|
||||
int64_t device_index = local_kv.get_device_index();
|
||||
STD_TORCH_CHECK(
|
||||
received_kv.get_device_index() == device_index &&
|
||||
received_signal.get_device_index() == device_index &&
|
||||
completion.get_device_index() == device_index &&
|
||||
epoch.get_device_index() == device_index &&
|
||||
gathered_kv.get_device_index() == device_index,
|
||||
"direct DCP kv-gather tensors must be on the same CUDA device");
|
||||
|
||||
int64_t slice_bytes = num_tokens * token_dim * element_size;
|
||||
int64_t slot_stride_bytes = max_gathered_tokens * token_dim * element_size;
|
||||
bool vectorized =
|
||||
reinterpret_cast<uintptr_t>(local_kv.data_ptr()) % alignof(uint4) == 0 &&
|
||||
reinterpret_cast<uintptr_t>(received_kv.data_ptr()) % alignof(uint4) ==
|
||||
0 &&
|
||||
reinterpret_cast<uintptr_t>(gathered_kv.data_ptr()) % alignof(uint4) ==
|
||||
0 &&
|
||||
slice_bytes % sizeof(uint4) == 0 &&
|
||||
slot_stride_bytes % sizeof(uint4) == 0;
|
||||
STD_TORCH_CHECK(
|
||||
vectorized,
|
||||
"direct DCP kv-gather requires 16-byte-aligned pointers and strides");
|
||||
STD_TORCH_CHECK(kv_mc_ptr != 0 && signal_mc_ptr != 0,
|
||||
"direct DCP kv-gather requires multicast pointers");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(device_index);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
increment_epoch_kernel<<<1, 1, 0, stream>>>(
|
||||
epoch.mutable_data_ptr<int64_t>());
|
||||
check_cuda_launch("direct DCP kv-gather");
|
||||
|
||||
int64_t item_count = slice_bytes / sizeof(uint4);
|
||||
int64_t blocks = (item_count + kThreads - 1) / kThreads;
|
||||
blocks = blocks < kMaxMulticastBlocks ? blocks : kMaxMulticastBlocks;
|
||||
direct_dcp_kv_gather_multimem_kernel<<<blocks, kThreads, 0, stream>>>(
|
||||
reinterpret_cast<const uint4*>(local_kv.data_ptr()),
|
||||
reinterpret_cast<uint4*>(static_cast<uintptr_t>(kv_mc_ptr)),
|
||||
reinterpret_cast<uint32_t*>(static_cast<uintptr_t>(signal_mc_ptr)),
|
||||
reinterpret_cast<const uint32_t*>(
|
||||
received_signal.const_data_ptr<int32_t>()),
|
||||
epoch.const_data_ptr<int64_t>(),
|
||||
reinterpret_cast<uint32_t*>(completion.mutable_data_ptr<int32_t>()),
|
||||
world_size, rank, slice_bytes, slot_stride_bytes);
|
||||
check_cuda_launch("direct DCP kv-gather");
|
||||
|
||||
int64_t gathered_item_count = world_size * item_count;
|
||||
int64_t copy_blocks = (gathered_item_count + kThreads - 1) / kThreads;
|
||||
copy_blocks = copy_blocks < kMaxCopyBlocks ? copy_blocks : kMaxCopyBlocks;
|
||||
materialize_kv_gather_kernel<uint4><<<copy_blocks, kThreads, 0, stream>>>(
|
||||
reinterpret_cast<const uint4*>(received_kv.data_ptr()),
|
||||
epoch.const_data_ptr<int64_t>(),
|
||||
reinterpret_cast<uint4*>(gathered_kv.mutable_data_ptr()),
|
||||
gathered_item_count, slot_stride_bytes / sizeof(uint4));
|
||||
check_cuda_launch("direct DCP kv-gather");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_kv_gather_ops) {
|
||||
direct_dcp_kv_gather_ops.def(
|
||||
"direct_dcp_kv_gather("
|
||||
"Tensor local_kv, Tensor! received_kv, Tensor! received_signal, "
|
||||
"Tensor! completion, "
|
||||
"Tensor! epoch, Tensor! gathered_kv, "
|
||||
"int world_size, int rank, int max_gathered_tokens, "
|
||||
"int kv_mc_ptr, int signal_mc_ptr) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_kv_gather_ops) {
|
||||
direct_dcp_kv_gather_ops.impl("direct_dcp_kv_gather",
|
||||
TORCH_BOX(&direct_dcp_kv_gather));
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
// Direct symmetric-memory DCP query gather into the consumer-final buffer.
|
||||
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include "dcp_direct_common.cuh"
|
||||
|
||||
namespace {
|
||||
|
||||
using vllm::direct_dcp::check_cuda_launch;
|
||||
using vllm::direct_dcp::increment_epoch_kernel;
|
||||
using vllm::direct_dcp::multimem_store_16;
|
||||
using vllm::direct_dcp::multimem_store_release_system;
|
||||
using vllm::direct_dcp::wait_for_epoch;
|
||||
|
||||
// Multicast each rank's head slice directly into every consumer's final query
|
||||
// buffer. Reuse is ordered by the downstream DCP output synchronization.
|
||||
__global__ void direct_dcp_q_gather_multimem_kernel(
|
||||
const uint4* local_query, uint4* mc_final_query, uint32_t* mc_signal,
|
||||
const uint32_t* received_signal, int64_t* epoch_ptr, uint32_t* completion,
|
||||
int64_t world_size, int64_t rank, int64_t num_tokens,
|
||||
int64_t bytes_per_token, int64_t query_token_stride_bytes,
|
||||
int64_t destination_token_stride_bytes) {
|
||||
// The common one-token decode case uses one block. Fold the epoch update
|
||||
// into that publication kernel to avoid an otherwise separate tiny launch.
|
||||
if (gridDim.x == 1 && threadIdx.x == 0) {
|
||||
epoch_ptr[0] += 1;
|
||||
}
|
||||
__syncthreads();
|
||||
uint32_t epoch = static_cast<uint32_t>(epoch_ptr[0]);
|
||||
int64_t signal_slot = static_cast<int64_t>(epoch & 1u);
|
||||
|
||||
int64_t items_per_token = bytes_per_token / sizeof(uint4);
|
||||
int64_t source_token_stride = query_token_stride_bytes / sizeof(uint4);
|
||||
int64_t destination_token_stride =
|
||||
destination_token_stride_bytes / sizeof(uint4);
|
||||
int64_t destination_head_offset = rank * items_per_token;
|
||||
for (int64_t token_idx = blockIdx.x; token_idx < num_tokens;
|
||||
token_idx += gridDim.x) {
|
||||
for (int64_t token_item = threadIdx.x; token_item < items_per_token;
|
||||
token_item += blockDim.x) {
|
||||
multimem_store_16(
|
||||
mc_final_query + token_idx * destination_token_stride +
|
||||
destination_head_offset + token_item,
|
||||
local_query[token_idx * source_token_stride + token_item]);
|
||||
}
|
||||
}
|
||||
|
||||
// Publish all multicast writes before incrementing completion.
|
||||
__threadfence_system();
|
||||
__syncthreads();
|
||||
if (threadIdx.x != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (gridDim.x > 1) {
|
||||
uint32_t completed = atomicAdd(completion, 1u);
|
||||
if (completed + 1u != gridDim.x) {
|
||||
return;
|
||||
}
|
||||
atomicExch(completion, 0u);
|
||||
}
|
||||
|
||||
multimem_store_release_system(mc_signal + signal_slot * world_size + rank,
|
||||
epoch);
|
||||
|
||||
for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) {
|
||||
int64_t signal_item = signal_slot * world_size + source_rank;
|
||||
if (!wait_for_epoch(received_signal + signal_item, epoch)) {
|
||||
printf("direct DCP q-gather multimem timeout source=%lld epoch=%u\n",
|
||||
static_cast<long long>(source_rank), epoch);
|
||||
asm volatile("trap;");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void direct_dcp_q_gather(const torch::stable::Tensor& local_query,
|
||||
torch::stable::Tensor& final_query,
|
||||
torch::stable::Tensor& received_signal,
|
||||
torch::stable::Tensor& completion,
|
||||
torch::stable::Tensor& epoch, int64_t world_size,
|
||||
int64_t rank, int64_t max_num_tokens,
|
||||
int64_t padded_num_heads, int64_t query_mc_ptr,
|
||||
int64_t signal_mc_ptr) {
|
||||
using torch::headeronly::ScalarType;
|
||||
|
||||
STD_TORCH_CHECK(local_query.is_cuda(), "local query must be a CUDA tensor");
|
||||
ScalarType dtype = local_query.scalar_type();
|
||||
STD_TORCH_CHECK(local_query.dim() == 3,
|
||||
"local query must have shape [T,H,D]");
|
||||
STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1");
|
||||
STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank");
|
||||
|
||||
int64_t num_tokens = local_query.size(0);
|
||||
int64_t heads_per_rank = local_query.size(1);
|
||||
int64_t head_dim = local_query.size(2);
|
||||
int64_t gathered_num_heads = world_size * heads_per_rank;
|
||||
int64_t element_size = local_query.element_size();
|
||||
STD_TORCH_CHECK(num_tokens > 0 && num_tokens <= max_num_tokens,
|
||||
"token count exceeds symmetric q-gather buffer capacity");
|
||||
STD_TORCH_CHECK(heads_per_rank > 0 && head_dim > 0,
|
||||
"query head dimensions must be positive");
|
||||
STD_TORCH_CHECK(padded_num_heads >= gathered_num_heads,
|
||||
"padded query heads must cover all gathered heads");
|
||||
STD_TORCH_CHECK(local_query.stride(2) == 1 &&
|
||||
local_query.stride(1) == head_dim &&
|
||||
local_query.stride(0) >= heads_per_rank * head_dim,
|
||||
"local query must have packed heads");
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
final_query.is_cuda() && final_query.scalar_type() == dtype &&
|
||||
final_query.is_contiguous() && final_query.dim() == 3 &&
|
||||
final_query.size(0) == num_tokens &&
|
||||
final_query.size(1) == gathered_num_heads &&
|
||||
final_query.size(2) == head_dim,
|
||||
"final query must be contiguous with shape [T,world_size*H,D]");
|
||||
STD_TORCH_CHECK(
|
||||
received_signal.is_cuda() && received_signal.is_contiguous() &&
|
||||
received_signal.scalar_type() == ScalarType::Int &&
|
||||
received_signal.dim() == 2 && received_signal.size(0) == 2 &&
|
||||
received_signal.size(1) == world_size,
|
||||
"received signal has the wrong symmetric buffer layout");
|
||||
STD_TORCH_CHECK(completion.is_cuda() && completion.is_contiguous() &&
|
||||
completion.scalar_type() == ScalarType::Int &&
|
||||
completion.numel() == 1,
|
||||
"completion counter must be one CUDA int32 tensor");
|
||||
STD_TORCH_CHECK(epoch.is_cuda() && epoch.is_contiguous() &&
|
||||
epoch.scalar_type() == ScalarType::Long &&
|
||||
epoch.numel() == 1,
|
||||
"epoch must be a one-element CUDA int64 tensor");
|
||||
int64_t device_index = local_query.get_device_index();
|
||||
STD_TORCH_CHECK(
|
||||
final_query.get_device_index() == device_index &&
|
||||
received_signal.get_device_index() == device_index &&
|
||||
completion.get_device_index() == device_index &&
|
||||
epoch.get_device_index() == device_index,
|
||||
"direct DCP q-gather tensors must be on the same CUDA device");
|
||||
|
||||
int64_t query_token_stride_bytes = local_query.stride(0) * element_size;
|
||||
int64_t bytes_per_token = heads_per_rank * head_dim * element_size;
|
||||
int64_t gathered_token_stride_bytes =
|
||||
gathered_num_heads * head_dim * element_size;
|
||||
bool vectorized =
|
||||
reinterpret_cast<uintptr_t>(local_query.data_ptr()) % alignof(uint4) ==
|
||||
0 &&
|
||||
reinterpret_cast<uintptr_t>(final_query.data_ptr()) % alignof(uint4) ==
|
||||
0 &&
|
||||
query_token_stride_bytes % sizeof(uint4) == 0 &&
|
||||
bytes_per_token % sizeof(uint4) == 0;
|
||||
STD_TORCH_CHECK(
|
||||
vectorized,
|
||||
"direct DCP q-gather requires 16-byte-aligned pointers and strides");
|
||||
STD_TORCH_CHECK(query_mc_ptr != 0 && signal_mc_ptr != 0,
|
||||
"direct DCP q-gather requires multicast pointers");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(device_index);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
constexpr int kThreads = 256;
|
||||
int64_t blocks = num_tokens < world_size ? num_tokens : world_size;
|
||||
if (blocks > 1) {
|
||||
increment_epoch_kernel<<<1, 1, 0, stream>>>(
|
||||
epoch.mutable_data_ptr<int64_t>());
|
||||
check_cuda_launch("direct DCP q-gather epoch");
|
||||
}
|
||||
|
||||
direct_dcp_q_gather_multimem_kernel<<<blocks, kThreads, 0, stream>>>(
|
||||
reinterpret_cast<const uint4*>(local_query.data_ptr()),
|
||||
reinterpret_cast<uint4*>(static_cast<uintptr_t>(query_mc_ptr)),
|
||||
reinterpret_cast<uint32_t*>(static_cast<uintptr_t>(signal_mc_ptr)),
|
||||
reinterpret_cast<const uint32_t*>(
|
||||
received_signal.const_data_ptr<int32_t>()),
|
||||
epoch.mutable_data_ptr<int64_t>(),
|
||||
reinterpret_cast<uint32_t*>(completion.mutable_data_ptr<int32_t>()),
|
||||
world_size, rank, num_tokens, bytes_per_token, query_token_stride_bytes,
|
||||
gathered_token_stride_bytes);
|
||||
check_cuda_launch("direct DCP q-gather");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_q_gather_ops) {
|
||||
direct_dcp_q_gather_ops.def(
|
||||
"direct_dcp_q_gather("
|
||||
"Tensor local_query, Tensor! final_query, Tensor! received_signal, "
|
||||
"Tensor! completion, Tensor! epoch, "
|
||||
"int world_size, int rank, int max_num_tokens, "
|
||||
"int padded_num_heads, int query_mc_ptr, int signal_mc_ptr) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_q_gather_ops) {
|
||||
direct_dcp_q_gather_ops.impl("direct_dcp_q_gather",
|
||||
TORCH_BOX(&direct_dcp_q_gather));
|
||||
}
|
||||
@@ -222,6 +222,49 @@ class TestLSEWeightedCombine:
|
||||
assert result.shape == (B, H, D)
|
||||
torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5)
|
||||
|
||||
def test_empty_shard_ignores_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
|
||||
outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]])
|
||||
lses = torch.tensor([[[-float("inf")]], [[0.0]]])
|
||||
|
||||
result = _lse_weighted_combine(outputs, lses)
|
||||
|
||||
torch.testing.assert_close(result, outputs[1])
|
||||
|
||||
def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch):
|
||||
import vllm.v1.attention.ops.common as common
|
||||
|
||||
class FakeGroup:
|
||||
world_size = 2
|
||||
rank_in_group = 0
|
||||
|
||||
def all_gather(self, tensor, dim):
|
||||
assert dim == 0
|
||||
return torch.cat((tensor, tensor), dim=dim)
|
||||
|
||||
monkeypatch.setattr(
|
||||
common,
|
||||
"correct_attn_out",
|
||||
lambda output, lses, *args, **kwargs: (output, lses[0]),
|
||||
)
|
||||
output = torch.ones(7, 1, 1)
|
||||
lse = torch.ones(7, 1)
|
||||
seq_lens = torch.tensor([0, 2], dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
|
||||
_, masked_lse = common._cp_lse_common(
|
||||
output,
|
||||
lse,
|
||||
FakeGroup(),
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
|
||||
assert torch.isneginf(masked_lse[:1]).all()
|
||||
torch.testing.assert_close(masked_lse[1:5], torch.ones_like(masked_lse[1:5]))
|
||||
assert torch.isneginf(masked_lse[5:]).all()
|
||||
|
||||
def test_mathematically_correct(self):
|
||||
"""Verify mathematical correctness of LSE combination."""
|
||||
from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine
|
||||
@@ -344,7 +387,7 @@ class TestPackedA2AKernels:
|
||||
world_size, B, h_per_rank, D = 4, 7, 2, 32
|
||||
H = world_size * h_per_rank
|
||||
cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32)
|
||||
cp_attn_lse = torch.randn(B, H, device=device, dtype=dtype)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype)
|
||||
send_buffer = torch.empty(
|
||||
(world_size, B, h_per_rank, D + lse_pack_dim),
|
||||
@@ -375,6 +418,67 @@ class TestPackedA2AKernels:
|
||||
else:
|
||||
_assert_packed_a2a_close(actual, expected_out, dtype)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 1, reason="CUDA is required."
|
||||
)
|
||||
def test_empty_seq_lens_ignore_undefined_output(self):
|
||||
from vllm.v1.attention.ops.dcp_alltoall import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
_dcp_a2a_unpack_combine,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
world_size, num_tokens, h_per_rank, head_dim = 2, 5, 1, 32
|
||||
num_heads = world_size * h_per_rank
|
||||
output = torch.randn(
|
||||
num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
output[:1] = float("nan")
|
||||
lse = torch.randn(num_tokens, num_heads, device=device, dtype=output.dtype)
|
||||
seq_lens = torch.tensor([0, 2], device=device, dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 5], device=device, dtype=torch.int32)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(output.dtype)
|
||||
send_buffer = torch.empty(
|
||||
(
|
||||
world_size,
|
||||
num_tokens,
|
||||
h_per_rank,
|
||||
head_dim + lse_pack_dim,
|
||||
),
|
||||
device=device,
|
||||
dtype=output.dtype,
|
||||
)
|
||||
|
||||
_dcp_a2a_pack_send(
|
||||
output,
|
||||
lse,
|
||||
send_buffer,
|
||||
world_size,
|
||||
h_per_rank,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
actual_output, actual_lse = _dcp_a2a_unpack_combine(
|
||||
send_buffer,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
return_lse=True,
|
||||
is_lse_base_on_e=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_output[:1], torch.zeros_like(actual_output[:1])
|
||||
)
|
||||
assert torch.isneginf(actual_lse[:1]).all()
|
||||
assert torch.isfinite(actual_output[1:]).all()
|
||||
|
||||
|
||||
def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
@@ -417,7 +521,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None:
|
||||
B,
|
||||
H,
|
||||
device=f"cuda:{local_rank}",
|
||||
dtype=torch.float32,
|
||||
dtype=dtype,
|
||||
generator=generator,
|
||||
)
|
||||
actual = dcp_a2a_lse_reduce(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
|
||||
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
|
||||
|
||||
|
||||
def _make_tiny_overrides() -> dict:
|
||||
linear_attn_config = {
|
||||
"full_attn_layers": [4],
|
||||
"head_dim": 128,
|
||||
"kda_layers": [1, 2, 3],
|
||||
"num_heads": 8,
|
||||
"short_conv_kernel_size": 4,
|
||||
"use_full_rank_gate": True,
|
||||
}
|
||||
|
||||
return {
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"head_dim": 32,
|
||||
"intermediate_size": 512,
|
||||
"num_hidden_layers": 4,
|
||||
"num_attention_heads": 8,
|
||||
"num_key_value_heads": 8,
|
||||
"num_experts": None,
|
||||
"num_experts_per_token": None,
|
||||
"num_shared_experts": 0,
|
||||
"q_lora_rank": 128,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"mla_use_nope": True,
|
||||
"mla_use_output_gate": True,
|
||||
"linear_attn_config": linear_attn_config,
|
||||
"max_position_embeddings": 512,
|
||||
"model_max_length": 512,
|
||||
}
|
||||
|
||||
|
||||
def _run_tiny_model(
|
||||
vllm_runner: type[VllmRunner], dcp_size: int
|
||||
) -> list[tuple[list[int], list[float]]]:
|
||||
with vllm_runner(
|
||||
model_name=MODEL,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
hf_overrides=_make_tiny_overrides(),
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
distributed_executor_backend="mp",
|
||||
dtype="bfloat16",
|
||||
seed=0,
|
||||
enforce_eager=True,
|
||||
max_model_len=512,
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=1024,
|
||||
gpu_memory_utilization=0.25,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
) as runner:
|
||||
lengths = [1, 32, 33, 129]
|
||||
prompts = [
|
||||
TokensPrompt(
|
||||
prompt_token_ids=[
|
||||
3 + ((request_idx + token_idx) % 251) for token_idx in range(length)
|
||||
]
|
||||
)
|
||||
for request_idx, length in enumerate(lengths)
|
||||
]
|
||||
outputs = runner.llm.generate(
|
||||
prompts,
|
||||
SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=8,
|
||||
seed=0,
|
||||
ignore_eos=True,
|
||||
logprobs=20,
|
||||
),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
results = []
|
||||
for request_output in outputs:
|
||||
completion = request_output.outputs[0]
|
||||
token_ids = list(completion.token_ids)
|
||||
assert completion.logprobs is not None
|
||||
selected_logprobs = [
|
||||
step_logprobs[token_id].logprob
|
||||
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
|
||||
]
|
||||
results.append((token_ids, selected_logprobs))
|
||||
return results
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
def test_kimi_linear_dcp_tiny(
|
||||
vllm_runner: type[VllmRunner],
|
||||
num_gpus_available: int,
|
||||
) -> None:
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip("Need at least 2 GPUs")
|
||||
|
||||
baseline = _run_tiny_model(vllm_runner, dcp_size=1)
|
||||
dcp = _run_tiny_model(vllm_runner, dcp_size=2)
|
||||
|
||||
assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline]
|
||||
logprob_drifts = []
|
||||
for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp):
|
||||
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
|
||||
assert math.isfinite(baseline_logprob)
|
||||
assert math.isfinite(dcp_logprob)
|
||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||
|
||||
assert max(logprob_drifts) <= 1e-2
|
||||
@@ -0,0 +1,246 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
fused_mla_decode_q_concat_kv_cache_insert,
|
||||
fused_mla_key_concat_ds_mla_insert,
|
||||
fused_mla_key_concat_kv_cache_insert,
|
||||
fused_mla_qkv_quant_kv_cache_fp8_insert,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
NUM_TOKENS = 4
|
||||
NUM_HEADS = 2
|
||||
KV_LORA_RANK = 512
|
||||
NOPE_HEAD_DIM = 128
|
||||
ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
|
||||
V_HEAD_DIM = 128
|
||||
CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM
|
||||
DS_MLA_CACHE_ENTRY = 656
|
||||
BLOCK_SIZE = 8
|
||||
OWNED_TOKEN_INDICES = [1, 3]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA"
|
||||
)
|
||||
|
||||
|
||||
def _inputs() -> dict[str, torch.Tensor]:
|
||||
torch.manual_seed(0)
|
||||
|
||||
def make(*shape: int) -> torch.Tensor:
|
||||
return torch.randn(*shape, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
return {
|
||||
"q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM),
|
||||
"k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM),
|
||||
"k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM),
|
||||
"kv_c": make(NUM_TOKENS, KV_LORA_RANK),
|
||||
"v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM),
|
||||
"ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK),
|
||||
"q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM),
|
||||
}
|
||||
|
||||
|
||||
def _slot_mapping() -> torch.Tensor:
|
||||
return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64)
|
||||
|
||||
|
||||
def _owned(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor[OWNED_TOKEN_INDICES]
|
||||
|
||||
|
||||
def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor:
|
||||
shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1)
|
||||
return torch.cat((k_nope, shared_pe), dim=-1)
|
||||
|
||||
|
||||
def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor:
|
||||
return torch.cat((ql_nope, q_pe), dim=-1)
|
||||
|
||||
|
||||
def _make_cache(cache_format: str) -> torch.Tensor:
|
||||
if cache_format == "bf16":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
-7,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if cache_format == "fp8":
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, CACHE_ENTRY),
|
||||
1.0,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
).to(torch.float8_e4m3fn)
|
||||
assert cache_format == "fp8_ds_mla"
|
||||
return torch.full(
|
||||
(1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
|
||||
|
||||
def _reference_cache(
|
||||
cache_format: str,
|
||||
inputs: dict[str, torch.Tensor],
|
||||
initial_cache: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
reference_cache = initial_cache.clone()
|
||||
valid_slots = _owned(slots)
|
||||
if cache_format == "fp8_ds_mla":
|
||||
ops.concat_and_cache_mla(
|
||||
_owned(inputs["kv_c"]),
|
||||
_owned(inputs["k_pe"]),
|
||||
reference_cache,
|
||||
valid_slots,
|
||||
cache_format,
|
||||
torch.ones(1, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
return reference_cache
|
||||
|
||||
latent = torch.cat(
|
||||
(_owned(inputs["kv_c"]), _owned(inputs["k_pe"])),
|
||||
dim=-1,
|
||||
)
|
||||
flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY)
|
||||
for row, slot in zip(latent, valid_slots.tolist()):
|
||||
flat_reference_cache[slot].copy_(row.to(reference_cache.dtype))
|
||||
return reference_cache
|
||||
|
||||
|
||||
def _assert_cache_matches_reference(
|
||||
cache: torch.Tensor,
|
||||
reference_cache: torch.Tensor,
|
||||
initial_cache: torch.Tensor,
|
||||
) -> None:
|
||||
assert torch.equal(cache, reference_cache)
|
||||
for slot in (0, 5):
|
||||
assert not torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
for slot in (1, 2, 3, 4, 6, 7):
|
||||
assert torch.equal(cache[0, slot], initial_cache[0, slot])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"])
|
||||
def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
|
||||
op = (
|
||||
fused_mla_key_concat_kv_cache_insert
|
||||
if cache_format == "bf16"
|
||||
else fused_mla_key_concat_ds_mla_insert
|
||||
)
|
||||
output = op(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0
|
||||
)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache("fp8")
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache("fp8", inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
|
||||
q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
inputs["q"],
|
||||
inputs["k_nope"],
|
||||
inputs["k_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["v"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
scale_inv,
|
||||
)
|
||||
|
||||
expected = (
|
||||
inputs["q"].to(torch.float8_e4m3fn),
|
||||
_full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn),
|
||||
inputs["v"].to(torch.float8_e4m3fn),
|
||||
)
|
||||
for actual, reference in zip((q, k, v), expected):
|
||||
torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"])
|
||||
def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None:
|
||||
inputs = _inputs()
|
||||
slots = _slot_mapping()
|
||||
initial_cache = _make_cache(cache_format)
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
kwargs = {
|
||||
"ds_mla": cache_format == "fp8_ds_mla",
|
||||
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
}
|
||||
|
||||
output = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
inputs["ql_nope"],
|
||||
inputs["q_pe"],
|
||||
inputs["kv_c"],
|
||||
inputs["k_pe"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
|
||||
if cache_format == "fp8":
|
||||
expected = expected.to(torch.float8_e4m3fn)
|
||||
torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0)
|
||||
else:
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
_assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache)
|
||||
|
||||
|
||||
def test_ds_mla_cache_insert_bit_compatible_with_reference() -> None:
|
||||
"""Fused ds_mla insertion must match the reference bit-for-bit."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_blocks = 33, 16
|
||||
dt = torch.bfloat16
|
||||
kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt)
|
||||
k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt)
|
||||
q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt)
|
||||
slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1
|
||||
ref = torch.full(
|
||||
(num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY),
|
||||
165,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
got = ref.clone()
|
||||
scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale)
|
||||
fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots)
|
||||
assert torch.equal(ref, got)
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
requires_flashinfer_mla = pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(100),
|
||||
reason="FlashInfer MLA requires compute capability 10 or above.",
|
||||
)
|
||||
|
||||
|
||||
def test_mla_dcp_gathered_query_reserves_backend_head_storage():
|
||||
from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage
|
||||
|
||||
query = torch.randn(3, 24, 576, dtype=torch.bfloat16)
|
||||
|
||||
padded = reserve_query_head_storage(query, 128)
|
||||
|
||||
assert padded.shape == query.shape
|
||||
assert padded.stride() == query.stride()
|
||||
assert padded.untyped_storage().nbytes() >= 3 * 128 * 576 * 2
|
||||
assert torch.equal(padded, query)
|
||||
|
||||
|
||||
@requires_flashinfer_mla
|
||||
def test_flashinfer_mla_selects_backend_for_gathered_heads():
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla import (
|
||||
_select_mla_decode_backend,
|
||||
)
|
||||
|
||||
assert _select_mla_decode_backend(6) is None
|
||||
assert _select_mla_decode_backend(24) == "cute-dsl"
|
||||
|
||||
|
||||
@requires_flashinfer_mla
|
||||
def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch):
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla as flashinfer_mla
|
||||
|
||||
impl = MagicMock()
|
||||
impl.bmm1_scale = 1.0
|
||||
impl.bmm2_scale = 1.0
|
||||
impl.need_to_return_lse_for_decode = True
|
||||
impl.num_heads = 6
|
||||
impl.qk_nope_head_dim = 128
|
||||
impl.kv_lora_rank = 512
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.kv_cache_dtype = "auto"
|
||||
|
||||
attn_metadata = MagicMock()
|
||||
attn_metadata.causal = True
|
||||
attn_metadata.num_decode_tokens = 2
|
||||
attn_metadata.num_decodes = 2
|
||||
attn_metadata.max_seq_len = 1
|
||||
attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32)
|
||||
attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32)
|
||||
|
||||
query = torch.ones(2, 24, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16)
|
||||
kernel = MagicMock(
|
||||
return_value=(
|
||||
torch.ones(2, 1, 24, 512, dtype=torch.bfloat16),
|
||||
torch.ones(2, 24, dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_mla, "_get_workspace_buffer", MagicMock())
|
||||
monkeypatch.setattr(flashinfer_mla, "trtllm_batch_decode_with_kv_cache_mla", kernel)
|
||||
layer = MagicMock()
|
||||
|
||||
output, lse = flashinfer_mla.FlashInferMLAImpl.forward_mqa(
|
||||
impl, query, kv_cache, attn_metadata, layer
|
||||
)
|
||||
|
||||
assert kernel.call_args.kwargs["backend"] == "cute-dsl"
|
||||
assert output.shape == (2, 24, 512)
|
||||
assert lse is not None and lse.shape == (2, 24)
|
||||
@@ -190,6 +190,9 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_DEEP_GEMM_E8M0: bool = True
|
||||
VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True
|
||||
VLLM_DCP_Q_REPLICATE: bool = False
|
||||
VLLM_USE_DIRECT_DCP_A2A: bool | None = None
|
||||
VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None
|
||||
VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None
|
||||
VLLM_DEEP_GEMM_WARMUP: Literal[
|
||||
"skip",
|
||||
"full",
|
||||
@@ -2057,6 +2060,16 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0"))
|
||||
),
|
||||
# Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable.
|
||||
"VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool(
|
||||
os.getenv("VLLM_USE_DIRECT_DCP_A2A")
|
||||
),
|
||||
"VLLM_USE_DIRECT_DCP_Q_GATHER": lambda: maybe_convert_bool(
|
||||
os.getenv("VLLM_USE_DIRECT_DCP_Q_GATHER")
|
||||
),
|
||||
"VLLM_USE_DIRECT_DCP_KV_GATHER": lambda: maybe_convert_bool(
|
||||
os.getenv("VLLM_USE_DIRECT_DCP_KV_GATHER")
|
||||
),
|
||||
# Whether to enable dual cuda streams for LoRA computation
|
||||
# (used by both BaseLinearLayerWithLoRA and FusedMoEWithLoRA to
|
||||
# overlap the base layer compute with the LoRA fast path).
|
||||
|
||||
@@ -206,6 +206,7 @@ from abc import abstractmethod
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from math import lcm
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -222,7 +223,6 @@ from vllm.config import (
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed.parallel_state import (
|
||||
@@ -264,7 +264,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.utils.math_utils import cdiv, round_down, round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -292,8 +292,7 @@ from vllm.v1.attention.backends.utils import (
|
||||
get_num_attention_heads_from_layers,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -579,12 +578,27 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
self.use_sparse = use_sparse
|
||||
|
||||
_vllm_config = get_current_vllm_config_or_none()
|
||||
self.dcp_a2a = (
|
||||
_vllm_config is not None
|
||||
and _vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
)
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.impl.dcp_world_size > 1:
|
||||
query_dtype = (
|
||||
current_platform.fp8_dtype()
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and self.kv_cache_dtype != "fp8_ds_mla"
|
||||
and self.impl.supports_quant_query_input
|
||||
else dtype
|
||||
)
|
||||
self.dcp_manager = MLADCPManager(
|
||||
vllm_config=vllm_config,
|
||||
device=next(kv_b_proj.parameters()).device,
|
||||
num_heads=self.num_heads,
|
||||
query_head_dim=self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
output_head_dim=self.kv_lora_rank,
|
||||
query_dtype=query_dtype,
|
||||
output_dtype=dtype,
|
||||
padded_num_heads=self.q_pad_num_heads,
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
use_pcp=self.use_pcp,
|
||||
)
|
||||
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
@@ -753,9 +767,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
return quant_output.fill_(0)
|
||||
return output.fill_(0)
|
||||
|
||||
if self.impl.dcp_world_size == -1:
|
||||
self.impl.dcp_world_size = get_dcp_group().world_size
|
||||
|
||||
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
@@ -914,6 +925,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
mqa_q = (mqa_ql_nope, mqa_q_pe)
|
||||
# concatenate nope + pe -> (B, N, L + P) (fp8 op above may have fused)
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert self.dcp_manager is not None
|
||||
if self.use_pcp:
|
||||
if self.impl.dcp_world_size > self.impl.pcp_world_size:
|
||||
if isinstance(mqa_q, tuple):
|
||||
@@ -924,8 +936,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
if not qrep_decode:
|
||||
# mqa_q do allgather in head dim.
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
assert self.dcp_manager.query_gather is not None
|
||||
mqa_q = self.dcp_manager.query_gather(mqa_q)
|
||||
|
||||
# call decode attn
|
||||
if not self.impl.is_sparse:
|
||||
@@ -935,27 +947,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# correct dcp attn_out with lse.
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert lse is not None
|
||||
if self.dcp_a2a:
|
||||
attn_out = dcp_a2a_lse_reduce(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
elif self.use_pcp:
|
||||
attn_out = cp_lse_ag_out_ar(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
else:
|
||||
attn_out = cp_lse_ag_out_rs(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
assert self.dcp_manager is not None
|
||||
seq_lens = (
|
||||
attn_metadata.decode.seq_lens
|
||||
if attn_metadata.decode is not None
|
||||
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
|
||||
: attn_metadata.num_decodes
|
||||
]
|
||||
)
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
if self.use_pcp:
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
@@ -1427,6 +1435,7 @@ class MLACommonPrefillMetadata:
|
||||
chunks: "list[MLACommonPrefillMetadata.ContextChunk]"
|
||||
context_lens_list: list[int]
|
||||
empty_token_slices: list[slice]
|
||||
dcp_manager: MLADCPManager | None = None
|
||||
|
||||
block_table: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
@@ -1684,6 +1693,25 @@ def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
|
||||
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
|
||||
|
||||
|
||||
def align_mla_chunked_context_workspace_size(
|
||||
vllm_config: VllmConfig,
|
||||
workspace_size: int,
|
||||
) -> int:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
alignment = vllm_config.cache_config.block_size
|
||||
if parallel_config.decode_context_parallel_size > 1:
|
||||
alignment = lcm(
|
||||
alignment,
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
workspace_size = max(
|
||||
workspace_size,
|
||||
vllm_config.scheduler_config.max_num_seqs * alignment,
|
||||
)
|
||||
return round_up(workspace_size, alignment)
|
||||
|
||||
|
||||
def build_mla_chunked_context_metadata(
|
||||
*,
|
||||
context_lens_cpu: torch.Tensor,
|
||||
@@ -1696,6 +1724,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_world_size: int,
|
||||
dcp_local_block_size: int,
|
||||
dcp_virtual_block_size: int,
|
||||
dcp_manager: MLADCPManager | None = None,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
@@ -1714,6 +1743,7 @@ def build_mla_chunked_context_metadata(
|
||||
dcp_world_size: Decode-context-parallel world size (1 if disabled).
|
||||
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||
dcp_manager: Shared MLA DCP collective manager.
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
@@ -1902,6 +1932,7 @@ def build_mla_chunked_context_metadata(
|
||||
chunks=chunks,
|
||||
context_lens_list=context_lens,
|
||||
empty_token_slices=empty_token_slices,
|
||||
dcp_manager=dcp_manager,
|
||||
)
|
||||
|
||||
|
||||
@@ -1954,7 +1985,10 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
64 * 1024,
|
||||
)
|
||||
|
||||
return max(chunked_prefill_workspace_size, cache_config.block_size)
|
||||
return align_mla_chunked_context_workspace_size(
|
||||
vllm_config,
|
||||
chunked_prefill_workspace_size,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def determine_prefill_query_data_type(
|
||||
@@ -2033,6 +2067,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.q_data_type = self.determine_prefill_query_data_type(
|
||||
vllm_config, self.model_config.dtype
|
||||
)
|
||||
attention_layer = self.compilation_config.static_forward_context[layer_names[0]]
|
||||
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
@@ -2050,6 +2085,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
)
|
||||
|
||||
use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.dcp_world_size > 1:
|
||||
# Note(hc): The local kvcache is incomplete when DCP is triggered,
|
||||
# an additional kvcache allgather across the DCP group is therefore
|
||||
@@ -2067,6 +2103,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
else self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
|
||||
assert isinstance(self.dcp_manager, MLADCPManager)
|
||||
self.dcp_manager.init_kv_gather(
|
||||
self.chunked_prefill_workspace,
|
||||
self.chunked_prefill_workspace_size,
|
||||
)
|
||||
else:
|
||||
self.chunked_prefill_workspace = torch.empty(
|
||||
(
|
||||
@@ -2080,9 +2122,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
# Metadata builders are created per ubatch when DBO is enabled. MLA
|
||||
# prefill backends keep the prepared metadata on the backend object, so
|
||||
# each builder needs its own backend instance to avoid cross-ubatch races.
|
||||
self._prefill_backend = self.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend.clone()
|
||||
self._prefill_backend = attention_layer.prefill_backend.clone()
|
||||
|
||||
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
|
||||
self._init_reorder_batch_threshold(
|
||||
@@ -2223,6 +2263,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
)
|
||||
|
||||
prefill_metadata = MLACommonPrefillMetadata(
|
||||
@@ -2660,9 +2701,8 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
]
|
||||
assert toks * dcp_world_size <= cur_allgather_workspace.shape[0]
|
||||
cur_allgather_kvcache = cur_allgather_workspace[: toks * dcp_world_size]
|
||||
cur_allgather_kvcache.copy_(
|
||||
get_dcp_group().all_gather(local_gathered_kvcache, dim=0)
|
||||
)
|
||||
dcp_manager = cast(MLADCPManager, chunked_context.dcp_manager)
|
||||
dcp_manager.kv_gather(cur_allgather_kvcache, local_gathered_kvcache)
|
||||
assert (
|
||||
cur_allgather_kvcache.shape[-1]
|
||||
== self.kv_lora_rank + self.qk_rope_head_dim
|
||||
@@ -2732,8 +2772,6 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
assert attn_metadata.prefill is not None
|
||||
assert self.dcp_world_size != -1
|
||||
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
|
||||
@@ -2877,10 +2915,11 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
|
||||
and (self.qk_rope_head_dim == 64)
|
||||
)
|
||||
|
||||
self.dcp_world_size: int = -1
|
||||
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
# Avoid requiring an initialized DCP group in tests.
|
||||
self.dcp_world_size: int = parallel_config.decode_context_parallel_size
|
||||
self.cp_kv_cache_interleave_size: int = (
|
||||
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
||||
parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonMetadata,
|
||||
MLACommonPrefillMetadata,
|
||||
accumulate_mla_context_chunk,
|
||||
align_mla_chunked_context_workspace_size,
|
||||
build_mla_chunked_context_metadata,
|
||||
get_mla_dims,
|
||||
init_mla_context_partial,
|
||||
@@ -31,6 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -162,9 +164,19 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
|
||||
attention_layer = vllm_config.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend
|
||||
]
|
||||
layer_prefill_backend = attention_layer.prefill_backend
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.dcp_world_size > 1:
|
||||
self.dcp_manager = getattr(attention_layer, "dcp_manager", None)
|
||||
assert isinstance(self.dcp_manager, MLADCPManager)
|
||||
if layer_prefill_backend is not None:
|
||||
self.dcp_manager.init_kv_gather(
|
||||
self.chunked_prefill_workspace,
|
||||
self.chunked_prefill_workspace_size,
|
||||
)
|
||||
self._prefill_backend = (
|
||||
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
|
||||
)
|
||||
@@ -184,7 +196,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
64 * 1024,
|
||||
scheduler_config.max_num_seqs * topk_tokens,
|
||||
)
|
||||
return max(workspace_size, cache_config.block_size)
|
||||
workspace_size = max(
|
||||
workspace_size,
|
||||
scheduler_config.max_num_seqs * cache_config.block_size,
|
||||
)
|
||||
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
return align_mla_chunked_context_workspace_size(vllm_config, workspace_size)
|
||||
return workspace_size
|
||||
|
||||
def _build_req_id_per_token(
|
||||
self,
|
||||
@@ -232,6 +250,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
dcp_manager=self.dcp_manager,
|
||||
)
|
||||
|
||||
def build(
|
||||
|
||||
@@ -24,8 +24,8 @@ KV cache, and absorbs ``kv_b_proj`` into ``W_UK_T`` / ``W_UV`` -- mirroring the
|
||||
K3 specifics: optional rotary embedding (disabled for the target model's NoPE
|
||||
layers, enabled for DSpark) and an optional sigmoid output gate (``g_proj``).
|
||||
|
||||
Out of scope (extension points, not wired here): context parallelism (DCP/PCP),
|
||||
sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths.
|
||||
Out of scope (extension points, not wired here): prefill context parallelism
|
||||
(PCP), sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths.
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -35,7 +35,11 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -78,6 +82,7 @@ from vllm.v1.attention.backend import (
|
||||
MLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
from vllm.v1.attention.ops.dcp_utils import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode
|
||||
@@ -310,10 +315,36 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
assert (
|
||||
parallel_config.decode_context_parallel_size <= 1
|
||||
and parallel_config.prefill_context_parallel_size <= 1
|
||||
), "Kimi-K3 MultiHeadLatentAttention does not support context parallelism."
|
||||
assert parallel_config.prefill_context_parallel_size == 1, (
|
||||
"Kimi-K3 MultiHeadLatentAttention does not support prefill context "
|
||||
"parallelism."
|
||||
)
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
assert self.dcp_world_size <= 1 or self.rotary_emb is None, (
|
||||
"Kimi-K3 MultiHeadLatentAttention does not support RoPE with decode "
|
||||
"context parallelism because gathered queries require gathered "
|
||||
"positions."
|
||||
)
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.dcp_world_size > 1:
|
||||
query_dtype = (
|
||||
torch.float8_e4m3fn
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and self.kv_cache_dtype != "fp8_ds_mla"
|
||||
else dtype
|
||||
)
|
||||
self.dcp_manager = MLADCPManager(
|
||||
vllm_config=vllm_config,
|
||||
device=next(self.kv_b_proj.parameters()).device,
|
||||
num_heads=self.num_local_heads,
|
||||
query_head_dim=self.head_size,
|
||||
output_head_dim=self.kv_lora_rank,
|
||||
query_dtype=query_dtype,
|
||||
output_dtype=dtype,
|
||||
padded_num_heads=self.q_pad_num_heads,
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
use_pcp=False,
|
||||
)
|
||||
self.prefill_backend = get_mla_prefill_backend(vllm_config)(
|
||||
num_heads=self.num_local_heads,
|
||||
scale=self.scale,
|
||||
@@ -584,9 +615,25 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
cos_sin_cache,
|
||||
slot_mapping[:num_mqa_tokens],
|
||||
)
|
||||
latent_out, _lse = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
if self.dcp_world_size > 1:
|
||||
assert self.dcp_manager is not None
|
||||
assert self.dcp_manager.query_gather is not None
|
||||
mqa_q = self.dcp_manager.query_gather(mqa_q)
|
||||
latent_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
mqa_q, self._attn_read_kv_cache(), attn_metadata, self
|
||||
)
|
||||
if self.dcp_world_size > 1:
|
||||
assert lse is not None
|
||||
assert self.dcp_manager is not None
|
||||
assert attn_metadata.decode is not None
|
||||
latent_out = self.dcp_manager.combine(
|
||||
latent_out,
|
||||
lse,
|
||||
seq_lens=attn_metadata.decode.seq_lens,
|
||||
query_start_loc=attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
],
|
||||
)
|
||||
self._v_up_proj(latent_out, out=attn_out[:num_mqa_tokens])
|
||||
|
||||
def _decode_concat_cache(
|
||||
@@ -752,9 +799,20 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
|
||||
if has_context:
|
||||
context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined]
|
||||
q, self._attn_read_kv_cache(), attn_metadata, self._k_scale
|
||||
)
|
||||
if self.dcp_world_size > 1:
|
||||
context_output, context_lse = (
|
||||
self.impl._context_parallel_compute_prefill_context( # type: ignore[attr-defined]
|
||||
q,
|
||||
self._attn_read_kv_cache(),
|
||||
attn_metadata,
|
||||
k_scale=self._k_scale,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
)
|
||||
)
|
||||
else:
|
||||
context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined]
|
||||
q, self._attn_read_kv_cache(), attn_metadata, self._k_scale
|
||||
)
|
||||
suffix_output, suffix_lse = output_prefill
|
||||
out = out.view(-1, self.num_local_heads, self.v_head_dim)
|
||||
merge_attn_states(
|
||||
|
||||
@@ -309,9 +309,11 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
|
||||
return_lse = self.need_to_return_lse_for_decode
|
||||
workspace_buffer = _get_workspace_buffer(return_lse)
|
||||
# Parallel gathers can change the runtime Q heads from TP-local num_heads.
|
||||
runtime_num_heads = q.shape[-2]
|
||||
# trtllm-gen rejects MLA head counts it can't tile (e.g. 96);
|
||||
# fall back to cute-dsl for those.
|
||||
decode_backend = _select_mla_decode_backend(self.num_heads)
|
||||
decode_backend = _select_mla_decode_backend(runtime_num_heads)
|
||||
extra_kwargs = {}
|
||||
if decode_backend:
|
||||
extra_kwargs["backend"] = decode_backend
|
||||
@@ -326,7 +328,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
if self._mla_counter_bytes is None:
|
||||
self._mla_counter_bytes = get_trtllm_gen_multi_ctas_kv_counter_bytes(
|
||||
self._mla_counter_max_batch,
|
||||
self.num_heads,
|
||||
runtime_num_heads,
|
||||
get_device_sm_count(q.device),
|
||||
)
|
||||
extra_kwargs["multi_ctas_kv_counter_buffer"] = (
|
||||
|
||||
@@ -289,8 +289,9 @@ class TokenspeedMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
self.output_scale = layer._k_scale_float
|
||||
|
||||
if self._workspace_buffer is None:
|
||||
# Parallelism can change the runtime query head count.
|
||||
self._workspace_buffer = _get_workspace(
|
||||
q.device, self.num_heads, self.kv_lora_rank
|
||||
q.device, q.shape[-2], self.kv_lora_rank
|
||||
)
|
||||
|
||||
# vLLM kv_c_and_k_pe_cache is already (num_blocks, block_size, head_size).
|
||||
|
||||
@@ -6,6 +6,34 @@ from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def mask_dcp_empty_shards_(
|
||||
lse: torch.Tensor,
|
||||
seq_lens: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
) -> None:
|
||||
if seq_lens is None and query_start_loc is None:
|
||||
return
|
||||
if seq_lens is None or query_start_loc is None:
|
||||
raise ValueError("seq_lens and query_start_loc must be provided together")
|
||||
if (
|
||||
seq_lens.ndim != 1
|
||||
or query_start_loc.ndim != 1
|
||||
or query_start_loc.shape[0] != seq_lens.shape[0] + 1
|
||||
):
|
||||
raise ValueError("query_start_loc must contain one boundary per sequence")
|
||||
|
||||
row_indices = torch.arange(
|
||||
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
|
||||
)
|
||||
sequence_indices = torch.searchsorted(
|
||||
query_start_loc[1:], row_indices, right=True
|
||||
).clamp_max(seq_lens.shape[0] - 1)
|
||||
empty_rows = (row_indices >= query_start_loc[-1]) | (
|
||||
seq_lens[sequence_indices] == 0
|
||||
)
|
||||
lse.masked_fill_(empty_rows[:, None], float("-inf"))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _correct_attn_cp_out_kernel(
|
||||
outputs_ptr,
|
||||
@@ -51,7 +79,7 @@ def _correct_attn_cp_out_kernel(
|
||||
)
|
||||
|
||||
# calc final lse
|
||||
lse = tl.load(lses_ptr + lse_offsets)
|
||||
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
|
||||
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
|
||||
lse_max = tl.max(lse, axis=0)
|
||||
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
|
||||
@@ -80,7 +108,7 @@ def _correct_attn_cp_out_kernel(
|
||||
lse_offset = (
|
||||
lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H
|
||||
)
|
||||
lse_tmp = tl.load(lses_ptr + lse_offset)
|
||||
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
|
||||
lse_finally = lse_tmp - lse
|
||||
lse_finally = tl.where(
|
||||
(lse_finally != lse_finally) | (lse_finally == float("inf")),
|
||||
@@ -185,6 +213,8 @@ def _cp_lse_common(
|
||||
cp_group: GroupCoordinator,
|
||||
ctx: CPTritonContext | None = None,
|
||||
is_lse_base_on_e=True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
cp_attn_out: [ B, H, D ]
|
||||
@@ -197,6 +227,7 @@ def _cp_lse_common(
|
||||
ctx = CPTritonContext()
|
||||
|
||||
cp_attn_lse = cp_attn_lse.contiguous()
|
||||
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
|
||||
lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape(
|
||||
(cp_group.world_size,) + cp_attn_lse.shape
|
||||
)
|
||||
@@ -217,13 +248,21 @@ def cp_lse_ag_out_rs(
|
||||
ctx: CPTritonContext | None = None,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e=True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
cp_attn_out: [ B, H, D ]
|
||||
cp_attn_lse: [ B, H ]
|
||||
"""
|
||||
out, lse = _cp_lse_common(
|
||||
cp_attn_out, cp_attn_lse, cp_group, ctx=ctx, is_lse_base_on_e=is_lse_base_on_e
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
cp_group,
|
||||
ctx=ctx,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
out = cp_group.reduce_scatter(out, dim=1)
|
||||
|
||||
@@ -242,13 +281,21 @@ def cp_lse_ag_out_ar(
|
||||
ctx: CPTritonContext | None = None,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e=True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
cp_attn_out: [ B, H, D ]
|
||||
cp_attn_lse: [ B, H ]
|
||||
"""
|
||||
out, lse = _cp_lse_common(
|
||||
cp_attn_out, cp_attn_lse, cp_group, ctx=ctx, is_lse_base_on_e=is_lse_base_on_e
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
cp_group,
|
||||
ctx=ctx,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
out = cp_group.all_reduce(out)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.common import mask_dcp_empty_shards_
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
@@ -87,7 +88,9 @@ def _lse_weighted_combine(
|
||||
weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H]
|
||||
|
||||
# Weighted combination: sum over N dimension
|
||||
result = (outputs * weights.unsqueeze(-1)).sum(dim=0) # [B, H, D]
|
||||
weights = weights.unsqueeze(-1)
|
||||
outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs)
|
||||
result = (outputs * weights).sum(dim=0) # [B, H, D]
|
||||
|
||||
if return_lse:
|
||||
if is_lse_base_on_e:
|
||||
@@ -172,7 +175,7 @@ def _dcp_a2a_pack_send_kernel(
|
||||
|
||||
lse_val = tl.load(
|
||||
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
|
||||
)
|
||||
).to(tl.float32)
|
||||
if LSE_PACK_DIM == 1:
|
||||
tl.store(
|
||||
send_ptr + send_base + HEAD_DIM * send_stride_D,
|
||||
@@ -301,10 +304,11 @@ def _dcp_a2a_unpack_combine_kernel(
|
||||
else:
|
||||
weight = tl.exp2(lse_val - global_lse)
|
||||
weight = tl.where(weight != weight, 0.0, weight)
|
||||
acc += (
|
||||
tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(tl.float32)
|
||||
* weight
|
||||
partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(
|
||||
tl.float32
|
||||
)
|
||||
partial = tl.where(weight == 0.0, 0.0, partial)
|
||||
acc += partial * weight
|
||||
|
||||
final_offsets = (
|
||||
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
|
||||
@@ -324,7 +328,10 @@ def _dcp_a2a_pack_send(
|
||||
h_per_rank: int,
|
||||
head_dim: int,
|
||||
lse_pack_dim: int,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
|
||||
grid = (cp_attn_out.shape[0], h_per_rank, 1)
|
||||
_dcp_a2a_pack_send_kernel[grid](
|
||||
cp_attn_out,
|
||||
@@ -396,20 +403,24 @@ def dcp_a2a_lse_reduce(
|
||||
ctx: CPTritonContext | None = None,
|
||||
return_lse: bool = False,
|
||||
is_lse_base_on_e: bool = True,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Combine partial attention outputs across DCP ranks using All-to-All.
|
||||
|
||||
The output and fp32 LSE are packed into a single output-dtype buffer, sent
|
||||
The output and LSE are packed into a single output-dtype buffer, sent
|
||||
with one All-to-All, then unpacked and combined with exact LSE weighting.
|
||||
|
||||
Args:
|
||||
cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim
|
||||
cp_attn_lse: [B, H] log-sum-exp values (fp32)
|
||||
cp_attn_lse: [B, H] floating-point log-sum-exp values
|
||||
cp_group: GroupCoordinator for DCP communication
|
||||
ctx: CPTritonContext (unused, for signature compatibility)
|
||||
return_lse: If True, also return the combined global LSE
|
||||
is_lse_base_on_e: If True, LSE is base e; if False, base 2
|
||||
seq_lens: Local KV lengths. Empty shards contribute zero weight.
|
||||
query_start_loc: Cumulative query-token offsets for each request.
|
||||
|
||||
Returns:
|
||||
Combined output [B, H/N, D] (head-scattered)
|
||||
@@ -426,10 +437,6 @@ def dcp_a2a_lse_reduce(
|
||||
if H % world_size != 0:
|
||||
raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.")
|
||||
H_per_rank = H // world_size
|
||||
# The pack kernel bit-casts the LSE as fp32; some MLA backends return it in
|
||||
# the activation dtype (bf16/fp16), so enforce the documented fp32 contract.
|
||||
if cp_attn_lse.dtype != torch.float32:
|
||||
cp_attn_lse = cp_attn_lse.to(torch.float32)
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype)
|
||||
|
||||
send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers(
|
||||
@@ -446,6 +453,8 @@ def dcp_a2a_lse_reduce(
|
||||
H_per_rank,
|
||||
D,
|
||||
lse_pack_dim,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
|
||||
work = dist.all_to_all_single(
|
||||
|
||||
@@ -0,0 +1,740 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MLA DCP collective selection and direct symmetric-memory implementations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.distributed.parallel_state import in_the_same_node_as
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
try:
|
||||
import torch.distributed._symmetric_memory as symm_mem
|
||||
|
||||
symm_mem_available = True
|
||||
except ImportError:
|
||||
symm_mem = None # type: ignore[assignment]
|
||||
symm_mem_available = False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _symm_mem_spans_group(group: GroupCoordinator) -> bool:
|
||||
"""Probe whether the group has NVLS symmetric memory."""
|
||||
if not symm_mem_available:
|
||||
return False
|
||||
try:
|
||||
from torch._C._autograd import DeviceType
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index):
|
||||
return False
|
||||
probe = symm_mem.empty(8, dtype=torch.uint8, device=device)
|
||||
probe.zero_()
|
||||
torch.accelerator.synchronize()
|
||||
handle = symm_mem.rendezvous(probe, group.device_group.group_name)
|
||||
spans = handle is not None and handle.multicast_ptr != 0
|
||||
except Exception as error:
|
||||
logger.debug("Direct DCP symmetric-memory probe failed: %s", error)
|
||||
return False
|
||||
logger.debug_once(
|
||||
"Direct DCP symmetric memory across %d ranks: %s",
|
||||
group.world_size,
|
||||
"available" if spans else "unavailable",
|
||||
)
|
||||
return spans
|
||||
|
||||
|
||||
def _direct_dcp_enabled(
|
||||
group: GroupCoordinator,
|
||||
dtype: torch.dtype,
|
||||
use_direct: bool | None,
|
||||
supported_dtypes: tuple[torch.dtype, ...] | None = None,
|
||||
) -> bool:
|
||||
if use_direct is not None:
|
||||
return use_direct
|
||||
return (
|
||||
symm_mem_available
|
||||
and current_platform.is_cuda()
|
||||
and (supported_dtypes is None or dtype in supported_dtypes)
|
||||
and (
|
||||
all(in_the_same_node_as(group.cpu_group, source_rank=0))
|
||||
or _symm_mem_spans_group(group)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _direct_dcp_multicast_enabled(
|
||||
group: GroupCoordinator,
|
||||
dtype: torch.dtype,
|
||||
use_direct: bool | None,
|
||||
supported_dtypes: tuple[torch.dtype, ...] | None = None,
|
||||
) -> bool:
|
||||
return _direct_dcp_enabled(
|
||||
group, dtype, use_direct, supported_dtypes
|
||||
) and _symm_mem_spans_group(group)
|
||||
|
||||
|
||||
def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int:
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
speculative_tokens = vllm_config.num_speculative_tokens
|
||||
tokens_per_seq = (
|
||||
1
|
||||
+ (
|
||||
2
|
||||
if speculative_config is not None and speculative_config.parallel_drafting
|
||||
else 1
|
||||
)
|
||||
* speculative_tokens
|
||||
)
|
||||
return min(
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
max(
|
||||
scheduler_config.max_num_seqs * tokens_per_seq,
|
||||
vllm_config.compilation_config.max_cudagraph_capture_size or 0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _DirectDCPWorkspace:
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
num_ubatches: int,
|
||||
) -> None:
|
||||
self.group = group
|
||||
self.world_size = group.size()
|
||||
self.rank = group.rank()
|
||||
self.device = torch.device(device)
|
||||
self.num_ubatches = num_ubatches
|
||||
self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device)
|
||||
self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = []
|
||||
|
||||
def _allocate(
|
||||
self, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
storage = symm_mem.empty(shape, device=self.device, dtype=dtype)
|
||||
storage.zero_()
|
||||
torch.accelerator.synchronize()
|
||||
handle = symm_mem.rendezvous(storage, self.group.group_name)
|
||||
assert handle is not None, "DCP symmetric memory rendezvous returned None"
|
||||
handle.barrier()
|
||||
views = [
|
||||
handle.get_buffer(peer, list(shape), dtype, 0)
|
||||
for peer in range(self.world_size)
|
||||
]
|
||||
self.device = storage.device
|
||||
peer_ptrs = torch.tensor(
|
||||
[
|
||||
[view[ubatch].data_ptr() for view in views]
|
||||
for ubatch in range(self.num_ubatches)
|
||||
],
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self._allocations.append((storage, handle, views))
|
||||
return storage, peer_ptrs
|
||||
|
||||
def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]:
|
||||
disabled = [0] * self.num_ubatches
|
||||
for allocated, handle, _ in self._allocations:
|
||||
if allocated is storage:
|
||||
break
|
||||
else:
|
||||
return disabled
|
||||
try:
|
||||
from torch._C._autograd import DeviceType
|
||||
from torch._C._distributed_c10d import _SymmetricMemory
|
||||
|
||||
if not _SymmetricMemory.has_multicast_support(
|
||||
DeviceType.CUDA, storage.device.index
|
||||
):
|
||||
return disabled
|
||||
multicast_base = handle.multicast_ptr
|
||||
except Exception:
|
||||
return disabled
|
||||
if not multicast_base:
|
||||
return disabled
|
||||
storage_base = storage.data_ptr()
|
||||
return [
|
||||
multicast_base + (storage[ubatch].data_ptr() - storage_base)
|
||||
for ubatch in range(self.num_ubatches)
|
||||
]
|
||||
|
||||
|
||||
def reserve_query_head_storage(
|
||||
query: torch.Tensor, padded_num_heads: int
|
||||
) -> torch.Tensor:
|
||||
"""Reserve backing storage for fixed-head decode kernels."""
|
||||
assert query.ndim == 3
|
||||
assert query.shape[1] <= padded_num_heads
|
||||
padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2]))
|
||||
padded.resize_(query.shape)
|
||||
padded.copy_(query)
|
||||
return padded
|
||||
|
||||
|
||||
_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
|
||||
|
||||
|
||||
class DirectDCPA2AWorkspace(_DirectDCPWorkspace):
|
||||
"""Persistent symmetric buffers for direct DCP output exchange."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_ubatches: int = 1,
|
||||
) -> None:
|
||||
if dtype not in _A2A_SUPPORTED_DTYPES:
|
||||
raise ValueError(f"Direct DCP A2A does not support {dtype}")
|
||||
if num_ubatches < 1:
|
||||
raise ValueError(
|
||||
f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}"
|
||||
)
|
||||
super().__init__(group, device, num_ubatches)
|
||||
self.max_num_tokens = max_num_tokens
|
||||
self.heads_per_rank = heads_per_rank
|
||||
self.head_dim = head_dim
|
||||
|
||||
output_shape = (
|
||||
num_ubatches,
|
||||
2,
|
||||
self.world_size,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
head_dim,
|
||||
)
|
||||
lse_shape = (
|
||||
num_ubatches,
|
||||
2,
|
||||
self.world_size,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
)
|
||||
signal_shape = (num_ubatches, 2, self.world_size)
|
||||
self.received_output, self.peer_output_ptrs = self._allocate(
|
||||
output_shape, dtype
|
||||
)
|
||||
self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32)
|
||||
self.received_signal, self.peer_signal_ptrs = self._allocate(
|
||||
signal_shape, torch.int32
|
||||
)
|
||||
|
||||
def lse_reduce(
|
||||
self,
|
||||
partial_output: torch.Tensor,
|
||||
partial_lse: torch.Tensor,
|
||||
is_lse_base_on_e: bool,
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
ubatch = dbo_current_ubatch_id()
|
||||
num_tokens = partial_output.shape[0]
|
||||
output = partial_output.new_empty(
|
||||
(num_tokens, self.heads_per_rank, self.head_dim)
|
||||
)
|
||||
torch.ops._C.direct_dcp_a2a_lse_reduce(
|
||||
partial_output,
|
||||
partial_lse,
|
||||
seq_lens,
|
||||
query_start_loc,
|
||||
self.peer_output_ptrs[ubatch],
|
||||
self.peer_lse_ptrs[ubatch],
|
||||
self.peer_signal_ptrs[ubatch],
|
||||
self.received_output[ubatch],
|
||||
self.received_lse[ubatch],
|
||||
self.received_signal[ubatch],
|
||||
self.epoch[ubatch : ubatch + 1],
|
||||
output,
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.max_num_tokens,
|
||||
is_lse_base_on_e,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_direct_dcp_a2a_workspace(
|
||||
group: GroupCoordinator,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
num_ubatches: int,
|
||||
) -> DirectDCPA2AWorkspace | None:
|
||||
if not _direct_dcp_enabled(
|
||||
group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES
|
||||
):
|
||||
return None
|
||||
return DirectDCPA2AWorkspace(
|
||||
group.device_group,
|
||||
device,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
head_dim,
|
||||
dtype,
|
||||
num_ubatches,
|
||||
)
|
||||
|
||||
|
||||
def _q_gather_layout_supported(
|
||||
world_size: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
padded_num_heads: int | None,
|
||||
) -> bool:
|
||||
element_size = torch.empty((), dtype=dtype).element_size()
|
||||
gathered_num_heads = world_size * heads_per_rank
|
||||
storage_num_heads = (
|
||||
gathered_num_heads if padded_num_heads is None else padded_num_heads
|
||||
)
|
||||
return (
|
||||
heads_per_rank * head_dim * element_size % 16 == 0
|
||||
and storage_num_heads * head_dim * element_size % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
class DirectDCPQGatherWorkspace(_DirectDCPWorkspace):
|
||||
"""Publish query shards directly into the consumer-final symmetric buffer.
|
||||
|
||||
The final buffer is reusable after the downstream DCP output combine. That
|
||||
combine orders all ranks after attention has consumed the gathered query.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_ubatches: int = 1,
|
||||
padded_num_heads: int | None = None,
|
||||
) -> None:
|
||||
if num_ubatches < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP q-gather requires at least one ubatch slot, "
|
||||
f"got {num_ubatches}"
|
||||
)
|
||||
if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP q-gather dimensions must be positive, got "
|
||||
f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}"
|
||||
)
|
||||
gathered_num_heads = group.size() * heads_per_rank
|
||||
if not _q_gather_layout_supported(
|
||||
group.size(), heads_per_rank, head_dim, dtype, padded_num_heads
|
||||
):
|
||||
raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.")
|
||||
super().__init__(group, device, num_ubatches)
|
||||
if self.world_size <= 1:
|
||||
raise ValueError("Direct DCP q-gather requires at least two ranks")
|
||||
self.max_num_tokens = max_num_tokens
|
||||
self.heads_per_rank = heads_per_rank
|
||||
self.gathered_num_heads = gathered_num_heads
|
||||
self.padded_num_heads = (
|
||||
self.gathered_num_heads if padded_num_heads is None else padded_num_heads
|
||||
)
|
||||
if self.padded_num_heads < self.gathered_num_heads:
|
||||
raise ValueError(
|
||||
"Direct DCP q-gather padded heads must cover gathered heads: "
|
||||
f"{self.padded_num_heads} < {self.gathered_num_heads}"
|
||||
)
|
||||
self.head_dim = head_dim
|
||||
|
||||
query_shape = (
|
||||
num_ubatches,
|
||||
max_num_tokens,
|
||||
self.padded_num_heads,
|
||||
head_dim,
|
||||
)
|
||||
signal_shape = (num_ubatches, 2, self.world_size)
|
||||
self.final_query, _ = self._allocate(query_shape, dtype)
|
||||
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
|
||||
query_multicast_ptrs = self._multicast_ptrs(self.final_query)
|
||||
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
|
||||
self.multicast_ptrs = list(
|
||||
zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True)
|
||||
)
|
||||
if not all(
|
||||
query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Direct DCP q-gather requires NVLS symmetric-memory multicast."
|
||||
)
|
||||
self.completion = self.received_signal.new_zeros((num_ubatches, 1))
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def gather(self, local_query: torch.Tensor) -> torch.Tensor:
|
||||
ubatch = dbo_current_ubatch_id()
|
||||
if not 0 <= ubatch < self.num_ubatches:
|
||||
raise ValueError(
|
||||
f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
|
||||
)
|
||||
if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank:
|
||||
raise ValueError(
|
||||
f"DCP q-gather expected {self.heads_per_rank} local query heads, "
|
||||
f"got {local_query.shape[1]}"
|
||||
)
|
||||
|
||||
num_tokens = local_query.shape[0]
|
||||
output = torch.as_strided(
|
||||
self.final_query[ubatch],
|
||||
size=(num_tokens, self.gathered_num_heads, self.head_dim),
|
||||
stride=(
|
||||
self.gathered_num_heads * self.head_dim,
|
||||
self.head_dim,
|
||||
1,
|
||||
),
|
||||
)
|
||||
query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
|
||||
torch.ops._C.direct_dcp_q_gather(
|
||||
local_query,
|
||||
output,
|
||||
self.received_signal[ubatch],
|
||||
self.completion[ubatch],
|
||||
self.epoch[ubatch : ubatch + 1],
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.max_num_tokens,
|
||||
self.padded_num_heads,
|
||||
query_multicast_ptr,
|
||||
signal_multicast_ptr,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_direct_dcp_q_gather_workspace(
|
||||
group: GroupCoordinator,
|
||||
device: torch.device,
|
||||
max_num_tokens: int,
|
||||
heads_per_rank: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
num_ubatches: int,
|
||||
padded_num_heads: int | None = None,
|
||||
) -> DirectDCPQGatherWorkspace | None:
|
||||
if not _direct_dcp_multicast_enabled(
|
||||
group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER
|
||||
):
|
||||
return None
|
||||
if not _q_gather_layout_supported(
|
||||
group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads
|
||||
):
|
||||
return None
|
||||
return DirectDCPQGatherWorkspace(
|
||||
group.device_group,
|
||||
device,
|
||||
max_num_tokens,
|
||||
heads_per_rank,
|
||||
head_dim,
|
||||
dtype,
|
||||
num_ubatches,
|
||||
padded_num_heads,
|
||||
)
|
||||
|
||||
|
||||
_KV_GATHER_SUPPORTED_DTYPES = (
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
torch.float8_e4m3fn,
|
||||
)
|
||||
|
||||
|
||||
def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool:
|
||||
return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0
|
||||
|
||||
|
||||
class DirectDCPKVGatherWorkspace(_DirectDCPWorkspace):
|
||||
"""Persistent symmetric buffers for direct DCP KV gather."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device,
|
||||
max_gathered_tokens: int,
|
||||
token_dim: int,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
num_ubatches: int = 1,
|
||||
) -> None:
|
||||
if dtype not in _KV_GATHER_SUPPORTED_DTYPES:
|
||||
raise ValueError(f"Direct DCP kv-gather does not support {dtype}")
|
||||
if num_ubatches < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP kv-gather requires at least one ubatch slot, "
|
||||
f"got {num_ubatches}"
|
||||
)
|
||||
if max_gathered_tokens < 1 or token_dim < 1:
|
||||
raise ValueError(
|
||||
"Direct DCP kv-gather dimensions must be positive, got "
|
||||
f"T={max_gathered_tokens}, D={token_dim}"
|
||||
)
|
||||
if not _kv_gather_layout_supported(token_dim, dtype):
|
||||
raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.")
|
||||
super().__init__(group, device, num_ubatches)
|
||||
if self.world_size <= 1:
|
||||
raise ValueError("Direct DCP kv-gather requires at least two ranks")
|
||||
if max_gathered_tokens % self.world_size != 0:
|
||||
raise ValueError(
|
||||
"Direct DCP kv-gather capacity must divide evenly across "
|
||||
f"ranks: {max_gathered_tokens} % {self.world_size} != 0"
|
||||
)
|
||||
self.max_gathered_tokens = max_gathered_tokens
|
||||
|
||||
kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim)
|
||||
signal_shape = (num_ubatches, 2, self.world_size)
|
||||
self.received_kv, _ = self._allocate(kv_shape, dtype)
|
||||
self.received_signal, _ = self._allocate(signal_shape, torch.int32)
|
||||
kv_multicast_ptrs = self._multicast_ptrs(self.received_kv)
|
||||
signal_multicast_ptrs = self._multicast_ptrs(self.received_signal)
|
||||
self.multicast_ptrs = list(
|
||||
zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True)
|
||||
)
|
||||
if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs):
|
||||
raise RuntimeError(
|
||||
"Direct DCP kv-gather requires NVLS symmetric-memory multicast."
|
||||
)
|
||||
self.completion = self.received_signal.new_zeros((num_ubatches, 2))
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None:
|
||||
ubatch = dbo_current_ubatch_id()
|
||||
if not 0 <= ubatch < self.num_ubatches:
|
||||
raise ValueError(
|
||||
f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots"
|
||||
)
|
||||
kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch]
|
||||
torch.ops._C.direct_dcp_kv_gather(
|
||||
local_kv,
|
||||
self.received_kv[ubatch],
|
||||
self.received_signal[ubatch],
|
||||
self.completion[ubatch],
|
||||
self.epoch[ubatch : ubatch + 1],
|
||||
gathered_kv,
|
||||
self.world_size,
|
||||
self.rank,
|
||||
self.max_gathered_tokens,
|
||||
kv_multicast_ptr,
|
||||
signal_multicast_ptr,
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_direct_dcp_kv_gather_workspace(
|
||||
group: GroupCoordinator,
|
||||
device: torch.device,
|
||||
max_gathered_tokens: int,
|
||||
token_dim: int,
|
||||
dtype: torch.dtype,
|
||||
num_ubatches: int,
|
||||
) -> DirectDCPKVGatherWorkspace | None:
|
||||
if not _direct_dcp_multicast_enabled(
|
||||
group,
|
||||
dtype,
|
||||
envs.VLLM_USE_DIRECT_DCP_KV_GATHER,
|
||||
_KV_GATHER_SUPPORTED_DTYPES,
|
||||
):
|
||||
return None
|
||||
if not _kv_gather_layout_supported(token_dim, dtype):
|
||||
return None
|
||||
return DirectDCPKVGatherWorkspace(
|
||||
group.device_group,
|
||||
device,
|
||||
max_gathered_tokens,
|
||||
token_dim,
|
||||
dtype,
|
||||
num_ubatches,
|
||||
)
|
||||
|
||||
|
||||
class DCPCombine(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
partial_output: torch.Tensor,
|
||||
partial_lse: torch.Tensor,
|
||||
*,
|
||||
seq_lens: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> torch.Tensor: ...
|
||||
|
||||
|
||||
class MLADCPManager:
|
||||
"""Select and own layer-level collective implementations for MLA DCP."""
|
||||
|
||||
_kv_gather: Callable[[torch.Tensor, torch.Tensor], object]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
num_heads: int,
|
||||
query_head_dim: int,
|
||||
output_head_dim: int,
|
||||
query_dtype: torch.dtype,
|
||||
output_dtype: torch.dtype,
|
||||
padded_num_heads: int | None,
|
||||
is_lse_base_on_e: bool,
|
||||
use_pcp: bool,
|
||||
) -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.group = get_dcp_group()
|
||||
self.device = torch.device(device)
|
||||
self.num_ubatches = max(parallel_config.num_ubatches, 1)
|
||||
self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config)
|
||||
self.use_a2a = parallel_config.dcp_comm_backend == "a2a"
|
||||
self.padded_num_heads = padded_num_heads
|
||||
|
||||
self.combine = self._init_combine(
|
||||
num_heads,
|
||||
output_head_dim,
|
||||
output_dtype,
|
||||
is_lse_base_on_e,
|
||||
use_pcp,
|
||||
)
|
||||
self.query_gather = (
|
||||
None
|
||||
if use_pcp
|
||||
else self._init_query_gather(
|
||||
num_heads,
|
||||
query_head_dim,
|
||||
query_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
def _init_combine(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
is_lse_base_on_e: bool,
|
||||
use_pcp: bool,
|
||||
) -> DCPCombine:
|
||||
direct_workspace = None
|
||||
if self.use_a2a:
|
||||
direct_workspace = get_direct_dcp_a2a_workspace(
|
||||
self.group,
|
||||
self.device,
|
||||
self.max_num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
dtype,
|
||||
self.num_ubatches,
|
||||
)
|
||||
if direct_workspace is not None:
|
||||
logger.info_once("Using direct symmetric-memory DCP A2A for MLA.")
|
||||
return functools.partial(
|
||||
direct_workspace.lse_reduce,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
combine_fn = (
|
||||
dcp_a2a_lse_reduce
|
||||
if self.use_a2a
|
||||
else cp_lse_ag_out_ar
|
||||
if use_pcp
|
||||
else cp_lse_ag_out_rs
|
||||
)
|
||||
return functools.partial(
|
||||
combine_fn,
|
||||
cp_group=self.group,
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
|
||||
def _init_query_gather(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
dtype: torch.dtype,
|
||||
) -> Callable[[torch.Tensor], torch.Tensor]:
|
||||
direct_workspace = get_direct_dcp_q_gather_workspace(
|
||||
self.group,
|
||||
self.device,
|
||||
self.max_num_tokens,
|
||||
num_heads,
|
||||
head_dim,
|
||||
dtype,
|
||||
self.num_ubatches,
|
||||
self.padded_num_heads,
|
||||
)
|
||||
if direct_workspace is not None:
|
||||
logger.info_once("Using direct symmetric-memory DCP query gather for MLA.")
|
||||
return direct_workspace.gather
|
||||
return self._gather_query
|
||||
|
||||
def _gather_query(self, query: torch.Tensor) -> torch.Tensor:
|
||||
query = self.group.all_gather(query, dim=1)
|
||||
if self.padded_num_heads is not None:
|
||||
query = reserve_query_head_storage(query, self.padded_num_heads)
|
||||
return query
|
||||
|
||||
def init_kv_gather(
|
||||
self,
|
||||
workspace: torch.Tensor,
|
||||
max_gathered_tokens: int,
|
||||
) -> None:
|
||||
world_size = self.group.world_size
|
||||
assert max_gathered_tokens > 0
|
||||
assert max_gathered_tokens % world_size == 0
|
||||
assert workspace.ndim == 2
|
||||
assert workspace.is_contiguous()
|
||||
assert workspace.shape[0] == (
|
||||
max_gathered_tokens + max_gathered_tokens // world_size
|
||||
)
|
||||
assert workspace.shape[1] > 0
|
||||
|
||||
direct_workspace = get_direct_dcp_kv_gather_workspace(
|
||||
self.group,
|
||||
workspace.device,
|
||||
max_gathered_tokens,
|
||||
workspace.shape[1],
|
||||
workspace.dtype,
|
||||
self.num_ubatches,
|
||||
)
|
||||
if direct_workspace is not None:
|
||||
logger.info_once(
|
||||
"Using direct symmetric-memory DCP chunked-context KV gather for MLA."
|
||||
)
|
||||
self._kv_gather = direct_workspace.gather
|
||||
else:
|
||||
self._kv_gather = functools.partial(
|
||||
torch.distributed.all_gather_into_tensor,
|
||||
group=self.group.device_group,
|
||||
)
|
||||
|
||||
def kv_gather(
|
||||
self,
|
||||
gathered_kv: torch.Tensor,
|
||||
local_kv: torch.Tensor,
|
||||
) -> object:
|
||||
return self._kv_gather(gathered_kv, local_kv)
|
||||
Reference in New Issue
Block a user