diff --git a/CMakeLists.txt b/CMakeLists.txt index bcc0fc53d5..baa4cf7dfa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -474,6 +474,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() if(VLLM_GPU_LANG STREQUAL "CUDA") + list(APPEND VLLM_STABLE_EXT_SRC + "csrc/libtorch_stable/attention/dcp_utils/dcp_direct_a2a_lse_reduce.cu" + "csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu" + "csrc/libtorch_stable/attention/dcp_utils/dcp_direct_q_gather.cu") + SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library") # Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building. diff --git a/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_a2a_lse_reduce.cu b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_a2a_lse_reduce.cu new file mode 100644 index 0000000000..bfc8e6c42a --- /dev/null +++ b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_a2a_lse_reduce.cu @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project +// Direct symmetric-memory DCP A2A LSE reduction. + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "dcp_direct_common.cuh" + +namespace { + +using vllm::direct_dcp::check_cuda_launch; +using vllm::direct_dcp::get_peer_ptr; +using vllm::direct_dcp::increment_epoch_kernel; +using vllm::direct_dcp::store_release_system; +using vllm::direct_dcp::wait_for_epoch; + +__device__ __forceinline__ int64_t find_sequence(const int32_t* query_start_loc, + int64_t token_idx, + int64_t num_seqs) { + int64_t left = 0; + int64_t right = num_seqs; + while (left < right) { + int64_t mid = (left + right) / 2; + if (query_start_loc[mid] <= token_idx) { + left = mid + 1; + } else { + right = mid; + } + } + return left - 1; +} + +template +__device__ __forceinline__ float to_float(scalar_t value) { + if constexpr (std::is_same_v) { + return value; + } else if constexpr (std::is_same_v) { + return __half2float(value); + } else { + return __bfloat162float(value); + } +} + +template +__device__ __forceinline__ scalar_t from_float(float value) { + if constexpr (std::is_same_v) { + return __float2half_rn(value); + } else { + return __float2bfloat16_rn(value); + } +} + +// Send each destination's head slice and LSE to its symmetric buffer. Mask +// undefined empty-shard rows at the source so peers give them zero weight. +template +__global__ void dispatch_output_lse_kernel( + const uint4* partial_output, const lse_t* partial_lse, + const int32_t* seq_lens, const int32_t* query_start_loc, + const int64_t* peer_output_ptrs, const int64_t* peer_lse_ptrs, + const int64_t* epoch_ptr, int64_t world_size, int64_t rank, + int64_t num_tokens, int64_t max_num_tokens, int64_t num_seqs, + int64_t heads_per_rank, int64_t head_dim, int64_t output_token_stride, + int64_t lse_token_stride) { + int64_t item = static_cast(blockIdx.x); + int64_t destination_rank = item / num_tokens; + int64_t token_idx = item - destination_rank * num_tokens; + + __shared__ bool empty_kv; + if (threadIdx.x == 0) { + empty_kv = false; + if (seq_lens != nullptr) { + int64_t seq_idx = find_sequence(query_start_loc, token_idx, num_seqs); + empty_kv = seq_lens[seq_idx] == 0; + } + } + __syncthreads(); + + uint32_t epoch = static_cast(epoch_ptr[0]); + int64_t parity = static_cast(epoch & 1u); + int64_t destination_item = + ((parity * world_size + rank) * max_num_tokens + token_idx) * + heads_per_rank; + int64_t source_head = destination_rank * heads_per_rank; + + if (!empty_kv) { + uint4* peer_output = + get_peer_ptr(peer_output_ptrs, destination_rank); + int64_t vectors_per_item = heads_per_rank * head_dim / 8; + int64_t source_vector = + (token_idx * output_token_stride + source_head * head_dim) / 8; + int64_t destination_vector = destination_item * head_dim / 8; + for (int64_t vector_idx = threadIdx.x; vector_idx < vectors_per_item; + vector_idx += blockDim.x) { + peer_output[destination_vector + vector_idx] = + partial_output[source_vector + vector_idx]; + } + } + + float* peer_lse = get_peer_ptr(peer_lse_ptrs, destination_rank); + int64_t source_lse = token_idx * lse_token_stride + source_head; + for (int64_t head_idx = threadIdx.x; head_idx < heads_per_rank; + head_idx += blockDim.x) { + peer_lse[destination_item + head_idx] = + empty_kv ? -CUDART_INF_F : to_float(partial_lse[source_lse + head_idx]); + } +} + +// Publish completion after the stream-ordered payload writes. +__global__ void signal_kernel(const int64_t* peer_signal_ptrs, + const int64_t* epoch_ptr, int64_t world_size, + int64_t rank) { + int64_t destination_rank = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (destination_rank >= world_size) { + return; + } + + uint32_t epoch = static_cast(epoch_ptr[0]); + int64_t parity = static_cast(epoch & 1u); + uint32_t* peer_signal = + get_peer_ptr(peer_signal_ptrs, destination_rank); + int64_t signal_item = parity * world_size + rank; + store_release_system(peer_signal + signal_item, epoch); +} + +// Form stable base-2 LSE weights and combine in FP32. +template +__global__ void wait_lse_combine_kernel( + const scalar_t* received_output, const float* received_lse, + const uint32_t* received_signal, const int64_t* epoch_ptr, + scalar_t* combined_output, int64_t world_size, int64_t num_tokens, + int64_t max_num_tokens, int64_t heads_per_rank, int64_t head_dim, + bool is_lse_base_on_e) { + extern __shared__ float weights[]; + + int64_t item = static_cast(blockIdx.x); + int64_t token_idx = item / heads_per_rank; + int64_t head_idx = item - token_idx * heads_per_rank; + uint32_t epoch = static_cast(epoch_ptr[0]); + int64_t parity = static_cast(epoch & 1u); + + if (threadIdx.x == 0) { + float lse_max = -CUDART_INF_F; + for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) { + int64_t source_item = + ((parity * world_size + source_rank) * max_num_tokens + token_idx) * + heads_per_rank + + head_idx; + int64_t signal_item = parity * world_size + source_rank; + if (!wait_for_epoch(received_signal + signal_item, epoch)) { + printf( + "direct DCP A2A timeout source=%lld token=%lld head=%lld " + "epoch=%u\n", + static_cast(source_rank), + static_cast(token_idx), static_cast(head_idx), + epoch); + asm volatile("trap;"); + } + float value = received_lse[source_item]; + if (isnan(value) || value == CUDART_INF_F) { + value = -CUDART_INF_F; + } + if (is_lse_base_on_e) { + value *= CUDART_L2E_F; + } + weights[source_rank] = value; + lse_max = fmaxf(lse_max, value); + } + if (lse_max == -CUDART_INF_F) { + lse_max = 0.0f; + } + + float lse_sum = 0.0f; + for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) { + lse_sum += exp2f(weights[source_rank] - lse_max); + } + float inverse_lse_sum = lse_sum > 0.0f ? 1.0f / lse_sum : 0.0f; + for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) { + weights[source_rank] = + exp2f(weights[source_rank] - lse_max) * inverse_lse_sum; + } + } + __syncthreads(); + + for (int64_t dim_idx = threadIdx.x; dim_idx < head_dim; + dim_idx += blockDim.x) { + float accumulator = 0.0f; + for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) { + // Empty shards leave stale payloads, so never read zero-weight sources. + float weight = weights[source_rank]; + if (weight == 0.0f) { + continue; + } + int64_t source_item = + ((parity * world_size + source_rank) * max_num_tokens + token_idx) * + heads_per_rank + + head_idx; + accumulator += + to_float(received_output[source_item * head_dim + dim_idx]) * weight; + } + combined_output[item * head_dim + dim_idx] = + from_float(accumulator); + } +} + +void direct_dcp_a2a_lse_reduce( + const torch::stable::Tensor& partial_output, + const torch::stable::Tensor& partial_lse, + const std::optional& seq_lens, + const std::optional& query_start_loc, + const torch::stable::Tensor& peer_output_ptrs, + const torch::stable::Tensor& peer_lse_ptrs, + const torch::stable::Tensor& peer_signal_ptrs, + torch::stable::Tensor& received_output, torch::stable::Tensor& received_lse, + torch::stable::Tensor& received_signal, torch::stable::Tensor& epoch, + torch::stable::Tensor& combined_output, int64_t world_size, int64_t rank, + int64_t max_num_tokens, bool is_lse_base_on_e) { + STD_TORCH_CHECK(partial_output.is_cuda() && partial_lse.is_cuda(), + "partial output and LSE must be CUDA tensors"); + auto output_dtype = partial_output.scalar_type(); + STD_TORCH_CHECK(output_dtype == torch::headeronly::ScalarType::Half || + output_dtype == torch::headeronly::ScalarType::BFloat16, + "direct DCP A2A only supports FP16 and BF16 output"); + auto lse_dtype = partial_lse.scalar_type(); + STD_TORCH_CHECK(lse_dtype == torch::headeronly::ScalarType::Float || + lse_dtype == torch::headeronly::ScalarType::Half || + lse_dtype == torch::headeronly::ScalarType::BFloat16, + "partial LSE must be FP32, FP16, or BF16"); + STD_TORCH_CHECK(partial_output.dim() == 3 && partial_lse.dim() == 2, + "expected output [T,H,D] and LSE [T,H]"); + STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1"); + STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank"); + + int64_t num_tokens = partial_output.size(0); + int64_t total_heads = partial_output.size(1); + int64_t head_dim = partial_output.size(2); + int64_t output_token_stride = partial_output.stride(0); + int64_t lse_token_stride = partial_lse.stride(0); + STD_TORCH_CHECK( + partial_output.stride(2) == 1 && partial_output.stride(1) == head_dim && + output_token_stride >= total_heads * head_dim && + output_token_stride % 8 == 0, + "partial output must have packed heads and an aligned token stride"); + STD_TORCH_CHECK(partial_lse.stride(1) == 1 && lse_token_stride >= total_heads, + "partial LSE must have packed heads"); + STD_TORCH_CHECK(num_tokens > 0 && num_tokens <= max_num_tokens, + "token count exceeds symmetric buffer capacity"); + STD_TORCH_CHECK(total_heads % world_size == 0, + "attention heads must divide evenly across DCP ranks"); + STD_TORCH_CHECK( + partial_lse.size(0) == num_tokens && partial_lse.size(1) == total_heads, + "LSE shape must match attention output"); + STD_TORCH_CHECK(head_dim % 8 == 0, + "head_dim must be divisible by 8 for 16-byte stores"); + int64_t heads_per_rank = total_heads / world_size; + STD_TORCH_CHECK(combined_output.scalar_type() == output_dtype && + combined_output.is_contiguous() && + combined_output.is_cuda(), + "combined output must match the contiguous CUDA input"); + STD_TORCH_CHECK(combined_output.size(0) == num_tokens && + combined_output.size(1) == heads_per_rank && + combined_output.size(2) == head_dim, + "combined output has the wrong shape"); + STD_TORCH_CHECK( + received_output.scalar_type() == output_dtype && + received_lse.scalar_type() == torch::headeronly::ScalarType::Float && + received_signal.scalar_type() == torch::headeronly::ScalarType::Int, + "invalid symmetric staging dtypes"); + STD_TORCH_CHECK( + peer_output_ptrs.scalar_type() == torch::headeronly::ScalarType::Long && + peer_lse_ptrs.scalar_type() == torch::headeronly::ScalarType::Long && + peer_signal_ptrs.scalar_type() == torch::headeronly::ScalarType::Long, + "peer pointer tables must be int64"); + const int32_t* seq_lens_ptr = nullptr; + const int32_t* query_start_loc_ptr = nullptr; + int64_t num_seqs = 0; + STD_TORCH_CHECK(seq_lens.has_value() == query_start_loc.has_value(), + "seq_lens and query_start_loc must be provided together"); + if (seq_lens.has_value() && query_start_loc.has_value()) { + STD_TORCH_CHECK( + seq_lens->is_cuda() && seq_lens->dim() == 1 && + seq_lens->stride(0) == 1 && + seq_lens->scalar_type() == torch::headeronly::ScalarType::Int, + "seq_lens must be a contiguous 1-D int32 CUDA tensor"); + STD_TORCH_CHECK( + query_start_loc->is_cuda() && query_start_loc->dim() == 1 && + query_start_loc->stride(0) == 1 && + query_start_loc->scalar_type() == + torch::headeronly::ScalarType::Int, + "query_start_loc must be a contiguous 1-D int32 CUDA tensor"); + num_seqs = seq_lens->size(0); + STD_TORCH_CHECK(num_seqs > 0 && query_start_loc->size(0) == num_seqs + 1, + "query_start_loc must contain one boundary per sequence"); + seq_lens_ptr = seq_lens->const_data_ptr(); + query_start_loc_ptr = query_start_loc->const_data_ptr(); + } + + const torch::stable::accelerator::DeviceGuard device_guard( + partial_output.get_device_index()); + cudaStream_t stream = get_current_cuda_stream(); + constexpr int kExchangeThreads = 256; + constexpr int kCombineThreads = 128; + + increment_epoch_kernel<<<1, 1, 0, stream>>>( + epoch.mutable_data_ptr()); + check_cuda_launch("direct DCP A2A"); + int64_t dispatch_blocks = world_size * num_tokens; + auto launch_dispatch = [&]() { + dispatch_output_lse_kernel + <<>>( + reinterpret_cast(partial_output.data_ptr()), + reinterpret_cast(partial_lse.data_ptr()), + seq_lens_ptr, query_start_loc_ptr, + peer_output_ptrs.const_data_ptr(), + peer_lse_ptrs.const_data_ptr(), + epoch.const_data_ptr(), world_size, rank, num_tokens, + max_num_tokens, num_seqs, heads_per_rank, head_dim, + output_token_stride, lse_token_stride); + }; + if (lse_dtype == torch::headeronly::ScalarType::Float) { + launch_dispatch.operator()(); + } else if (lse_dtype == torch::headeronly::ScalarType::BFloat16) { + launch_dispatch.operator()<__nv_bfloat16>(); + } else { + launch_dispatch.operator()<__half>(); + } + check_cuda_launch("direct DCP A2A"); + int64_t signal_items = world_size; + int64_t signal_blocks = + (signal_items + kExchangeThreads - 1) / kExchangeThreads; + signal_kernel<<>>( + peer_signal_ptrs.const_data_ptr(), + epoch.const_data_ptr(), world_size, rank); + check_cuda_launch("direct DCP A2A"); + int64_t combine_blocks = num_tokens * heads_per_rank; + size_t shared_memory_bytes = world_size * sizeof(float); + auto launch_combine = [&]() { + wait_lse_combine_kernel + <<>>( + reinterpret_cast(received_output.data_ptr()), + received_lse.const_data_ptr(), + reinterpret_cast( + received_signal.const_data_ptr()), + epoch.const_data_ptr(), + reinterpret_cast(combined_output.mutable_data_ptr()), + world_size, num_tokens, max_num_tokens, heads_per_rank, head_dim, + is_lse_base_on_e); + }; + if (output_dtype == torch::headeronly::ScalarType::BFloat16) { + launch_combine.operator()<__nv_bfloat16>(); + } else { + launch_combine.operator()<__half>(); + } + check_cuda_launch("direct DCP A2A"); +} + +} // namespace + +STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_a2a_ops) { + direct_dcp_a2a_ops.def( + "direct_dcp_a2a_lse_reduce(" + "Tensor partial_output, Tensor partial_lse, Tensor? seq_lens, " + "Tensor? query_start_loc, Tensor peer_output_ptrs, " + "Tensor peer_lse_ptrs, Tensor peer_signal_ptrs, Tensor! received_output, " + "Tensor! received_lse, Tensor! received_signal, Tensor! epoch, " + "Tensor! combined_output, int world_size, int rank, " + "int max_num_tokens, bool is_lse_base_on_e) -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_a2a_ops) { + direct_dcp_a2a_ops.impl("direct_dcp_a2a_lse_reduce", + TORCH_BOX(&direct_dcp_a2a_lse_reduce)); +} diff --git a/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_common.cuh b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_common.cuh new file mode 100644 index 0000000000..968f8c6d73 --- /dev/null +++ b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_common.cuh @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#pragma once + +#include + +#include +#include + +#include "../../torch_utils.h" + +namespace vllm::direct_dcp { + +constexpr uint64_t kSpinLimit = 100000000; + +// Advance the invocation ID; its low bit selects one of two staging slots. +static __global__ void increment_epoch_kernel(int64_t* epoch) { + if (blockIdx.x == 0 && threadIdx.x == 0) { + epoch[0] += 1; + } +} + +template +__device__ __forceinline__ T* get_peer_ptr(const int64_t* peer_ptrs, + int64_t peer) { + return reinterpret_cast(static_cast(peer_ptrs[peer])); +} + +// Replicate one 16-byte payload to every symmetric-buffer replica. +__device__ __forceinline__ void multimem_store_16(uint4* mc_ptr, uint4 value) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("multimem.st.relaxed.sys.global.v4.f32 [%0], {%1,%2,%3,%4};" + : + : "l"(mc_ptr), "r"(value.x), "r"(value.y), "r"(value.z), + "r"(value.w) + : "memory"); +#else + asm volatile("trap;"); +#endif +} + +// Publish prior system-scope writes and signal every replica. +__device__ __forceinline__ void multimem_store_release_system(uint32_t* mc_ptr, + uint32_t value) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("multimem.st.release.sys.global.u32 [%0], %1;" + : + : "l"(mc_ptr), "r"(value) + : "memory"); +#else + asm volatile("trap;"); +#endif +} + +__device__ __forceinline__ void store_release_system(uint32_t* ptr, + uint32_t value) { + uint64_t address = reinterpret_cast(ptr); + asm volatile("st.global.release.sys.u32 [%0], %1;" + : + : "l"(address), "r"(value) + : "memory"); +} + +__device__ __forceinline__ uint32_t load_acquire_system(const uint32_t* ptr) { + uint32_t value; + uint64_t address = reinterpret_cast(ptr); + asm volatile("ld.global.acquire.sys.u32 %0, [%1];" + : "=r"(value) + : "l"(address) + : "memory"); + return value; +} + +__device__ __forceinline__ bool wait_for_epoch(const uint32_t* ptr, + uint32_t epoch) { + for (uint64_t spins = 0; spins < kSpinLimit; ++spins) { + if (load_acquire_system(ptr) == epoch) { + return true; + } + } + return false; +} + +inline void check_cuda_launch(const char* operation) { + cudaError_t error = cudaGetLastError(); + STD_TORCH_CHECK(error == cudaSuccess, + std::string(operation) + + " kernel launch failed: " + cudaGetErrorString(error)); +} + +} // namespace vllm::direct_dcp diff --git a/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu new file mode 100644 index 0000000000..809f759b6a --- /dev/null +++ b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_kv_gather.cu @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project +// Direct symmetric-memory DCP KV gather. + +#include +#include + +#include + +#include "dcp_direct_common.cuh" + +namespace { + +using vllm::direct_dcp::check_cuda_launch; +using vllm::direct_dcp::increment_epoch_kernel; +using vllm::direct_dcp::multimem_store_16; +using vllm::direct_dcp::multimem_store_release_system; +using vllm::direct_dcp::wait_for_epoch; + +constexpr int kThreads = 256; +// KV chunks need many blocks in flight to saturate the fabric. +constexpr int64_t kMaxMulticastBlocks = 128; +constexpr int64_t kMaxCopyBlocks = 1024; + +// Multicast each rank's KV slice and completion epoch to every replica. +__global__ void direct_dcp_kv_gather_multimem_kernel( + const uint4* local_kv, uint4* mc_kv, uint32_t* mc_signal, + const uint32_t* received_signal, const int64_t* epoch_ptr, + uint32_t* completion, int64_t world_size, int64_t rank, int64_t slice_bytes, + int64_t slot_stride_bytes) { + uint32_t epoch = static_cast(epoch_ptr[0]); + int64_t buffer_slot = static_cast(epoch & 1u); + int64_t item_count = slice_bytes / sizeof(uint4); + mc_kv += buffer_slot * slot_stride_bytes / sizeof(uint4); + mc_kv += rank * item_count; + + int64_t item_stride = static_cast(gridDim.x) * blockDim.x; + for (int64_t item = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + item < item_count; item += item_stride) { + multimem_store_16(mc_kv + item, local_kv[item]); + } + + // Publish all multicast writes before incrementing completion. + __threadfence_system(); + __syncthreads(); + if (threadIdx.x != 0) { + return; + } + + uint32_t completed = atomicAdd(completion + buffer_slot, 1u); + if (completed + 1u != gridDim.x) { + return; + } + atomicExch(completion + buffer_slot, 0u); + + multimem_store_release_system(mc_signal + buffer_slot * world_size + rank, + epoch); + + for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) { + int64_t signal_item = buffer_slot * world_size + source_rank; + if (!wait_for_epoch(received_signal + signal_item, epoch)) { + printf("direct DCP kv-gather multimem timeout source=%lld epoch=%u\n", + static_cast(source_rank), epoch); + asm volatile("trap;"); + } + } +} + +// Materialize the acquired slot into the caller's workspace. +template +__global__ void materialize_kv_gather_kernel(const copy_t* received_kv, + const int64_t* epoch_ptr, + copy_t* gathered_kv, + int64_t gathered_item_count, + int64_t slot_stride_items) { + uint32_t epoch = static_cast(epoch_ptr[0]); + int64_t buffer_slot = static_cast(epoch & 1u); + received_kv += buffer_slot * slot_stride_items; + int64_t item = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + int64_t item_stride = static_cast(gridDim.x) * blockDim.x; + for (; item < gathered_item_count; item += item_stride) { + gathered_kv[item] = received_kv[item]; + } +} + +void direct_dcp_kv_gather( + const torch::stable::Tensor& local_kv, torch::stable::Tensor& received_kv, + torch::stable::Tensor& received_signal, torch::stable::Tensor& completion, + torch::stable::Tensor& epoch, torch::stable::Tensor& gathered_kv, + int64_t world_size, int64_t rank, int64_t max_gathered_tokens, + int64_t kv_mc_ptr, int64_t signal_mc_ptr) { + using torch::headeronly::ScalarType; + + STD_TORCH_CHECK(local_kv.is_cuda(), "local kv must be a CUDA tensor"); + ScalarType dtype = local_kv.scalar_type(); + STD_TORCH_CHECK(dtype == ScalarType::Half || dtype == ScalarType::BFloat16 || + dtype == ScalarType::Float8_e4m3fn, + "direct DCP kv-gather only supports FP16, BF16, and FP8"); + STD_TORCH_CHECK(local_kv.dim() == 2 && local_kv.is_contiguous(), + "local kv must be a contiguous [T,D] tensor"); + STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1"); + STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank"); + + int64_t num_tokens = local_kv.size(0); + int64_t token_dim = local_kv.size(1); + int64_t element_size = local_kv.element_size(); + STD_TORCH_CHECK(num_tokens > 0 && token_dim > 0, + "local kv dimensions must be positive"); + STD_TORCH_CHECK(num_tokens * world_size <= max_gathered_tokens, + "gathered kv exceeds symmetric kv-gather buffer capacity"); + + STD_TORCH_CHECK(received_kv.is_cuda() && received_kv.scalar_type() == dtype && + received_kv.is_contiguous() && received_kv.dim() == 3 && + received_kv.size(0) == 2 && + received_kv.size(1) == max_gathered_tokens && + received_kv.size(2) == token_dim, + "received kv has the wrong symmetric buffer layout"); + STD_TORCH_CHECK( + received_signal.is_cuda() && received_signal.is_contiguous() && + received_signal.scalar_type() == ScalarType::Int && + received_signal.dim() == 2 && received_signal.size(0) == 2 && + received_signal.size(1) == world_size, + "received signal has the wrong symmetric buffer layout"); + STD_TORCH_CHECK(completion.is_cuda() && completion.is_contiguous() && + completion.scalar_type() == ScalarType::Int && + completion.numel() == 2, + "completion counter must be a two-element CUDA int32 tensor"); + STD_TORCH_CHECK(epoch.is_cuda() && epoch.is_contiguous() && + epoch.scalar_type() == ScalarType::Long && + epoch.numel() == 1, + "epoch must be a one-element CUDA int64 tensor"); + STD_TORCH_CHECK(gathered_kv.is_cuda() && gathered_kv.scalar_type() == dtype && + gathered_kv.is_contiguous() && gathered_kv.dim() == 2 && + gathered_kv.size(0) == num_tokens * world_size && + gathered_kv.size(1) == token_dim, + "gathered kv must be contiguous with shape [world_size*T,D]"); + int64_t device_index = local_kv.get_device_index(); + STD_TORCH_CHECK( + received_kv.get_device_index() == device_index && + received_signal.get_device_index() == device_index && + completion.get_device_index() == device_index && + epoch.get_device_index() == device_index && + gathered_kv.get_device_index() == device_index, + "direct DCP kv-gather tensors must be on the same CUDA device"); + + int64_t slice_bytes = num_tokens * token_dim * element_size; + int64_t slot_stride_bytes = max_gathered_tokens * token_dim * element_size; + bool vectorized = + reinterpret_cast(local_kv.data_ptr()) % alignof(uint4) == 0 && + reinterpret_cast(received_kv.data_ptr()) % alignof(uint4) == + 0 && + reinterpret_cast(gathered_kv.data_ptr()) % alignof(uint4) == + 0 && + slice_bytes % sizeof(uint4) == 0 && + slot_stride_bytes % sizeof(uint4) == 0; + STD_TORCH_CHECK( + vectorized, + "direct DCP kv-gather requires 16-byte-aligned pointers and strides"); + STD_TORCH_CHECK(kv_mc_ptr != 0 && signal_mc_ptr != 0, + "direct DCP kv-gather requires multicast pointers"); + + const torch::stable::accelerator::DeviceGuard device_guard(device_index); + cudaStream_t stream = get_current_cuda_stream(); + increment_epoch_kernel<<<1, 1, 0, stream>>>( + epoch.mutable_data_ptr()); + check_cuda_launch("direct DCP kv-gather"); + + int64_t item_count = slice_bytes / sizeof(uint4); + int64_t blocks = (item_count + kThreads - 1) / kThreads; + blocks = blocks < kMaxMulticastBlocks ? blocks : kMaxMulticastBlocks; + direct_dcp_kv_gather_multimem_kernel<<>>( + reinterpret_cast(local_kv.data_ptr()), + reinterpret_cast(static_cast(kv_mc_ptr)), + reinterpret_cast(static_cast(signal_mc_ptr)), + reinterpret_cast( + received_signal.const_data_ptr()), + epoch.const_data_ptr(), + reinterpret_cast(completion.mutable_data_ptr()), + world_size, rank, slice_bytes, slot_stride_bytes); + check_cuda_launch("direct DCP kv-gather"); + + int64_t gathered_item_count = world_size * item_count; + int64_t copy_blocks = (gathered_item_count + kThreads - 1) / kThreads; + copy_blocks = copy_blocks < kMaxCopyBlocks ? copy_blocks : kMaxCopyBlocks; + materialize_kv_gather_kernel<<>>( + reinterpret_cast(received_kv.data_ptr()), + epoch.const_data_ptr(), + reinterpret_cast(gathered_kv.mutable_data_ptr()), + gathered_item_count, slot_stride_bytes / sizeof(uint4)); + check_cuda_launch("direct DCP kv-gather"); +} + +} // namespace + +STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_kv_gather_ops) { + direct_dcp_kv_gather_ops.def( + "direct_dcp_kv_gather(" + "Tensor local_kv, Tensor! received_kv, Tensor! received_signal, " + "Tensor! completion, " + "Tensor! epoch, Tensor! gathered_kv, " + "int world_size, int rank, int max_gathered_tokens, " + "int kv_mc_ptr, int signal_mc_ptr) -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_kv_gather_ops) { + direct_dcp_kv_gather_ops.impl("direct_dcp_kv_gather", + TORCH_BOX(&direct_dcp_kv_gather)); +} diff --git a/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_q_gather.cu b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_q_gather.cu new file mode 100644 index 0000000000..c6d36e2b54 --- /dev/null +++ b/csrc/libtorch_stable/attention/dcp_utils/dcp_direct_q_gather.cu @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project +// Direct symmetric-memory DCP query gather into the consumer-final buffer. + +#include +#include + +#include + +#include "dcp_direct_common.cuh" + +namespace { + +using vllm::direct_dcp::check_cuda_launch; +using vllm::direct_dcp::increment_epoch_kernel; +using vllm::direct_dcp::multimem_store_16; +using vllm::direct_dcp::multimem_store_release_system; +using vllm::direct_dcp::wait_for_epoch; + +// Multicast each rank's head slice directly into every consumer's final query +// buffer. Reuse is ordered by the downstream DCP output synchronization. +__global__ void direct_dcp_q_gather_multimem_kernel( + const uint4* local_query, uint4* mc_final_query, uint32_t* mc_signal, + const uint32_t* received_signal, int64_t* epoch_ptr, uint32_t* completion, + int64_t world_size, int64_t rank, int64_t num_tokens, + int64_t bytes_per_token, int64_t query_token_stride_bytes, + int64_t destination_token_stride_bytes) { + // The common one-token decode case uses one block. Fold the epoch update + // into that publication kernel to avoid an otherwise separate tiny launch. + if (gridDim.x == 1 && threadIdx.x == 0) { + epoch_ptr[0] += 1; + } + __syncthreads(); + uint32_t epoch = static_cast(epoch_ptr[0]); + int64_t signal_slot = static_cast(epoch & 1u); + + int64_t items_per_token = bytes_per_token / sizeof(uint4); + int64_t source_token_stride = query_token_stride_bytes / sizeof(uint4); + int64_t destination_token_stride = + destination_token_stride_bytes / sizeof(uint4); + int64_t destination_head_offset = rank * items_per_token; + for (int64_t token_idx = blockIdx.x; token_idx < num_tokens; + token_idx += gridDim.x) { + for (int64_t token_item = threadIdx.x; token_item < items_per_token; + token_item += blockDim.x) { + multimem_store_16( + mc_final_query + token_idx * destination_token_stride + + destination_head_offset + token_item, + local_query[token_idx * source_token_stride + token_item]); + } + } + + // Publish all multicast writes before incrementing completion. + __threadfence_system(); + __syncthreads(); + if (threadIdx.x != 0) { + return; + } + + if (gridDim.x > 1) { + uint32_t completed = atomicAdd(completion, 1u); + if (completed + 1u != gridDim.x) { + return; + } + atomicExch(completion, 0u); + } + + multimem_store_release_system(mc_signal + signal_slot * world_size + rank, + epoch); + + for (int64_t source_rank = 0; source_rank < world_size; ++source_rank) { + int64_t signal_item = signal_slot * world_size + source_rank; + if (!wait_for_epoch(received_signal + signal_item, epoch)) { + printf("direct DCP q-gather multimem timeout source=%lld epoch=%u\n", + static_cast(source_rank), epoch); + asm volatile("trap;"); + } + } +} + +void direct_dcp_q_gather(const torch::stable::Tensor& local_query, + torch::stable::Tensor& final_query, + torch::stable::Tensor& received_signal, + torch::stable::Tensor& completion, + torch::stable::Tensor& epoch, int64_t world_size, + int64_t rank, int64_t max_num_tokens, + int64_t padded_num_heads, int64_t query_mc_ptr, + int64_t signal_mc_ptr) { + using torch::headeronly::ScalarType; + + STD_TORCH_CHECK(local_query.is_cuda(), "local query must be a CUDA tensor"); + ScalarType dtype = local_query.scalar_type(); + STD_TORCH_CHECK(local_query.dim() == 3, + "local query must have shape [T,H,D]"); + STD_TORCH_CHECK(world_size > 1, "world_size must be greater than 1"); + STD_TORCH_CHECK(rank >= 0 && rank < world_size, "invalid rank"); + + int64_t num_tokens = local_query.size(0); + int64_t heads_per_rank = local_query.size(1); + int64_t head_dim = local_query.size(2); + int64_t gathered_num_heads = world_size * heads_per_rank; + int64_t element_size = local_query.element_size(); + STD_TORCH_CHECK(num_tokens > 0 && num_tokens <= max_num_tokens, + "token count exceeds symmetric q-gather buffer capacity"); + STD_TORCH_CHECK(heads_per_rank > 0 && head_dim > 0, + "query head dimensions must be positive"); + STD_TORCH_CHECK(padded_num_heads >= gathered_num_heads, + "padded query heads must cover all gathered heads"); + STD_TORCH_CHECK(local_query.stride(2) == 1 && + local_query.stride(1) == head_dim && + local_query.stride(0) >= heads_per_rank * head_dim, + "local query must have packed heads"); + + STD_TORCH_CHECK( + final_query.is_cuda() && final_query.scalar_type() == dtype && + final_query.is_contiguous() && final_query.dim() == 3 && + final_query.size(0) == num_tokens && + final_query.size(1) == gathered_num_heads && + final_query.size(2) == head_dim, + "final query must be contiguous with shape [T,world_size*H,D]"); + STD_TORCH_CHECK( + received_signal.is_cuda() && received_signal.is_contiguous() && + received_signal.scalar_type() == ScalarType::Int && + received_signal.dim() == 2 && received_signal.size(0) == 2 && + received_signal.size(1) == world_size, + "received signal has the wrong symmetric buffer layout"); + STD_TORCH_CHECK(completion.is_cuda() && completion.is_contiguous() && + completion.scalar_type() == ScalarType::Int && + completion.numel() == 1, + "completion counter must be one CUDA int32 tensor"); + STD_TORCH_CHECK(epoch.is_cuda() && epoch.is_contiguous() && + epoch.scalar_type() == ScalarType::Long && + epoch.numel() == 1, + "epoch must be a one-element CUDA int64 tensor"); + int64_t device_index = local_query.get_device_index(); + STD_TORCH_CHECK( + final_query.get_device_index() == device_index && + received_signal.get_device_index() == device_index && + completion.get_device_index() == device_index && + epoch.get_device_index() == device_index, + "direct DCP q-gather tensors must be on the same CUDA device"); + + int64_t query_token_stride_bytes = local_query.stride(0) * element_size; + int64_t bytes_per_token = heads_per_rank * head_dim * element_size; + int64_t gathered_token_stride_bytes = + gathered_num_heads * head_dim * element_size; + bool vectorized = + reinterpret_cast(local_query.data_ptr()) % alignof(uint4) == + 0 && + reinterpret_cast(final_query.data_ptr()) % alignof(uint4) == + 0 && + query_token_stride_bytes % sizeof(uint4) == 0 && + bytes_per_token % sizeof(uint4) == 0; + STD_TORCH_CHECK( + vectorized, + "direct DCP q-gather requires 16-byte-aligned pointers and strides"); + STD_TORCH_CHECK(query_mc_ptr != 0 && signal_mc_ptr != 0, + "direct DCP q-gather requires multicast pointers"); + + const torch::stable::accelerator::DeviceGuard device_guard(device_index); + cudaStream_t stream = get_current_cuda_stream(); + constexpr int kThreads = 256; + int64_t blocks = num_tokens < world_size ? num_tokens : world_size; + if (blocks > 1) { + increment_epoch_kernel<<<1, 1, 0, stream>>>( + epoch.mutable_data_ptr()); + check_cuda_launch("direct DCP q-gather epoch"); + } + + direct_dcp_q_gather_multimem_kernel<<>>( + reinterpret_cast(local_query.data_ptr()), + reinterpret_cast(static_cast(query_mc_ptr)), + reinterpret_cast(static_cast(signal_mc_ptr)), + reinterpret_cast( + received_signal.const_data_ptr()), + epoch.mutable_data_ptr(), + reinterpret_cast(completion.mutable_data_ptr()), + world_size, rank, num_tokens, bytes_per_token, query_token_stride_bytes, + gathered_token_stride_bytes); + check_cuda_launch("direct DCP q-gather"); +} + +} // namespace + +STABLE_TORCH_LIBRARY_FRAGMENT(_C, direct_dcp_q_gather_ops) { + direct_dcp_q_gather_ops.def( + "direct_dcp_q_gather(" + "Tensor local_query, Tensor! final_query, Tensor! received_signal, " + "Tensor! completion, Tensor! epoch, " + "int world_size, int rank, int max_num_tokens, " + "int padded_num_heads, int query_mc_ptr, int signal_mc_ptr) -> ()"); +} + +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, direct_dcp_q_gather_ops) { + direct_dcp_q_gather_ops.impl("direct_dcp_q_gather", + TORCH_BOX(&direct_dcp_q_gather)); +} diff --git a/tests/distributed/test_dcp_a2a.py b/tests/distributed/test_dcp_a2a.py index 5ab0f3de97..6dc39a166c 100644 --- a/tests/distributed/test_dcp_a2a.py +++ b/tests/distributed/test_dcp_a2a.py @@ -222,6 +222,49 @@ class TestLSEWeightedCombine: assert result.shape == (B, H, D) torch.testing.assert_close(result, outputs[1], atol=1e-5, rtol=1e-5) + def test_empty_shard_ignores_undefined_output(self): + from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + + outputs = torch.tensor([[[[float("nan")]]], [[[3.0]]]]) + lses = torch.tensor([[[-float("inf")]], [[0.0]]]) + + result = _lse_weighted_combine(outputs, lses) + + torch.testing.assert_close(result, outputs[1]) + + def test_ag_rs_masks_empty_shard_and_padded_lse(self, monkeypatch): + import vllm.v1.attention.ops.common as common + + class FakeGroup: + world_size = 2 + rank_in_group = 0 + + def all_gather(self, tensor, dim): + assert dim == 0 + return torch.cat((tensor, tensor), dim=dim) + + monkeypatch.setattr( + common, + "correct_attn_out", + lambda output, lses, *args, **kwargs: (output, lses[0]), + ) + output = torch.ones(7, 1, 1) + lse = torch.ones(7, 1) + seq_lens = torch.tensor([0, 2], dtype=torch.int32) + query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32) + + _, masked_lse = common._cp_lse_common( + output, + lse, + FakeGroup(), + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + + assert torch.isneginf(masked_lse[:1]).all() + torch.testing.assert_close(masked_lse[1:5], torch.ones_like(masked_lse[1:5])) + assert torch.isneginf(masked_lse[5:]).all() + def test_mathematically_correct(self): """Verify mathematical correctness of LSE combination.""" from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine @@ -344,7 +387,7 @@ class TestPackedA2AKernels: world_size, B, h_per_rank, D = 4, 7, 2, 32 H = world_size * h_per_rank cp_attn_out = torch.randn(B, H, D, device=device, dtype=dtype) - cp_attn_lse = torch.randn(B, H, device=device, dtype=torch.float32) + cp_attn_lse = torch.randn(B, H, device=device, dtype=dtype) lse_pack_dim = _dcp_a2a_lse_pack_dim(dtype) send_buffer = torch.empty( (world_size, B, h_per_rank, D + lse_pack_dim), @@ -375,6 +418,67 @@ class TestPackedA2AKernels: else: _assert_packed_a2a_close(actual, expected_out, dtype) + @pytest.mark.skipif( + torch.accelerator.device_count() < 1, reason="CUDA is required." + ) + def test_empty_seq_lens_ignore_undefined_output(self): + from vllm.v1.attention.ops.dcp_alltoall import ( + _dcp_a2a_lse_pack_dim, + _dcp_a2a_pack_send, + _dcp_a2a_unpack_combine, + ) + + device = torch.device("cuda") + world_size, num_tokens, h_per_rank, head_dim = 2, 5, 1, 32 + num_heads = world_size * h_per_rank + output = torch.randn( + num_tokens, + num_heads, + head_dim, + device=device, + dtype=torch.bfloat16, + ) + output[:1] = float("nan") + lse = torch.randn(num_tokens, num_heads, device=device, dtype=output.dtype) + seq_lens = torch.tensor([0, 2], device=device, dtype=torch.int32) + query_start_loc = torch.tensor([0, 1, 5], device=device, dtype=torch.int32) + lse_pack_dim = _dcp_a2a_lse_pack_dim(output.dtype) + send_buffer = torch.empty( + ( + world_size, + num_tokens, + h_per_rank, + head_dim + lse_pack_dim, + ), + device=device, + dtype=output.dtype, + ) + + _dcp_a2a_pack_send( + output, + lse, + send_buffer, + world_size, + h_per_rank, + head_dim, + lse_pack_dim, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + actual_output, actual_lse = _dcp_a2a_unpack_combine( + send_buffer, + head_dim, + lse_pack_dim, + return_lse=True, + is_lse_base_on_e=True, + ) + + torch.testing.assert_close( + actual_output[:1], torch.zeros_like(actual_output[:1]) + ) + assert torch.isneginf(actual_lse[:1]).all() + assert torch.isfinite(actual_output[1:]).all() + def _distributed_packed_a2a_worker(env: dict[str, str]) -> None: update_environment_variables(env) @@ -417,7 +521,7 @@ def _distributed_packed_a2a_worker(env: dict[str, str]) -> None: B, H, device=f"cuda:{local_rank}", - dtype=torch.float32, + dtype=dtype, generator=generator, ) actual = dcp_a2a_lse_reduce( diff --git a/tests/distributed/test_dcp_direct_a2a_lse_reduce.py b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py new file mode 100644 index 0000000000..b05a54825d --- /dev/null +++ b/tests/distributed/test_dcp_direct_a2a_lse_reduce.py @@ -0,0 +1,1077 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for direct symmetric-memory DCP collectives.""" + +import functools +from unittest.mock import MagicMock + +import multiprocess as mp +import pytest +import torch +import torch.distributed as dist + +import vllm.v1.attention.ops.dcp_utils as dcp_utils +from vllm.utils.network_utils import get_open_port +from vllm.utils.system_utils import update_environment_variables + +mp.set_start_method("spawn", force=True) + + +def _has_multicast_support() -> bool: + if not torch.cuda.is_available(): + return False + try: + from torch._C._autograd import DeviceType + from torch._C._distributed_c10d import _SymmetricMemory + + return _SymmetricMemory.has_multicast_support(DeviceType.CUDA, 0) + except Exception: + return False + + +def _dtype_from_name(dtype_name: str) -> torch.dtype: + return { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float8_e4m3fn": torch.float8_e4m3fn, + "float32": torch.float32, + }[dtype_name] + + +def _assert_close( + actual: torch.Tensor, + expected: torch.Tensor, + dtype: torch.dtype, +) -> None: + torch.testing.assert_close(actual.float(), expected.float(), rtol=3e-2, atol=3e-2) + + +def _q_gather_reference( + local_query: torch.Tensor, + world_size: int, + padded_num_heads: int | None, +) -> torch.Tensor: + num_tokens, heads_per_rank, head_dim = local_query.shape + gathered = torch.empty( + (world_size * num_tokens, heads_per_rank, head_dim), + dtype=local_query.dtype, + device=local_query.device, + ) + dist.all_gather_into_tensor(gathered, local_query.contiguous()) + expected = ( + gathered.view(world_size, num_tokens, heads_per_rank, head_dim) + .movedim(0, 1) + .reshape(num_tokens, world_size * heads_per_rank, head_dim) + ) + if padded_num_heads is not None: + reserved = expected.new_empty((num_tokens, padded_num_heads, head_dim)) + reserved.resize_(expected.shape) + reserved.copy_(expected) + expected = reserved + return expected + + +def _assert_q_gather_matches_reference( + actual: torch.Tensor, + local_query: torch.Tensor, + world_size: int, + padded_num_heads: int | None, +) -> None: + expected = _q_gather_reference(local_query, world_size, padded_num_heads) + assert actual.shape == expected.shape + assert actual.dtype == expected.dtype + assert actual.is_contiguous() + assert actual.stride() == expected.stride() + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + + storage_num_heads = padded_num_heads or world_size * local_query.shape[1] + remaining_storage_bytes = ( + actual.untyped_storage().nbytes() + - actual.storage_offset() * actual.element_size() + ) + required_storage_bytes = ( + local_query.shape[0] + * storage_num_heads + * local_query.shape[2] + * actual.element_size() + ) + assert remaining_storage_bytes >= required_storage_bytes + + +def _distributed_run(fn, world_size: int, extra_env: dict[str, str]) -> None: + port = str(get_open_port()) + processes: list[mp.Process] = [] + for rank in range(world_size): + env = { + "RANK": str(rank), + "LOCAL_RANK": str(rank), + "WORLD_SIZE": str(world_size), + "LOCAL_WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": port, + **extra_env, + } + process = mp.Process(target=fn, args=(env,)) + processes.append(process) + process.start() + + for process in processes: + process.join(timeout=120) + + for process in processes: + if process.is_alive(): + process.kill() + process.join() + assert process.exitcode == 0 + + +class _FakeGroupCoordinator: + device_group = None + cpu_group = None + world_size = 4 + + +class _FakeProcessGroup: + def size(self) -> int: + return 4 + + def rank(self) -> int: + return 0 + + +class TestDirectDCPGating: + def test_env_disabled_returns_none(self, monkeypatch): + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0") + dcp_utils.get_direct_dcp_a2a_workspace.cache_clear() + workspace = dcp_utils.get_direct_dcp_a2a_workspace( + _FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.bfloat16, 1 + ) + assert workspace is None + + def test_forced_with_unsupported_dtype_raises(self, monkeypatch): + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1") + dcp_utils.get_direct_dcp_a2a_workspace.cache_clear() + with pytest.raises(ValueError, match="does not support"): + dcp_utils.get_direct_dcp_a2a_workspace( + _FakeGroupCoordinator(), + torch.device("cpu"), + 16, + 2, + 32, + torch.float32, + 1, + ) + + def test_zero_ubatches_raises(self): + with pytest.raises(ValueError, match="ubatch"): + dcp_utils.DirectDCPA2AWorkspace( + None, torch.device("cpu"), 16, 2, 32, torch.bfloat16, num_ubatches=0 + ) + + def test_auto_with_unsupported_dtype_returns_none(self, monkeypatch): + monkeypatch.delenv("VLLM_USE_DIRECT_DCP_A2A", raising=False) + dcp_utils.get_direct_dcp_a2a_workspace.cache_clear() + workspace = dcp_utils.get_direct_dcp_a2a_workspace( + _FakeGroupCoordinator(), torch.device("cpu"), 16, 2, 32, torch.float32, 1 + ) + assert workspace is None + + def test_q_gather_env_disabled_returns_none(self, monkeypatch): + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "0") + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1") + dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear() + workspace = dcp_utils.get_direct_dcp_q_gather_workspace( + _FakeGroupCoordinator(), + torch.device("cpu"), + 16, + 2, + 32, + torch.bfloat16, + 1, + ) + assert workspace is None + + def test_q_gather_flag_is_independent(self, monkeypatch): + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_Q_GATHER", "1") + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0") + monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True) + dcp_utils.get_direct_dcp_q_gather_workspace.cache_clear() + workspace = object() + init_workspace = MagicMock(return_value=workspace) + monkeypatch.setattr( + dcp_utils, + "DirectDCPQGatherWorkspace", + init_workspace, + ) + + result = dcp_utils.get_direct_dcp_q_gather_workspace( + _FakeGroupCoordinator(), + torch.device("cpu"), + 16, + 2, + 32, + torch.float32, + 1, + ) + + assert result is workspace + assert init_workspace.call_args.args[5] == torch.float32 + + def test_kv_gather_env_disabled_returns_none(self, monkeypatch): + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "0") + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "1") + dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear() + workspace = dcp_utils.get_direct_dcp_kv_gather_workspace( + _FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1 + ) + assert workspace is None + + def test_kv_gather_flag_is_independent(self, monkeypatch): + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_KV_GATHER", "1") + monkeypatch.setenv("VLLM_USE_DIRECT_DCP_A2A", "0") + monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: True) + dcp_utils.get_direct_dcp_kv_gather_workspace.cache_clear() + workspace = object() + init_workspace = MagicMock(return_value=workspace) + monkeypatch.setattr( + dcp_utils, + "DirectDCPKVGatherWorkspace", + init_workspace, + ) + + result = dcp_utils.get_direct_dcp_kv_gather_workspace( + _FakeGroupCoordinator(), torch.device("cpu"), 64, 576, torch.bfloat16, 1 + ) + + assert result is workspace + + @pytest.mark.parametrize( + ("flag_name", "factory_name", "factory_args"), + [ + ( + "VLLM_USE_DIRECT_DCP_Q_GATHER", + "get_direct_dcp_q_gather_workspace", + (16, 2, 32, torch.bfloat16, 1), + ), + ( + "VLLM_USE_DIRECT_DCP_KV_GATHER", + "get_direct_dcp_kv_gather_workspace", + (64, 576, torch.bfloat16, 1), + ), + ], + ) + def test_gather_requires_multicast( + self, + monkeypatch, + flag_name, + factory_name, + factory_args, + ): + factory = getattr(dcp_utils, factory_name) + monkeypatch.setenv(flag_name, "1") + monkeypatch.setattr(dcp_utils, "_symm_mem_spans_group", lambda group: False) + factory.cache_clear() + + assert ( + factory( + _FakeGroupCoordinator(), + torch.device("cpu"), + *factory_args, + ) + is None + ) + + def test_kv_gather_rejects_invalid_workspace_geometry(self): + with pytest.raises(ValueError, match="ubatch"): + dcp_utils.DirectDCPKVGatherWorkspace( + None, torch.device("cpu"), 64, 576, num_ubatches=0 + ) + with pytest.raises(ValueError, match="divide evenly"): + dcp_utils.DirectDCPKVGatherWorkspace( + _FakeProcessGroup(), torch.device("cpu"), 63, 576 + ) + with pytest.raises(ValueError, match="16-byte"): + dcp_utils.DirectDCPKVGatherWorkspace( + _FakeProcessGroup(), torch.device("cpu"), 64, 3 + ) + + def test_q_gather_rejects_invalid_workspace_geometry(self): + with pytest.raises(ValueError, match="ubatch"): + dcp_utils.DirectDCPQGatherWorkspace( + None, torch.device("cpu"), 16, 2, 32, num_ubatches=0 + ) + with pytest.raises(ValueError, match="padded heads"): + dcp_utils.DirectDCPQGatherWorkspace( + _FakeProcessGroup(), + torch.device("cpu"), + 16, + 2, + 32, + padded_num_heads=7, + ) + with pytest.raises(ValueError, match="16-byte"): + dcp_utils.DirectDCPQGatherWorkspace( + _FakeProcessGroup(), + torch.device("cpu"), + 16, + 1, + 3, + ) + + +def _manager_config(dcp_comm_backend: str = "a2a"): + config = MagicMock() + config.parallel_config.num_ubatches = 1 + config.parallel_config.dcp_comm_backend = dcp_comm_backend + config.scheduler_config.max_num_batched_tokens = 16 + config.scheduler_config.max_num_seqs = 4 + config.num_speculative_tokens = 0 + config.speculative_config = None + config.compilation_config.max_cudagraph_capture_size = 0 + return config + + +def test_mla_dcp_manager_selects_direct_backends(monkeypatch): + import vllm.v1.attention.ops.dcp_utils as dcp_manager + + group = MagicMock(world_size=2) + monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: group) + direct_a2a = MagicMock() + direct_query = MagicMock() + direct_kv = MagicMock() + monkeypatch.setattr( + dcp_manager, "get_direct_dcp_a2a_workspace", MagicMock(return_value=direct_a2a) + ) + monkeypatch.setattr( + dcp_manager, + "get_direct_dcp_q_gather_workspace", + MagicMock(return_value=direct_query), + ) + monkeypatch.setattr( + dcp_manager, + "get_direct_dcp_kv_gather_workspace", + MagicMock(return_value=direct_kv), + ) + + manager = dcp_manager.MLADCPManager( + vllm_config=_manager_config(), + device=torch.device("cpu"), + num_heads=2, + query_head_dim=8, + output_head_dim=4, + query_dtype=torch.bfloat16, + output_dtype=torch.bfloat16, + padded_num_heads=None, + is_lse_base_on_e=False, + use_pcp=False, + ) + workspace = torch.empty(96, 8) + + assert manager.query_gather == direct_query.gather + manager.init_kv_gather(workspace, 64) + gathered_kv, local_kv = torch.empty(4, 8), torch.empty(2, 8) + manager.kv_gather(gathered_kv, local_kv) + direct_kv.gather.assert_called_once_with(gathered_kv, local_kv) + output, lse = torch.empty(1), torch.empty(1) + seq_lens = torch.ones(1, dtype=torch.int32) + query_start_loc = torch.tensor([0, 1], dtype=torch.int32) + manager.combine( + output, + lse, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + direct_a2a.lse_reduce.assert_called_once_with( + output, + lse, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + is_lse_base_on_e=False, + ) + + +def test_mla_dcp_manager_selects_fallback_backends(monkeypatch): + import vllm.v1.attention.ops.dcp_utils as dcp_manager + + group = MagicMock(world_size=2) + gathered_query = torch.empty(1, 4, 8) + group.all_gather.return_value = gathered_query + monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: group) + monkeypatch.setattr( + dcp_manager, "get_direct_dcp_a2a_workspace", MagicMock(return_value=None) + ) + monkeypatch.setattr( + dcp_manager, "get_direct_dcp_q_gather_workspace", MagicMock(return_value=None) + ) + monkeypatch.setattr( + dcp_manager, "get_direct_dcp_kv_gather_workspace", MagicMock(return_value=None) + ) + fallback_combine = MagicMock(return_value=torch.empty(1)) + monkeypatch.setattr(dcp_manager, "dcp_a2a_lse_reduce", fallback_combine) + + manager = dcp_manager.MLADCPManager( + vllm_config=_manager_config(), + device=torch.device("cpu"), + num_heads=2, + query_head_dim=8, + output_head_dim=4, + query_dtype=torch.bfloat16, + output_dtype=torch.bfloat16, + padded_num_heads=None, + is_lse_base_on_e=True, + use_pcp=False, + ) + + all_gather = MagicMock() + monkeypatch.setattr(torch.distributed, "all_gather_into_tensor", all_gather) + workspace = torch.empty(96, 8) + manager.init_kv_gather(workspace, 64) + output, local = torch.empty(4, 8), torch.empty(2, 8) + manager.kv_gather(output, local) + all_gather.assert_called_once_with(output, local, group=group.device_group) + + query = torch.empty(1, 2, 8) + assert manager.query_gather is not None + assert manager.query_gather(query) is gathered_query + group.all_gather.assert_called_once_with(query, dim=1) + + partial_output, partial_lse = torch.empty(1), torch.empty(1) + seq_lens = torch.ones(1, dtype=torch.int32) + query_start_loc = torch.tensor([0, 1], dtype=torch.int32) + manager.combine( + partial_output, + partial_lse, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + fallback_combine.assert_called_once_with( + partial_output, + partial_lse, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + cp_group=group, + is_lse_base_on_e=True, + ) + + +def test_dcp_workspace_covers_parallel_drafting(): + config = _manager_config() + config.scheduler_config.max_num_batched_tokens = 128 + config.num_speculative_tokens = 3 + config.speculative_config = MagicMock(parallel_drafting=True) + + assert dcp_utils.get_dcp_workspace_max_num_tokens(config) == 28 + + +def test_mla_dcp_manager_selects_pcp_combine(monkeypatch): + import vllm.v1.attention.ops.dcp_utils as dcp_manager + + monkeypatch.setattr(dcp_manager, "get_dcp_group", lambda: MagicMock(world_size=2)) + manager = dcp_manager.MLADCPManager( + vllm_config=_manager_config(dcp_comm_backend="ag_rs"), + device=torch.device("cpu"), + num_heads=2, + query_head_dim=8, + output_head_dim=4, + query_dtype=torch.bfloat16, + output_dtype=torch.bfloat16, + padded_num_heads=None, + is_lse_base_on_e=True, + use_pcp=True, + ) + + assert isinstance(manager.combine, functools.partial) + assert manager.combine.func is dcp_manager.cp_lse_ag_out_ar + assert manager.query_gather is None + + +def test_dcp_chunk_workspace_alignment_covers_interleave(): + from vllm.model_executor.layers.attention.mla_attention import ( + align_mla_chunked_context_workspace_size, + ) + + config = MagicMock() + config.cache_config.block_size = 32 + config.parallel_config.decode_context_parallel_size = 8 + config.parallel_config.cp_kv_cache_interleave_size = 8 + config.scheduler_config.max_num_seqs = 3 + + assert align_mla_chunked_context_workspace_size(config, 100) == 192 + + +def test_sparse_mla_builder_initializes_dcp_manager(monkeypatch): + import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla + + monkeypatch.setattr( + sparse_mla.AttentionMetadataBuilder, + "__init__", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + sparse_mla, + "get_dcp_group", + lambda: MagicMock(world_size=2), + ) + monkeypatch.setattr( + sparse_mla, + "get_mla_dims", + lambda _: MagicMock(kv_lora_rank=8, qk_rope_head_dim=4), + ) + + manager = object.__new__(dcp_utils.MLADCPManager) + manager.init_kv_gather = MagicMock() + layer = MagicMock(dcp_manager=manager) + config = MagicMock() + config.model_config.dtype = torch.bfloat16 + config.model_config.max_model_len = 64 + config.model_config.hf_config.index_topk = 8 + config.scheduler_config.max_num_batched_tokens = 64 + config.scheduler_config.max_num_seqs = 2 + config.cache_config.block_size = 4 + config.parallel_config.prefill_context_parallel_size = 1 + config.parallel_config.decode_context_parallel_size = 2 + config.parallel_config.cp_kv_cache_interleave_size = 1 + config.compilation_config.static_forward_context = {"layer": layer} + + builder = sparse_mla.SparseMLACommonMetadataBuilder( + MagicMock(), + ["layer"], + config, + torch.device("cpu"), + ) + + assert builder.dcp_manager is manager + manager.init_kv_gather.assert_called_once_with( + builder.chunked_prefill_workspace, + builder.chunked_prefill_workspace_size, + ) + + +def test_sparse_mla_workspace_preserves_non_dcp_size(): + from vllm.model_executor.layers.attention.sparse_mla_attention import ( + SparseMLACommonMetadataBuilder, + ) + + config = MagicMock() + config.model_config.max_model_len = 1 + config.model_config.hf_config.index_topk = 7 + config.scheduler_config.max_num_seqs = 3 + config.cache_config.block_size = 4 + config.parallel_config.decode_context_parallel_size = 1 + + assert ( + SparseMLACommonMetadataBuilder.determine_chunked_prefill_workspace_size(config) + == 21 + ) + + +def _distributed_direct_q_gather_worker(env: dict[str, str]) -> None: + update_environment_variables(env) + local_rank = int(env["LOCAL_RANK"]) + device = torch.device(f"cuda:{local_rank}") + torch.accelerator.set_device_index(local_rank) + dist.init_process_group(backend="nccl") + try: + rank = dist.get_rank() + world_size = dist.get_world_size() + heads_per_rank, head_dim, max_num_tokens = 6, 576, 128 + padded_num_heads = 128 if world_size == 4 else None + active_ubatch = [0] + dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0] + for dtype_idx, dtype_name in enumerate( + ("bfloat16", "float8_e4m3fn", "float32") + ): + dtype = _dtype_from_name(dtype_name) + workspace = dcp_utils.DirectDCPQGatherWorkspace( + dist.group.WORLD, + device, + max_num_tokens, + heads_per_rank, + head_dim, + dtype, + num_ubatches=2, + padded_num_heads=padded_num_heads, + ) + + cases = ( + ((1, False), (128, True), (17, True), (5, False)) + if dtype == torch.bfloat16 + else ((17, True),) + ) + for iteration, (num_tokens, noncontiguous) in enumerate(cases): + generator = torch.Generator(device=device) + generator.manual_seed(9000 + rank * 101 + dtype_idx * 1009 + iteration) + source_num_heads = ( + heads_per_rank + 2 if noncontiguous else heads_per_rank + ) + query_storage = torch.randn( + num_tokens, + source_num_heads, + head_dim, + device=device, + dtype=torch.bfloat16, + generator=generator, + ).to(dtype) + local_query = query_storage[:, :heads_per_rank] + assert local_query.is_contiguous() is not noncontiguous + + active_ubatch[0] = iteration % 2 + actual = workspace.gather(local_query) + torch.accelerator.synchronize() + + assert ( + actual.data_ptr() + == workspace.final_query[active_ubatch[0]].data_ptr() + ) + assert workspace.completion[active_ubatch[0]].numel() == 1 + _assert_q_gather_matches_reference( + actual, local_query, world_size, padded_num_heads + ) + + if env.get("TEST_CUDA_GRAPH") != "1" or dtype != torch.bfloat16: + continue + + capture_num_tokens = 17 + capture_storage = torch.empty( + capture_num_tokens, + heads_per_rank + 2, + head_dim, + dtype=dtype, + device=device, + ) + captured_input = capture_storage[:, :heads_per_rank] + assert not captured_input.is_contiguous() + input_pattern = ( + torch.arange(captured_input.numel(), device=device, dtype=torch.int32) + .remainder(31) + .view(captured_input.shape) + .to(dtype) + ) + + active_ubatch[0] = 1 + torch.accelerator.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured_output = workspace.gather(captured_input) + torch.accelerator.synchronize() + dist.barrier() + + eager_input = torch.full( + captured_input.shape, + 96 + rank, + dtype=dtype, + device=device, + ) + eager_output = workspace.gather(eager_input) + torch.accelerator.synchronize() + _assert_q_gather_matches_reference( + eager_output, eager_input, world_size, padded_num_heads + ) + epoch_before_replays = int(workspace.epoch[1].item()) + + for replay in range(2): + captured_input.copy_(input_pattern + rank * 32 + replay) + torch.accelerator.synchronize() + graph.replay() + torch.accelerator.synchronize() + assert int(workspace.epoch[1].item()) == ( + epoch_before_replays + replay + 1 + ) + _assert_q_gather_matches_reference( + captured_output, + captured_input, + world_size, + padded_num_heads, + ) + + interleaved_eager_input = torch.full( + captured_input.shape, + 160 + rank, + dtype=dtype, + device=device, + ) + interleaved_eager_output = workspace.gather(interleaved_eager_input) + torch.accelerator.synchronize() + _assert_q_gather_matches_reference( + interleaved_eager_output, + interleaved_eager_input, + world_size, + padded_num_heads, + ) + + captured_input.copy_(input_pattern + rank * 32 + 2) + torch.accelerator.synchronize() + epoch_before_mixed_replay = int(workspace.epoch[1].item()) + graph.replay() + torch.accelerator.synchronize() + assert int(workspace.epoch[1].item()) == epoch_before_mixed_replay + 1 + _assert_q_gather_matches_reference( + captured_output, captured_input, world_size, padded_num_heads + ) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize( + "world_size", + [ + pytest.param( + 4, + marks=pytest.mark.skipif( + torch.accelerator.device_count() < 4 or not _has_multicast_support(), + reason="Need 4 GPUs with symmetric-memory multicast.", + ), + ), + ], +) +def test_distributed_direct_q_gather_cuda_graph_replay(world_size: int): + _distributed_run( + _distributed_direct_q_gather_worker, + world_size=world_size, + extra_env={"TEST_CUDA_GRAPH": "1"}, + ) + + +def _distributed_direct_kv_gather_worker(env: dict[str, str]) -> None: + update_environment_variables(env) + local_rank = int(env["LOCAL_RANK"]) + device = torch.device(f"cuda:{local_rank}") + torch.accelerator.set_device_index(local_rank) + dist.init_process_group(backend="nccl") + try: + rank = dist.get_rank() + world_size = dist.get_world_size() + token_dim = 576 + max_gathered_tokens = 128 * world_size + active_ubatch = [0] + dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0] + + for dtype_idx, dtype_name in enumerate(("bfloat16", "float16")): + dtype = _dtype_from_name(dtype_name) + workspace = dcp_utils.DirectDCPKVGatherWorkspace( + dist.group.WORLD, + device, + max_gathered_tokens, + token_dim, + dtype, + num_ubatches=2, + ) + + # Use disjoint slices of one persistent chunked-context workspace. + storage = torch.zeros( + (world_size + 1) * 128, token_dim, device=device, dtype=dtype + ) + for iteration, num_tokens in enumerate((1, 128, 17)): + generator = torch.Generator(device=device) + generator.manual_seed(7000 + rank * 101 + dtype_idx * 977 + iteration) + local_kv = storage[:num_tokens] + local_kv.copy_( + torch.randn( + num_tokens, + token_dim, + device=device, + dtype=torch.float32, + generator=generator, + ).to(dtype) + ) + gathered = storage[128 : 128 + num_tokens * world_size] + active_ubatch[0] = iteration % 2 + workspace.gather(gathered, local_kv) + torch.accelerator.synchronize() + + expected = torch.empty_like(gathered) + dist.all_gather_into_tensor(expected, local_kv.contiguous()) + assert torch.equal( + gathered.view(torch.uint8), expected.view(torch.uint8) + ) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize( + "world_size", + [ + pytest.param( + 4, + marks=pytest.mark.skipif( + torch.accelerator.device_count() < 4 or not _has_multicast_support(), + reason="Need 4 GPUs with symmetric-memory multicast.", + ), + ), + ], +) +def test_distributed_direct_kv_gather_matches_reference(world_size: int): + _distributed_run( + _distributed_direct_kv_gather_worker, + world_size=world_size, + extra_env={}, + ) + + +def _distributed_direct_a2a_worker(env: dict[str, str]) -> None: + update_environment_variables(env) + local_rank = int(env["LOCAL_RANK"]) + device = torch.device(f"cuda:{local_rank}") + torch.accelerator.set_device_index(local_rank) + dist.init_process_group(backend="nccl") + try: + from vllm.v1.attention.ops.dcp_alltoall import _lse_weighted_combine + + rank = dist.get_rank() + world_size = dist.get_world_size() + dtype = _dtype_from_name(env["TEST_DTYPE"]) + lse_dtype = _dtype_from_name(env["TEST_LSE_DTYPE"]) + is_lse_base_on_e = env["LSE_BASE_E"] == "1" + # Match Kimi-K3's six heads per DCP rank. + heads_per_rank, head_dim, max_num_tokens = 6, 512, 128 + total_heads = world_size * heads_per_rank + active_ubatch = [0] + dcp_utils.dbo_current_ubatch_id = lambda: active_ubatch[0] + workspace = dcp_utils.DirectDCPA2AWorkspace( + dist.group.WORLD, + device, + max_num_tokens, + heads_per_rank, + head_dim, + dtype, + num_ubatches=2, + ) + + def check(num_tokens: int, iteration: int, padded: bool) -> None: + generator = torch.Generator(device=device) + generator.manual_seed(1234 + rank + iteration * world_size) + storage_heads = 128 if padded else total_heads + partial_output_storage = torch.randn( + num_tokens, + storage_heads, + head_dim, + device=device, + dtype=dtype, + generator=generator, + ) + partial_lse_storage = torch.randn( + num_tokens, + storage_heads, + device=device, + dtype=lse_dtype, + generator=generator, + ) + partial_output = partial_output_storage[:, :total_heads, :] + partial_lse = partial_lse_storage[:, :total_heads] + if padded: + assert not partial_output.is_contiguous() + assert not partial_lse.is_contiguous() + active_ubatch[0] = iteration % 2 + actual = workspace.lse_reduce(partial_output, partial_lse, is_lse_base_on_e) + torch.accelerator.synchronize() + + reference_output = partial_output.contiguous() + reference_lse = partial_lse.contiguous() + gathered_output = [ + torch.empty_like(reference_output) for _ in range(world_size) + ] + gathered_lse = [torch.empty_like(reference_lse) for _ in range(world_size)] + dist.all_gather(gathered_output, reference_output) + dist.all_gather(gathered_lse, reference_lse) + outputs = torch.stack( + [ + value[ + :, + rank * heads_per_rank : (rank + 1) * heads_per_rank, + :, + ] + for value in gathered_output + ] + ).float() + lses = torch.stack( + [ + value[:, rank * heads_per_rank : (rank + 1) * heads_per_rank] + for value in gathered_lse + ] + ) + expected = _lse_weighted_combine( + outputs, lses, is_lse_base_on_e=is_lse_base_on_e + ) + _assert_close(actual, expected, dtype) + + def check_empty_shards(query_lens: list[int], iteration: int) -> None: + """Verify empty local shards contribute zero weight.""" + num_seqs = len(query_lens) + num_tokens = sum(query_lens) + generator = torch.Generator(device=device) + generator.manual_seed(777 + rank + iteration * world_size) + partial_output = torch.randn( + num_tokens, + total_heads, + head_dim, + device=device, + dtype=dtype, + generator=generator, + ) + partial_lse = torch.randn( + num_tokens, + total_heads, + device=device, + dtype=lse_dtype, + generator=generator, + ) + + # Cover globally empty, rank-local empty, and non-empty sequences. + def is_empty(seq_idx: int, source_rank: int) -> bool: + return seq_idx == 0 or seq_idx == 1 + source_rank + + seq_lens = torch.tensor( + [ + 0 if is_empty(seq_idx, rank) else seq_idx + 3 + for seq_idx in range(num_seqs) + ], + dtype=torch.int32, + device=device, + ) + query_lens_tensor = torch.tensor( + query_lens, + dtype=torch.int32, + device=device, + ) + query_start_loc = torch.cat( + ( + query_lens_tensor.new_zeros(1), + query_lens_tensor.cumsum(0), + ) + ) + empty_rows = torch.repeat_interleave(seq_lens == 0, query_lens_tensor) + # Model undefined attention rows with NaNs. + partial_output[empty_rows] = float("nan") + partial_lse[empty_rows] = float("nan") + + active_ubatch[0] = iteration % 2 + actual = workspace.lse_reduce( + partial_output, + partial_lse, + is_lse_base_on_e, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + torch.accelerator.synchronize() + + gathered_output = [ + torch.empty_like(partial_output) for _ in range(world_size) + ] + gathered_lse = [torch.empty_like(partial_lse) for _ in range(world_size)] + dist.all_gather(gathered_output, partial_output.contiguous()) + dist.all_gather(gathered_lse, partial_lse.contiguous()) + head_slice = slice(rank * heads_per_rank, (rank + 1) * heads_per_rank) + outputs = torch.stack( + [value[:, head_slice, :] for value in gathered_output] + ).float() + lses = torch.stack([value[:, head_slice] for value in gathered_lse]) + for source_rank in range(world_size): + source_empty = torch.tensor( + [is_empty(seq_idx, source_rank) for seq_idx in range(num_seqs)], + device=device, + ).repeat_interleave(query_lens_tensor) + outputs[source_rank][source_empty] = 0.0 + lses[source_rank][source_empty] = float("-inf") + expected = _lse_weighted_combine( + outputs, lses, is_lse_base_on_e=is_lse_base_on_e + ) + all_empty = torch.tensor( + [ + all(is_empty(seq_idx, r) for r in range(world_size)) + for seq_idx in range(num_seqs) + ], + device=device, + ).repeat_interleave(query_lens_tensor) + assert torch.equal(actual[all_empty], torch.zeros_like(actual[all_empty])) + assert not torch.isnan(actual.float()).any() + _assert_close(actual, expected, dtype) + + cases = ((1, False), (17, True), (128, True)) + for iteration, (num_tokens, padded) in enumerate(cases): + check(num_tokens, iteration, padded) + check_empty_shards( + query_lens=[1, 3, 2, *([1] * (world_size - 1))], + iteration=len(cases), + ) + generator = torch.Generator(device=device) + generator.manual_seed(4321 + rank) + partial_output_storage = torch.randn( + 128, + 128, + head_dim, + device=device, + dtype=dtype, + generator=generator, + ) + partial_lse_storage = torch.randn( + 128, + 128, + device=device, + dtype=lse_dtype, + generator=generator, + ) + partial_output = partial_output_storage[:, :total_heads, :] + partial_lse = partial_lse_storage[:, :total_heads] + assert not partial_output.is_contiguous() + assert not partial_lse.is_contiguous() + torch.accelerator.synchronize() + active_ubatch[0] = 1 + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = workspace.lse_reduce(partial_output, partial_lse, is_lse_base_on_e) + for _ in range(3): + graph.replay() + torch.accelerator.synchronize() + + reference_output = partial_output.contiguous() + reference_lse = partial_lse.contiguous() + gathered_output = [ + torch.empty_like(reference_output) for _ in range(world_size) + ] + gathered_lse = [torch.empty_like(reference_lse) for _ in range(world_size)] + dist.all_gather(gathered_output, reference_output) + dist.all_gather(gathered_lse, reference_lse) + head_slice = slice(rank * heads_per_rank, (rank + 1) * heads_per_rank) + outputs = torch.stack( + [value[:, head_slice, :] for value in gathered_output] + ).float() + lses = torch.stack([value[:, head_slice] for value in gathered_lse]) + expected = _lse_weighted_combine( + outputs, lses, is_lse_base_on_e=is_lse_base_on_e + ) + _assert_close(actual, expected, dtype) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize( + "world_size", + [ + pytest.param( + 2, + marks=pytest.mark.skipif( + torch.accelerator.device_count() < 2, reason="Need at least 2 GPUs." + ), + ), + pytest.param( + 4, + marks=pytest.mark.skipif( + torch.accelerator.device_count() < 4, reason="Need at least 4 GPUs." + ), + ), + ], +) +def test_distributed_direct_a2a_matches_reference(world_size: int): + _distributed_run( + _distributed_direct_a2a_worker, + world_size=world_size, + extra_env={ + "TEST_DTYPE": "bfloat16", + "TEST_LSE_DTYPE": "bfloat16", + "LSE_BASE_E": "0", + }, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/distributed/test_kimi_linear_context_parallel.py b/tests/distributed/test_kimi_linear_context_parallel.py new file mode 100644 index 0000000000..d37e19d0bd --- /dev/null +++ b/tests/distributed/test_kimi_linear_context_parallel.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math + +import pytest + +from tests.conftest import VllmRunner +from tests.utils import create_new_process_for_each_test +from vllm import SamplingParams, TokensPrompt + +MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct" + + +def _make_tiny_overrides() -> dict: + linear_attn_config = { + "full_attn_layers": [4], + "head_dim": 128, + "kda_layers": [1, 2, 3], + "num_heads": 8, + "short_conv_kernel_size": 4, + "use_full_rank_gate": True, + } + + return { + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "vocab_size": 256, + "hidden_size": 256, + "head_dim": 32, + "intermediate_size": 512, + "num_hidden_layers": 4, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "num_experts": None, + "num_experts_per_token": None, + "num_shared_experts": 0, + "q_lora_rank": 128, + "kv_lora_rank": 512, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "v_head_dim": 128, + "mla_use_nope": True, + "mla_use_output_gate": True, + "linear_attn_config": linear_attn_config, + "max_position_embeddings": 512, + "model_max_length": 512, + } + + +def _run_tiny_model( + vllm_runner: type[VllmRunner], dcp_size: int +) -> list[tuple[list[int], list[float]]]: + with vllm_runner( + model_name=MODEL, + skip_tokenizer_init=True, + load_format="dummy", + hf_overrides=_make_tiny_overrides(), + tensor_parallel_size=2, + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=1, + distributed_executor_backend="mp", + dtype="bfloat16", + seed=0, + enforce_eager=True, + max_model_len=512, + max_num_seqs=8, + max_num_batched_tokens=1024, + gpu_memory_utilization=0.25, + enable_chunked_prefill=True, + enable_prefix_caching=False, + ) as runner: + lengths = [1, 32, 33, 129] + prompts = [ + TokensPrompt( + prompt_token_ids=[ + 3 + ((request_idx + token_idx) % 251) for token_idx in range(length) + ] + ) + for request_idx, length in enumerate(lengths) + ] + outputs = runner.llm.generate( + prompts, + SamplingParams( + temperature=0, + max_tokens=8, + seed=0, + ignore_eos=True, + logprobs=20, + ), + use_tqdm=False, + ) + + results = [] + for request_output in outputs: + completion = request_output.outputs[0] + token_ids = list(completion.token_ids) + assert completion.logprobs is not None + selected_logprobs = [ + step_logprobs[token_id].logprob + for token_id, step_logprobs in zip(token_ids, completion.logprobs) + ] + results.append((token_ids, selected_logprobs)) + return results + + +@create_new_process_for_each_test() +@pytest.mark.distributed(num_gpus=2) +def test_kimi_linear_dcp_tiny( + vllm_runner: type[VllmRunner], + num_gpus_available: int, +) -> None: + if num_gpus_available < 2: + pytest.skip("Need at least 2 GPUs") + + baseline = _run_tiny_model(vllm_runner, dcp_size=1) + dcp = _run_tiny_model(vllm_runner, dcp_size=2) + + assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline] + logprob_drifts = [] + for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp): + for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs): + assert math.isfinite(baseline_logprob) + assert math.isfinite(dcp_logprob) + logprob_drifts.append(abs(baseline_logprob - dcp_logprob)) + + assert max(logprob_drifts) <= 1e-2 diff --git a/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py b/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py new file mode 100644 index 0000000000..e1582b7621 --- /dev/null +++ b/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +import vllm._custom_ops as ops +from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import ( + fused_mla_decode_q_concat_kv_cache_insert, + fused_mla_key_concat_ds_mla_insert, + fused_mla_key_concat_kv_cache_insert, + fused_mla_qkv_quant_kv_cache_fp8_insert, +) +from vllm.platforms import current_platform + +NUM_TOKENS = 4 +NUM_HEADS = 2 +KV_LORA_RANK = 512 +NOPE_HEAD_DIM = 128 +ROPE_HEAD_DIM = 64 +QK_HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM +V_HEAD_DIM = 128 +CACHE_ENTRY = KV_LORA_RANK + ROPE_HEAD_DIM +DS_MLA_CACHE_ENTRY = 656 +BLOCK_SIZE = 8 +OWNED_TOKEN_INDICES = [1, 3] + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), reason="Kimi-K3 fused MLA ops require CUDA" +) + + +def _inputs() -> dict[str, torch.Tensor]: + torch.manual_seed(0) + + def make(*shape: int) -> torch.Tensor: + return torch.randn(*shape, device="cuda", dtype=torch.bfloat16) + + return { + "q": make(NUM_TOKENS, NUM_HEADS, QK_HEAD_DIM), + "k_nope": make(NUM_TOKENS, NUM_HEADS, NOPE_HEAD_DIM), + "k_pe": make(NUM_TOKENS, ROPE_HEAD_DIM), + "kv_c": make(NUM_TOKENS, KV_LORA_RANK), + "v": make(NUM_TOKENS, NUM_HEADS, V_HEAD_DIM), + "ql_nope": make(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK), + "q_pe": make(NUM_TOKENS, NUM_HEADS, ROPE_HEAD_DIM), + } + + +def _slot_mapping() -> torch.Tensor: + return torch.tensor([-1, 0, -1, 5], device="cuda", dtype=torch.int64) + + +def _owned(tensor: torch.Tensor) -> torch.Tensor: + return tensor[OWNED_TOKEN_INDICES] + + +def _full_key(k_nope: torch.Tensor, k_pe: torch.Tensor) -> torch.Tensor: + shared_pe = k_pe[:, None, :].expand(-1, k_nope.shape[1], -1) + return torch.cat((k_nope, shared_pe), dim=-1) + + +def _latent_query(ql_nope: torch.Tensor, q_pe: torch.Tensor) -> torch.Tensor: + return torch.cat((ql_nope, q_pe), dim=-1) + + +def _make_cache(cache_format: str) -> torch.Tensor: + if cache_format == "bf16": + return torch.full( + (1, BLOCK_SIZE, CACHE_ENTRY), + -7, + device="cuda", + dtype=torch.bfloat16, + ) + if cache_format == "fp8": + return torch.full( + (1, BLOCK_SIZE, CACHE_ENTRY), + 1.0, + device="cuda", + dtype=torch.float32, + ).to(torch.float8_e4m3fn) + assert cache_format == "fp8_ds_mla" + return torch.full( + (1, BLOCK_SIZE, DS_MLA_CACHE_ENTRY), + 165, + device="cuda", + dtype=torch.uint8, + ) + + +def _reference_cache( + cache_format: str, + inputs: dict[str, torch.Tensor], + initial_cache: torch.Tensor, + slots: torch.Tensor, +) -> torch.Tensor: + reference_cache = initial_cache.clone() + valid_slots = _owned(slots) + if cache_format == "fp8_ds_mla": + ops.concat_and_cache_mla( + _owned(inputs["kv_c"]), + _owned(inputs["k_pe"]), + reference_cache, + valid_slots, + cache_format, + torch.ones(1, device="cuda", dtype=torch.float32), + ) + return reference_cache + + latent = torch.cat( + (_owned(inputs["kv_c"]), _owned(inputs["k_pe"])), + dim=-1, + ) + flat_reference_cache = reference_cache.view(-1, CACHE_ENTRY) + for row, slot in zip(latent, valid_slots.tolist()): + flat_reference_cache[slot].copy_(row.to(reference_cache.dtype)) + return reference_cache + + +def _assert_cache_matches_reference( + cache: torch.Tensor, + reference_cache: torch.Tensor, + initial_cache: torch.Tensor, +) -> None: + assert torch.equal(cache, reference_cache) + for slot in (0, 5): + assert not torch.equal(cache[0, slot], initial_cache[0, slot]) + for slot in (1, 2, 3, 4, 6, 7): + assert torch.equal(cache[0, slot], initial_cache[0, slot]) + + +@pytest.mark.parametrize("cache_format", ["bf16", "fp8_ds_mla"]) +def test_prefill_concat_ignores_negative_slots_for_cache(cache_format: str) -> None: + inputs = _inputs() + slots = _slot_mapping() + initial_cache = _make_cache(cache_format) + mixed_cache = initial_cache.clone() + reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots) + + op = ( + fused_mla_key_concat_kv_cache_insert + if cache_format == "bf16" + else fused_mla_key_concat_ds_mla_insert + ) + output = op( + inputs["q"], + inputs["k_nope"], + inputs["k_pe"], + inputs["kv_c"], + mixed_cache, + slots, + ) + + torch.testing.assert_close( + output, _full_key(inputs["k_nope"], inputs["k_pe"]), rtol=0, atol=0 + ) + _assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache) + + +def test_fp8_prefill_outputs_ignore_negative_slots_for_cache() -> None: + inputs = _inputs() + slots = _slot_mapping() + initial_cache = _make_cache("fp8") + mixed_cache = initial_cache.clone() + reference_cache = _reference_cache("fp8", inputs, initial_cache, slots) + scale_inv = torch.ones(1, device="cuda", dtype=torch.float32) + + q, k, v = fused_mla_qkv_quant_kv_cache_fp8_insert( + inputs["q"], + inputs["k_nope"], + inputs["k_pe"], + inputs["kv_c"], + inputs["v"], + mixed_cache, + slots, + scale_inv, + scale_inv, + scale_inv, + scale_inv, + ) + + expected = ( + inputs["q"].to(torch.float8_e4m3fn), + _full_key(inputs["k_nope"], inputs["k_pe"]).to(torch.float8_e4m3fn), + inputs["v"].to(torch.float8_e4m3fn), + ) + for actual, reference in zip((q, k, v), expected): + torch.testing.assert_close(actual.float(), reference.float(), rtol=0, atol=0) + _assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache) + + +@pytest.mark.parametrize("cache_format", ["bf16", "fp8", "fp8_ds_mla"]) +def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> None: + inputs = _inputs() + slots = _slot_mapping() + initial_cache = _make_cache(cache_format) + mixed_cache = initial_cache.clone() + reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots) + scale_inv = torch.ones(1, device="cuda", dtype=torch.float32) + kwargs = { + "ds_mla": cache_format == "fp8_ds_mla", + "q_scale_inv": scale_inv if cache_format == "fp8" else None, + "cache_scale_inv": scale_inv if cache_format == "fp8" else None, + } + + output = fused_mla_decode_q_concat_kv_cache_insert( + inputs["ql_nope"], + inputs["q_pe"], + inputs["kv_c"], + inputs["k_pe"], + mixed_cache, + slots, + **kwargs, + ) + + expected = _latent_query(inputs["ql_nope"], inputs["q_pe"]) + if cache_format == "fp8": + expected = expected.to(torch.float8_e4m3fn) + torch.testing.assert_close(output.float(), expected.float(), rtol=0, atol=0) + else: + torch.testing.assert_close(output, expected, rtol=0, atol=0) + _assert_cache_matches_reference(mixed_cache, reference_cache, initial_cache) + + +def test_ds_mla_cache_insert_bit_compatible_with_reference() -> None: + """Fused ds_mla insertion must match the reference bit-for-bit.""" + torch.manual_seed(0) + num_tokens, num_blocks = 33, 16 + dt = torch.bfloat16 + kv_c = torch.randn(num_tokens, KV_LORA_RANK, device="cuda", dtype=dt) + k_pe = torch.randn(num_tokens, 1, ROPE_HEAD_DIM, device="cuda", dtype=dt) + k_nope = torch.randn(num_tokens, NUM_HEADS, NOPE_HEAD_DIM, device="cuda", dtype=dt) + q = torch.randn(num_tokens, NUM_HEADS, QK_HEAD_DIM, device="cuda", dtype=dt) + slots = torch.randperm(num_blocks * BLOCK_SIZE, device="cuda")[:num_tokens].long() + slots[torch.rand(num_tokens, device="cuda") < 0.3] = -1 + ref = torch.full( + (num_blocks, BLOCK_SIZE, DS_MLA_CACHE_ENTRY), + 165, + device="cuda", + dtype=torch.uint8, + ) + got = ref.clone() + scale = torch.ones(1, device="cuda", dtype=torch.float32) + ops.concat_and_cache_mla(kv_c, k_pe.squeeze(1), ref, slots, "fp8_ds_mla", scale) + fused_mla_key_concat_ds_mla_insert(q, k_nope, k_pe, kv_c, got, slots) + assert torch.equal(ref, got) diff --git a/tests/v1/attention/test_flashinfer_mla_dcp.py b/tests/v1/attention/test_flashinfer_mla_dcp.py new file mode 100644 index 0000000000..acc38919b8 --- /dev/null +++ b/tests/v1/attention/test_flashinfer_mla_dcp.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import MagicMock + +import pytest +import torch + +from vllm.platforms import current_platform + +requires_flashinfer_mla = pytest.mark.skipif( + not current_platform.has_device_capability(100), + reason="FlashInfer MLA requires compute capability 10 or above.", +) + + +def test_mla_dcp_gathered_query_reserves_backend_head_storage(): + from vllm.v1.attention.ops.dcp_utils import reserve_query_head_storage + + query = torch.randn(3, 24, 576, dtype=torch.bfloat16) + + padded = reserve_query_head_storage(query, 128) + + assert padded.shape == query.shape + assert padded.stride() == query.stride() + assert padded.untyped_storage().nbytes() >= 3 * 128 * 576 * 2 + assert torch.equal(padded, query) + + +@requires_flashinfer_mla +def test_flashinfer_mla_selects_backend_for_gathered_heads(): + from vllm.v1.attention.backends.mla.flashinfer_mla import ( + _select_mla_decode_backend, + ) + + assert _select_mla_decode_backend(6) is None + assert _select_mla_decode_backend(24) == "cute-dsl" + + +@requires_flashinfer_mla +def test_flashinfer_mla_forward_uses_gathered_head_count(monkeypatch): + import vllm.v1.attention.backends.mla.flashinfer_mla as flashinfer_mla + + impl = MagicMock() + impl.bmm1_scale = 1.0 + impl.bmm2_scale = 1.0 + impl.need_to_return_lse_for_decode = True + impl.num_heads = 6 + impl.qk_nope_head_dim = 128 + impl.kv_lora_rank = 512 + impl.qk_rope_head_dim = 64 + impl.kv_cache_dtype = "auto" + + attn_metadata = MagicMock() + attn_metadata.causal = True + attn_metadata.num_decode_tokens = 2 + attn_metadata.num_decodes = 2 + attn_metadata.max_seq_len = 1 + attn_metadata.decode.block_table = torch.zeros(2, 1, dtype=torch.int32) + attn_metadata.decode.seq_lens = torch.ones(2, dtype=torch.int32) + + query = torch.ones(2, 24, 576, dtype=torch.bfloat16) + kv_cache = torch.ones(1, 128, 576, dtype=torch.bfloat16) + kernel = MagicMock( + return_value=( + torch.ones(2, 1, 24, 512, dtype=torch.bfloat16), + torch.ones(2, 24, dtype=torch.float32), + ) + ) + monkeypatch.setattr(flashinfer_mla, "_get_workspace_buffer", MagicMock()) + monkeypatch.setattr(flashinfer_mla, "trtllm_batch_decode_with_kv_cache_mla", kernel) + layer = MagicMock() + + output, lse = flashinfer_mla.FlashInferMLAImpl.forward_mqa( + impl, query, kv_cache, attn_metadata, layer + ) + + assert kernel.call_args.kwargs["backend"] == "cute-dsl" + assert output.shape == (2, 24, 512) + assert lse is not None and lse.shape == (2, 24) diff --git a/vllm/envs.py b/vllm/envs.py index 1460fa1a89..1b941db970 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -190,6 +190,9 @@ if TYPE_CHECKING: VLLM_USE_DEEP_GEMM_E8M0: bool = True VLLM_USE_DEEP_GEMM_TMA_ALIGNED_SCALES: bool = True VLLM_DCP_Q_REPLICATE: bool = False + VLLM_USE_DIRECT_DCP_A2A: bool | None = None + VLLM_USE_DIRECT_DCP_Q_GATHER: bool | None = None + VLLM_USE_DIRECT_DCP_KV_GATHER: bool | None = None VLLM_DEEP_GEMM_WARMUP: Literal[ "skip", "full", @@ -2057,6 +2060,16 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_USE_SIMPLE_KV_OFFLOAD": lambda: bool( int(os.getenv("VLLM_USE_SIMPLE_KV_OFFLOAD", "0")) ), + # Direct DCP ops default on when applicable; set to 1 to enforce or 0 to disable. + "VLLM_USE_DIRECT_DCP_A2A": lambda: maybe_convert_bool( + os.getenv("VLLM_USE_DIRECT_DCP_A2A") + ), + "VLLM_USE_DIRECT_DCP_Q_GATHER": lambda: maybe_convert_bool( + os.getenv("VLLM_USE_DIRECT_DCP_Q_GATHER") + ), + "VLLM_USE_DIRECT_DCP_KV_GATHER": lambda: maybe_convert_bool( + os.getenv("VLLM_USE_DIRECT_DCP_KV_GATHER") + ), # Whether to enable dual cuda streams for LoRA computation # (used by both BaseLinearLayerWithLoRA and FusedMoEWithLoRA to # overlap the base layer compute with the LoRA fast path). diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 978ee27ed7..b21f967732 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -206,6 +206,7 @@ from abc import abstractmethod from collections.abc import Callable from dataclasses import dataclass from enum import Enum +from math import lcm from typing import ClassVar, Generic, TypeVar, cast import numpy as np @@ -222,7 +223,6 @@ from vllm.config import ( ModelConfig, VllmConfig, get_current_vllm_config, - get_current_vllm_config_or_none, ) from vllm.config.cache import CacheDType from vllm.distributed.parallel_state import ( @@ -264,7 +264,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer -from vllm.utils.math_utils import cdiv, round_down +from vllm.utils.math_utils import cdiv, round_down, round_up from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, @@ -292,8 +292,7 @@ from vllm.v1.attention.backends.utils import ( get_num_attention_heads_from_layers, split_decodes_and_prefills, ) -from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs -from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce +from vllm.v1.attention.ops.dcp_utils import MLADCPManager from vllm.v1.attention.ops.merge_attn_states import merge_attn_states from vllm.v1.attention.selector import get_attn_backend from vllm.v1.kv_cache_interface import ( @@ -579,12 +578,27 @@ class MLAAttention(nn.Module, AttentionLayerBase): self.use_sparse = use_sparse - _vllm_config = get_current_vllm_config_or_none() - self.dcp_a2a = ( - _vllm_config is not None - and _vllm_config.parallel_config.decode_context_parallel_size > 1 - and _vllm_config.parallel_config.dcp_comm_backend == "a2a" - ) + self.dcp_manager: MLADCPManager | None = None + if self.impl.dcp_world_size > 1: + query_dtype = ( + current_platform.fp8_dtype() + if is_quantized_kv_cache(self.kv_cache_dtype) + and self.kv_cache_dtype != "fp8_ds_mla" + and self.impl.supports_quant_query_input + else dtype + ) + self.dcp_manager = MLADCPManager( + vllm_config=vllm_config, + device=next(kv_b_proj.parameters()).device, + num_heads=self.num_heads, + query_head_dim=self.kv_lora_rank + self.qk_rope_head_dim, + output_head_dim=self.kv_lora_rank, + query_dtype=query_dtype, + output_dtype=dtype, + padded_num_heads=self.q_pad_num_heads, + is_lse_base_on_e=self.impl.lse_base_on_e, + use_pcp=self.use_pcp, + ) self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled() @@ -753,9 +767,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): return quant_output.fill_(0) return output.fill_(0) - if self.impl.dcp_world_size == -1: - self.impl.dcp_world_size = get_dcp_group().world_size - fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype) num_actual_toks = attn_metadata.num_actual_tokens @@ -914,6 +925,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): mqa_q = (mqa_ql_nope, mqa_q_pe) # concatenate nope + pe -> (B, N, L + P) (fp8 op above may have fused) if self.impl.dcp_world_size > 1: + assert self.dcp_manager is not None if self.use_pcp: if self.impl.dcp_world_size > self.impl.pcp_world_size: if isinstance(mqa_q, tuple): @@ -924,8 +936,8 @@ class MLAAttention(nn.Module, AttentionLayerBase): # concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P) mqa_q = torch.cat(mqa_q, dim=-1) if not qrep_decode: - # mqa_q do allgather in head dim. - mqa_q = get_dcp_group().all_gather(mqa_q, dim=1) + assert self.dcp_manager.query_gather is not None + mqa_q = self.dcp_manager.query_gather(mqa_q) # call decode attn if not self.impl.is_sparse: @@ -935,27 +947,23 @@ class MLAAttention(nn.Module, AttentionLayerBase): # correct dcp attn_out with lse. if self.impl.dcp_world_size > 1: assert lse is not None - if self.dcp_a2a: - attn_out = dcp_a2a_lse_reduce( - attn_out, - lse, - get_dcp_group(), - is_lse_base_on_e=self.impl.lse_base_on_e, - ) - elif self.use_pcp: - attn_out = cp_lse_ag_out_ar( - attn_out, - lse, - get_dcp_group(), - is_lse_base_on_e=self.impl.lse_base_on_e, - ) - else: - attn_out = cp_lse_ag_out_rs( - attn_out, - lse, - get_dcp_group(), - is_lse_base_on_e=self.impl.lse_base_on_e, - ) + assert self.dcp_manager is not None + seq_lens = ( + attn_metadata.decode.seq_lens + if attn_metadata.decode is not None + else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined] + : attn_metadata.num_decodes + ] + ) + query_start_loc = attn_metadata.query_start_loc[ + : attn_metadata.num_decodes + 1 + ] + attn_out = self.dcp_manager.combine( + attn_out, + lse, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) if self.use_pcp: attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads) @@ -1427,6 +1435,7 @@ class MLACommonPrefillMetadata: chunks: "list[MLACommonPrefillMetadata.ContextChunk]" context_lens_list: list[int] empty_token_slices: list[slice] + dcp_manager: MLADCPManager | None = None block_table: torch.Tensor query_start_loc: torch.Tensor @@ -1684,6 +1693,25 @@ def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor: return np_to_pinned_tensor(np.asarray(values, dtype=np.int32)) +def align_mla_chunked_context_workspace_size( + vllm_config: VllmConfig, + workspace_size: int, +) -> int: + parallel_config = vllm_config.parallel_config + alignment = vllm_config.cache_config.block_size + if parallel_config.decode_context_parallel_size > 1: + alignment = lcm( + alignment, + parallel_config.decode_context_parallel_size + * parallel_config.cp_kv_cache_interleave_size, + ) + workspace_size = max( + workspace_size, + vllm_config.scheduler_config.max_num_seqs * alignment, + ) + return round_up(workspace_size, alignment) + + def build_mla_chunked_context_metadata( *, context_lens_cpu: torch.Tensor, @@ -1696,6 +1724,7 @@ def build_mla_chunked_context_metadata( dcp_world_size: int, dcp_local_block_size: int, dcp_virtual_block_size: int, + dcp_manager: MLADCPManager | None = None, ) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None": """Build chunked-context metadata for an MLA prefill. @@ -1714,6 +1743,7 @@ def build_mla_chunked_context_metadata( dcp_world_size: Decode-context-parallel world size (1 if disabled). dcp_local_block_size: Per-rank interleave block size for DCP. dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``. + dcp_manager: Shared MLA DCP collective manager. Returns: The chunked-context metadata, or None when no prefill has any context. @@ -1902,6 +1932,7 @@ def build_mla_chunked_context_metadata( chunks=chunks, context_lens_list=context_lens, empty_token_slices=empty_token_slices, + dcp_manager=dcp_manager, ) @@ -1954,7 +1985,10 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): 64 * 1024, ) - return max(chunked_prefill_workspace_size, cache_config.block_size) + return align_mla_chunked_context_workspace_size( + vllm_config, + chunked_prefill_workspace_size, + ) @staticmethod def determine_prefill_query_data_type( @@ -2033,6 +2067,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): self.q_data_type = self.determine_prefill_query_data_type( vllm_config, self.model_config.dtype ) + attention_layer = self.compilation_config.static_forward_context[layer_names[0]] try: self.dcp_world_size = get_dcp_group().world_size @@ -2050,6 +2085,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): ) use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla" + self.dcp_manager: MLADCPManager | None = None if self.dcp_world_size > 1: # Note(hc): The local kvcache is incomplete when DCP is triggered, # an additional kvcache allgather across the DCP group is therefore @@ -2067,6 +2103,12 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): else self.model_config.dtype, device=device, ) + self.dcp_manager = getattr(attention_layer, "dcp_manager", None) + assert isinstance(self.dcp_manager, MLADCPManager) + self.dcp_manager.init_kv_gather( + self.chunked_prefill_workspace, + self.chunked_prefill_workspace_size, + ) else: self.chunked_prefill_workspace = torch.empty( ( @@ -2080,9 +2122,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): # Metadata builders are created per ubatch when DBO is enabled. MLA # prefill backends keep the prepared metadata on the backend object, so # each builder needs its own backend instance to avoid cross-ubatch races. - self._prefill_backend = self.compilation_config.static_forward_context[ - layer_names[0] - ].prefill_backend.clone() + self._prefill_backend = attention_layer.prefill_backend.clone() supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY self._init_reorder_batch_threshold( @@ -2223,6 +2263,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): dcp_world_size=self.dcp_world_size, dcp_local_block_size=self.dcp_local_block_size, dcp_virtual_block_size=self.dcp_virtual_block_size, + dcp_manager=self.dcp_manager, ) prefill_metadata = MLACommonPrefillMetadata( @@ -2660,9 +2701,8 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]): ] assert toks * dcp_world_size <= cur_allgather_workspace.shape[0] cur_allgather_kvcache = cur_allgather_workspace[: toks * dcp_world_size] - cur_allgather_kvcache.copy_( - get_dcp_group().all_gather(local_gathered_kvcache, dim=0) - ) + dcp_manager = cast(MLADCPManager, chunked_context.dcp_manager) + dcp_manager.kv_gather(cur_allgather_kvcache, local_gathered_kvcache) assert ( cur_allgather_kvcache.shape[-1] == self.kv_lora_rank + self.qk_rope_head_dim @@ -2732,8 +2772,6 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]): output_scale: torch.Tensor | None = None, ) -> None: assert attn_metadata.prefill is not None - assert self.dcp_world_size != -1 - prefill_metadata = attn_metadata.prefill assert prefill_metadata.prefill_backend is not None use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype() @@ -2877,10 +2915,11 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]): and (self.qk_rope_head_dim == 64) ) - self.dcp_world_size: int = -1 - + parallel_config = get_current_vllm_config().parallel_config + # Avoid requiring an initialized DCP group in tests. + self.dcp_world_size: int = parallel_config.decode_context_parallel_size self.cp_kv_cache_interleave_size: int = ( - get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size + parallel_config.cp_kv_cache_interleave_size ) @abstractmethod diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index ec3a86bbb3..132bb5f920 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( MLACommonMetadata, MLACommonPrefillMetadata, accumulate_mla_context_chunk, + align_mla_chunked_context_workspace_size, build_mla_chunked_context_metadata, get_mla_dims, init_mla_context_partial, @@ -31,6 +32,7 @@ from vllm.utils.torch_utils import is_quantized_kv_cache, np_to_pinned_tensor from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder from vllm.v1.attention.backends.fa_utils import get_flash_attn_version from vllm.v1.attention.backends.utils import split_decodes_and_prefills +from vllm.v1.attention.ops.dcp_utils import MLADCPManager from vllm.v1.attention.ops.merge_attn_states import merge_attn_states if TYPE_CHECKING: @@ -162,9 +164,19 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): dtype=torch.int32, device=device, ) - layer_prefill_backend = vllm_config.compilation_config.static_forward_context[ + attention_layer = vllm_config.compilation_config.static_forward_context[ layer_names[0] - ].prefill_backend + ] + layer_prefill_backend = attention_layer.prefill_backend + self.dcp_manager: MLADCPManager | None = None + if self.dcp_world_size > 1: + self.dcp_manager = getattr(attention_layer, "dcp_manager", None) + assert isinstance(self.dcp_manager, MLADCPManager) + if layer_prefill_backend is not None: + self.dcp_manager.init_kv_gather( + self.chunked_prefill_workspace, + self.chunked_prefill_workspace_size, + ) self._prefill_backend = ( layer_prefill_backend.clone() if layer_prefill_backend is not None else None ) @@ -184,7 +196,13 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): 64 * 1024, scheduler_config.max_num_seqs * topk_tokens, ) - return max(workspace_size, cache_config.block_size) + workspace_size = max( + workspace_size, + scheduler_config.max_num_seqs * cache_config.block_size, + ) + if vllm_config.parallel_config.decode_context_parallel_size > 1: + return align_mla_chunked_context_workspace_size(vllm_config, workspace_size) + return workspace_size def _build_req_id_per_token( self, @@ -232,6 +250,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): dcp_world_size=self.dcp_world_size, dcp_local_block_size=self.dcp_local_block_size, dcp_virtual_block_size=self.dcp_virtual_block_size, + dcp_manager=self.dcp_manager, ) def build( diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index c4e3ce90e5..654f83b452 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -24,8 +24,8 @@ KV cache, and absorbs ``kv_b_proj`` into ``W_UK_T`` / ``W_UV`` -- mirroring the K3 specifics: optional rotary embedding (disabled for the target model's NoPE layers, enabled for DSpark) and an optional sigmoid output gate (``g_proj``). -Out of scope (extension points, not wired here): context parallelism (DCP/PCP), -sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths. +Out of scope (extension points, not wired here): prefill context parallelism +(PCP), sparse/indexer MLA, and the ROCm/aiter fp8/fp4 BMM fast paths. """ import math @@ -35,7 +35,11 @@ import torch from torch import nn from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config +from vllm.config import ( + CacheConfig, + VllmConfig, + get_current_vllm_config, +) from vllm.distributed import get_tensor_model_parallel_world_size from vllm.forward_context import get_forward_context from vllm.logger import init_logger @@ -78,6 +82,7 @@ from vllm.v1.attention.backend import ( MLAAttentionImpl, ) from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend +from vllm.v1.attention.ops.dcp_utils import MLADCPManager from vllm.v1.attention.ops.merge_attn_states import merge_attn_states from vllm.v1.attention.selector import get_attn_backend from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode @@ -310,10 +315,36 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): vllm_config = get_current_vllm_config() parallel_config = vllm_config.parallel_config - assert ( - parallel_config.decode_context_parallel_size <= 1 - and parallel_config.prefill_context_parallel_size <= 1 - ), "Kimi-K3 MultiHeadLatentAttention does not support context parallelism." + assert parallel_config.prefill_context_parallel_size == 1, ( + "Kimi-K3 MultiHeadLatentAttention does not support prefill context " + "parallelism." + ) + self.dcp_world_size = parallel_config.decode_context_parallel_size + assert self.dcp_world_size <= 1 or self.rotary_emb is None, ( + "Kimi-K3 MultiHeadLatentAttention does not support RoPE with decode " + "context parallelism because gathered queries require gathered " + "positions." + ) + self.dcp_manager: MLADCPManager | None = None + if self.dcp_world_size > 1: + query_dtype = ( + torch.float8_e4m3fn + if is_quantized_kv_cache(self.kv_cache_dtype) + and self.kv_cache_dtype != "fp8_ds_mla" + else dtype + ) + self.dcp_manager = MLADCPManager( + vllm_config=vllm_config, + device=next(self.kv_b_proj.parameters()).device, + num_heads=self.num_local_heads, + query_head_dim=self.head_size, + output_head_dim=self.kv_lora_rank, + query_dtype=query_dtype, + output_dtype=dtype, + padded_num_heads=self.q_pad_num_heads, + is_lse_base_on_e=self.impl.lse_base_on_e, + use_pcp=False, + ) self.prefill_backend = get_mla_prefill_backend(vllm_config)( num_heads=self.num_local_heads, scale=self.scale, @@ -584,9 +615,25 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): cos_sin_cache, slot_mapping[:num_mqa_tokens], ) - latent_out, _lse = self.impl.forward_mqa( # type: ignore[attr-defined] + if self.dcp_world_size > 1: + assert self.dcp_manager is not None + assert self.dcp_manager.query_gather is not None + mqa_q = self.dcp_manager.query_gather(mqa_q) + latent_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined] mqa_q, self._attn_read_kv_cache(), attn_metadata, self ) + if self.dcp_world_size > 1: + assert lse is not None + assert self.dcp_manager is not None + assert attn_metadata.decode is not None + latent_out = self.dcp_manager.combine( + latent_out, + lse, + seq_lens=attn_metadata.decode.seq_lens, + query_start_loc=attn_metadata.query_start_loc[ + : attn_metadata.num_decodes + 1 + ], + ) self._v_up_proj(latent_out, out=attn_out[:num_mqa_tokens]) def _decode_concat_cache( @@ -752,9 +799,20 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): ) if has_context: - context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined] - q, self._attn_read_kv_cache(), attn_metadata, self._k_scale - ) + if self.dcp_world_size > 1: + context_output, context_lse = ( + self.impl._context_parallel_compute_prefill_context( # type: ignore[attr-defined] + q, + self._attn_read_kv_cache(), + attn_metadata, + k_scale=self._k_scale, + dcp_world_size=self.dcp_world_size, + ) + ) + else: + context_output, context_lse = self.impl._compute_prefill_context( # type: ignore[attr-defined] + q, self._attn_read_kv_cache(), attn_metadata, self._k_scale + ) suffix_output, suffix_lse = output_prefill out = out.view(-1, self.num_local_heads, self.v_head_dim) merge_attn_states( diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index acd3b81816..bf92bf692b 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -309,9 +309,11 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): return_lse = self.need_to_return_lse_for_decode workspace_buffer = _get_workspace_buffer(return_lse) + # Parallel gathers can change the runtime Q heads from TP-local num_heads. + runtime_num_heads = q.shape[-2] # trtllm-gen rejects MLA head counts it can't tile (e.g. 96); # fall back to cute-dsl for those. - decode_backend = _select_mla_decode_backend(self.num_heads) + decode_backend = _select_mla_decode_backend(runtime_num_heads) extra_kwargs = {} if decode_backend: extra_kwargs["backend"] = decode_backend @@ -326,7 +328,7 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): if self._mla_counter_bytes is None: self._mla_counter_bytes = get_trtllm_gen_multi_ctas_kv_counter_bytes( self._mla_counter_max_batch, - self.num_heads, + runtime_num_heads, get_device_sm_count(q.device), ) extra_kwargs["multi_ctas_kv_counter_buffer"] = ( diff --git a/vllm/v1/attention/backends/mla/tokenspeed_mla.py b/vllm/v1/attention/backends/mla/tokenspeed_mla.py index df4e5c45d1..8ab81c438f 100644 --- a/vllm/v1/attention/backends/mla/tokenspeed_mla.py +++ b/vllm/v1/attention/backends/mla/tokenspeed_mla.py @@ -289,8 +289,9 @@ class TokenspeedMLAImpl(MLACommonImpl[MLACommonMetadata]): self.output_scale = layer._k_scale_float if self._workspace_buffer is None: + # Parallelism can change the runtime query head count. self._workspace_buffer = _get_workspace( - q.device, self.num_heads, self.kv_lora_rank + q.device, q.shape[-2], self.kv_lora_rank ) # vLLM kv_c_and_k_pe_cache is already (num_blocks, block_size, head_size). diff --git a/vllm/v1/attention/ops/common.py b/vllm/v1/attention/ops/common.py index 901d6bb30b..cbe24cf87c 100644 --- a/vllm/v1/attention/ops/common.py +++ b/vllm/v1/attention/ops/common.py @@ -6,6 +6,34 @@ from vllm.distributed.parallel_state import GroupCoordinator from vllm.triton_utils import tl, triton +def mask_dcp_empty_shards_( + lse: torch.Tensor, + seq_lens: torch.Tensor | None, + query_start_loc: torch.Tensor | None, +) -> None: + if seq_lens is None and query_start_loc is None: + return + if seq_lens is None or query_start_loc is None: + raise ValueError("seq_lens and query_start_loc must be provided together") + if ( + seq_lens.ndim != 1 + or query_start_loc.ndim != 1 + or query_start_loc.shape[0] != seq_lens.shape[0] + 1 + ): + raise ValueError("query_start_loc must contain one boundary per sequence") + + row_indices = torch.arange( + lse.shape[0], device=lse.device, dtype=query_start_loc.dtype + ) + sequence_indices = torch.searchsorted( + query_start_loc[1:], row_indices, right=True + ).clamp_max(seq_lens.shape[0] - 1) + empty_rows = (row_indices >= query_start_loc[-1]) | ( + seq_lens[sequence_indices] == 0 + ) + lse.masked_fill_(empty_rows[:, None], float("-inf")) + + @triton.jit def _correct_attn_cp_out_kernel( outputs_ptr, @@ -51,7 +79,7 @@ def _correct_attn_cp_out_kernel( ) # calc final lse - lse = tl.load(lses_ptr + lse_offsets) + lse = tl.load(lses_ptr + lse_offsets).to(tl.float32) lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse) lse_max = tl.max(lse, axis=0) lse_max = tl.where(lse_max == -float("inf"), 0, lse_max) @@ -80,7 +108,7 @@ def _correct_attn_cp_out_kernel( lse_offset = ( lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H ) - lse_tmp = tl.load(lses_ptr + lse_offset) + lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32) lse_finally = lse_tmp - lse lse_finally = tl.where( (lse_finally != lse_finally) | (lse_finally == float("inf")), @@ -185,6 +213,8 @@ def _cp_lse_common( cp_group: GroupCoordinator, ctx: CPTritonContext | None = None, is_lse_base_on_e=True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, ): """ cp_attn_out: [ B, H, D ] @@ -197,6 +227,7 @@ def _cp_lse_common( ctx = CPTritonContext() cp_attn_lse = cp_attn_lse.contiguous() + mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) lses = cp_group.all_gather(cp_attn_lse, dim=0).reshape( (cp_group.world_size,) + cp_attn_lse.shape ) @@ -217,13 +248,21 @@ def cp_lse_ag_out_rs( ctx: CPTritonContext | None = None, return_lse: bool = False, is_lse_base_on_e=True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, ): """ cp_attn_out: [ B, H, D ] cp_attn_lse: [ B, H ] """ out, lse = _cp_lse_common( - cp_attn_out, cp_attn_lse, cp_group, ctx=ctx, is_lse_base_on_e=is_lse_base_on_e + cp_attn_out, + cp_attn_lse, + cp_group, + ctx=ctx, + is_lse_base_on_e=is_lse_base_on_e, + seq_lens=seq_lens, + query_start_loc=query_start_loc, ) out = cp_group.reduce_scatter(out, dim=1) @@ -242,13 +281,21 @@ def cp_lse_ag_out_ar( ctx: CPTritonContext | None = None, return_lse: bool = False, is_lse_base_on_e=True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, ): """ cp_attn_out: [ B, H, D ] cp_attn_lse: [ B, H ] """ out, lse = _cp_lse_common( - cp_attn_out, cp_attn_lse, cp_group, ctx=ctx, is_lse_base_on_e=is_lse_base_on_e + cp_attn_out, + cp_attn_lse, + cp_group, + ctx=ctx, + is_lse_base_on_e=is_lse_base_on_e, + seq_lens=seq_lens, + query_start_loc=query_start_loc, ) out = cp_group.all_reduce(out) diff --git a/vllm/v1/attention/ops/dcp_alltoall.py b/vllm/v1/attention/ops/dcp_alltoall.py index e100dbc79e..5939b61a8d 100644 --- a/vllm/v1/attention/ops/dcp_alltoall.py +++ b/vllm/v1/attention/ops/dcp_alltoall.py @@ -26,6 +26,7 @@ import torch import torch.distributed as dist from vllm.triton_utils import tl, triton +from vllm.v1.attention.ops.common import mask_dcp_empty_shards_ if TYPE_CHECKING: from vllm.distributed.parallel_state import GroupCoordinator @@ -87,7 +88,9 @@ def _lse_weighted_combine( weights = weights / weight_sum.clamp(min=1e-10) # [N, B, H] # Weighted combination: sum over N dimension - result = (outputs * weights.unsqueeze(-1)).sum(dim=0) # [B, H, D] + weights = weights.unsqueeze(-1) + outputs = torch.where(weights == 0, torch.zeros_like(outputs), outputs) + result = (outputs * weights).sum(dim=0) # [B, H, D] if return_lse: if is_lse_base_on_e: @@ -172,7 +175,7 @@ def _dcp_a2a_pack_send_kernel( lse_val = tl.load( lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H - ) + ).to(tl.float32) if LSE_PACK_DIM == 1: tl.store( send_ptr + send_base + HEAD_DIM * send_stride_D, @@ -301,10 +304,11 @@ def _dcp_a2a_unpack_combine_kernel( else: weight = tl.exp2(lse_val - global_lse) weight = tl.where(weight != weight, 0.0, weight) - acc += ( - tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to(tl.float32) - * weight + partial = tl.load(recv_ptr + recv_base + d_offsets * recv_stride_D).to( + tl.float32 ) + partial = tl.where(weight == 0.0, 0.0, partial) + acc += partial * weight final_offsets = ( batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D @@ -324,7 +328,10 @@ def _dcp_a2a_pack_send( h_per_rank: int, head_dim: int, lse_pack_dim: int, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, ) -> None: + mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) grid = (cp_attn_out.shape[0], h_per_rank, 1) _dcp_a2a_pack_send_kernel[grid]( cp_attn_out, @@ -396,20 +403,24 @@ def dcp_a2a_lse_reduce( ctx: CPTritonContext | None = None, return_lse: bool = False, is_lse_base_on_e: bool = True, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """ Combine partial attention outputs across DCP ranks using All-to-All. - The output and fp32 LSE are packed into a single output-dtype buffer, sent + The output and LSE are packed into a single output-dtype buffer, sent with one All-to-All, then unpacked and combined with exact LSE weighting. Args: cp_attn_out: [B, H, D] where B=num_tokens, H=total_heads, D=head_dim - cp_attn_lse: [B, H] log-sum-exp values (fp32) + cp_attn_lse: [B, H] floating-point log-sum-exp values cp_group: GroupCoordinator for DCP communication ctx: CPTritonContext (unused, for signature compatibility) return_lse: If True, also return the combined global LSE is_lse_base_on_e: If True, LSE is base e; if False, base 2 + seq_lens: Local KV lengths. Empty shards contribute zero weight. + query_start_loc: Cumulative query-token offsets for each request. Returns: Combined output [B, H/N, D] (head-scattered) @@ -426,10 +437,6 @@ def dcp_a2a_lse_reduce( if H % world_size != 0: raise ValueError(f"H={H} must be divisible by DCP world size {world_size}.") H_per_rank = H // world_size - # The pack kernel bit-casts the LSE as fp32; some MLA backends return it in - # the activation dtype (bf16/fp16), so enforce the documented fp32 contract. - if cp_attn_lse.dtype != torch.float32: - cp_attn_lse = cp_attn_lse.to(torch.float32) lse_pack_dim = _dcp_a2a_lse_pack_dim(cp_attn_out.dtype) send_buffer, recv_buffer = _dcp_a2a_send_recv_buffers( @@ -446,6 +453,8 @@ def dcp_a2a_lse_reduce( H_per_rank, D, lse_pack_dim, + seq_lens=seq_lens, + query_start_loc=query_start_loc, ) work = dist.all_to_all_single( diff --git a/vllm/v1/attention/ops/dcp_utils.py b/vllm/v1/attention/ops/dcp_utils.py new file mode 100644 index 0000000000..b28d337a10 --- /dev/null +++ b/vllm/v1/attention/ops/dcp_utils.py @@ -0,0 +1,740 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MLA DCP collective selection and direct symmetric-memory implementations.""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Protocol + +import torch + +import vllm.envs as envs +from vllm.config import VllmConfig +from vllm.distributed import get_dcp_group +from vllm.distributed.parallel_state import in_the_same_node_as +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs +from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce +from vllm.v1.worker.ubatching import dbo_current_ubatch_id + +logger = init_logger(__name__) + +if TYPE_CHECKING: + from torch.distributed import ProcessGroup + + from vllm.distributed.parallel_state import GroupCoordinator + +try: + import torch.distributed._symmetric_memory as symm_mem + + symm_mem_available = True +except ImportError: + symm_mem = None # type: ignore[assignment] + symm_mem_available = False + + +@functools.cache +def _symm_mem_spans_group(group: GroupCoordinator) -> bool: + """Probe whether the group has NVLS symmetric memory.""" + if not symm_mem_available: + return False + try: + from torch._C._autograd import DeviceType + from torch._C._distributed_c10d import _SymmetricMemory + + device = torch.device("cuda", torch.accelerator.current_device_index()) + if not _SymmetricMemory.has_multicast_support(DeviceType.CUDA, device.index): + return False + probe = symm_mem.empty(8, dtype=torch.uint8, device=device) + probe.zero_() + torch.accelerator.synchronize() + handle = symm_mem.rendezvous(probe, group.device_group.group_name) + spans = handle is not None and handle.multicast_ptr != 0 + except Exception as error: + logger.debug("Direct DCP symmetric-memory probe failed: %s", error) + return False + logger.debug_once( + "Direct DCP symmetric memory across %d ranks: %s", + group.world_size, + "available" if spans else "unavailable", + ) + return spans + + +def _direct_dcp_enabled( + group: GroupCoordinator, + dtype: torch.dtype, + use_direct: bool | None, + supported_dtypes: tuple[torch.dtype, ...] | None = None, +) -> bool: + if use_direct is not None: + return use_direct + return ( + symm_mem_available + and current_platform.is_cuda() + and (supported_dtypes is None or dtype in supported_dtypes) + and ( + all(in_the_same_node_as(group.cpu_group, source_rank=0)) + or _symm_mem_spans_group(group) + ) + ) + + +def _direct_dcp_multicast_enabled( + group: GroupCoordinator, + dtype: torch.dtype, + use_direct: bool | None, + supported_dtypes: tuple[torch.dtype, ...] | None = None, +) -> bool: + return _direct_dcp_enabled( + group, dtype, use_direct, supported_dtypes + ) and _symm_mem_spans_group(group) + + +def get_dcp_workspace_max_num_tokens(vllm_config: VllmConfig) -> int: + scheduler_config = vllm_config.scheduler_config + speculative_config = vllm_config.speculative_config + speculative_tokens = vllm_config.num_speculative_tokens + tokens_per_seq = ( + 1 + + ( + 2 + if speculative_config is not None and speculative_config.parallel_drafting + else 1 + ) + * speculative_tokens + ) + return min( + scheduler_config.max_num_batched_tokens, + max( + scheduler_config.max_num_seqs * tokens_per_seq, + vllm_config.compilation_config.max_cudagraph_capture_size or 0, + ), + ) + + +class _DirectDCPWorkspace: + def __init__( + self, + group: ProcessGroup, + device: torch.device, + num_ubatches: int, + ) -> None: + self.group = group + self.world_size = group.size() + self.rank = group.rank() + self.device = torch.device(device) + self.num_ubatches = num_ubatches + self.epoch = torch.zeros(num_ubatches, dtype=torch.int64, device=self.device) + self._allocations: list[tuple[torch.Tensor, Any, list[torch.Tensor]]] = [] + + def _allocate( + self, shape: tuple[int, ...], dtype: torch.dtype + ) -> tuple[torch.Tensor, torch.Tensor]: + storage = symm_mem.empty(shape, device=self.device, dtype=dtype) + storage.zero_() + torch.accelerator.synchronize() + handle = symm_mem.rendezvous(storage, self.group.group_name) + assert handle is not None, "DCP symmetric memory rendezvous returned None" + handle.barrier() + views = [ + handle.get_buffer(peer, list(shape), dtype, 0) + for peer in range(self.world_size) + ] + self.device = storage.device + peer_ptrs = torch.tensor( + [ + [view[ubatch].data_ptr() for view in views] + for ubatch in range(self.num_ubatches) + ], + dtype=torch.int64, + device=self.device, + ) + self._allocations.append((storage, handle, views)) + return storage, peer_ptrs + + def _multicast_ptrs(self, storage: torch.Tensor) -> list[int]: + disabled = [0] * self.num_ubatches + for allocated, handle, _ in self._allocations: + if allocated is storage: + break + else: + return disabled + try: + from torch._C._autograd import DeviceType + from torch._C._distributed_c10d import _SymmetricMemory + + if not _SymmetricMemory.has_multicast_support( + DeviceType.CUDA, storage.device.index + ): + return disabled + multicast_base = handle.multicast_ptr + except Exception: + return disabled + if not multicast_base: + return disabled + storage_base = storage.data_ptr() + return [ + multicast_base + (storage[ubatch].data_ptr() - storage_base) + for ubatch in range(self.num_ubatches) + ] + + +def reserve_query_head_storage( + query: torch.Tensor, padded_num_heads: int +) -> torch.Tensor: + """Reserve backing storage for fixed-head decode kernels.""" + assert query.ndim == 3 + assert query.shape[1] <= padded_num_heads + padded = query.new_empty((query.shape[0], padded_num_heads, query.shape[2])) + padded.resize_(query.shape) + padded.copy_(query) + return padded + + +_A2A_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16) + + +class DirectDCPA2AWorkspace(_DirectDCPWorkspace): + """Persistent symmetric buffers for direct DCP output exchange.""" + + def __init__( + self, + group: ProcessGroup, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype = torch.bfloat16, + num_ubatches: int = 1, + ) -> None: + if dtype not in _A2A_SUPPORTED_DTYPES: + raise ValueError(f"Direct DCP A2A does not support {dtype}") + if num_ubatches < 1: + raise ValueError( + f"Direct DCP A2A requires at least one ubatch slot, got {num_ubatches}" + ) + super().__init__(group, device, num_ubatches) + self.max_num_tokens = max_num_tokens + self.heads_per_rank = heads_per_rank + self.head_dim = head_dim + + output_shape = ( + num_ubatches, + 2, + self.world_size, + max_num_tokens, + heads_per_rank, + head_dim, + ) + lse_shape = ( + num_ubatches, + 2, + self.world_size, + max_num_tokens, + heads_per_rank, + ) + signal_shape = (num_ubatches, 2, self.world_size) + self.received_output, self.peer_output_ptrs = self._allocate( + output_shape, dtype + ) + self.received_lse, self.peer_lse_ptrs = self._allocate(lse_shape, torch.float32) + self.received_signal, self.peer_signal_ptrs = self._allocate( + signal_shape, torch.int32 + ) + + def lse_reduce( + self, + partial_output: torch.Tensor, + partial_lse: torch.Tensor, + is_lse_base_on_e: bool, + seq_lens: torch.Tensor | None = None, + query_start_loc: torch.Tensor | None = None, + ) -> torch.Tensor: + ubatch = dbo_current_ubatch_id() + num_tokens = partial_output.shape[0] + output = partial_output.new_empty( + (num_tokens, self.heads_per_rank, self.head_dim) + ) + torch.ops._C.direct_dcp_a2a_lse_reduce( + partial_output, + partial_lse, + seq_lens, + query_start_loc, + self.peer_output_ptrs[ubatch], + self.peer_lse_ptrs[ubatch], + self.peer_signal_ptrs[ubatch], + self.received_output[ubatch], + self.received_lse[ubatch], + self.received_signal[ubatch], + self.epoch[ubatch : ubatch + 1], + output, + self.world_size, + self.rank, + self.max_num_tokens, + is_lse_base_on_e, + ) + return output + + +@functools.cache +def get_direct_dcp_a2a_workspace( + group: GroupCoordinator, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype, + num_ubatches: int, +) -> DirectDCPA2AWorkspace | None: + if not _direct_dcp_enabled( + group, dtype, envs.VLLM_USE_DIRECT_DCP_A2A, _A2A_SUPPORTED_DTYPES + ): + return None + return DirectDCPA2AWorkspace( + group.device_group, + device, + max_num_tokens, + heads_per_rank, + head_dim, + dtype, + num_ubatches, + ) + + +def _q_gather_layout_supported( + world_size: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype, + padded_num_heads: int | None, +) -> bool: + element_size = torch.empty((), dtype=dtype).element_size() + gathered_num_heads = world_size * heads_per_rank + storage_num_heads = ( + gathered_num_heads if padded_num_heads is None else padded_num_heads + ) + return ( + heads_per_rank * head_dim * element_size % 16 == 0 + and storage_num_heads * head_dim * element_size % 16 == 0 + ) + + +class DirectDCPQGatherWorkspace(_DirectDCPWorkspace): + """Publish query shards directly into the consumer-final symmetric buffer. + + The final buffer is reusable after the downstream DCP output combine. That + combine orders all ranks after attention has consumed the gathered query. + """ + + def __init__( + self, + group: ProcessGroup, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype = torch.bfloat16, + num_ubatches: int = 1, + padded_num_heads: int | None = None, + ) -> None: + if num_ubatches < 1: + raise ValueError( + "Direct DCP q-gather requires at least one ubatch slot, " + f"got {num_ubatches}" + ) + if max_num_tokens < 1 or heads_per_rank < 1 or head_dim < 1: + raise ValueError( + "Direct DCP q-gather dimensions must be positive, got " + f"T={max_num_tokens}, H={heads_per_rank}, D={head_dim}" + ) + gathered_num_heads = group.size() * heads_per_rank + if not _q_gather_layout_supported( + group.size(), heads_per_rank, head_dim, dtype, padded_num_heads + ): + raise ValueError("Direct DCP q-gather requires 16-byte-aligned query rows.") + super().__init__(group, device, num_ubatches) + if self.world_size <= 1: + raise ValueError("Direct DCP q-gather requires at least two ranks") + self.max_num_tokens = max_num_tokens + self.heads_per_rank = heads_per_rank + self.gathered_num_heads = gathered_num_heads + self.padded_num_heads = ( + self.gathered_num_heads if padded_num_heads is None else padded_num_heads + ) + if self.padded_num_heads < self.gathered_num_heads: + raise ValueError( + "Direct DCP q-gather padded heads must cover gathered heads: " + f"{self.padded_num_heads} < {self.gathered_num_heads}" + ) + self.head_dim = head_dim + + query_shape = ( + num_ubatches, + max_num_tokens, + self.padded_num_heads, + head_dim, + ) + signal_shape = (num_ubatches, 2, self.world_size) + self.final_query, _ = self._allocate(query_shape, dtype) + self.received_signal, _ = self._allocate(signal_shape, torch.int32) + query_multicast_ptrs = self._multicast_ptrs(self.final_query) + signal_multicast_ptrs = self._multicast_ptrs(self.received_signal) + self.multicast_ptrs = list( + zip(query_multicast_ptrs, signal_multicast_ptrs, strict=True) + ) + if not all( + query_ptr and signal_ptr for query_ptr, signal_ptr in self.multicast_ptrs + ): + raise RuntimeError( + "Direct DCP q-gather requires NVLS symmetric-memory multicast." + ) + self.completion = self.received_signal.new_zeros((num_ubatches, 1)) + torch.accelerator.synchronize() + + def gather(self, local_query: torch.Tensor) -> torch.Tensor: + ubatch = dbo_current_ubatch_id() + if not 0 <= ubatch < self.num_ubatches: + raise ValueError( + f"DCP q-gather ubatch {ubatch} exceeds {self.num_ubatches} slots" + ) + if local_query.ndim == 3 and local_query.shape[1] != self.heads_per_rank: + raise ValueError( + f"DCP q-gather expected {self.heads_per_rank} local query heads, " + f"got {local_query.shape[1]}" + ) + + num_tokens = local_query.shape[0] + output = torch.as_strided( + self.final_query[ubatch], + size=(num_tokens, self.gathered_num_heads, self.head_dim), + stride=( + self.gathered_num_heads * self.head_dim, + self.head_dim, + 1, + ), + ) + query_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch] + torch.ops._C.direct_dcp_q_gather( + local_query, + output, + self.received_signal[ubatch], + self.completion[ubatch], + self.epoch[ubatch : ubatch + 1], + self.world_size, + self.rank, + self.max_num_tokens, + self.padded_num_heads, + query_multicast_ptr, + signal_multicast_ptr, + ) + return output + + +@functools.cache +def get_direct_dcp_q_gather_workspace( + group: GroupCoordinator, + device: torch.device, + max_num_tokens: int, + heads_per_rank: int, + head_dim: int, + dtype: torch.dtype, + num_ubatches: int, + padded_num_heads: int | None = None, +) -> DirectDCPQGatherWorkspace | None: + if not _direct_dcp_multicast_enabled( + group, dtype, envs.VLLM_USE_DIRECT_DCP_Q_GATHER + ): + return None + if not _q_gather_layout_supported( + group.world_size, heads_per_rank, head_dim, dtype, padded_num_heads + ): + return None + return DirectDCPQGatherWorkspace( + group.device_group, + device, + max_num_tokens, + heads_per_rank, + head_dim, + dtype, + num_ubatches, + padded_num_heads, + ) + + +_KV_GATHER_SUPPORTED_DTYPES = ( + torch.float16, + torch.bfloat16, + torch.float8_e4m3fn, +) + + +def _kv_gather_layout_supported(token_dim: int, dtype: torch.dtype) -> bool: + return token_dim * torch.empty((), dtype=dtype).element_size() % 16 == 0 + + +class DirectDCPKVGatherWorkspace(_DirectDCPWorkspace): + """Persistent symmetric buffers for direct DCP KV gather.""" + + def __init__( + self, + group: ProcessGroup, + device: torch.device, + max_gathered_tokens: int, + token_dim: int, + dtype: torch.dtype = torch.bfloat16, + num_ubatches: int = 1, + ) -> None: + if dtype not in _KV_GATHER_SUPPORTED_DTYPES: + raise ValueError(f"Direct DCP kv-gather does not support {dtype}") + if num_ubatches < 1: + raise ValueError( + "Direct DCP kv-gather requires at least one ubatch slot, " + f"got {num_ubatches}" + ) + if max_gathered_tokens < 1 or token_dim < 1: + raise ValueError( + "Direct DCP kv-gather dimensions must be positive, got " + f"T={max_gathered_tokens}, D={token_dim}" + ) + if not _kv_gather_layout_supported(token_dim, dtype): + raise ValueError("Direct DCP kv-gather requires 16-byte-aligned KV rows.") + super().__init__(group, device, num_ubatches) + if self.world_size <= 1: + raise ValueError("Direct DCP kv-gather requires at least two ranks") + if max_gathered_tokens % self.world_size != 0: + raise ValueError( + "Direct DCP kv-gather capacity must divide evenly across " + f"ranks: {max_gathered_tokens} % {self.world_size} != 0" + ) + self.max_gathered_tokens = max_gathered_tokens + + kv_shape = (num_ubatches, 2, max_gathered_tokens, token_dim) + signal_shape = (num_ubatches, 2, self.world_size) + self.received_kv, _ = self._allocate(kv_shape, dtype) + self.received_signal, _ = self._allocate(signal_shape, torch.int32) + kv_multicast_ptrs = self._multicast_ptrs(self.received_kv) + signal_multicast_ptrs = self._multicast_ptrs(self.received_signal) + self.multicast_ptrs = list( + zip(kv_multicast_ptrs, signal_multicast_ptrs, strict=True) + ) + if not all(kv_ptr and signal_ptr for kv_ptr, signal_ptr in self.multicast_ptrs): + raise RuntimeError( + "Direct DCP kv-gather requires NVLS symmetric-memory multicast." + ) + self.completion = self.received_signal.new_zeros((num_ubatches, 2)) + torch.accelerator.synchronize() + + def gather(self, gathered_kv: torch.Tensor, local_kv: torch.Tensor) -> None: + ubatch = dbo_current_ubatch_id() + if not 0 <= ubatch < self.num_ubatches: + raise ValueError( + f"DCP kv-gather ubatch {ubatch} exceeds {self.num_ubatches} slots" + ) + kv_multicast_ptr, signal_multicast_ptr = self.multicast_ptrs[ubatch] + torch.ops._C.direct_dcp_kv_gather( + local_kv, + self.received_kv[ubatch], + self.received_signal[ubatch], + self.completion[ubatch], + self.epoch[ubatch : ubatch + 1], + gathered_kv, + self.world_size, + self.rank, + self.max_gathered_tokens, + kv_multicast_ptr, + signal_multicast_ptr, + ) + + +@functools.cache +def get_direct_dcp_kv_gather_workspace( + group: GroupCoordinator, + device: torch.device, + max_gathered_tokens: int, + token_dim: int, + dtype: torch.dtype, + num_ubatches: int, +) -> DirectDCPKVGatherWorkspace | None: + if not _direct_dcp_multicast_enabled( + group, + dtype, + envs.VLLM_USE_DIRECT_DCP_KV_GATHER, + _KV_GATHER_SUPPORTED_DTYPES, + ): + return None + if not _kv_gather_layout_supported(token_dim, dtype): + return None + return DirectDCPKVGatherWorkspace( + group.device_group, + device, + max_gathered_tokens, + token_dim, + dtype, + num_ubatches, + ) + + +class DCPCombine(Protocol): + def __call__( + self, + partial_output: torch.Tensor, + partial_lse: torch.Tensor, + *, + seq_lens: torch.Tensor, + query_start_loc: torch.Tensor, + ) -> torch.Tensor: ... + + +class MLADCPManager: + """Select and own layer-level collective implementations for MLA DCP.""" + + _kv_gather: Callable[[torch.Tensor, torch.Tensor], object] + + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + num_heads: int, + query_head_dim: int, + output_head_dim: int, + query_dtype: torch.dtype, + output_dtype: torch.dtype, + padded_num_heads: int | None, + is_lse_base_on_e: bool, + use_pcp: bool, + ) -> None: + parallel_config = vllm_config.parallel_config + self.group = get_dcp_group() + self.device = torch.device(device) + self.num_ubatches = max(parallel_config.num_ubatches, 1) + self.max_num_tokens = get_dcp_workspace_max_num_tokens(vllm_config) + self.use_a2a = parallel_config.dcp_comm_backend == "a2a" + self.padded_num_heads = padded_num_heads + + self.combine = self._init_combine( + num_heads, + output_head_dim, + output_dtype, + is_lse_base_on_e, + use_pcp, + ) + self.query_gather = ( + None + if use_pcp + else self._init_query_gather( + num_heads, + query_head_dim, + query_dtype, + ) + ) + + def _init_combine( + self, + num_heads: int, + head_dim: int, + dtype: torch.dtype, + is_lse_base_on_e: bool, + use_pcp: bool, + ) -> DCPCombine: + direct_workspace = None + if self.use_a2a: + direct_workspace = get_direct_dcp_a2a_workspace( + self.group, + self.device, + self.max_num_tokens, + num_heads, + head_dim, + dtype, + self.num_ubatches, + ) + if direct_workspace is not None: + logger.info_once("Using direct symmetric-memory DCP A2A for MLA.") + return functools.partial( + direct_workspace.lse_reduce, + is_lse_base_on_e=is_lse_base_on_e, + ) + + combine_fn = ( + dcp_a2a_lse_reduce + if self.use_a2a + else cp_lse_ag_out_ar + if use_pcp + else cp_lse_ag_out_rs + ) + return functools.partial( + combine_fn, + cp_group=self.group, + is_lse_base_on_e=is_lse_base_on_e, + ) + + def _init_query_gather( + self, + num_heads: int, + head_dim: int, + dtype: torch.dtype, + ) -> Callable[[torch.Tensor], torch.Tensor]: + direct_workspace = get_direct_dcp_q_gather_workspace( + self.group, + self.device, + self.max_num_tokens, + num_heads, + head_dim, + dtype, + self.num_ubatches, + self.padded_num_heads, + ) + if direct_workspace is not None: + logger.info_once("Using direct symmetric-memory DCP query gather for MLA.") + return direct_workspace.gather + return self._gather_query + + def _gather_query(self, query: torch.Tensor) -> torch.Tensor: + query = self.group.all_gather(query, dim=1) + if self.padded_num_heads is not None: + query = reserve_query_head_storage(query, self.padded_num_heads) + return query + + def init_kv_gather( + self, + workspace: torch.Tensor, + max_gathered_tokens: int, + ) -> None: + world_size = self.group.world_size + assert max_gathered_tokens > 0 + assert max_gathered_tokens % world_size == 0 + assert workspace.ndim == 2 + assert workspace.is_contiguous() + assert workspace.shape[0] == ( + max_gathered_tokens + max_gathered_tokens // world_size + ) + assert workspace.shape[1] > 0 + + direct_workspace = get_direct_dcp_kv_gather_workspace( + self.group, + workspace.device, + max_gathered_tokens, + workspace.shape[1], + workspace.dtype, + self.num_ubatches, + ) + if direct_workspace is not None: + logger.info_once( + "Using direct symmetric-memory DCP chunked-context KV gather for MLA." + ) + self._kv_gather = direct_workspace.gather + else: + self._kv_gather = functools.partial( + torch.distributed.all_gather_into_tensor, + group=self.group.device_group, + ) + + def kv_gather( + self, + gathered_kv: torch.Tensor, + local_kv: torch.Tensor, + ) -> object: + return self._kv_gather(gathered_kv, local_kv)