Compare commits
112
Commits
main
...
31bee2c2fd
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
31bee2c2fd | ||
|
|
bf3e4af6b1 | ||
|
|
f5d9021745 | ||
|
|
633c758d4c | ||
|
|
95ba99ed29 | ||
|
|
944007d9e5 | ||
|
|
db2851e54a | ||
|
|
69aed12cf8 | ||
|
|
86d314050f | ||
|
|
b5f9fdbbac | ||
|
|
72485816d9 | ||
|
|
21fb7de7bf | ||
|
|
010521f10d | ||
|
|
0f72060d66 | ||
|
|
7c0a916c5d | ||
|
|
8c518d4f09 | ||
|
|
4cfd09ebda | ||
|
|
812d662602 | ||
|
|
8918d40b59 | ||
|
|
7e9cd428a1 | ||
|
|
e5e541446f | ||
|
|
2c6efaabe8 | ||
|
|
eb68fb9105 | ||
|
|
cb27582783 | ||
|
|
24bcbc5087 | ||
|
|
5df639bb4e | ||
|
|
ef55e095c3 | ||
|
|
036608b012 | ||
|
|
6c956ee3bd | ||
|
|
eeea036c12 | ||
|
|
a4aecf9ec7 | ||
|
|
3575cdb126 | ||
|
|
e3fcd7e3ae | ||
|
|
7f6055caa2 | ||
|
|
fb52250c11 | ||
|
|
fba504d5bb | ||
|
|
ad80cac06d | ||
|
|
dec8be19de | ||
|
|
281491646d | ||
|
|
bef464a69d | ||
|
|
cff46af086 | ||
|
|
2918e3fda1 | ||
|
|
512ca2fc0b | ||
|
|
63e2d8be20 | ||
|
|
daa56356c8 | ||
|
|
66c3d11016 | ||
|
|
7f4385a422 | ||
|
|
54d4ab1b9a | ||
|
|
09ea3a66e9 | ||
|
|
e2eaefdec9 | ||
|
|
6705330924 | ||
|
|
2e68aba2a7 | ||
|
|
23349e43b2 | ||
|
|
3b5fc7c7be | ||
|
|
025fd08603 | ||
|
|
bc0a1cee82 | ||
|
|
7351759ecf | ||
|
|
f1e5437a3a | ||
|
|
31ad7e113f | ||
|
|
754fb3e29d | ||
|
|
17c1c70447 | ||
|
|
03c8ebae34 | ||
|
|
4c6c7af076 | ||
|
|
c841dce4a4 | ||
|
|
73ab1259c1 | ||
|
|
5de9f4a6ae | ||
|
|
9bc82b7ceb | ||
|
|
5ac158fbe6 | ||
|
|
c646a6c792 | ||
|
|
7118c2f999 | ||
|
|
8ca58f1714 | ||
|
|
b1fdcd6e27 | ||
|
|
62acb3dab8 | ||
|
|
41f6691c26 | ||
|
|
66b3d8c108 | ||
|
|
a902e402c6 | ||
|
|
7e5f658d75 | ||
|
|
461a8a0f82 | ||
|
|
03f5313f0f | ||
|
|
4a3ebc6b5b | ||
|
|
f83ecfac16
|
||
|
|
85c31d47ad
|
||
|
|
2877121b2d
|
||
|
|
ae0350c0de
|
||
|
|
a1d3c2cf1e
|
||
|
|
cb275fdf20
|
||
|
|
a63dca4afb
|
||
|
|
76af434852
|
||
|
|
521235d6df | ||
|
|
5567cc1b25 | ||
|
|
ed2a9665ea | ||
|
|
5da84df6b7
|
||
|
|
5e6656e7a0
|
||
|
|
43fbd6510b
|
||
|
|
5af48b498c
|
||
|
|
835c44af15
|
||
|
|
758cc4f9e1
|
||
|
|
03cae7837f | ||
|
|
a80e41d690 | ||
|
|
848dbf89ec | ||
|
|
31d78a21b9 | ||
|
|
045467c102 | ||
|
|
51d5b3ac8a | ||
|
|
5780c0125a | ||
|
|
1eacb41e0f | ||
|
|
b8adc7710e | ||
|
|
77b8783b25 | ||
|
|
6c5552776b | ||
|
|
e582b9f4a2 | ||
|
|
21d2dd56d8 | ||
|
|
a66a3e1b4e | ||
|
|
3b1911c0ca |
@@ -20,7 +20,8 @@
|
|||||||
const int32_t *__restrict__ num_tokens_past_padded_ptr, \
|
const int32_t *__restrict__ num_tokens_past_padded_ptr, \
|
||||||
const float *__restrict__ topk_weights_ptr, int top_k, \
|
const float *__restrict__ topk_weights_ptr, int top_k, \
|
||||||
bool mul_topk_weights, int prob_m, int prob_n, int prob_k, int *locks, \
|
bool mul_topk_weights, int prob_m, int prob_n, int prob_k, int *locks, \
|
||||||
bool has_bias, bool use_atomic_add, bool use_fp32_reduce
|
bool has_bias, bool use_atomic_add, bool use_fp32_reduce, \
|
||||||
|
bool use_full_k
|
||||||
|
|
||||||
namespace MARLIN_NAMESPACE_NAME {
|
namespace MARLIN_NAMESPACE_NAME {
|
||||||
template <const vllm::ScalarTypeId a_type_id, // A ScalarType id
|
template <const vllm::ScalarTypeId a_type_id, // A ScalarType id
|
||||||
|
|||||||
@@ -74,7 +74,8 @@ __global__ void Marlin(
|
|||||||
int prob_k, // reduction dimension k
|
int prob_k, // reduction dimension k
|
||||||
int* locks, // extra global storage for barrier synchronization
|
int* locks, // extra global storage for barrier synchronization
|
||||||
bool use_atomic_add, // whether to use atomic add to reduce
|
bool use_atomic_add, // whether to use atomic add to reduce
|
||||||
bool use_fp32_reduce // whether to use fp32 global reduce
|
bool use_fp32_reduce, // whether to use fp32 global reduce
|
||||||
|
bool use_full_k // whether to reduce over full K per block
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
} // namespace MARLIN_NAMESPACE_NAME
|
} // namespace MARLIN_NAMESPACE_NAME
|
||||||
@@ -250,7 +251,8 @@ __global__ void Marlin(
|
|||||||
int* locks, // extra global storage for barrier synchronization
|
int* locks, // extra global storage for barrier synchronization
|
||||||
bool has_bias,
|
bool has_bias,
|
||||||
bool use_atomic_add, // whether to use atomic add to reduce
|
bool use_atomic_add, // whether to use atomic add to reduce
|
||||||
bool use_fp32_reduce // whether to use fp32 global reduce
|
bool use_fp32_reduce, // whether to use fp32 global reduce
|
||||||
|
bool use_full_k // whether to reduce over full K per block
|
||||||
) {
|
) {
|
||||||
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
|
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
|
||||||
// same size, which might involve multiple column "slices" (of width 16 *
|
// same size, which might involve multiple column "slices" (of width 16 *
|
||||||
@@ -370,11 +372,13 @@ __global__ void Marlin(
|
|||||||
// see https://github.com/vllm-project/vllm/pull/24722 for more details
|
// see https://github.com/vllm-project/vllm/pull/24722 for more details
|
||||||
if (global_mn_tiles > gridDim.x) {
|
if (global_mn_tiles > gridDim.x) {
|
||||||
part2_mn_tiles = global_mn_tiles % gridDim.x;
|
part2_mn_tiles = global_mn_tiles % gridDim.x;
|
||||||
if (part2_mn_tiles * 3 <= gridDim.x) part2_mn_tiles += gridDim.x;
|
if (!use_full_k && part2_mn_tiles * 3 <= gridDim.x)
|
||||||
|
part2_mn_tiles += gridDim.x;
|
||||||
part1_mn_iters = (global_mn_tiles - part2_mn_tiles) / gridDim.x;
|
part1_mn_iters = (global_mn_tiles - part2_mn_tiles) / gridDim.x;
|
||||||
}
|
}
|
||||||
|
|
||||||
int iters = div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
|
int iters =
|
||||||
|
use_full_k ? k_tiles : div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
|
||||||
|
|
||||||
if constexpr (group_blocks != -1) {
|
if constexpr (group_blocks != -1) {
|
||||||
if (group_blocks >= thread_k_blocks) {
|
if (group_blocks >= thread_k_blocks) {
|
||||||
|
|||||||
@@ -33,6 +33,53 @@
|
|||||||
#include <torch/headeronly/util/Exception.h>
|
#include <torch/headeronly/util/Exception.h>
|
||||||
|
|
||||||
#include "libtorch_stable/torch_utils.h"
|
#include "libtorch_stable/torch_utils.h"
|
||||||
|
#include <atomic>
|
||||||
|
#include <cstdlib>
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// MERLIN_FULL_K: the CUDA half of the unmerged upstream PR #46639
|
||||||
|
// (`use_full_k`), which makes the Marlin MoE split-K reduction order
|
||||||
|
// independent of how many blocks the launch was split across.
|
||||||
|
//
|
||||||
|
// Our own toggle rather than upstream's VLLM_BATCH_INVARIANT, because that
|
||||||
|
// one aborts engine startup on GDN models (upstream #42960) and so can never
|
||||||
|
// reach this kernel on Flash-Next.
|
||||||
|
//
|
||||||
|
// Resolved ONCE per process, not per launch, so it is a launch constant in
|
||||||
|
// exactly the sense topk.cu's exact_medium_enabled() is: a cudagraph bakes in
|
||||||
|
// the value that was live at capture time and every replay agrees with it.
|
||||||
|
// Code default off; vllm/env_override.py sets 1 (the shipped value) when unset.
|
||||||
|
//
|
||||||
|
// This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw
|
||||||
|
// passthrough entry for validate_environ and the compile-factor hash only;
|
||||||
|
// nothing in Python may parse this name a second time.
|
||||||
|
//
|
||||||
|
// The initialiser also
|
||||||
|
// publishes the value it cached into g_cached_full_k (-1 = not initialised
|
||||||
|
// yet; nothing else writes it), and marlin_moe_full_k_state() below only
|
||||||
|
// LOADS it -- asking never fills the cache. Host code only.
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
std::atomic<int> g_cached_full_k{-1};
|
||||||
|
|
||||||
|
bool merlin_full_k() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_FULL_K");
|
||||||
|
const bool value =
|
||||||
|
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||||
|
g_cached_full_k.store(value ? 1 : 0, std::memory_order_release);
|
||||||
|
return value;
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
// MERLIN_FULL_K as this process cached it: -1 = not initialised yet (no Marlin
|
||||||
|
// MoE launch has happened in this process), 0 = off, 1 = on. Tensor-free, no
|
||||||
|
// CUDA call.
|
||||||
|
int64_t marlin_moe_full_k_state() {
|
||||||
|
return static_cast<int64_t>(g_cached_full_k.load(std::memory_order_acquire));
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
|
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
|
||||||
static_assert(std::is_same<scalar_t, half>::value || \
|
static_assert(std::is_same<scalar_t, half>::value || \
|
||||||
@@ -183,7 +230,8 @@ exec_config_t determine_exec_config(
|
|||||||
const vllm::ScalarType& c_type, const vllm::ScalarType& s_type, int prob_m,
|
const vllm::ScalarType& c_type, const vllm::ScalarType& s_type, int prob_m,
|
||||||
int prob_n, int prob_k, int num_experts, int top_k, int thread_m_blocks,
|
int prob_n, int prob_k, int num_experts, int top_k, int thread_m_blocks,
|
||||||
bool m_block_size_8, int num_bits, int group_size, bool has_zp,
|
bool m_block_size_8, int num_bits, int group_size, bool has_zp,
|
||||||
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms) {
|
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms,
|
||||||
|
bool use_full_k) {
|
||||||
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
|
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
|
||||||
thread_config_t* thread_configs = thread_m_blocks > 1
|
thread_config_t* thread_configs = thread_m_blocks > 1
|
||||||
? large_batch_thread_configs
|
? large_batch_thread_configs
|
||||||
@@ -228,7 +276,8 @@ exec_config_t determine_exec_config(
|
|||||||
else
|
else
|
||||||
allow_count = max(min(allow_count, 2), 1);
|
allow_count = max(min(allow_count, 2), 1);
|
||||||
|
|
||||||
if (prob_n / th_config.thread_n * prob_m * top_k * 4 < sms * allow_count) {
|
if (!use_full_k &&
|
||||||
|
prob_n / th_config.thread_n * prob_m * top_k * 4 < sms * allow_count) {
|
||||||
allow_count =
|
allow_count =
|
||||||
max(prob_n / th_config.thread_n * prob_m * top_k * 4 / sms, 1);
|
max(prob_n / th_config.thread_n * prob_m * top_k * 4 / sms, 1);
|
||||||
}
|
}
|
||||||
@@ -254,6 +303,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
|||||||
int group_size, int dev, cudaStream_t stream, int thread_k,
|
int group_size, int dev, cudaStream_t stream, int thread_k,
|
||||||
int thread_n, int sms, int blocks_per_sm, bool use_atomic_add,
|
int thread_n, int sms, int blocks_per_sm, bool use_atomic_add,
|
||||||
bool use_fp32_reduce, bool is_zp_float) {
|
bool use_fp32_reduce, bool is_zp_float) {
|
||||||
|
const bool use_full_k = merlin_full_k();
|
||||||
int thread_m_blocks = div_ceil(moe_block_size, 16);
|
int thread_m_blocks = div_ceil(moe_block_size, 16);
|
||||||
bool m_block_size_8 = moe_block_size == 8;
|
bool m_block_size_8 = moe_block_size == 8;
|
||||||
bool is_a_8bit = a_type.size_bits() == 8;
|
bool is_a_8bit = a_type.size_bits() == 8;
|
||||||
@@ -331,7 +381,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
|||||||
exec_cfg = determine_exec_config(
|
exec_cfg = determine_exec_config(
|
||||||
a_type, b_type, c_type, s_type, prob_m, prob_n, prob_k, num_experts,
|
a_type, b_type, c_type, s_type, prob_m, prob_n, prob_k, num_experts,
|
||||||
top_k, thread_m_blocks, m_block_size_8, num_bits, group_size, has_zp,
|
top_k, thread_m_blocks, m_block_size_8, num_bits, group_size, has_zp,
|
||||||
is_zp_float, is_a_8bit, stages, max_shared_mem, sms);
|
is_zp_float, is_a_8bit, stages, max_shared_mem, sms, use_full_k);
|
||||||
thread_tfg = exec_cfg.tb_cfg;
|
thread_tfg = exec_cfg.tb_cfg;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -383,7 +433,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
|||||||
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr,
|
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr,
|
||||||
sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr,
|
sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr,
|
||||||
topk_weights_ptr, top_k, mul_topk_weights, prob_m, prob_n, prob_k, locks,
|
topk_weights_ptr, top_k, mul_topk_weights, prob_m, prob_n, prob_k, locks,
|
||||||
has_bias, use_atomic_add, use_fp32_reduce);
|
has_bias, use_atomic_add, use_fp32_reduce, use_full_k);
|
||||||
// clang-format on
|
// clang-format on
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -704,3 +754,14 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
|||||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||||
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
|
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// v2.2 -- defined and registered in this TU so the query exists exactly when
|
||||||
|
// the static it reads does. Tensor-free, so CompositeExplicitAutograd (a CUDA
|
||||||
|
// registration could never be dispatched to without a tensor argument).
|
||||||
|
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||||
|
m.def("marlin_moe_full_k_state() -> int");
|
||||||
|
}
|
||||||
|
|
||||||
|
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
|
||||||
|
m.impl("marlin_moe_full_k_state", TORCH_BOX(&marlin_moe_full_k_state));
|
||||||
|
}
|
||||||
|
|||||||
@@ -481,6 +481,18 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
torch::stable::Tensor& workspace, int64_t k,
|
torch::stable::Tensor& workspace, int64_t k,
|
||||||
int64_t max_seq_len);
|
int64_t max_seq_len);
|
||||||
|
|
||||||
|
// Medium-path exact-restart counters; see topk.cu for the slot layout.
|
||||||
|
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||||
|
int64_t sync);
|
||||||
|
|
||||||
|
// The slot count of THIS build's counter array, so the Python
|
||||||
|
// reader never has to guess it from an environment variable.
|
||||||
|
int64_t persistent_topk_exact_counters_slots();
|
||||||
|
|
||||||
|
// The persistent_topk switches as this
|
||||||
|
// process cached them, packed; see topk.cu for the layout.
|
||||||
|
int64_t persistent_topk_toggle_state();
|
||||||
|
|
||||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||||
void cooperative_topk(const torch::stable::Tensor& logits,
|
void cooperative_topk(const torch::stable::Tensor& logits,
|
||||||
const torch::stable::Tensor& lengths,
|
const torch::stable::Tensor& lengths,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,11 @@
|
|||||||
|
|
||||||
#include <cuda_runtime.h>
|
#include <cuda_runtime.h>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <atomic>
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <cstring>
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
#include "ops.h"
|
#include "ops.h"
|
||||||
#include "torch_utils.h"
|
#include "torch_utils.h"
|
||||||
@@ -15,6 +20,388 @@
|
|||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
#ifndef USE_ROCM
|
#ifndef USE_ROCM
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MERLIN_TOPK_EXACT_MEDIUM switch and its counters.
|
||||||
|
//
|
||||||
|
// The switch is resolved ONCE per process, not per launch, so it is a launch
|
||||||
|
// constant: a cudagraph captures the value that was live at capture time and
|
||||||
|
// every replay agrees with it. Default ON in this release; set
|
||||||
|
// MERLIN_TOPK_EXACT_MEDIUM=0 for the A/B control arm, which restores v2.0.1
|
||||||
|
// behaviour exactly (the counters still record, so the control arm reports the
|
||||||
|
// overflow incidence it is the control for).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// MERLIN_TIE_DET: deterministic tie-break at the k-th cutoff in
|
||||||
|
// persistent_topk. Resolved once per process, exactly like
|
||||||
|
// exact_medium_enabled() below, so it is a launch constant that a cudagraph
|
||||||
|
// bakes in at capture.
|
||||||
|
//
|
||||||
|
// Our own toggle rather than upstream's VLLM_BATCH_INVARIANT, which aborts
|
||||||
|
// engine startup on GDN models (upstream #42960) and so can never reach this
|
||||||
|
// kernel on Flash-Next. Code default off; vllm/env_override.py sets 1 (the shipped value) when unset.
|
||||||
|
//
|
||||||
|
// This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw
|
||||||
|
// passthrough for validate_environ and the compile-factor hash only.
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// A READ-ONLY MIRROR of the
|
||||||
|
// switches as this process CACHED them, for persistent_topk_toggle_state().
|
||||||
|
//
|
||||||
|
// Each switch below is a function-local static filled once by its lambda. The
|
||||||
|
// lambda now also stores the value it cached into the matching atomic here
|
||||||
|
// (publish_cached), and nothing else ever writes these atomics, so a mirror
|
||||||
|
// either says "not initialised yet" (-1) or holds exactly the cached value.
|
||||||
|
// The query only LOADS them: it never calls a getter, so asking cannot fill a
|
||||||
|
// cache from an environment that differs from the one the kernel saw. The
|
||||||
|
// parse expressions are unchanged; this is host code only.
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
std::atomic<int> g_cached_tie_recent{-1};
|
||||||
|
std::atomic<int> g_cached_tie_det{-1};
|
||||||
|
std::atomic<int> g_cached_exact_medium{-1};
|
||||||
|
std::atomic<int> g_cached_det_large{-1};
|
||||||
|
std::atomic<int> g_cached_exact_decode{-1};
|
||||||
|
std::atomic<int> g_cached_sorted_emit{-1};
|
||||||
|
|
||||||
|
bool publish_cached(std::atomic<int>& mirror, bool value) {
|
||||||
|
mirror.store(value ? 1 : 0, std::memory_order_release);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
// MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above.
|
||||||
|
// Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode,
|
||||||
|
// most-recent-wins is +1.5 %. Both deterministic. Also the direction
|
||||||
|
// medium_exact_restart() already implements, so setting this alongside
|
||||||
|
// MERLIN_TIE_DET makes the medium path agree with itself on overflowing
|
||||||
|
// rows. Only read when MERLIN_TIE_DET is set; code default OFF; vllm/env_override.py sets 1 (the shipped value) when unset.
|
||||||
|
bool tie_recent_enabled() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TIE_RECENT");
|
||||||
|
return publish_cached(
|
||||||
|
g_cached_tie_recent,
|
||||||
|
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool det_ties_enabled() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TIE_DET");
|
||||||
|
return publish_cached(
|
||||||
|
g_cached_tie_det,
|
||||||
|
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool exact_medium_enabled() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
|
||||||
|
return publish_cached(
|
||||||
|
g_cached_exact_medium,
|
||||||
|
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MERLIN_TOPK_DET_LARGE switch.
|
||||||
|
//
|
||||||
|
// Same contract as the medium-path switch: resolved once per process, so it is a
|
||||||
|
// launch constant and a cudagraph captures one value. Default ON on this
|
||||||
|
// branch. MERLIN_TOPK_DET_LARGE=0 is the A/B control arm and runs the
|
||||||
|
// v2.0.1 `radix_topk` tail verbatim -- the deterministic emission sits behind
|
||||||
|
// one `if (det_large)` in front of it and touches nothing when off, including
|
||||||
|
// the large-path counters, which is what makes the control arm the real baseline.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
bool det_large_enabled() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TOPK_DET_LARGE");
|
||||||
|
return publish_cached(
|
||||||
|
g_cached_det_large,
|
||||||
|
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
// MERLIN_TOPK_EXACT_DECODE: repair an overflowing decode
|
||||||
|
// row with the same exact restart the medium path uses. Resolved once per
|
||||||
|
// process like the others, so it is a launch constant a cudagraph bakes in.
|
||||||
|
//
|
||||||
|
// Default OFF, unlike MERLIN_TOPK_EXACT_MEDIUM. The medium path's overflow
|
||||||
|
// incidence was measured (0.1-0.2 % of medium rows at 38-51K) before that
|
||||||
|
// switch defaulted on; the decode path's has never been measured at all,
|
||||||
|
// which is precisely what the [10] counter now exists to establish. Detection
|
||||||
|
// is unconditional and free; the repair waits on a number.
|
||||||
|
bool exact_decode_enabled() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE");
|
||||||
|
return publish_cached(
|
||||||
|
g_cached_exact_decode,
|
||||||
|
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MERLIN_TOPK_SORTED_EMIT: every persistent_topk
|
||||||
|
// row leaves the op in the canonical order the Python MERLIN_QSA_SORT imposes
|
||||||
|
// (ascending index, padding last as -1), so qsa_indexer.py can drop that sort.
|
||||||
|
// See sorted_emit_row() in persistent_topk.cuh for the contract; the selected
|
||||||
|
// SET is never touched. Resolved once per process like every switch here, so
|
||||||
|
// it is a launch constant a cudagraph bakes in. Code default OFF; vllm/env_override.py
|
||||||
|
// sets 1 (the shipped value) when unset.
|
||||||
|
//
|
||||||
|
// This std::getenv is the single source of truth. Python does not parse the
|
||||||
|
// variable to decide anything: qsa_indexer.py asks THIS cache, through
|
||||||
|
// persistent_topk_toggle_state() field 6, after its first launch -- so the
|
||||||
|
// Python sort is skipped only when the kernel that ran says it sorted.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
bool sorted_emit_enabled() {
|
||||||
|
static const bool enabled = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TOPK_SORTED_EMIT");
|
||||||
|
return publish_cached(
|
||||||
|
g_cached_sorted_emit,
|
||||||
|
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||||
|
}();
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MERLIN_TOPK_LARGE_FAULT_INJECT. TEST ONLY.
|
||||||
|
//
|
||||||
|
// -1 (the default, and the value of an unset or negative variable) is off.
|
||||||
|
// Any other value names the `cta_in_group` whose CTA-LOCAL fault predicate the
|
||||||
|
// large path forces, which is the only way to reach the fail-closed arm: no
|
||||||
|
// legal input can make a CTA's own pivot-equal count disagree with the one it
|
||||||
|
// published, so those branches are otherwise untestable (as with the
|
||||||
|
// medium-path restart's own fault path). Resolved once per process, so
|
||||||
|
// it is a launch constant and a cudagraph captures one value. Setting it also
|
||||||
|
// suppresses __trap(), because the point of injecting is to OBSERVE the
|
||||||
|
// trap-off row-failure policy (all -1 row, counter += 1) without a second
|
||||||
|
// build.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
int large_fault_inject_cta() {
|
||||||
|
static const int cta = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TOPK_LARGE_FAULT_INJECT");
|
||||||
|
if (raw == nullptr || raw[0] == '\0') return -1;
|
||||||
|
const int v = std::atoi(raw);
|
||||||
|
return (v < 0) ? -1 : v;
|
||||||
|
}();
|
||||||
|
return cta;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// MERLIN_TOPK_LARGE_FAULT_KIND. TEST ONLY, companion to
|
||||||
|
// the variable above; ignored unless MERLIN_TOPK_LARGE_FAULT_INJECT names a
|
||||||
|
// CTA.
|
||||||
|
//
|
||||||
|
// unset / "pre" (default) force the PRE-EMISSION predicate, caught at X3.
|
||||||
|
// "emit" force the EMISSION-BOUND slot check, caught at X4.
|
||||||
|
//
|
||||||
|
// The two are mutually exclusive by construction: a pre-emission fault makes
|
||||||
|
// `row_fault` nonzero, the emission block is skipped wholesale, and the X4
|
||||||
|
// code -- including `suffix_sum[6]`, the word whose concurrent non-atomic
|
||||||
|
// write this change is about -- is never reached. The shipped injector could
|
||||||
|
// therefore only ever exercise the arm that was already correct, which is why
|
||||||
|
// the X4 arm had never been exercised. Resolved once per process like
|
||||||
|
// every other switch here, so it is a launch constant a cudagraph bakes in.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
bool large_fault_inject_emit() {
|
||||||
|
static const bool emit = [] {
|
||||||
|
const char* raw = std::getenv("MERLIN_TOPK_LARGE_FAULT_KIND");
|
||||||
|
if (raw == nullptr || raw[0] == '\0') return false;
|
||||||
|
return std::strncmp(raw, "emit", 5) == 0;
|
||||||
|
}();
|
||||||
|
return emit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Say the effective tie policy out loud, once.
|
||||||
|
//
|
||||||
|
// "TIE_DET=1 TIE_RECENT=0" used to mean three different things on the three
|
||||||
|
// paths, and nothing printed the difference. One stderr line at the first
|
||||||
|
// launch of the process naming the policy the kernel will actually apply on
|
||||||
|
// EVERY path. It is a plain fprintf on the host: no CUDA call, so it is safe
|
||||||
|
// from a capturing thread, and the `static` initialiser makes it exactly once
|
||||||
|
// per process regardless of how many workers share the log.
|
||||||
|
//
|
||||||
|
// The line covers all FIVE paths. The large
|
||||||
|
// path is no longer a fixed rule: `radix_emit_deterministic` takes the same
|
||||||
|
// `effective_tie_recent()` value as the medium restart, so `large=` reports
|
||||||
|
// the real direction when MERLIN_TOPK_DET_LARGE is on and the inherited
|
||||||
|
// v2.0.1 atomic-arrival race when it is off. The flag printed here is the
|
||||||
|
// ENV value; a launch whose ctas_per_group overflows the staging row degrades
|
||||||
|
// to atomic-arrival for that launch and leaves counter [4] at 0, which is the
|
||||||
|
// separate warning below.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
void report_tie_policy_once() {
|
||||||
|
static const bool reported = [] {
|
||||||
|
const bool det = det_ties_enabled();
|
||||||
|
const bool recent =
|
||||||
|
vllm::persistent::effective_tie_recent(det, tie_recent_enabled());
|
||||||
|
const char* dir = recent ? "most-recent-wins" : "oldest-wins";
|
||||||
|
std::fprintf(
|
||||||
|
stderr,
|
||||||
|
"[merlin] persistent_topk tie policy: MERLIN_TIE_DET=%d "
|
||||||
|
"MERLIN_TIE_RECENT=%d MERLIN_TOPK_EXACT_MEDIUM=%d "
|
||||||
|
"MERLIN_TOPK_EXACT_DECODE=%d MERLIN_TOPK_DET_LARGE=%d "
|
||||||
|
"MERLIN_TOPK_SORTED_EMIT=%d -> "
|
||||||
|
"decode-buffered=%s decode-restart=%s medium-buffered=%s "
|
||||||
|
"medium-restart=%s large=%s emission=%s\n",
|
||||||
|
det ? 1 : 0, tie_recent_enabled() ? 1 : 0,
|
||||||
|
exact_medium_enabled() ? 1 : 0, exact_decode_enabled() ? 1 : 0,
|
||||||
|
det_large_enabled() ? 1 : 0, sorted_emit_enabled() ? 1 : 0,
|
||||||
|
det ? dir : "atomic-arrival",
|
||||||
|
exact_decode_enabled() ? dir : "n/a (detect-only; counter [10])",
|
||||||
|
det ? dir : "atomic-arrival",
|
||||||
|
exact_medium_enabled() ? dir : "n/a (exact restart disabled)",
|
||||||
|
det_large_enabled() ? dir : "atomic-arrival (non-deterministic)",
|
||||||
|
sorted_emit_enabled()
|
||||||
|
? "ascending-in-kernel (Python QSA sort skipped)"
|
||||||
|
: "arrival-order on buffered paths (Python QSA sort decides)");
|
||||||
|
return true;
|
||||||
|
}();
|
||||||
|
(void)reported;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if MERLIN_TOPK_DIAG_ANY
|
||||||
|
// DIAGNOSTIC BUILD ONLY (persistent_topk.cuh, "DIAGNOSTIC BUILDS ONLY"). Say
|
||||||
|
// so once per process, on stderr, before the first launch: the diagnostic-build tests
|
||||||
|
// key on this line to prove which build they ran against, and the smoke's
|
||||||
|
// startup-warning scan must reject any process that prints it.
|
||||||
|
void report_diag_build_once() {
|
||||||
|
static const bool reported = [] {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"[merlin] persistent_topk DIAGNOSTIC BUILD: "
|
||||||
|
"MERLIN_TOPK_DIAG_DELAY=%lld MERLIN_TOPK_DIAG_PREREPAIR=%d "
|
||||||
|
"MERLIN_TOPK_DIAG_FORCE_FAULT=%d "
|
||||||
|
"MERLIN_TOPK_DIAG_FORCE_FAULT_CTA=%d -- NOT FOR SERVING\n",
|
||||||
|
static_cast<long long>(MERLIN_TOPK_DIAG_DELAY),
|
||||||
|
static_cast<int>(MERLIN_TOPK_DIAG_PREREPAIR),
|
||||||
|
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT),
|
||||||
|
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA));
|
||||||
|
return true;
|
||||||
|
}();
|
||||||
|
(void)reported;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Process-lifetime device counters, one block per device. The kernel only
|
||||||
|
// ever atomicAdds into them, so the launch path needs no host sync and the
|
||||||
|
// values accumulate across cudagraph replays.
|
||||||
|
//
|
||||||
|
// Allocation, however, is NOT capture-safe: cudaMalloc during stream capture
|
||||||
|
// is at best not captured and at worst invalidates the capture. So:
|
||||||
|
// - the allocation only ever happens on a call that is explicitly allowed to
|
||||||
|
// allocate, which is a non-capturing launch or the reader op;
|
||||||
|
// - a call made *during* capture never allocates and never latches a result,
|
||||||
|
// so a capture that happens to be first does not permanently poison the
|
||||||
|
// instrument;
|
||||||
|
// - the launch path fails loudly if it is capturing with no counters, rather
|
||||||
|
// than baking a graph that runs the patched kernel with no instrument at
|
||||||
|
// all. A silently uninstrumented graph is the failure this is here to
|
||||||
|
// prevent, so the check is deliberately not a warning.
|
||||||
|
// Read (and, on the first call, allocate) with
|
||||||
|
// torch.ops._C.persistent_topk_exact_counters, which is the explicit
|
||||||
|
// pre-capture init path.
|
||||||
|
constexpr int kMaxCounterDevices = 16;
|
||||||
|
|
||||||
|
enum CounterState : int {
|
||||||
|
kCountersUninit = 0,
|
||||||
|
kCountersReady = 1,
|
||||||
|
kCountersFailed = 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Atomics, not plain globals: the fast path reads these without the mutex, and
|
||||||
|
// a second host thread may be publishing them from the slow path. Release on
|
||||||
|
// publish, acquire on read, and the pointer is always published BEFORE the
|
||||||
|
// state that advertises it.
|
||||||
|
std::atomic<unsigned long long*> g_exact_counters[kMaxCounterDevices] = {};
|
||||||
|
std::atomic<int> g_exact_counters_state[kMaxCounterDevices] = {};
|
||||||
|
std::mutex g_exact_counters_mutex;
|
||||||
|
|
||||||
|
// Lookup only. Never allocates, never blocks, safe from any context.
|
||||||
|
unsigned long long* get_exact_counters(int device_index) {
|
||||||
|
if (device_index < 0 || device_index >= kMaxCounterDevices) return nullptr;
|
||||||
|
if (g_exact_counters_state[device_index].load(std::memory_order_acquire) !=
|
||||||
|
kCountersReady) {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
return g_exact_counters[device_index].load(std::memory_order_acquire);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allocate, zero, and publish. MUST NOT be called on a capturing stream, and
|
||||||
|
// should be called in a quiescent lifecycle phase: it allocates and blocks,
|
||||||
|
// and neither is safe while any thread in the process is capturing a graph.
|
||||||
|
// `stream_is_capturing` only proves the CALLING stream is idle of capture, so
|
||||||
|
// the intended entry point is the reader op called explicitly before capture;
|
||||||
|
// the non-capturing launch path calls this as a fallback.
|
||||||
|
unsigned long long* ensure_exact_counters(int device_index, cudaStream_t stream) {
|
||||||
|
if (device_index < 0 || device_index >= kMaxCounterDevices) return nullptr;
|
||||||
|
|
||||||
|
const int state =
|
||||||
|
g_exact_counters_state[device_index].load(std::memory_order_acquire);
|
||||||
|
if (state == kCountersReady) {
|
||||||
|
return g_exact_counters[device_index].load(std::memory_order_acquire);
|
||||||
|
}
|
||||||
|
// A previous attempt failed for real (OOM); do not retry on every launch.
|
||||||
|
// Only a call that was actually allowed to allocate can reach this state, so
|
||||||
|
// a capture-time call never latches it.
|
||||||
|
if (state == kCountersFailed) return nullptr;
|
||||||
|
|
||||||
|
std::lock_guard<std::mutex> lock(g_exact_counters_mutex);
|
||||||
|
// Re-check under the lock; another thread may have published meanwhile.
|
||||||
|
const int locked_state =
|
||||||
|
g_exact_counters_state[device_index].load(std::memory_order_relaxed);
|
||||||
|
if (locked_state != kCountersUninit) {
|
||||||
|
return (locked_state == kCountersReady)
|
||||||
|
? g_exact_counters[device_index].load(std::memory_order_acquire)
|
||||||
|
: nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
constexpr size_t bytes =
|
||||||
|
vllm::persistent::kNumTopKExactCounters * sizeof(unsigned long long);
|
||||||
|
unsigned long long* ptr = nullptr;
|
||||||
|
if (cudaMalloc(&ptr, bytes) == cudaSuccess && ptr != nullptr) {
|
||||||
|
// Host acquire/release orders the HOST state word; it says
|
||||||
|
// nothing about the device. cudaMemset on device memory is asynchronous
|
||||||
|
// with respect to the host, so publishing Ready straight after it would
|
||||||
|
// let a first launch -- on this or any other stream -- increment counters
|
||||||
|
// that had not been zeroed yet, or read uninitialised bytes. Zero on a
|
||||||
|
// known stream and WAIT for it, so initialisation is complete before the
|
||||||
|
// pointer becomes visible to any consumer.
|
||||||
|
if (cudaMemsetAsync(ptr, 0, bytes, stream) != cudaSuccess ||
|
||||||
|
cudaStreamSynchronize(stream) != cudaSuccess) {
|
||||||
|
cudaFree(ptr);
|
||||||
|
ptr = nullptr;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ptr = nullptr;
|
||||||
|
}
|
||||||
|
// A failed allocation leaves the error state set; clear it so the launch's
|
||||||
|
// own cudaGetLastError() gate does not report this as a kernel failure.
|
||||||
|
if (ptr == nullptr) cudaGetLastError();
|
||||||
|
|
||||||
|
g_exact_counters[device_index].store(ptr, std::memory_order_release);
|
||||||
|
g_exact_counters_state[device_index].store(
|
||||||
|
ptr != nullptr ? kCountersReady : kCountersFailed,
|
||||||
|
std::memory_order_release);
|
||||||
|
return ptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
// True when the stream is mid-capture, in which case nothing on this path may
|
||||||
|
// allocate. A query failure is treated as "capturing": the conservative
|
||||||
|
// direction is to refuse to allocate, not to risk invalidating a capture.
|
||||||
|
bool stream_is_capturing(cudaStream_t stream) {
|
||||||
|
cudaStreamCaptureStatus status = cudaStreamCaptureStatusNone;
|
||||||
|
if (cudaStreamIsCapturing(stream, &status) != cudaSuccess) {
|
||||||
|
cudaGetLastError(); // do not leak this into the launch's error gate
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return status != cudaStreamCaptureStatusNone;
|
||||||
|
}
|
||||||
|
|
||||||
template <int TopK>
|
template <int TopK>
|
||||||
void launch_persistent_topk(const torch::stable::Tensor& logits,
|
void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||||
const torch::stable::Tensor& lengths,
|
const torch::stable::Tensor& lengths,
|
||||||
@@ -37,6 +424,19 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
|
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MERLIN_TOPK_SORTED_EMIT on the two fallbacks that bypass
|
||||||
|
// persistent_topk_kernel: re-emit every row ascending on the same stream.
|
||||||
|
// A plain launch with a fixed configuration, so it is capture-safe.
|
||||||
|
const auto sorted_emit_fallback_rows = [&]() {
|
||||||
|
if (!sorted_emit_enabled()) return;
|
||||||
|
report_tie_policy_once();
|
||||||
|
P::sorted_emit_rows_kernel<TopK>
|
||||||
|
<<<static_cast<unsigned>(num_rows), P::kThreadsPerBlock,
|
||||||
|
P::sorted_emit_smem_bytes<TopK>(), stream>>>(
|
||||||
|
output.mutable_data_ptr<int32_t>(),
|
||||||
|
static_cast<uint32_t>(num_rows));
|
||||||
|
};
|
||||||
|
|
||||||
// Allow static fallback storage in addition to the 128 KiB dynamic buffer.
|
// Allow static fallback storage in addition to the 128 KiB dynamic buffer.
|
||||||
if (num_rows > 64 &&
|
if (num_rows > 64 &&
|
||||||
max_seq_len >= vllm::sampled_topk::kMinSampledLength<TopK> &&
|
max_seq_len >= vllm::sampled_topk::kMinSampledLength<TopK> &&
|
||||||
@@ -52,6 +452,7 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
logits.const_data_ptr<float>(), lengths.const_data_ptr<int32_t>(),
|
logits.const_data_ptr<float>(), lengths.const_data_ptr<int32_t>(),
|
||||||
output.mutable_data_ptr<int32_t>(), stride,
|
output.mutable_data_ptr<int32_t>(), stride,
|
||||||
static_cast<int>(std::min(max_seq_len, logits.size(1))));
|
static_cast<int>(std::min(max_seq_len, logits.size(1))));
|
||||||
|
sorted_emit_fallback_rows();
|
||||||
} else if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
|
} else if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
|
||||||
cudaError_t status =
|
cudaError_t status =
|
||||||
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
|
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
|
||||||
@@ -60,6 +461,7 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
static_cast<uint32_t>(TopK), static_cast<uint32_t>(stride), stream);
|
static_cast<uint32_t>(TopK), static_cast<uint32_t>(stride), stream);
|
||||||
STD_TORCH_CHECK(status == cudaSuccess,
|
STD_TORCH_CHECK(status == cudaSuccess,
|
||||||
"FilteredTopK failed: ", cudaGetErrorString(status));
|
"FilteredTopK failed: ", cudaGetErrorString(status));
|
||||||
|
sorted_emit_fallback_rows();
|
||||||
} else {
|
} else {
|
||||||
STD_TORCH_CHECK(workspace.is_cuda(), "workspace must be CUDA tensor");
|
STD_TORCH_CHECK(workspace.is_cuda(), "workspace must be CUDA tensor");
|
||||||
STD_TORCH_CHECK(
|
STD_TORCH_CHECK(
|
||||||
@@ -156,6 +558,10 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
const int64_t next_n = lengths.dim() == 2 ? lengths.size(1) : 1;
|
const int64_t next_n = lengths.dim() == 2 ? lengths.size(1) : 1;
|
||||||
top_k_per_row_decode(logits, next_n, lengths, output, num_rows,
|
top_k_per_row_decode(logits, next_n, lengths, output, num_rows,
|
||||||
logits.stride(0), logits.stride(1), TopK);
|
logits.stride(0), logits.stride(1), TopK);
|
||||||
|
sorted_emit_fallback_rows();
|
||||||
|
const cudaError_t fb_err = cudaGetLastError();
|
||||||
|
STD_TORCH_CHECK(fb_err == cudaSuccess, "sorted emission failed: ",
|
||||||
|
cudaGetErrorString(fb_err));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
cudaError_t status =
|
cudaError_t status =
|
||||||
@@ -167,6 +573,10 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
static_cast<uint32_t>(stride), stream);
|
static_cast<uint32_t>(stride), stream);
|
||||||
STD_TORCH_CHECK(status == cudaSuccess, "FilteredTopK fallback failed: ",
|
STD_TORCH_CHECK(status == cudaSuccess, "FilteredTopK fallback failed: ",
|
||||||
cudaGetErrorString(status));
|
cudaGetErrorString(status));
|
||||||
|
sorted_emit_fallback_rows();
|
||||||
|
const cudaError_t fb_err = cudaGetLastError();
|
||||||
|
STD_TORCH_CHECK(fb_err == cudaSuccess, "sorted emission failed: ",
|
||||||
|
cudaGetErrorString(fb_err));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,6 +628,81 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
workspace.mutable_data_ptr<uint8_t>());
|
workspace.mutable_data_ptr<uint8_t>());
|
||||||
params.ctas_per_group = ctas_per_group;
|
params.ctas_per_group = ctas_per_group;
|
||||||
params.max_seq_len = static_cast<uint32_t>(max_seq_len);
|
params.max_seq_len = static_cast<uint32_t>(max_seq_len);
|
||||||
|
params.exact_medium = exact_medium_enabled();
|
||||||
|
// The per-CTA selected / pivot-equal counts are staged in ONE
|
||||||
|
// 256-slot RadixRowState histogram row (the slot round 3 has just finished
|
||||||
|
// reading). Two more per-CTA regions were added for the
|
||||||
|
// group-uniform fault verdicts, so the row now holds 4 * ctas_per_group
|
||||||
|
// uint32s. That is still the whole reason the workspace contract is
|
||||||
|
// unchanged, and it is also the only
|
||||||
|
// limit: above 64 CTAs per group the deterministic path cannot be
|
||||||
|
// expressed there, so the launch degrades to the v2.0.1 tail rather than
|
||||||
|
// failing. Unreachable at any shape this kernel serves (ctas_per_group is
|
||||||
|
// 2-3 across our 135K-262K band and <= 16 at the smallest chunk size), but
|
||||||
|
// a silent loss of determinism is exactly what this branch exists to stop,
|
||||||
|
// so say it once.
|
||||||
|
const bool det_large_fits =
|
||||||
|
4u * ctas_per_group <= static_cast<uint32_t>(P::RADIX);
|
||||||
|
params.det_large = det_large_enabled() && det_large_fits;
|
||||||
|
params.fault_inject_cta = large_fault_inject_cta();
|
||||||
|
params.fault_inject_emit = large_fault_inject_emit();
|
||||||
|
if (det_large_enabled() && !det_large_fits) {
|
||||||
|
static std::atomic<bool> det_large_warned{false};
|
||||||
|
if (!det_large_warned.exchange(true)) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"[vllm] MERLIN_TOPK_DET_LARGE is on but ctas_per_group=%u "
|
||||||
|
"needs %u of %d staging slots; the large path falls back "
|
||||||
|
"to nondeterministic placement at this shape (counter [4] "
|
||||||
|
"stays 0).\n",
|
||||||
|
ctas_per_group, 4u * ctas_per_group, P::RADIX);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (params.fault_inject_cta >= 0) {
|
||||||
|
static std::atomic<bool> inject_warned{false};
|
||||||
|
if (!inject_warned.exchange(true)) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"[vllm] MERLIN_TOPK_LARGE_FAULT_INJECT=%d kind=%s is "
|
||||||
|
"set: the large path FORCES a %s fault in that CTA of "
|
||||||
|
"every deterministic large row and returns an all -1 row. "
|
||||||
|
"This is a test mode and must not be set in "
|
||||||
|
"production.\n",
|
||||||
|
params.fault_inject_cta,
|
||||||
|
params.fault_inject_emit ? "emit" : "pre",
|
||||||
|
params.fault_inject_emit ? "emission-bound (X4)"
|
||||||
|
: "pre-emission (X3)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
params.det_ties = det_ties_enabled();
|
||||||
|
params.tie_recent = tie_recent_enabled();
|
||||||
|
params.exact_decode = exact_decode_enabled();
|
||||||
|
params.sorted_emit = sorted_emit_enabled();
|
||||||
|
report_tie_policy_once();
|
||||||
|
#if MERLIN_TOPK_DIAG_ANY
|
||||||
|
report_diag_build_once();
|
||||||
|
#endif
|
||||||
|
// Allocate only off the capture path; in the ordinary engine lifecycle the
|
||||||
|
// eager warmup lands here first, long before capture. The explicit,
|
||||||
|
// quiescent-phase entry point is the reader op.
|
||||||
|
const int device_index = static_cast<int>(logits.get_device_index());
|
||||||
|
params.exact_counters = stream_is_capturing(stream)
|
||||||
|
? get_exact_counters(device_index)
|
||||||
|
: ensure_exact_counters(device_index, stream);
|
||||||
|
// Note: this check is UNCONDITIONAL -- eager as well as
|
||||||
|
// capturing, and in both arms. Previously only a capturing launch was
|
||||||
|
// rejected, so an eager launch ran with nullptr and the reader then
|
||||||
|
// reported [0,0,0,0,1]: an unavailable instrument encoded as healthy zero
|
||||||
|
// counts, which is the exact failure the counters exist to prevent.
|
||||||
|
// MERLIN_TOPK_EXACT_MEDIUM=0 is NOT an escape: the control arm's whole job
|
||||||
|
// is to report the overflow incidence it is the control for, so it needs
|
||||||
|
// the counters just as much.
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
params.exact_counters != nullptr,
|
||||||
|
"persistent_topk: the medium-path exact-restart counters are unavailable on "
|
||||||
|
"device ", device_index,
|
||||||
|
" (not initialised before cudagraph capture, or the one-time "
|
||||||
|
"allocation failed). Call "
|
||||||
|
"torch.ops._C.persistent_topk_exact_counters(out, 0, 1) once per "
|
||||||
|
"device in a quiescent phase before capture.");
|
||||||
|
|
||||||
#define LAUNCH_PERSISTENT(TOPK_VAL, VS) \
|
#define LAUNCH_PERSISTENT(TOPK_VAL, VS) \
|
||||||
do { \
|
do { \
|
||||||
@@ -294,3 +779,249 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
|||||||
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read (and, on first call, allocate) the medium-path exact-restart
|
||||||
|
// counters. This is also the explicit pre-capture init path: call it once per
|
||||||
|
// device before any cudagraph capture.
|
||||||
|
//
|
||||||
|
// PROTOCOL VERSION 2 -- BREAKING. `out` is an **int64**
|
||||||
|
// CUDA tensor with EXACTLY kNumTopKExactCounters (11) elements; int32, and any
|
||||||
|
// other length, are REJECTED rather than silently accepted. The device counters are 64-bit unsigned
|
||||||
|
// and are copied D2D without conversion, so the reader's element width must
|
||||||
|
// match the device's.
|
||||||
|
//
|
||||||
|
// Why the break rather than a reinterpreting reader: the counters were u32
|
||||||
|
// copied into a SIGNED int32 tensor. At 65,536 columns with k=512 an
|
||||||
|
// all-equal row drops 65,024 ties, so [6] goes negative in Python after 33,027
|
||||||
|
// rows and wraps modulo 2^32 after 66,053 -- and a subtraction across the
|
||||||
|
// signed boundary gives a wrong delta, which is worse than a missing number
|
||||||
|
// because it looks like a measurement. Keeping u32 and reinterpreting in the
|
||||||
|
// reader would need an enforced per-window increment bound plus quiescent
|
||||||
|
// resets, i.e. a contract every future caller has to remember. 64-bit device
|
||||||
|
// counters make the failure unreachable instead: `atomicAdd(unsigned long
|
||||||
|
// long*)` is native on sm_86 (it is a single ATOM.E.ADD.S64, not a CAS loop),
|
||||||
|
// so the instrument costs the same, and a 64-bit slot cannot be driven to the
|
||||||
|
// signed boundary by any run this kernel will ever see.
|
||||||
|
//
|
||||||
|
// The layout is the UNION of the tie-determinism and large-path slot
|
||||||
|
// maps -- the large path's ten slots plus the decode restart's [10] decode_overflow, eleven in all,
|
||||||
|
// with nothing renumbered. The two switch flags are slots INSIDE the array at
|
||||||
|
// [8] and [9], not two elements appended after it, so a probe sized to the
|
||||||
|
// old ten- or five-element layout is rejected by the length check below rather
|
||||||
|
// than being filled with the wrong meanings. Ask the build for its own answer
|
||||||
|
// with `torch.ops._C.persistent_topk_exact_counters_slots()`.
|
||||||
|
//
|
||||||
|
// Everything that reads these counters must move together: the Python reader
|
||||||
|
// `vllm/models/qwen4_exp/nvidia/ops/topk_counters.py` allocates int64 and
|
||||||
|
// eleven slots, and the medium- and large-path tests do too. An int32 caller gets a loud
|
||||||
|
// error rather than half a number. The OPERATOR SCHEMA is unchanged (dtype
|
||||||
|
// and length are not part of it), so a stale caller fails at the check, not at
|
||||||
|
// the binding.
|
||||||
|
//
|
||||||
|
// Slots:
|
||||||
|
// [0] medium-path rows entered (denominator)
|
||||||
|
// [1] rows where a 4,096-slot guard would drop a candidate
|
||||||
|
// [2] rows that took the exact restart (0 in the control arm)
|
||||||
|
// [3] exact restarts that could not complete (must stay 0)
|
||||||
|
// [4] large-path rows emitted deterministically (0 in the control arm)
|
||||||
|
// [5] large-path rows whose pivot-equal group was cut at the k-th value
|
||||||
|
// [6] large-path pivot-equal elements discarded by those cuts
|
||||||
|
// [7] large-path invariant violations (must stay 0)
|
||||||
|
// [8] 1 if MERLIN_TOPK_EXACT_MEDIUM is on, else 0 (host-written)
|
||||||
|
// [9] 1 if MERLIN_TOPK_DET_LARGE is on, else 0 (host-written)
|
||||||
|
// [10] decode rows whose buffered candidates exceeded DBUF (3,708)
|
||||||
|
// before deterministic ranking -- MUST be read before any decode-path
|
||||||
|
// determinism claim
|
||||||
|
//
|
||||||
|
// COHERENCY -- stated precisely, because the earlier wording overclaimed.
|
||||||
|
// With `sync == 0` the snapshot is ordered ONLY against work already enqueued
|
||||||
|
// on the current stream; a kernel running on another stream can land
|
||||||
|
// increments it misses. With `sync != 0` the call synchronizes the device
|
||||||
|
// before the copy AND again after the copy/reset have been issued, so on
|
||||||
|
// return every increment submitted before the call is included and any reset
|
||||||
|
// has COMPLETED. That is still conditional on the CALLER HOLDING EXCLUSIVE
|
||||||
|
// SUBMISSION OWNERSHIP: nothing stops another thread enqueueing work after the
|
||||||
|
// leading synchronize, and no claim of unconditional global coherence is made
|
||||||
|
// here. A stronger contract would need an event every later producer waits
|
||||||
|
// on; this op deliberately implements the simpler half and says so.
|
||||||
|
//
|
||||||
|
// `reset != 0` REQUIRES `sync != 0`. Reset between measurements, never during
|
||||||
|
// one.
|
||||||
|
//
|
||||||
|
// Must not be called during cudagraph capture (it allocates, synchronizes and
|
||||||
|
// issues a host-to-device copy). It IS the explicit, quiescent-phase
|
||||||
|
// initialization path: call it once per device before any capture.
|
||||||
|
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||||
|
int64_t sync) {
|
||||||
|
#ifndef USE_ROCM
|
||||||
|
STD_TORCH_CHECK(out.is_cuda(), "out must be a CUDA tensor");
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
out.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||||
|
"out must be int64: the exact-restart counters are 64-bit as of the "
|
||||||
|
"large-path counter protocol change, and an int32 buffer would be filled "
|
||||||
|
"from half of each counter. Allocate "
|
||||||
|
"torch.zeros(persistent_topk_exact_counters_slots(), dtype=torch.int64)."
|
||||||
|
" (int32 was protocol version 1 and is deliberately rejected.)");
|
||||||
|
// Element count alone is not enough. A CUDA scalar expanded
|
||||||
|
// to ten elements (stride 0) satisfies numel but owns one element's storage,
|
||||||
|
// and the raw int64 writes below would run past it; an ordinary strided view
|
||||||
|
// would receive the values in the wrong slots. Require a contiguous 1-D
|
||||||
|
// buffer before taking the pointer.
|
||||||
|
STD_TORCH_CHECK(out.dim() == 1, "out must be 1-D");
|
||||||
|
STD_TORCH_CHECK(out.is_contiguous(),
|
||||||
|
"out must be contiguous (expanded or strided views would be "
|
||||||
|
"written out of bounds or into the wrong elements)");
|
||||||
|
// EXACTLY the build's slot count, not "at least". `>=` accepted an
|
||||||
|
// 11-element buffer on the ten-slot large-path build and quietly left the extra
|
||||||
|
// element reading 0 -- a caller compiled against one layout got a plausible
|
||||||
|
// answer from a build with another, which is the failure mode the
|
||||||
|
// self-describing `..._slots` op exists to prevent. Ask the build, size the
|
||||||
|
// buffer to what it says, and a mismatch is an error rather than a wrong
|
||||||
|
// number.
|
||||||
|
STD_TORCH_CHECK(out.numel() == vllm::persistent::kNumTopKExactCounters,
|
||||||
|
"out must hold exactly ",
|
||||||
|
vllm::persistent::kNumTopKExactCounters,
|
||||||
|
" elements (this build's persistent_topk_exact_counters_"
|
||||||
|
"slots(); the switch flags are slots inside the array, not "
|
||||||
|
"appended after it), got ", out.numel());
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
reset == 0 || sync != 0,
|
||||||
|
"persistent_topk_exact_counters: reset requires sync != 0, otherwise the "
|
||||||
|
"zeroing races in-flight kernel atomics and silently drops events");
|
||||||
|
|
||||||
|
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||||
|
out.get_device_index());
|
||||||
|
const cudaStream_t stream = get_current_cuda_stream();
|
||||||
|
STD_TORCH_CHECK(!stream_is_capturing(stream),
|
||||||
|
"persistent_topk_exact_counters must not be called during "
|
||||||
|
"cudagraph capture");
|
||||||
|
constexpr size_t bytes =
|
||||||
|
vllm::persistent::kNumTopKExactCounters * sizeof(unsigned long long);
|
||||||
|
|
||||||
|
int64_t* dst = out.mutable_data_ptr<int64_t>();
|
||||||
|
unsigned long long* src =
|
||||||
|
ensure_exact_counters(static_cast<int>(out.get_device_index()), stream);
|
||||||
|
|
||||||
|
// An unavailable buffer used to be reported as four zeros
|
||||||
|
// plus "enabled", i.e. an instrument failure indistinguishable from "no
|
||||||
|
// medium work and no faults". Fail instead.
|
||||||
|
STD_TORCH_CHECK(src != nullptr,
|
||||||
|
"persistent_topk_exact_counters: the counter buffer could "
|
||||||
|
"not be allocated on this device; the instrument is "
|
||||||
|
"unavailable and its absence must not read as zero counts");
|
||||||
|
|
||||||
|
if (sync != 0) {
|
||||||
|
// Everything that could still be adding to the block must have retired
|
||||||
|
// before the snapshot (and before a reset) for the numbers to mean
|
||||||
|
// anything across streams.
|
||||||
|
STD_TORCH_CHECK(cudaDeviceSynchronize() == cudaSuccess,
|
||||||
|
"exact counter synchronize failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToDevice, stream) ==
|
||||||
|
cudaSuccess,
|
||||||
|
"exact counter read failed");
|
||||||
|
if (reset != 0) {
|
||||||
|
STD_TORCH_CHECK(cudaMemsetAsync(src, 0, bytes, stream) == cudaSuccess,
|
||||||
|
"exact counter reset failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Static storage: the source of an async H2D copy must outlive the copy.
|
||||||
|
// Written AFTER the device copy, which has already zeroed both slots: they
|
||||||
|
// are slots [8] and [9] INSIDE the array and the device never touches them.
|
||||||
|
// The large flag reports the ENV value, not the per-launch effective value:
|
||||||
|
// a launch whose ctas_per_group overflowed the staging row still reports 1
|
||||||
|
// here and leaves counter [4] at 0, which is how that degradation is
|
||||||
|
// diagnosed. kSlotSwitchLarge == kSlotSwitchMedium + 1, so one copy does
|
||||||
|
// both.
|
||||||
|
static_assert(vllm::persistent::kSlotSwitchLarge ==
|
||||||
|
vllm::persistent::kSlotSwitchMedium + 1,
|
||||||
|
"the two host-written flags must stay adjacent for this copy");
|
||||||
|
static const int64_t enabled[2] = {exact_medium_enabled() ? 1 : 0,
|
||||||
|
det_large_enabled() ? 1 : 0};
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
cudaMemcpyAsync(dst + vllm::persistent::kSlotSwitchMedium, enabled,
|
||||||
|
2 * sizeof(int64_t), cudaMemcpyHostToDevice,
|
||||||
|
stream) == cudaSuccess,
|
||||||
|
"exact counter flag write failed");
|
||||||
|
|
||||||
|
if (sync != 0) {
|
||||||
|
// The leading synchronize alone left the copy and the reset
|
||||||
|
// merely ENQUEUED, so the caller could return and immediately race the
|
||||||
|
// outstanding reset from another stream. Complete the handoff before
|
||||||
|
// returning.
|
||||||
|
STD_TORCH_CHECK(cudaDeviceSynchronize() == cudaSuccess,
|
||||||
|
"exact counter completion synchronize failed");
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// How many int32 slots THIS build's counter array carries.
|
||||||
|
//
|
||||||
|
// The Python reader used to guess from MERLIN_TOPK_COUNTER_SLOTS (default 5)
|
||||||
|
// and name the slots by that length. The decode restart makes the array 11 and the large path makes
|
||||||
|
// it 10, so a guess is now wrong on at least one branch, and a reader that
|
||||||
|
// allocates the wrong length is rejected by the length check above -- the
|
||||||
|
// instrument goes dark exactly when it starts mattering. Ask the build
|
||||||
|
// instead. No CUDA call, no allocation, safe from anywhere.
|
||||||
|
int64_t persistent_topk_exact_counters_slots() {
|
||||||
|
#ifndef USE_ROCM
|
||||||
|
return static_cast<int64_t>(vllm::persistent::kNumTopKExactCounters);
|
||||||
|
#else
|
||||||
|
return 0;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// The persistent_topk
|
||||||
|
// switches AS THIS PROCESS CACHED THEM -- the values every launch of this
|
||||||
|
// process uses -- read from the mirrors above. Tensor-free, no CUDA call, no
|
||||||
|
// allocation, never initialises a cache (see publish_cached), so it is safe
|
||||||
|
// from anywhere and asking changes nothing.
|
||||||
|
//
|
||||||
|
// Packed into one int64 so the op needs no tensor and no dispatch key.
|
||||||
|
// 2 bits per field, field i at bits [2i, 2i+1]: 0 = off, 1 = on,
|
||||||
|
// 2 = the static has not been initialised in this process yet.
|
||||||
|
// field 0 MERLIN_TIE_DET (det_ties_enabled)
|
||||||
|
// field 1 MERLIN_TIE_RECENT as parsed (tie_recent_enabled)
|
||||||
|
// field 2 effective direction, 1 = most-recent (effective_tie_recent)
|
||||||
|
// field 3 MERLIN_TOPK_EXACT_MEDIUM (exact_medium_enabled)
|
||||||
|
// field 4 MERLIN_TOPK_EXACT_DECODE (exact_decode_enabled)
|
||||||
|
// field 5 MERLIN_TOPK_DET_LARGE (det_large_enabled)
|
||||||
|
// field 6 MERLIN_TOPK_SORTED_EMIT (sorted_emit_enabled)
|
||||||
|
// bits [48, 55] = field count (7), bits [56, 62] = layout version (2). A
|
||||||
|
// reader must check both and refuse any other layout rather than guess.
|
||||||
|
// Layout 2 = layout 1 plus field 6; fields 0-5 are unchanged, so
|
||||||
|
// a reader that knows both can decode either. Field 6 is what
|
||||||
|
// qsa_indexer.py reads to decide whether its Python sort may be skipped.
|
||||||
|
int64_t persistent_topk_toggle_state() {
|
||||||
|
#ifndef USE_ROCM
|
||||||
|
const auto field = [](int cached) -> int64_t {
|
||||||
|
return cached < 0 ? 2 : (cached != 0 ? 1 : 0);
|
||||||
|
};
|
||||||
|
const int det = g_cached_tie_det.load(std::memory_order_acquire);
|
||||||
|
const int recent = g_cached_tie_recent.load(std::memory_order_acquire);
|
||||||
|
// The kernel's own rule, from the kernel's own header, on the cached values.
|
||||||
|
const int effective =
|
||||||
|
(det < 0 || recent < 0)
|
||||||
|
? -1
|
||||||
|
: (vllm::persistent::effective_tie_recent(det != 0, recent != 0)
|
||||||
|
? 1
|
||||||
|
: 0);
|
||||||
|
int64_t state = 0;
|
||||||
|
state |= field(det) << 0;
|
||||||
|
state |= field(recent) << 2;
|
||||||
|
state |= field(effective) << 4;
|
||||||
|
state |= field(g_cached_exact_medium.load(std::memory_order_acquire)) << 6;
|
||||||
|
state |= field(g_cached_exact_decode.load(std::memory_order_acquire)) << 8;
|
||||||
|
state |= field(g_cached_det_large.load(std::memory_order_acquire)) << 10;
|
||||||
|
state |= field(g_cached_sorted_emit.load(std::memory_order_acquire)) << 12;
|
||||||
|
state |= int64_t{7} << 48;
|
||||||
|
state |= int64_t{2} << 56;
|
||||||
|
return state;
|
||||||
|
#else
|
||||||
|
return 0; // layout version 0: no persistent_topk on ROCm
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|||||||
@@ -586,6 +586,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
|||||||
"persistent_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
"persistent_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||||
"Tensor workspace, int k, int max_seq_len) -> ()");
|
"Tensor workspace, int k, int max_seq_len) -> ()");
|
||||||
|
|
||||||
|
// Read (and optionally reset) the medium-path exact-restart counters.
|
||||||
|
ops.def(
|
||||||
|
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
|
||||||
|
ops.def("persistent_topk_exact_counters_slots() -> int");
|
||||||
|
// v2.2 -- the switches as this process cached them (topk.cu for the layout).
|
||||||
|
ops.def("persistent_topk_toggle_state() -> int");
|
||||||
|
|
||||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||||
ops.def(
|
ops.def(
|
||||||
"cooperative_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
"cooperative_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||||
@@ -858,6 +865,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
|||||||
ops.impl("top_k_per_row_prefill", TORCH_BOX(&top_k_per_row_prefill));
|
ops.impl("top_k_per_row_prefill", TORCH_BOX(&top_k_per_row_prefill));
|
||||||
ops.impl("top_k_per_row_decode", TORCH_BOX(&top_k_per_row_decode));
|
ops.impl("top_k_per_row_decode", TORCH_BOX(&top_k_per_row_decode));
|
||||||
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
|
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
|
||||||
|
ops.impl("persistent_topk_exact_counters",
|
||||||
|
TORCH_BOX(&persistent_topk_exact_counters));
|
||||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||||
ops.impl("cooperative_topk", TORCH_BOX(&cooperative_topk));
|
ops.impl("cooperative_topk", TORCH_BOX(&cooperative_topk));
|
||||||
#endif
|
#endif
|
||||||
@@ -934,6 +943,15 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
|
|||||||
ops.impl("cutlass_scaled_mm_supports_fp4",
|
ops.impl("cutlass_scaled_mm_supports_fp4",
|
||||||
TORCH_BOX(&cutlass_scaled_mm_supports_fp4));
|
TORCH_BOX(&cutlass_scaled_mm_supports_fp4));
|
||||||
#endif
|
#endif
|
||||||
|
// Tensor-free: the counter layout the BUILD carries. Registered here, not
|
||||||
|
// in the CUDA block -- with no tensor argument there is no dispatch key, so
|
||||||
|
// a CUDA-only registration can never be called (reference host:
|
||||||
|
// NotImplementedError "no fallback function is registered").
|
||||||
|
ops.impl("persistent_topk_exact_counters_slots",
|
||||||
|
TORCH_BOX(&persistent_topk_exact_counters_slots));
|
||||||
|
// Tensor-free for the same reason as the slot count above.
|
||||||
|
ops.impl("persistent_topk_toggle_state",
|
||||||
|
TORCH_BOX(&persistent_topk_toggle_state));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cache ops
|
// Cache ops
|
||||||
|
|||||||
+37
-4
@@ -83,7 +83,7 @@ directly from the Triton kernel through unified virtual addressing (UVA),
|
|||||||
prefetching on a side CUDA stream so transfers overlap with the compute of
|
prefetching on a side CUDA stream so transfers overlap with the compute of
|
||||||
preceding layers. The freed GPU memory goes to the KV cache.
|
preceding layers. The freed GPU memory goes to the KV cache.
|
||||||
|
|
||||||
`cpu_offload` defaults to `true` (on)
|
`cpu_offload` defaults to `true` and is set only through `--engram-config`.
|
||||||
|
|
||||||
To keep the tables resident on GPU instead (e.g. when host memory is
|
To keep the tables resident on GPU instead (e.g. when host memory is
|
||||||
scarce):
|
scarce):
|
||||||
@@ -93,8 +93,40 @@ vllm serve deepseek-ai/DeepSeek-V4.1-Flash \
|
|||||||
--engram-config.cpu_offload false
|
--engram-config.cpu_offload false
|
||||||
```
|
```
|
||||||
|
|
||||||
CPU offload requires a GPU with UVA support;
|
CPU offload requires a GPU with UVA support; vLLM fails fast when it is
|
||||||
vLLM fails fast when it is unavailable.
|
unavailable.
|
||||||
|
|
||||||
|
## Host file gather (Qwen4Exp)
|
||||||
|
|
||||||
|
`host_file_gather` serves Qwen4Exp PLE rows from the loader's own file
|
||||||
|
mapping of the safetensors shards instead of allocating the table. Set
|
||||||
|
`--engram-config '{"host_file_gather": true}'`. It applies only to Qwen4Exp
|
||||||
|
and takes precedence over `cpu_offload`.
|
||||||
|
|
||||||
|
Rows stay in the checkpoint dtype: an FP8 table stays FP8 and is dequantized
|
||||||
|
on the GPU, and an unquantized table must match `--dtype`. CPU offload
|
||||||
|
converts on load; this mode rejects the mismatch at load.
|
||||||
|
|
||||||
|
Faulted pages are reclaimable page cache, shared across co-located processes.
|
||||||
|
They appear in each process's RSS as file-backed (`RssFile`), and PSS is
|
||||||
|
where they are split across processes. A write to the mapping copies that
|
||||||
|
page into anonymous RSS. Each step pays a host sync, and every TP rank
|
||||||
|
gathers all rows. Co-located ranks share the cache; ranks on different
|
||||||
|
machines do not.
|
||||||
|
|
||||||
|
Serving depends on the checkpoint files staying unchanged while the server
|
||||||
|
runs: an in-place overwrite serves the new bytes, truncation kills the worker
|
||||||
|
with SIGBUS, and deleting or renaming the files is safe.
|
||||||
|
|
||||||
|
The first touch of a row costs a disk read, and a page evicted under memory
|
||||||
|
pressure is read again on the next miss. By default no prewarm is needed; for
|
||||||
|
a fast first request, read the PLE files once after the server has started
|
||||||
|
and allocated its KV cache.
|
||||||
|
|
||||||
|
Reload is unsupported and fails closed. ROCm and elastic EP reject the key.
|
||||||
|
Load paths that do not hand over file-backed safetensors tensors fail at bind
|
||||||
|
(`eager`, `.bin`, runai, `fastsafetensors`, `instanttensor`). The multithread
|
||||||
|
loader is file-backed and is admitted. `--load-format dummy` serves zeros.
|
||||||
|
|
||||||
## Data-parallel topologies (DeepSeek V4.1)
|
## Data-parallel topologies (DeepSeek V4.1)
|
||||||
|
|
||||||
@@ -115,7 +147,8 @@ vllm serve deepseek-ai/DeepSeek-V4.1-Flash \
|
|||||||
```
|
```
|
||||||
|
|
||||||
`embedding_across_dp` is not supported with elastic expert parallelism yet.
|
`embedding_across_dp` is not supported with elastic expert parallelism yet.
|
||||||
Qwen4Exp PLE tables are ETP-sharded instead and honor `cpu_offload` only.
|
Qwen4Exp PLE tables are ETP-sharded instead and honor only `cpu_offload` and
|
||||||
|
`host_file_gather`.
|
||||||
|
|
||||||
## Sharing host tables across DP replicas
|
## Sharing host tables across DP replicas
|
||||||
|
|
||||||
|
|||||||
@@ -132,6 +132,7 @@ markers = [
|
|||||||
"cpu_test: mark test as CPU-only test",
|
"cpu_test: mark test as CPU-only test",
|
||||||
"split: run this test as part of a split",
|
"split: run this test as part of a split",
|
||||||
"distributed: run this test only in distributed GPU tests",
|
"distributed: run this test only in distributed GPU tests",
|
||||||
|
"gpu: requires a CUDA device; auto-skipped when none is present, and deselectable as a tier with -m 'not gpu'",
|
||||||
"optional: optional tests that are automatically skipped, include --optional to run them",
|
"optional: optional tests that are automatically skipped, include --optional to run them",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,173 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Configuration-level regression tests for Qwen4Exp GDN/PLE RecoverSSM selection.
|
||||||
|
|
||||||
|
``--use-replayssm`` with speculative decoding selects RecoverSSM for Qwen4Exp, as it
|
||||||
|
does for Kimi-K3 KDA. The MTP drafter's derived config shares ``cache_config`` and must
|
||||||
|
keep the choice. These tests run the real validator function against stand-in configs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.config.mamba import MambaBackendEnum
|
||||||
|
from vllm.config.vllm import VllmConfig
|
||||||
|
from vllm.model_executor.layers.mamba.recoverssm_utils import uses_recoverssm
|
||||||
|
from vllm.v1.attention.backend import AttentionCGSupport
|
||||||
|
|
||||||
|
TARGET = "Qwen4ExpForConditionalGeneration"
|
||||||
|
DRAFTER = "Qwen4ExpMTP"
|
||||||
|
|
||||||
|
|
||||||
|
def _validator():
|
||||||
|
validators = VllmConfig.__pydantic_decorators__.model_validators # type: ignore[attr-defined]
|
||||||
|
return validators["validate_mamba_cached_kernel"].func
|
||||||
|
|
||||||
|
|
||||||
|
def _config(
|
||||||
|
arch: str,
|
||||||
|
cache_config=None,
|
||||||
|
num_spec: int = 3,
|
||||||
|
use_replayssm: bool = True,
|
||||||
|
supports_replayssm: bool = True,
|
||||||
|
**overrides,
|
||||||
|
):
|
||||||
|
cfg = SimpleNamespace(
|
||||||
|
num_speculative_tokens=num_spec,
|
||||||
|
model_config=SimpleNamespace(
|
||||||
|
architecture=arch, supports_replayssm=supports_replayssm
|
||||||
|
),
|
||||||
|
cache_config=cache_config
|
||||||
|
or SimpleNamespace(
|
||||||
|
use_recoverssm=False,
|
||||||
|
use_replayssm=use_replayssm,
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
),
|
||||||
|
mamba_config=SimpleNamespace(
|
||||||
|
enable_stochastic_rounding=False, backend=MambaBackendEnum.TRITON
|
||||||
|
),
|
||||||
|
use_v2_model_runner=True,
|
||||||
|
parallel_config=SimpleNamespace(pipeline_parallel_size=1),
|
||||||
|
kv_transfer_config=None,
|
||||||
|
)
|
||||||
|
for key, value in overrides.items():
|
||||||
|
target, _, attr = key.partition("__")
|
||||||
|
if attr:
|
||||||
|
setattr(getattr(cfg, target), attr, value)
|
||||||
|
else:
|
||||||
|
setattr(cfg, target, value)
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def test_validator_is_registered():
|
||||||
|
assert "validate_mamba_cached_kernel" in (
|
||||||
|
VllmConfig.__pydantic_decorators__.model_validators # type: ignore[attr-defined]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"arch", ["Qwen4ExpForConditionalGeneration", "Qwen4ExpForCausalLM"]
|
||||||
|
)
|
||||||
|
def test_replayssm_with_spec_selects_recoverssm(arch):
|
||||||
|
cfg = _config(arch)
|
||||||
|
_validator()(cfg)
|
||||||
|
assert cfg.cache_config.use_recoverssm is True
|
||||||
|
assert uses_recoverssm(cfg.cache_config, 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_derived_drafter_keeps_the_choice_and_runs_the_checks():
|
||||||
|
target = _config(TARGET)
|
||||||
|
_validator()(target)
|
||||||
|
drafter = _config(DRAFTER, cache_config=target.cache_config)
|
||||||
|
_validator()(drafter)
|
||||||
|
assert target.cache_config.use_recoverssm is True
|
||||||
|
bad = _config(
|
||||||
|
DRAFTER,
|
||||||
|
cache_config=target.cache_config,
|
||||||
|
parallel_config__pipeline_parallel_size=2,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="pipeline_parallel_size"):
|
||||||
|
_validator()(bad)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("override", "value", "match"),
|
||||||
|
[
|
||||||
|
("parallel_config__pipeline_parallel_size", 2, "pipeline_parallel_size"),
|
||||||
|
("mamba_config__enable_stochastic_rounding", True, "stochastic"),
|
||||||
|
("mamba_config__backend", "flashinfer", "mamba-backend triton"),
|
||||||
|
("use_v2_model_runner", False, "V2_MODEL_RUNNER"),
|
||||||
|
("cache_config__mamba_cache_mode", "all", "none and align"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_runtime_checks_reject(override, value, match):
|
||||||
|
cfg = _config(TARGET, **{override: value})
|
||||||
|
with pytest.raises(ValueError, match=match):
|
||||||
|
_validator()(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
def test_flag_unset_keeps_the_stock_path():
|
||||||
|
cfg = _config(TARGET, use_replayssm=False)
|
||||||
|
_validator()(cfg)
|
||||||
|
assert cfg.cache_config.use_recoverssm is False
|
||||||
|
assert not uses_recoverssm(cfg.cache_config, 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_qwen4exp_replayssm_requires_speculative_decoding():
|
||||||
|
with pytest.raises(ValueError, match="requires speculative decoding"):
|
||||||
|
_validator()(_config(TARGET, num_spec=0))
|
||||||
|
|
||||||
|
|
||||||
|
def test_architecture_without_replayssm_support_is_rejected():
|
||||||
|
cfg = _config("Qwen3NextForCausalLM", supports_replayssm=False)
|
||||||
|
with pytest.raises(ValueError, match="not supported for architecture"):
|
||||||
|
_validator()(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
def test_uses_recoverssm_predicate():
|
||||||
|
on = SimpleNamespace(use_recoverssm=True)
|
||||||
|
assert uses_recoverssm(on, 3)
|
||||||
|
assert not uses_recoverssm(on, 0)
|
||||||
|
assert not uses_recoverssm(SimpleNamespace(use_recoverssm=False), 3)
|
||||||
|
assert not uses_recoverssm(SimpleNamespace(), 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_builders_fall_back_to_piecewise():
|
||||||
|
from vllm.v1.attention.backends.gdn_recoverssm import GDNRecoverSSMMetadataBuilder
|
||||||
|
from vllm.v1.attention.backends.ple_recoverssm import PleRecoverSSMMetadataBuilder
|
||||||
|
|
||||||
|
for builder in (GDNRecoverSSMMetadataBuilder, PleRecoverSSMMetadataBuilder):
|
||||||
|
assert (
|
||||||
|
builder.get_cudagraph_support(cast(Any, None), cast(Any, None))
|
||||||
|
== AttentionCGSupport.NEVER
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("compile_mode", ["VLLM_COMPILE", "NONE"])
|
||||||
|
@pytest.mark.parametrize(("breakable", "expected"), [("1", "PIECEWISE"), ("0", "NONE")])
|
||||||
|
def test_never_support_falls_back_to_piecewise_with_breakable_cudagraphs(
|
||||||
|
monkeypatch, compile_mode, breakable, expected
|
||||||
|
):
|
||||||
|
"""A backend without full-graph support must keep piecewise graphs when breakable
|
||||||
|
CUDA graphs split at attention (splitting_ops is empty then, and torch.compile may
|
||||||
|
be off)."""
|
||||||
|
from vllm.config.compilation import (
|
||||||
|
CompilationConfig,
|
||||||
|
CompilationMode,
|
||||||
|
CUDAGraphMode,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", breakable)
|
||||||
|
compilation_config = CompilationConfig(
|
||||||
|
mode=CompilationMode[compile_mode],
|
||||||
|
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||||
|
splitting_ops=[],
|
||||||
|
)
|
||||||
|
resolved = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||||
|
AttentionCGSupport.NEVER,
|
||||||
|
"GDNRecoverSSMAttentionBackend",
|
||||||
|
uniform_decode_query_len=4,
|
||||||
|
)
|
||||||
|
assert resolved.name == expected
|
||||||
@@ -0,0 +1,602 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""GDN RecoverSSM: verify against the native fused kernel, commit against the native
|
||||||
|
per-token states, and the boundary checks (malformed metadata must raise ValueError
|
||||||
|
before any kernel reads or writes out of bounds)."""
|
||||||
|
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||||
|
GDNRecoverSSMCommitContext,
|
||||||
|
gdn_recoverssm_verify,
|
||||||
|
)
|
||||||
|
from vllm.third_party.flash_linear_attention.ops.fused_sigmoid_gating import (
|
||||||
|
fused_sigmoid_gating_delta_rule_update,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||||
|
H, HV, K, V, T = 16, 48, 128, 128, 4
|
||||||
|
QLENS = (4, 4, 3)
|
||||||
|
|
||||||
|
|
||||||
|
def _inputs(nb: int = 10) -> dict:
|
||||||
|
torch.manual_seed(0)
|
||||||
|
dev = "cuda"
|
||||||
|
tot = sum(QLENS)
|
||||||
|
qsl = torch.tensor([0, 4, 8, 11], dtype=torch.int32, device=dev)
|
||||||
|
return dict(
|
||||||
|
A_log=(torch.rand(HV, device=dev) * 2 - 1).float(),
|
||||||
|
a=torch.randn(tot, HV, device=dev, dtype=torch.bfloat16),
|
||||||
|
b=torch.randn(tot, HV, device=dev, dtype=torch.bfloat16),
|
||||||
|
dt_bias=(torch.randn(HV, device=dev) * 0.5).bfloat16(),
|
||||||
|
q=torch.randn(1, tot, H, K, device=dev, dtype=torch.bfloat16),
|
||||||
|
k=torch.randn(1, tot, H, K, device=dev, dtype=torch.bfloat16),
|
||||||
|
v=torch.randn(1, tot, HV, V, device=dev, dtype=torch.bfloat16),
|
||||||
|
checkpoint_state=(torch.randn(nb, HV, V, K, device=dev) * 0.05).float(),
|
||||||
|
replay_cache=torch.zeros(nb, HV, T, V + K + 1, device=dev),
|
||||||
|
query_start_loc=qsl,
|
||||||
|
state_indices=torch.tensor([2, 5, 7], dtype=torch.int32, device=dev),
|
||||||
|
spec_query_len=T,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify(inp: dict, **override):
|
||||||
|
d = {**inp, **override}
|
||||||
|
return gdn_recoverssm_verify(
|
||||||
|
d["A_log"],
|
||||||
|
d["a"],
|
||||||
|
d["b"],
|
||||||
|
d["dt_bias"],
|
||||||
|
d["q"],
|
||||||
|
d["k"],
|
||||||
|
d["v"],
|
||||||
|
checkpoint_state=d["checkpoint_state"],
|
||||||
|
replay_cache=d["replay_cache"],
|
||||||
|
query_start_loc=d["query_start_loc"],
|
||||||
|
state_indices=d["state_indices"],
|
||||||
|
spec_query_len=d["spec_query_len"],
|
||||||
|
out=d.get("out"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _native(inp: dict):
|
||||||
|
return fused_sigmoid_gating_delta_rule_update(
|
||||||
|
A_log=inp["A_log"],
|
||||||
|
a=inp["a"],
|
||||||
|
b=inp["b"],
|
||||||
|
dt_bias=inp["dt_bias"],
|
||||||
|
q=inp["q"],
|
||||||
|
k=inp["k"],
|
||||||
|
v=inp["v"],
|
||||||
|
initial_state=inp["checkpoint_state"].clone(),
|
||||||
|
inplace_final_state=False,
|
||||||
|
cu_seqlens=inp["query_start_loc"].long(),
|
||||||
|
ssm_state_indices=inp["state_indices"].long(),
|
||||||
|
use_qk_l2norm_in_kernel=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _context(inp: dict, ckpt: torch.Tensor) -> GDNRecoverSSMCommitContext:
|
||||||
|
nb = ckpt.shape[0]
|
||||||
|
conv = [torch.zeros(nb, 8, 3 + T - 1, device="cuda", dtype=torch.bfloat16)]
|
||||||
|
return GDNRecoverSSMCommitContext.from_tensors(
|
||||||
|
conv, [ckpt], [inp["replay_cache"]], spec_query_len=T, max_num_reqs=8
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_matches_native_and_keeps_the_checkpoint():
|
||||||
|
inp = _inputs()
|
||||||
|
before = inp["checkpoint_state"].clone()
|
||||||
|
out = _verify(inp)
|
||||||
|
ref, _ = _native(inp)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
out.float(), ref.reshape(out.shape).float(), rtol=0, atol=0
|
||||||
|
)
|
||||||
|
assert torch.equal(inp["checkpoint_state"], before)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("accepted", [1, 2, 3, 4])
|
||||||
|
def test_commit_matches_native_per_token_state(accepted):
|
||||||
|
inp = _inputs()
|
||||||
|
_verify(inp)
|
||||||
|
_, states = _native(inp)
|
||||||
|
ckpt = inp["checkpoint_state"].clone()
|
||||||
|
ctx = _context(inp, ckpt)
|
||||||
|
acc = torch.tensor(
|
||||||
|
[min(accepted, q) for q in QLENS], dtype=torch.int32, device="cuda"
|
||||||
|
)
|
||||||
|
ctx.commit(acc, inp["state_indices"], inp["query_start_loc"])
|
||||||
|
for i, blk in enumerate(inp["state_indices"].tolist()):
|
||||||
|
tok = int(inp["query_start_loc"][i]) + int(acc[i]) - 1
|
||||||
|
torch.testing.assert_close(ckpt[blk], states[tok], rtol=1e-5, atol=1e-6)
|
||||||
|
others = [b for b in range(ckpt.shape[0]) if b not in inp["state_indices"].tolist()]
|
||||||
|
assert torch.equal(ckpt[others], inp["checkpoint_state"][others])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"case", ["short_qsl", "capacity", "hv_lt_h", "a_log", "out", "replay", "device"]
|
||||||
|
)
|
||||||
|
def test_verify_rejects_malformed_inputs(case):
|
||||||
|
inp = _inputs()
|
||||||
|
over, match = {
|
||||||
|
"short_qsl": (
|
||||||
|
dict(query_start_loc=inp["query_start_loc"][:-1]),
|
||||||
|
"query metadata",
|
||||||
|
),
|
||||||
|
"capacity": (
|
||||||
|
dict(
|
||||||
|
state_indices=inp["state_indices"][:2],
|
||||||
|
query_start_loc=inp["query_start_loc"][:3],
|
||||||
|
),
|
||||||
|
"activation capacity",
|
||||||
|
),
|
||||||
|
"hv_lt_h": (
|
||||||
|
dict(
|
||||||
|
v=inp["v"][:, :, :8].contiguous(),
|
||||||
|
a=inp["a"][:, :8].contiguous(),
|
||||||
|
b=inp["b"][:, :8].contiguous(),
|
||||||
|
),
|
||||||
|
"positive multiple",
|
||||||
|
),
|
||||||
|
"a_log": (dict(A_log=torch.zeros(HV + 1, device="cuda")), "A_log"),
|
||||||
|
"out": (
|
||||||
|
dict(
|
||||||
|
out=torch.empty(1, 11, HV, V + 1, device="cuda", dtype=torch.bfloat16)
|
||||||
|
),
|
||||||
|
"output shape",
|
||||||
|
),
|
||||||
|
"replay": (
|
||||||
|
dict(replay_cache=torch.zeros(10, HV, T, V + K, device="cuda")),
|
||||||
|
"replay buffer",
|
||||||
|
),
|
||||||
|
"device": (dict(A_log=inp["A_log"].cpu()), "same device"),
|
||||||
|
}[case]
|
||||||
|
with pytest.raises(ValueError, match=match):
|
||||||
|
_verify(inp, **over)
|
||||||
|
|
||||||
|
|
||||||
|
def test_commit_rejects_partial_or_short_metadata():
|
||||||
|
inp = _inputs()
|
||||||
|
ctx = _context(inp, inp["checkpoint_state"].clone())
|
||||||
|
acc = torch.ones(3, dtype=torch.int32, device="cuda")
|
||||||
|
bt = torch.zeros(3, 4, dtype=torch.int32, device="cuda")
|
||||||
|
with pytest.raises(ValueError, match="align metadata is incomplete"):
|
||||||
|
ctx.commit(
|
||||||
|
acc,
|
||||||
|
inp["state_indices"],
|
||||||
|
inp["query_start_loc"],
|
||||||
|
block_table=bt,
|
||||||
|
mamba_block_size=16,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="commit metadata"):
|
||||||
|
ctx.commit(acc, inp["state_indices"], inp["query_start_loc"][:-1])
|
||||||
|
with pytest.raises(ValueError, match="request mapping"):
|
||||||
|
ctx.commit(
|
||||||
|
acc,
|
||||||
|
inp["state_indices"],
|
||||||
|
inp["query_start_loc"],
|
||||||
|
request_indices=torch.zeros(2, dtype=torch.int32, device="cuda"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_context_rejects_inconsistent_layers():
|
||||||
|
inp = _inputs()
|
||||||
|
ck = inp["checkpoint_state"]
|
||||||
|
conv = torch.zeros(ck.shape[0], 8, 3 + T - 1, device="cuda", dtype=torch.bfloat16)
|
||||||
|
with pytest.raises(ValueError, match="at least one layer"):
|
||||||
|
GDNRecoverSSMCommitContext.from_tensors(
|
||||||
|
[], [], [], spec_query_len=T, max_num_reqs=8
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="differ"):
|
||||||
|
GDNRecoverSSMCommitContext.from_tensors(
|
||||||
|
[conv], [ck, ck], [inp["replay_cache"]], spec_query_len=T, max_num_reqs=8
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="block count"):
|
||||||
|
GDNRecoverSSMCommitContext.from_tensors(
|
||||||
|
[conv[:5]], [ck], [inp["replay_cache"]], spec_query_len=T, max_num_reqs=8
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ review round 2:
|
||||||
|
# mapping, align, extreme gates
|
||||||
|
from vllm.v1.attention.backends.recoverssm_metadata import ( # noqa: E402
|
||||||
|
recoverssm_request_indices,
|
||||||
|
recoverssm_spec_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
# (A_log range, a scale, b offset): drive g = -exp(A_log) * softplus(a + dt_bias)
|
||||||
|
# and beta = sigmoid(b)
|
||||||
|
GATES = {
|
||||||
|
"typical": (0.0, 1.0, 0.0),
|
||||||
|
"g_near_0": (-12.0, 1.0, 0.0),
|
||||||
|
"g_very_negative": (2.5, 4.0, 0.0),
|
||||||
|
"beta_near_0": (0.0, 1.0, -12.0),
|
||||||
|
"beta_near_1": (0.0, 1.0, 12.0),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _make(qlens, nb=16, seed=0, gate="typical"):
|
||||||
|
g = torch.Generator(device="cuda").manual_seed(seed)
|
||||||
|
rnd = lambda *s: torch.randn(*s, device="cuda", generator=g) # noqa: E731
|
||||||
|
a_log_shift, ascale, boff = GATES[gate]
|
||||||
|
tot = sum(qlens)
|
||||||
|
qsl = torch.tensor(
|
||||||
|
[0, *torch.tensor(qlens).cumsum(0).tolist()], dtype=torch.int32, device="cuda"
|
||||||
|
)
|
||||||
|
return dict(
|
||||||
|
A_log=(
|
||||||
|
torch.rand(HV, device="cuda", generator=g) * 2 - 1 + a_log_shift
|
||||||
|
).float(),
|
||||||
|
a=(rnd(tot, HV) * ascale).bfloat16(),
|
||||||
|
b=(rnd(tot, HV) + boff).bfloat16(),
|
||||||
|
dt_bias=(rnd(HV) * 0.5).bfloat16(),
|
||||||
|
q=rnd(1, tot, H, K).bfloat16(),
|
||||||
|
k=rnd(1, tot, H, K).bfloat16(),
|
||||||
|
v=rnd(1, tot, HV, V).bfloat16(),
|
||||||
|
checkpoint_state=(rnd(nb, HV, V, K) * 0.05).float(),
|
||||||
|
replay_cache=torch.zeros(nb, HV, T, V + K + 1, device="cuda"),
|
||||||
|
query_start_loc=qsl,
|
||||||
|
spec_query_len=T,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_spec_rows_mixed_batch():
|
||||||
|
# rows: 0 spec K=3 | 1 prefill | 2 draft-less decode | 3 spec K=1 | 4 padded
|
||||||
|
# zero-query | 5,6 beyond num_reqs
|
||||||
|
is_prefilling = torch.tensor([False, True, False, False, False, False, False])
|
||||||
|
qsl = torch.tensor([0, 4, 104, 105, 107, 107], dtype=torch.int32)
|
||||||
|
drafts_in = torch.tensor(
|
||||||
|
[3, -1, -1, 1, -1, 2, 0], dtype=torch.int32
|
||||||
|
) # 5, 6: garbage padding
|
||||||
|
drafts, rows = recoverssm_spec_rows(is_prefilling, qsl, drafts_in, num_reqs=5)
|
||||||
|
assert rows.tolist() == [0, 2, 3]
|
||||||
|
assert drafts.tolist() == [1, -1, 1, 1, -1, -1, -1]
|
||||||
|
idx = recoverssm_request_indices(rows, 3, torch.device("cuda"))
|
||||||
|
assert idx is not None and idx.tolist() == [0, 2, 3]
|
||||||
|
assert (
|
||||||
|
recoverssm_request_indices(torch.tensor([0, 1, 2]), 3, torch.device("cuda"))
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="spec rows classified"):
|
||||||
|
recoverssm_request_indices(rows, 2, torch.device("cuda"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_commit_maps_noncontiguous_requests_in_align_mode():
|
||||||
|
"""Spec rows 0, 2, 3 of a 5-request batch; accepted counts, block table and
|
||||||
|
num_computed are indexed by request row. Every spec request must commit its own
|
||||||
|
state into its own block."""
|
||||||
|
qlens = [4, 1, 2] # the spec rows' windows (K=3, draft-less, K=1)
|
||||||
|
inp = _make(qlens, nb=40)
|
||||||
|
bs = 8
|
||||||
|
bt = torch.arange(1, 1 + 5 * 6, dtype=torch.int32, device="cuda").reshape(
|
||||||
|
5, 6
|
||||||
|
) # request row -> blocks
|
||||||
|
nc = torch.tensor(
|
||||||
|
[5, 0, 16, 9, 0], dtype=torch.int32, device="cuda"
|
||||||
|
) # per request row
|
||||||
|
req_rows = torch.tensor([0, 2, 3], dtype=torch.int32, device="cuda")
|
||||||
|
src = torch.stack([bt[r, int(nc[r]) // bs] for r in req_rows.tolist()]).contiguous()
|
||||||
|
inp["state_indices"] = src
|
||||||
|
_verify(inp)
|
||||||
|
_, states = _native(inp)
|
||||||
|
ckpt = inp["checkpoint_state"].clone()
|
||||||
|
ctx = _context(inp, ckpt)
|
||||||
|
acc = torch.tensor(
|
||||||
|
[3, 9, 1, 2, 9], dtype=torch.int32, device="cuda"
|
||||||
|
) # rows 1, 4 must be ignored
|
||||||
|
ctx.commit(
|
||||||
|
acc,
|
||||||
|
src,
|
||||||
|
inp["query_start_loc"],
|
||||||
|
request_indices=req_rows,
|
||||||
|
block_table=bt,
|
||||||
|
num_computed_tokens=nc,
|
||||||
|
mamba_block_size=bs,
|
||||||
|
)
|
||||||
|
for i, r in enumerate(req_rows.tolist()):
|
||||||
|
n = min(int(acc[r]), qlens[i])
|
||||||
|
c0 = int(nc[r])
|
||||||
|
final_blk = int(bt[r, max(c0 + n - 1, 0) // bs])
|
||||||
|
tok = int(inp["query_start_loc"][i]) + n - 1
|
||||||
|
torch.testing.assert_close(ckpt[final_blk], states[tok], rtol=1e-5, atol=1e-6)
|
||||||
|
|
||||||
|
|
||||||
|
# (num_computed, accepted, block size): boundary after 1 / after 2 / ending exactly on
|
||||||
|
# it / crossing and continuing / accepted = 1 without crossing / the full window
|
||||||
|
# crossing after 3. A window ending exactly on a boundary keeps its state in the block
|
||||||
|
# holding its last token: the next block may still be unallocated.
|
||||||
|
ALIGN_CASES = [
|
||||||
|
(3, 2, 4),
|
||||||
|
(2, 3, 4),
|
||||||
|
(0, 4, 4),
|
||||||
|
(6, 4, 4),
|
||||||
|
(1, 1, 4),
|
||||||
|
(5, 4, 8),
|
||||||
|
(7, 4, 8),
|
||||||
|
(4, 4, 8),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("nc,accepted,bs", ALIGN_CASES)
|
||||||
|
def test_align_boundary_and_final_state_match_native(nc, accepted, bs):
|
||||||
|
inp = _make([T], nb=12)
|
||||||
|
bt = torch.arange(2, 8, dtype=torch.int32, device="cuda").reshape(1, 6)
|
||||||
|
src = bt[0, nc // bs].reshape(1).contiguous()
|
||||||
|
inp["state_indices"] = src
|
||||||
|
_verify(inp)
|
||||||
|
_, states = _native(inp)
|
||||||
|
ckpt = inp["checkpoint_state"].clone()
|
||||||
|
ctx = _context(inp, ckpt)
|
||||||
|
ctx.commit(
|
||||||
|
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
|
||||||
|
src,
|
||||||
|
inp["query_start_loc"],
|
||||||
|
block_table=bt,
|
||||||
|
num_computed_tokens=torch.tensor([nc], dtype=torch.int32, device="cuda"),
|
||||||
|
mamba_block_size=bs,
|
||||||
|
)
|
||||||
|
next_boundary = (nc // bs + 1) * bs
|
||||||
|
final_blk = int(bt[0, max(nc + accepted - 1, 0) // bs])
|
||||||
|
if (
|
||||||
|
nc + accepted >= next_boundary
|
||||||
|
): # crosses: the boundary state goes into the source block
|
||||||
|
torch.testing.assert_close(
|
||||||
|
ckpt[int(src)], states[next_boundary - nc - 1], rtol=1e-5, atol=1e-6
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
ckpt[final_blk], states[accepted - 1], rtol=1e-5, atol=1e-6
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("gate", list(GATES))
|
||||||
|
@pytest.mark.parametrize("seed", [0, 1, 2])
|
||||||
|
def test_commit_matches_native_under_extreme_gates(gate, seed):
|
||||||
|
inp = _make(list(QLENS), nb=10, seed=seed, gate=gate)
|
||||||
|
inp["state_indices"] = torch.tensor([2, 5, 7], dtype=torch.int32, device="cuda")
|
||||||
|
_verify(inp)
|
||||||
|
_, states = _native(inp)
|
||||||
|
for accepted in (1, 2, 3, 4):
|
||||||
|
ckpt = inp["checkpoint_state"].clone()
|
||||||
|
ctx = _context(inp, ckpt)
|
||||||
|
acc = torch.tensor(
|
||||||
|
[min(accepted, q) for q in QLENS], dtype=torch.int32, device="cuda"
|
||||||
|
)
|
||||||
|
ctx.commit(acc, inp["state_indices"], inp["query_start_loc"])
|
||||||
|
for i, blk in enumerate(inp["state_indices"].tolist()):
|
||||||
|
tok = int(inp["query_start_loc"][i]) + int(acc[i]) - 1
|
||||||
|
torch.testing.assert_close(ckpt[blk], states[tok], rtol=1e-5, atol=1e-6)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("batch", [1, 2, 4])
|
||||||
|
def test_verify_on_strided_views_writes_in_place(batch):
|
||||||
|
"""The layer passes views of the packed conv output (token stride = the packed
|
||||||
|
width) and the layer's output buffer; result and replay record must equal the
|
||||||
|
contiguous-copy path."""
|
||||||
|
torch.manual_seed(batch)
|
||||||
|
H, HV, K, V, SQ, NB = 16, 48, 128, 128, 4, 6
|
||||||
|
T = batch * SQ
|
||||||
|
qd, vd = H * K, HV * V
|
||||||
|
packed = torch.randn(T, 2 * qd + 2 * vd, device="cuda", dtype=torch.bfloat16)
|
||||||
|
mixed = packed[:, : 2 * qd + vd]
|
||||||
|
a = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
|
||||||
|
b = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
|
||||||
|
A_log = torch.randn(HV, device="cuda")
|
||||||
|
dt_bias = torch.randn(HV, device="cuda")
|
||||||
|
checkpoint = torch.randn(NB, HV, V, K, device="cuda") * 0.05
|
||||||
|
qsl = torch.arange(0, T + 1, SQ, device="cuda", dtype=torch.int32)
|
||||||
|
si = torch.arange(1, batch + 1, device="cuda", dtype=torch.int32)
|
||||||
|
|
||||||
|
def run(q, k, v, out=None):
|
||||||
|
replay = torch.zeros(NB, HV, SQ, V + K + 1, device="cuda")
|
||||||
|
o = gdn_recoverssm_verify(
|
||||||
|
A_log,
|
||||||
|
a,
|
||||||
|
b,
|
||||||
|
dt_bias,
|
||||||
|
q,
|
||||||
|
k,
|
||||||
|
v,
|
||||||
|
checkpoint_state=checkpoint,
|
||||||
|
replay_cache=replay,
|
||||||
|
query_start_loc=qsl,
|
||||||
|
state_indices=si,
|
||||||
|
spec_query_len=SQ,
|
||||||
|
out=out,
|
||||||
|
)
|
||||||
|
return o, replay
|
||||||
|
|
||||||
|
q, k, v = (
|
||||||
|
x.contiguous().view(1, T, -1, d)
|
||||||
|
for x, d in zip(torch.split(mixed, [qd, qd, vd], dim=-1), (K, K, V))
|
||||||
|
)
|
||||||
|
want_out, want_replay = run(q, k, v)
|
||||||
|
|
||||||
|
core = torch.zeros(T + 3, HV, V, device="cuda", dtype=torch.bfloat16)
|
||||||
|
views = (
|
||||||
|
mixed[:, :qd].view(1, T, -1, K),
|
||||||
|
mixed[:, qd : 2 * qd].view(1, T, -1, K),
|
||||||
|
mixed[:, 2 * qd :].view(1, T, -1, V),
|
||||||
|
)
|
||||||
|
got_out, got_replay = run(*views, out=core[:T].unsqueeze(0))
|
||||||
|
assert got_out.data_ptr() == core.data_ptr()
|
||||||
|
assert torch.equal(core[:T].unsqueeze(0), want_out)
|
||||||
|
assert torch.equal(got_replay, want_replay)
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_on_strided_views_with_variable_query_lengths():
|
||||||
|
"""Token-strided views with ragged windows (query lengths 4, 1, 3), as MTP
|
||||||
|
scheduling produces them."""
|
||||||
|
torch.manual_seed(7)
|
||||||
|
H, HV, K, V, SQ, NB = 16, 48, 128, 128, 4, 6
|
||||||
|
qsl_list = [0, 4, 5, 8]
|
||||||
|
T = qsl_list[-1]
|
||||||
|
qd, vd = H * K, HV * V
|
||||||
|
packed = torch.randn(T, 2 * qd + 2 * vd, device="cuda", dtype=torch.bfloat16)
|
||||||
|
mixed = packed[:, : 2 * qd + vd]
|
||||||
|
a = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
|
||||||
|
b = torch.randn(T, HV, device="cuda", dtype=torch.bfloat16)
|
||||||
|
A_log = torch.randn(HV, device="cuda")
|
||||||
|
dt_bias = torch.randn(HV, device="cuda")
|
||||||
|
checkpoint = torch.randn(NB, HV, V, K, device="cuda") * 0.05
|
||||||
|
qsl = torch.tensor(qsl_list, device="cuda", dtype=torch.int32)
|
||||||
|
si = torch.tensor([1, 3, 5], device="cuda", dtype=torch.int32)
|
||||||
|
|
||||||
|
def run(q, k, v, out=None):
|
||||||
|
replay = torch.zeros(NB, HV, SQ, V + K + 1, device="cuda")
|
||||||
|
o = gdn_recoverssm_verify(
|
||||||
|
A_log,
|
||||||
|
a,
|
||||||
|
b,
|
||||||
|
dt_bias,
|
||||||
|
q,
|
||||||
|
k,
|
||||||
|
v,
|
||||||
|
checkpoint_state=checkpoint,
|
||||||
|
replay_cache=replay,
|
||||||
|
query_start_loc=qsl,
|
||||||
|
state_indices=si,
|
||||||
|
spec_query_len=SQ,
|
||||||
|
out=out,
|
||||||
|
)
|
||||||
|
return o, replay
|
||||||
|
|
||||||
|
q, k, v = (
|
||||||
|
x.contiguous().view(1, T, -1, d)
|
||||||
|
for x, d in zip(torch.split(mixed, [qd, qd, vd], dim=-1), (K, K, V))
|
||||||
|
)
|
||||||
|
want_out, want_replay = run(q, k, v)
|
||||||
|
core = torch.zeros(T + 2, HV, V, device="cuda", dtype=torch.bfloat16)
|
||||||
|
views = (
|
||||||
|
mixed[:, :qd].view(1, T, -1, K),
|
||||||
|
mixed[:, qd : 2 * qd].view(1, T, -1, K),
|
||||||
|
mixed[:, 2 * qd :].view(1, T, -1, V),
|
||||||
|
)
|
||||||
|
_, got_replay = run(*views, out=core[:T].unsqueeze(0))
|
||||||
|
assert torch.equal(core[:T].unsqueeze(0), want_out)
|
||||||
|
assert torch.equal(got_replay, want_replay)
|
||||||
|
|
||||||
|
|
||||||
|
def test_layer_verify_qkv_views_and_fallback():
|
||||||
|
"""The layer's guard: views for a [tokens, q|k|v] conv output with a contiguous
|
||||||
|
last dimension, the contiguous-copy fallback for any other layout; same values
|
||||||
|
either way."""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||||
|
QwenGatedDeltaNetAttention as Layer,
|
||||||
|
)
|
||||||
|
|
||||||
|
H, HV, K, V, T = 4, 8, 16, 16, 5
|
||||||
|
layer = SimpleNamespace(
|
||||||
|
key_dim=H * K, value_dim=HV * V, tp_size=1, head_k_dim=K, head_v_dim=V
|
||||||
|
)
|
||||||
|
layer.rearrange_mixed_qkv = lambda x: Layer.rearrange_mixed_qkv(cast(Any, layer), x)
|
||||||
|
width = 2 * H * K + HV * V
|
||||||
|
packed = torch.randn(T, width + HV * V, dtype=torch.bfloat16) # q|k|v|z, like qkvz
|
||||||
|
strided = packed[:, :width]
|
||||||
|
q, k, v, is_view = Layer._recoverssm_verify_qkv(cast(Any, layer), strided)
|
||||||
|
assert is_view
|
||||||
|
assert q.data_ptr() == packed.data_ptr()
|
||||||
|
rq, rk, rv = Layer.rearrange_mixed_qkv(cast(Any, layer), strided)
|
||||||
|
assert torch.equal(q, rq) and torch.equal(k, rk) and torch.equal(v, rv)
|
||||||
|
|
||||||
|
# Column-major storage: last dimension not contiguous -> copies, same values.
|
||||||
|
col_major = strided.t().contiguous().t()
|
||||||
|
assert col_major.stride(-1) != 1
|
||||||
|
q2, k2, v2, is_view2 = Layer._recoverssm_verify_qkv(cast(Any, layer), col_major)
|
||||||
|
assert not is_view2
|
||||||
|
assert torch.equal(q2, rq) and torch.equal(k2, rk) and torch.equal(v2, rv)
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- BF16 recurrent state
|
||||||
|
# RecoverSSM upcasts the BF16 checkpoint, carries the accepted window in FP32 and rounds
|
||||||
|
# once when it writes the committed state. The reference is the FP32 trajectory from the
|
||||||
|
# same (already BF16-rounded) checkpoint; the committed state must equal its BF16
|
||||||
|
# rounding up to one BF16 ulp, and be no further from it than the native path is.
|
||||||
|
BF16_ULP = 2**-7
|
||||||
|
|
||||||
|
|
||||||
|
def _bf16_case(qlens, nb=12, seed=0):
|
||||||
|
inp = _make(qlens, nb=nb, seed=seed)
|
||||||
|
inp["checkpoint_state"] = inp["checkpoint_state"].bfloat16()
|
||||||
|
ref = dict(inp)
|
||||||
|
ref["checkpoint_state"] = inp["checkpoint_state"].float()
|
||||||
|
return inp, ref
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_bf16_state_matches_native():
|
||||||
|
inp, _ = _bf16_case(list(QLENS))
|
||||||
|
inp["state_indices"] = torch.tensor([2, 5, 7], dtype=torch.int32, device="cuda")
|
||||||
|
before = inp["checkpoint_state"].clone()
|
||||||
|
out = _verify(inp)
|
||||||
|
ref, _ = _native(inp)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
out.float(), ref.reshape(out.shape).float(), rtol=0, atol=0
|
||||||
|
)
|
||||||
|
assert torch.equal(inp["checkpoint_state"], before)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("accepted", [1, 2, 3, 4])
|
||||||
|
@pytest.mark.parametrize("seed", [0, 1])
|
||||||
|
def test_commit_bf16_state_rounds_once(accepted, seed):
|
||||||
|
inp, ref_inp = _bf16_case(list(QLENS), seed=seed)
|
||||||
|
idx = torch.tensor([2, 5, 7], dtype=torch.int32, device="cuda")
|
||||||
|
inp["state_indices"] = ref_inp["state_indices"] = idx
|
||||||
|
_verify(inp)
|
||||||
|
_, native_states = _native(inp)
|
||||||
|
_, ref_states = _native(ref_inp)
|
||||||
|
ckpt = inp["checkpoint_state"].clone()
|
||||||
|
_context(inp, ckpt).commit(
|
||||||
|
torch.tensor(
|
||||||
|
[min(accepted, q) for q in QLENS], dtype=torch.int32, device="cuda"
|
||||||
|
),
|
||||||
|
idx,
|
||||||
|
inp["query_start_loc"],
|
||||||
|
)
|
||||||
|
for i, blk in enumerate(idx.tolist()):
|
||||||
|
tok = int(inp["query_start_loc"][i]) + min(accepted, QLENS[i]) - 1
|
||||||
|
ref = ref_states[tok].float()
|
||||||
|
got = ckpt[blk].float()
|
||||||
|
torch.testing.assert_close(
|
||||||
|
got, ref.bfloat16().float(), rtol=BF16_ULP, atol=1e-5
|
||||||
|
)
|
||||||
|
err_rssm = (got - ref).abs().mean()
|
||||||
|
err_native = (native_states[tok].bfloat16().float() - ref).abs().mean()
|
||||||
|
assert err_rssm <= err_native * 1.05 + 1e-7, (err_rssm, err_native)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("nc,accepted,bs", ALIGN_CASES)
|
||||||
|
def test_align_boundary_bf16_state(nc, accepted, bs):
|
||||||
|
inp, ref_inp = _bf16_case([T])
|
||||||
|
bt = torch.arange(2, 8, dtype=torch.int32, device="cuda").reshape(1, 6)
|
||||||
|
src = bt[0, nc // bs].reshape(1).contiguous()
|
||||||
|
inp["state_indices"] = ref_inp["state_indices"] = src
|
||||||
|
_verify(inp)
|
||||||
|
_, ref_states = _native(ref_inp)
|
||||||
|
ckpt = inp["checkpoint_state"].clone()
|
||||||
|
_context(inp, ckpt).commit(
|
||||||
|
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
|
||||||
|
src,
|
||||||
|
inp["query_start_loc"],
|
||||||
|
block_table=bt,
|
||||||
|
num_computed_tokens=torch.tensor([nc], dtype=torch.int32, device="cuda"),
|
||||||
|
mamba_block_size=bs,
|
||||||
|
)
|
||||||
|
next_boundary = (nc // bs + 1) * bs
|
||||||
|
final_blk = int(bt[0, max(nc + accepted - 1, 0) // bs])
|
||||||
|
if nc + accepted >= next_boundary:
|
||||||
|
torch.testing.assert_close(
|
||||||
|
ckpt[int(src)].float(),
|
||||||
|
ref_states[next_boundary - nc - 1].float().bfloat16().float(),
|
||||||
|
rtol=BF16_ULP,
|
||||||
|
atol=1e-5,
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
ckpt[final_blk].float(),
|
||||||
|
ref_states[accepted - 1].float().bfloat16().float(),
|
||||||
|
rtol=BF16_ULP,
|
||||||
|
atol=1e-5,
|
||||||
|
)
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""PLE short-conv RecoverSSM against the sequential reference.
|
||||||
|
|
||||||
|
Reference: the stock decode path fed the accepted tokens one at a time.
|
||||||
|
RecoverSSM: one spec-mode conv over the whole window from the checkpoint
|
||||||
|
(offset 0), then ``_PleConvCommit`` compacts the accepted suffix to the front.
|
||||||
|
The conv history is pure data movement, so the states must be bit-identical,
|
||||||
|
and the outputs of the accepted tokens must match as well.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||||
|
from vllm.models.qwen4_exp.nvidia.ops.ple import ple_conv
|
||||||
|
from vllm.v1.attention.backends.ple_recoverssm import _PleConvCommit
|
||||||
|
|
||||||
|
C, KSIZE, DIL, SQ, NB = 512, 4, 3, 4, 8
|
||||||
|
STATE_LEN = (KSIZE - 1) * DIL
|
||||||
|
WIDTH = STATE_LEN + SQ - 1
|
||||||
|
DEV = "cuda"
|
||||||
|
|
||||||
|
|
||||||
|
def _state():
|
||||||
|
if is_conv_state_dim_first():
|
||||||
|
raw = torch.zeros(NB, C, WIDTH, dtype=torch.bfloat16, device=DEV)
|
||||||
|
return raw, raw
|
||||||
|
raw = torch.zeros(NB, WIDTH, C, dtype=torch.bfloat16, device=DEV)
|
||||||
|
return raw, raw.transpose(-1, -2)
|
||||||
|
|
||||||
|
|
||||||
|
def _i32(xs):
|
||||||
|
return torch.tensor(xs, dtype=torch.int32, device=DEV)
|
||||||
|
|
||||||
|
|
||||||
|
def _setup(seed, blocks):
|
||||||
|
g = torch.Generator(device=DEV).manual_seed(seed)
|
||||||
|
w = torch.randn(C, KSIZE, generator=g, device=DEV).to(torch.bfloat16)
|
||||||
|
x = torch.randn(len(blocks) * SQ, C, generator=g, device=DEV).to(torch.bfloat16)
|
||||||
|
outer = torch.randn_like(x)
|
||||||
|
raw, view = _state()
|
||||||
|
for b in blocks:
|
||||||
|
view[b, :, :STATE_LEN] = torch.randn(C, STATE_LEN, generator=g, device=DEV).to(
|
||||||
|
torch.bfloat16
|
||||||
|
)
|
||||||
|
return w, x, outer, raw, view
|
||||||
|
|
||||||
|
|
||||||
|
def _sequential(w, x, outer, view, block, first, count):
|
||||||
|
"""Stock decode, one token at a time, on a copy of the checkpoint block."""
|
||||||
|
ref = view.clone()
|
||||||
|
res = torch.zeros(count, C, dtype=torch.bfloat16, device=DEV)
|
||||||
|
for i in range(count):
|
||||||
|
t = first + i
|
||||||
|
ple_conv(
|
||||||
|
inputs=x[t : t + 1],
|
||||||
|
residual=res[i : i + 1],
|
||||||
|
conv_state=ref,
|
||||||
|
conv_weights=w,
|
||||||
|
state_indices=_i32([block]),
|
||||||
|
outer_residual=outer[t : t + 1],
|
||||||
|
mode="decode",
|
||||||
|
dilation=DIL,
|
||||||
|
)
|
||||||
|
return ref[block, :, :STATE_LEN].clone(), res
|
||||||
|
|
||||||
|
|
||||||
|
def _spec(w, x, outer, view, blocks):
|
||||||
|
res = torch.zeros(len(blocks) * SQ, C, dtype=torch.bfloat16, device=DEV)
|
||||||
|
ple_conv(
|
||||||
|
inputs=x,
|
||||||
|
residual=res,
|
||||||
|
conv_state=view,
|
||||||
|
conv_weights=w,
|
||||||
|
state_indices=_i32(blocks),
|
||||||
|
outer_residual=outer,
|
||||||
|
mode="spec",
|
||||||
|
dilation=DIL,
|
||||||
|
query_start_loc=_i32(list(range(0, len(blocks) * SQ + 1, SQ))),
|
||||||
|
num_accepted_tokens=_i32([1] * len(blocks)),
|
||||||
|
spec_query_len=SQ,
|
||||||
|
)
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("accepted", [1, 2, SQ])
|
||||||
|
@pytest.mark.parametrize("seed", [0, 1])
|
||||||
|
def test_ple_commit_matches_sequential(accepted, seed):
|
||||||
|
block = 3
|
||||||
|
w, x, outer, raw, view = _setup(seed, [block])
|
||||||
|
want_state, want_out = _sequential(w, x, outer, view, block, 0, accepted)
|
||||||
|
history = torch.cat([view[block, :, :STATE_LEN], x[:accepted].T], dim=1)
|
||||||
|
|
||||||
|
got_out = _spec(w, x, outer, view, [block])
|
||||||
|
ctx = _PleConvCommit([raw], spec_query_len=SQ, max_num_reqs=NB)
|
||||||
|
ctx.commit(_i32([accepted]), _i32([block]), _i32([0, SQ]))
|
||||||
|
|
||||||
|
got_state = view[block, :, :STATE_LEN]
|
||||||
|
assert torch.equal(got_state, want_state)
|
||||||
|
assert torch.equal(got_state, history[:, -STATE_LEN:])
|
||||||
|
assert torch.equal(got_out[:accepted], want_out)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("num_computed", "accepted", "crosses"),
|
||||||
|
[(6, 3, True), (6, 2, True), (1, 2, False), (7, SQ, True)],
|
||||||
|
)
|
||||||
|
def test_ple_commit_align_boundary(num_computed, accepted, crosses):
|
||||||
|
block_size, src, nxt = 8, 2, 5
|
||||||
|
w, x, outer, raw, view = _setup(7, [src])
|
||||||
|
boundary_len = block_size - num_computed
|
||||||
|
want_final, _ = _sequential(w, x, outer, view, src, 0, accepted)
|
||||||
|
want_boundary, _ = _sequential(w, x, outer, view, src, 0, min(boundary_len, SQ))
|
||||||
|
|
||||||
|
_spec(w, x, outer, view, [src])
|
||||||
|
ctx = _PleConvCommit([raw], spec_query_len=SQ, max_num_reqs=NB)
|
||||||
|
ctx.commit(
|
||||||
|
_i32([accepted]),
|
||||||
|
_i32([src]),
|
||||||
|
_i32([0, SQ]),
|
||||||
|
block_table=_i32([[src, nxt]]),
|
||||||
|
num_computed_tokens=_i32([num_computed]),
|
||||||
|
mamba_block_size=block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
final_block = nxt if num_computed + accepted > block_size else src
|
||||||
|
assert torch.equal(view[final_block, :, :STATE_LEN], want_final)
|
||||||
|
if crosses:
|
||||||
|
assert torch.equal(view[src, :, :STATE_LEN], want_boundary)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ple_commit_non_contiguous_request_rows():
|
||||||
|
blocks, accepted = [1, 6], [3, 1]
|
||||||
|
w, x, outer, raw, view = _setup(11, blocks)
|
||||||
|
want = [
|
||||||
|
_sequential(w, x, outer, view, b, i * SQ, a)[0]
|
||||||
|
for i, (b, a) in enumerate(zip(blocks, accepted))
|
||||||
|
]
|
||||||
|
|
||||||
|
_spec(w, x, outer, view, blocks)
|
||||||
|
# Spec rows 0 and 1 belong to requests 1 and 3 of a four-request batch.
|
||||||
|
counts = _i32([9, accepted[0], 9, accepted[1]])
|
||||||
|
ctx = _PleConvCommit([raw], spec_query_len=SQ, max_num_reqs=NB)
|
||||||
|
ctx.commit(
|
||||||
|
counts,
|
||||||
|
_i32(blocks),
|
||||||
|
_i32([0, SQ, 2 * SQ]),
|
||||||
|
request_indices=_i32([1, 3]),
|
||||||
|
)
|
||||||
|
|
||||||
|
for b, ref in zip(blocks, want):
|
||||||
|
assert torch.equal(view[b, :, :STATE_LEN], ref)
|
||||||
@@ -1,9 +1,12 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
import vllm.model_executor.warmup.qwen_triton_warmup as qwen_triton_warmup_module
|
||||||
from vllm.model_executor.warmup.qwen_triton_warmup import (
|
from vllm.model_executor.warmup.qwen_triton_warmup import (
|
||||||
_FLA_POST_CONV_WARMUP_LENGTHS,
|
_FLA_POST_CONV_WARMUP_LENGTHS,
|
||||||
_QwenGDNWarmupConfig,
|
_QwenGDNWarmupConfig,
|
||||||
@@ -53,3 +56,57 @@ def test_qwen_gdn_prefill_warmup_kernels_compile_on_gpu() -> None:
|
|||||||
_warm_fused_post_conv_kernel(device, config)
|
_warm_fused_post_conv_kernel(device, config)
|
||||||
assert _FLA_POST_CONV_WARMUP_LENGTHS == (1, 2, 16)
|
assert _FLA_POST_CONV_WARMUP_LENGTHS == (1, 2, 16)
|
||||||
torch.accelerator.synchronize(device)
|
torch.accelerator.synchronize(device)
|
||||||
|
|
||||||
|
|
||||||
|
def _warmup_runner() -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
device=torch.device("cpu"),
|
||||||
|
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||||
|
max_num_tokens=1,
|
||||||
|
is_pooling_model=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _warmup_model_config(model_type: str) -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
hf_text_config=SimpleNamespace(model_type=model_type),
|
||||||
|
hf_config=SimpleNamespace(model_type=model_type),
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("model_type", "dispatched"),
|
||||||
|
[
|
||||||
|
("qwen4_exp", True),
|
||||||
|
("qwen4_exp_text", True),
|
||||||
|
("qwen3_next", True),
|
||||||
|
# The drafter never reaches this warmup: the worker passes its own
|
||||||
|
# model_config, which is the target's.
|
||||||
|
("qwen4_exp_mtp", False),
|
||||||
|
("llama", False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_qwen_triton_warmup_dispatch_gate(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
model_type: str,
|
||||||
|
dispatched: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Which model types reach the Qwen Triton warmup at all.
|
||||||
|
|
||||||
|
Stops at the gate: the kernels themselves need a GPU and are covered by the
|
||||||
|
CUDA test above.
|
||||||
|
"""
|
||||||
|
reached: list[object] = []
|
||||||
|
|
||||||
|
def _record(static_forward_context: object) -> None:
|
||||||
|
reached.append(static_forward_context)
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(qwen_triton_warmup_module, "_qwen_gdn_warmup_config", _record)
|
||||||
|
|
||||||
|
qwen_triton_warmup_module.qwen_triton_warmup(
|
||||||
|
_warmup_runner(), _warmup_model_config(model_type)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert bool(reached) is dispatched
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ def test_kda_recoverssm_config_state_layout():
|
|||||||
cache_config=SimpleNamespace(
|
cache_config=SimpleNamespace(
|
||||||
mamba_cache_dtype="auto",
|
mamba_cache_dtype="auto",
|
||||||
mamba_ssm_cache_dtype="auto",
|
mamba_ssm_cache_dtype="auto",
|
||||||
use_kda_recoverssm=True,
|
use_recoverssm=True,
|
||||||
),
|
),
|
||||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||||
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ def _make_builder(
|
|||||||
)
|
)
|
||||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||||
vllm_config.cache_config.use_replayssm = use_recoverssm
|
vllm_config.cache_config.use_replayssm = use_recoverssm
|
||||||
vllm_config.cache_config.use_kda_recoverssm = use_recoverssm
|
vllm_config.cache_config.use_recoverssm = use_recoverssm
|
||||||
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
||||||
builder = builder_cls(
|
builder = builder_cls(
|
||||||
kv_cache_spec=MambaSpec(
|
kv_cache_spec=MambaSpec(
|
||||||
@@ -162,7 +162,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
|||||||
cache_config=SimpleNamespace(
|
cache_config=SimpleNamespace(
|
||||||
mamba_cache_dtype="auto",
|
mamba_cache_dtype="auto",
|
||||||
mamba_ssm_cache_dtype="auto",
|
mamba_ssm_cache_dtype="auto",
|
||||||
use_kda_recoverssm=True,
|
use_recoverssm=True,
|
||||||
),
|
),
|
||||||
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
parallel_config=SimpleNamespace(tensor_parallel_size=1),
|
||||||
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
speculative_config=SimpleNamespace(num_speculative_tokens=2),
|
||||||
@@ -195,7 +195,7 @@ def test_kda_recoverssm_startup_metadata_flow_without_model(monkeypatch):
|
|||||||
),
|
),
|
||||||
cache_config=SimpleNamespace(
|
cache_config=SimpleNamespace(
|
||||||
mamba_cache_mode="align",
|
mamba_cache_mode="align",
|
||||||
use_kda_recoverssm=True,
|
use_recoverssm=True,
|
||||||
prefix_match_unit=None,
|
prefix_match_unit=None,
|
||||||
),
|
),
|
||||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Align-mode RecoverSSM state columns at recurrent block boundaries.
|
||||||
|
|
||||||
|
A speculative window whose accepted tokens end exactly on a recurrent block
|
||||||
|
boundary must keep its state in the block holding the last committed token.
|
||||||
|
The following block can still be unallocated, so selecting it would drop the
|
||||||
|
commit while the bookkeeping recorded the state as advanced.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.models.kimi_k3.nvidia.ops.recoverssm import _prepare_commit_plan_kernel
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||||
|
from vllm.v1.worker.gpu.model_states.recoverssm import (
|
||||||
|
_postprocess_recoverssm_align_kernel,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.skipif(
|
||||||
|
not current_platform.is_cuda_alike(),
|
||||||
|
reason="The RecoverSSM kernels require a CUDA-alike device.",
|
||||||
|
)
|
||||||
|
|
||||||
|
BLOCK_SIZE = 2048
|
||||||
|
SPEC_QUERY_LEN = 4
|
||||||
|
BLOCKS = [100, 101, 102, 103, 104, 105]
|
||||||
|
|
||||||
|
# (computed before the step, committed tokens, allocated block columns,
|
||||||
|
# final column, boundary column or None)
|
||||||
|
CASES = [
|
||||||
|
# 8,188 + 4 ends exactly on the 8,192 boundary; column 4 is unallocated.
|
||||||
|
(8188, 4, 4, 3, 3),
|
||||||
|
# The window crosses the boundary: state moves on, checkpoint stays behind.
|
||||||
|
(8190, 4, 5, 4, 3),
|
||||||
|
# The window stays inside column 3.
|
||||||
|
(8180, 4, 4, 3, None),
|
||||||
|
# The first token after a boundary belongs to the new column.
|
||||||
|
(8192, 1, 5, 4, None),
|
||||||
|
# Single-token commit ending on the first boundary.
|
||||||
|
(2047, 1, 1, 0, 0),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
|
||||||
|
)
|
||||||
|
def test_commit_plan_keeps_boundary_state_in_last_computed_block(
|
||||||
|
num_computed, commit_len, allocated, final_col, boundary_col
|
||||||
|
):
|
||||||
|
device = current_platform.device_type
|
||||||
|
columns = [
|
||||||
|
block if col < allocated else NULL_BLOCK_ID for col, block in enumerate(BLOCKS)
|
||||||
|
]
|
||||||
|
block_table = torch.tensor([columns], dtype=torch.int32, device=device)
|
||||||
|
num_accepted = torch.tensor([commit_len], dtype=torch.int32, device=device)
|
||||||
|
state_indices = torch.tensor(
|
||||||
|
[BLOCKS[(num_computed - 1) // BLOCK_SIZE]], dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
query_start_loc = torch.tensor(
|
||||||
|
[0, SPEC_QUERY_LEN], dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
|
||||||
|
commit_lens, final_indices, boundary_indices, recovery_lens = (
|
||||||
|
torch.full((1,), -7, dtype=torch.int32, device=device) for _ in range(4)
|
||||||
|
)
|
||||||
|
|
||||||
|
_prepare_commit_plan_kernel[(1,)](
|
||||||
|
num_accepted,
|
||||||
|
None,
|
||||||
|
state_indices,
|
||||||
|
query_start_loc,
|
||||||
|
block_table,
|
||||||
|
computed,
|
||||||
|
commit_lens,
|
||||||
|
final_indices,
|
||||||
|
boundary_indices,
|
||||||
|
recovery_lens,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
BLOCK_SIZE,
|
||||||
|
block_table.shape[1],
|
||||||
|
num_accepted.stride(0),
|
||||||
|
0,
|
||||||
|
state_indices.stride(0),
|
||||||
|
query_start_loc.stride(0),
|
||||||
|
block_table.stride(0),
|
||||||
|
block_table.stride(1),
|
||||||
|
computed.stride(0),
|
||||||
|
SPEC_QUERY_LEN=SPEC_QUERY_LEN,
|
||||||
|
num_warps=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert commit_lens.item() == commit_len
|
||||||
|
assert final_indices.item() == BLOCKS[final_col]
|
||||||
|
expected_boundary = NULL_BLOCK_ID if boundary_col is None else BLOCKS[boundary_col]
|
||||||
|
assert boundary_indices.item() == expected_boundary
|
||||||
|
if boundary_col is not None:
|
||||||
|
next_boundary = (num_computed // BLOCK_SIZE + 1) * BLOCK_SIZE
|
||||||
|
assert recovery_lens.item() == next_boundary - num_computed
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
|
||||||
|
)
|
||||||
|
def test_postprocess_records_the_committed_state_column(
|
||||||
|
num_computed, commit_len, allocated, final_col, boundary_col
|
||||||
|
):
|
||||||
|
device = current_platform.device_type
|
||||||
|
idx_mapping = torch.tensor([2], dtype=torch.int32, device=device)
|
||||||
|
num_sampled = torch.tensor([commit_len], dtype=torch.int32, device=device)
|
||||||
|
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
|
||||||
|
state_idx = torch.full((4,), -7, dtype=torch.int32, device=device)
|
||||||
|
num_accepted = torch.full((4,), -7, dtype=torch.int32, device=device)
|
||||||
|
|
||||||
|
_postprocess_recoverssm_align_kernel[(1,)](
|
||||||
|
idx_mapping,
|
||||||
|
num_sampled,
|
||||||
|
None,
|
||||||
|
computed,
|
||||||
|
state_idx,
|
||||||
|
num_accepted,
|
||||||
|
MAMBA_BLOCK_SIZE=BLOCK_SIZE,
|
||||||
|
BLOCK_TABLE_WIDTH=len(BLOCKS),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert state_idx[2].item() == final_col
|
||||||
|
assert num_accepted[2].item() == 1
|
||||||
|
assert state_idx[[0, 1, 3]].tolist() == [-7, -7, -7]
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from contextlib import contextmanager
|
||||||
from importlib import import_module
|
from importlib import import_module
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
@@ -14,10 +17,12 @@ from vllm.model_executor.models.config import (
|
|||||||
Qwen3_5ForConditionalGenerationConfig,
|
Qwen3_5ForConditionalGenerationConfig,
|
||||||
Qwen4ExpForConditionalGenerationConfig,
|
Qwen4ExpForConditionalGenerationConfig,
|
||||||
)
|
)
|
||||||
|
from vllm.models.qwen4_exp.common.mtp import make_mtp_hidden_buffer
|
||||||
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
|
||||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||||
|
|
||||||
from ...utils import spawn_new_process_for_each_test
|
from ...utils import spawn_new_process_for_each_test
|
||||||
|
from .test_ple import _make_file_gather_embedding
|
||||||
|
|
||||||
|
|
||||||
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
def _text_config(**kwargs) -> Qwen4ExpTextConfig:
|
||||||
@@ -175,6 +180,7 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
|||||||
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
||||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||||
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
||||||
|
model_state.file_gather_modules = []
|
||||||
|
|
||||||
input_batch = SimpleNamespace(
|
input_batch = SimpleNamespace(
|
||||||
num_reqs=2,
|
num_reqs=2,
|
||||||
@@ -224,6 +230,39 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
|||||||
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
|
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
|
||||||
|
|
||||||
|
|
||||||
|
def test_qwen4_exp_model_state_stages_file_gather_rows(monkeypatch, tmp_path) -> None:
|
||||||
|
"""Rows are staged from the same PLE inputs the forward receives."""
|
||||||
|
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
||||||
|
model_state = object.__new__(Qwen4ExpModelState)
|
||||||
|
model_state.uses_ngram_embedding = True
|
||||||
|
model_state.ngram_eos_token_id = 0
|
||||||
|
model_state.ngram_context = torch.empty((4, 1), dtype=torch.int32)
|
||||||
|
model_state.ngram_context_offsets = torch.arange(-1, 0, dtype=torch.int64)
|
||||||
|
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||||
|
model_state.file_gather_modules = [module]
|
||||||
|
input_batch = SimpleNamespace(
|
||||||
|
num_reqs=2,
|
||||||
|
num_reqs_after_padding=2,
|
||||||
|
idx_mapping=torch.tensor([0, 1]),
|
||||||
|
query_start_loc=torch.tensor([0, 4, 6], dtype=torch.int32),
|
||||||
|
input_ids=torch.tensor([5, 9, 3, 7, 2, 11]),
|
||||||
|
)
|
||||||
|
req_states = SimpleNamespace(
|
||||||
|
num_computed_tokens=SimpleNamespace(gpu=torch.tensor([2, 1])),
|
||||||
|
all_token_ids=SimpleNamespace(gpu=torch.tensor([[1, 8, 6], [4, 3, 3]])),
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||||
|
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||||
|
|
||||||
|
ids = module.compute_ngram_ids(
|
||||||
|
input_batch.input_ids,
|
||||||
|
model_inputs["query_start_loc"],
|
||||||
|
model_inputs["ngram_context"],
|
||||||
|
)
|
||||||
|
assert torch.equal(module.ngram_embedding._staging[:6], table[ids])
|
||||||
|
|
||||||
|
|
||||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||||
model_state = object.__new__(Qwen4ExpModelState)
|
model_state = object.__new__(Qwen4ExpModelState)
|
||||||
model_state.uses_ngram_embedding = True
|
model_state.uses_ngram_embedding = True
|
||||||
@@ -246,3 +285,107 @@ def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
|||||||
)
|
)
|
||||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _captured_transformers_warnings() -> Iterator[list[str]]:
|
||||||
|
"""Records what transformers logs, without relying on propagation.
|
||||||
|
|
||||||
|
transformers keeps its own logger hierarchy, and the rope validator reaches
|
||||||
|
it through ``warning_once``, so a root-level capture can miss the record
|
||||||
|
entirely.
|
||||||
|
"""
|
||||||
|
records: list[str] = []
|
||||||
|
|
||||||
|
class _Collect(logging.Handler):
|
||||||
|
def emit(self, record: logging.LogRecord) -> None:
|
||||||
|
records.append(record.getMessage())
|
||||||
|
|
||||||
|
handler = _Collect()
|
||||||
|
logger = logging.getLogger("transformers")
|
||||||
|
previous_level = logger.level
|
||||||
|
logger.addHandler(handler)
|
||||||
|
logger.setLevel(logging.WARNING)
|
||||||
|
try:
|
||||||
|
yield records
|
||||||
|
finally:
|
||||||
|
logger.removeHandler(handler)
|
||||||
|
logger.setLevel(previous_level)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("wrapped_config", [False, True])
|
||||||
|
def test_rope_validation_accepts_mrope_keys_and_still_warns_for_unknown_keys(
|
||||||
|
wrapped_config: bool,
|
||||||
|
) -> None:
|
||||||
|
"""M-RoPE is driven through two rope_parameters keys transformers does not
|
||||||
|
know for rope_type="default". Declaring them must not silence the
|
||||||
|
unrecognized-key warning for every other key."""
|
||||||
|
|
||||||
|
def build(rope_parameters: dict) -> None:
|
||||||
|
text_config = _text_config(
|
||||||
|
rope_parameters=dict(rope_parameters), rope_theta=10_000.0
|
||||||
|
)
|
||||||
|
if wrapped_config:
|
||||||
|
Qwen4ExpConfig(
|
||||||
|
architectures=["Qwen4ExpForConditionalGeneration"],
|
||||||
|
text_config=text_config.to_dict(),
|
||||||
|
)
|
||||||
|
|
||||||
|
with _captured_transformers_warnings() as records:
|
||||||
|
build(
|
||||||
|
{
|
||||||
|
"rope_type": "default",
|
||||||
|
"mrope_section": [16, 24, 24],
|
||||||
|
"mrope_interleaved": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert not [line for line in records if "Unrecognized keys" in line], records
|
||||||
|
|
||||||
|
# warning_once dedupes on the message, so each parametrization needs a key
|
||||||
|
# no earlier run has reported.
|
||||||
|
unknown_key = f"not_a_rope_key_{int(wrapped_config)}"
|
||||||
|
with _captured_transformers_warnings() as records:
|
||||||
|
build({"rope_type": "default", unknown_key: 1})
|
||||||
|
assert [line for line in records if unknown_key in line], records
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("is_last_rank", [True, False])
|
||||||
|
def test_mtp_hidden_buffer_is_allocated_on_the_configured_device(
|
||||||
|
is_last_rank: bool,
|
||||||
|
) -> None:
|
||||||
|
"""The drafter's hidden buffer belongs on the device the config names, not
|
||||||
|
on whichever device happens to be current."""
|
||||||
|
vllm_config = SimpleNamespace(
|
||||||
|
speculative_config=SimpleNamespace(method="mtp"),
|
||||||
|
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||||
|
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||||
|
# "meta" is never the ambient default, so an implicit allocation lands
|
||||||
|
# somewhere else and this test sees it.
|
||||||
|
device_config=SimpleNamespace(device=torch.device("meta")),
|
||||||
|
)
|
||||||
|
config = SimpleNamespace(hc_count=2, hidden_size=4)
|
||||||
|
|
||||||
|
buffer = make_mtp_hidden_buffer(vllm_config, config, is_last_rank=is_last_rank)
|
||||||
|
|
||||||
|
if not is_last_rank:
|
||||||
|
assert buffer is None
|
||||||
|
return
|
||||||
|
assert buffer is not None
|
||||||
|
assert buffer.device.type == "meta"
|
||||||
|
assert buffer.shape == (8, 8)
|
||||||
|
assert buffer.dtype is torch.bfloat16
|
||||||
|
|
||||||
|
|
||||||
|
def test_mtp_hidden_buffer_is_absent_without_mtp_speculation() -> None:
|
||||||
|
vllm_config = SimpleNamespace(
|
||||||
|
speculative_config=SimpleNamespace(method="eagle"),
|
||||||
|
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||||
|
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||||
|
device_config=SimpleNamespace(device=torch.device("meta")),
|
||||||
|
)
|
||||||
|
config = SimpleNamespace(hc_count=2, hidden_size=4)
|
||||||
|
|
||||||
|
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
|
||||||
|
|
||||||
|
vllm_config.speculative_config = None
|
||||||
|
assert make_mtp_hidden_buffer(vllm_config, config, is_last_rank=True) is None
|
||||||
|
|||||||
@@ -1,13 +1,18 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import mmap
|
||||||
|
import os
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from itertools import accumulate
|
from itertools import accumulate
|
||||||
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
from safetensors import safe_open
|
||||||
|
from safetensors.torch import load, load_file, save_file
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from torch.nn import functional as F
|
from torch.nn import functional as F
|
||||||
from transformers import Qwen4ExpTextConfig
|
from transformers import Qwen4ExpTextConfig
|
||||||
@@ -15,6 +20,7 @@ from transformers import Qwen4ExpTextConfig
|
|||||||
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
|
||||||
import vllm.model_executor.parameter as parameter_module
|
import vllm.model_executor.parameter as parameter_module
|
||||||
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
|
import vllm.models.qwen4_exp.common.ngram_embedding as ngram_embedding_module
|
||||||
|
import vllm.models.qwen4_exp.nvidia.ngram_embedding as nvidia_ngram_embedding_module
|
||||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||||
from vllm.config import SpeculativeConfig, VllmConfig
|
from vllm.config import SpeculativeConfig, VllmConfig
|
||||||
from vllm.config.compilation import CUDAGraphMode
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
@@ -39,6 +45,7 @@ from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
|
|||||||
Qwen4ExpNGramEmbedding,
|
Qwen4ExpNGramEmbedding,
|
||||||
Qwen4ExpPLEDeviceEmbedding,
|
Qwen4ExpPLEDeviceEmbedding,
|
||||||
Qwen4ExpPLEEmbeddingMethod,
|
Qwen4ExpPLEEmbeddingMethod,
|
||||||
|
Qwen4ExpPLEFileGatherEmbedding,
|
||||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||||
Qwen4ExpPLEPinnedHostEmbedding,
|
Qwen4ExpPLEPinnedHostEmbedding,
|
||||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||||
@@ -860,6 +867,350 @@ def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_ple_construction(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
*,
|
||||||
|
engram_config: SimpleNamespace | None = None,
|
||||||
|
load_format: str = "auto",
|
||||||
|
) -> None:
|
||||||
|
_mock_etp_group(monkeypatch)
|
||||||
|
for module in (embedding_module, parameter_module):
|
||||||
|
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: 0)
|
||||||
|
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||||
|
vllm_config = SimpleNamespace(
|
||||||
|
load_config=SimpleNamespace(load_format=load_format),
|
||||||
|
engram_config=engram_config,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
nvidia_ngram_embedding_module, "get_current_vllm_config", lambda: vllm_config
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_file_gather_embedding(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
*,
|
||||||
|
fp8: bool = False,
|
||||||
|
load_format: str = "auto",
|
||||||
|
bind: bool = False,
|
||||||
|
) -> tuple[Qwen4ExpNGramEmbedding, torch.Tensor]:
|
||||||
|
"""Build a two-head, 12-token host-file-gather embedding and its saved table.
|
||||||
|
|
||||||
|
The 21-row table is saved to ``tmp_path / "model.safetensors"`` as four
|
||||||
|
checkpoint shards of stride 6, so the last shard holds 3 rows.
|
||||||
|
"""
|
||||||
|
_patch_ple_construction(monkeypatch, load_format=load_format)
|
||||||
|
method = (
|
||||||
|
Qwen4ExpPLEFp8EmbeddingMethod()
|
||||||
|
if fp8
|
||||||
|
else Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||||
|
)
|
||||||
|
module = Qwen4ExpNGramEmbedding.__new__(Qwen4ExpNGramEmbedding)
|
||||||
|
nn.Module.__init__(module)
|
||||||
|
module.split_ngram_parts = 4
|
||||||
|
module.eos_token_id = 0
|
||||||
|
module.ngram_size = 2
|
||||||
|
module.heads_per_ngram = 2
|
||||||
|
module.register_buffer("layer_multipliers", torch.tensor([3, 5]))
|
||||||
|
module.register_buffer("ngram_heads_vocab_sizes", torch.tensor([13, 7]))
|
||||||
|
module.register_buffer("ngram_heads_offsets", torch.tensor([0, 13]))
|
||||||
|
module.ngram_embedding = Qwen4ExpPLEFileGatherEmbedding(
|
||||||
|
21,
|
||||||
|
2,
|
||||||
|
params_dtype=torch.bfloat16,
|
||||||
|
padding_size=1,
|
||||||
|
prefix="test.ple_embedding",
|
||||||
|
embedding_method=method,
|
||||||
|
num_ngram_heads=2,
|
||||||
|
max_total_tokens=12,
|
||||||
|
)
|
||||||
|
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
|
||||||
|
table = torch.arange(42, dtype=torch.float32).reshape(21, 2).to(dtype)
|
||||||
|
checkpoint = {
|
||||||
|
f"ngram_embedding.shard_{index}.weight": shard.clone()
|
||||||
|
for index, shard in enumerate(table.split(6))
|
||||||
|
}
|
||||||
|
if fp8:
|
||||||
|
checkpoint["ngram_embedding.weight_scale"] = torch.tensor([0.25]).bfloat16()
|
||||||
|
save_file(checkpoint, tmp_path / "model.safetensors")
|
||||||
|
if bind:
|
||||||
|
_load_file_shards(module, tmp_path / "model.safetensors")
|
||||||
|
module.ngram_embedding.bind_file_shards()
|
||||||
|
return module, table
|
||||||
|
|
||||||
|
|
||||||
|
def _load_file_shards(module: Qwen4ExpNGramEmbedding, path: Path) -> set[str]:
|
||||||
|
"""Load ``path`` through ``load_weights`` as the lazy safetensors iterator does."""
|
||||||
|
with safe_open(path, framework="pt") as checkpoint:
|
||||||
|
return module.load_weights(
|
||||||
|
(name, checkpoint.get_tensor(name))
|
||||||
|
for name in checkpoint.keys() # noqa: SIM118
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _dev_zero_shards() -> list[tuple[str, torch.Tensor]]:
|
||||||
|
"""Return shards in a ``/dev/zero`` mapping, which is how pinned memory maps."""
|
||||||
|
with open("/dev/zero", "r+b") as dev_zero:
|
||||||
|
mapping = mmap.mmap(dev_zero.fileno(), mmap.PAGESIZE)
|
||||||
|
table = torch.frombuffer(mapping, dtype=torch.uint8)[:84].view(torch.bfloat16)
|
||||||
|
return [
|
||||||
|
(f"ngram_embedding.shard_{index}.weight", shard)
|
||||||
|
for index, shard in enumerate(table.reshape(21, 2).split(6))
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("engram_config", "backend", "rows"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
SimpleNamespace(host_file_gather=True, cpu_offload=True),
|
||||||
|
Qwen4ExpPLEFileGatherEmbedding,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
SimpleNamespace(host_file_gather=False, cpu_offload=False),
|
||||||
|
Qwen4ExpPLEDeviceEmbedding,
|
||||||
|
24,
|
||||||
|
),
|
||||||
|
(None, Qwen4ExpPLEDeviceEmbedding, 24),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_ngram_embedding_selects_backend_from_engram_config(
|
||||||
|
monkeypatch, engram_config, backend, rows
|
||||||
|
) -> None:
|
||||||
|
"""File gather wins with no rows; ``cpu_offload=False`` keeps rows on device."""
|
||||||
|
_patch_ple_construction(monkeypatch, engram_config=engram_config)
|
||||||
|
config = SimpleNamespace(
|
||||||
|
ngram_size=2,
|
||||||
|
heads_per_ngram=2,
|
||||||
|
eos_token_id=0,
|
||||||
|
vocab_size=64,
|
||||||
|
ngram_vocab_size_base=8,
|
||||||
|
make_ngram_vocab_size_divisible_by=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
module = Qwen4ExpNGramEmbedding(config, 4, 0, 4, data_parallel_rank=0, prefix="t")
|
||||||
|
|
||||||
|
assert tuple(module.ngram_embedding.weight.shape) == (rows, 2)
|
||||||
|
assert isinstance(module.ngram_embedding, backend)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ngram_embedding_default_cpu_offload_selects_pinned_backend(
|
||||||
|
monkeypatch,
|
||||||
|
) -> None:
|
||||||
|
"""Default ``cpu_offload=True`` selects pinned storage and requires UVA."""
|
||||||
|
_patch_ple_construction(
|
||||||
|
monkeypatch,
|
||||||
|
engram_config=SimpleNamespace(host_file_gather=False, cpu_offload=True),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(ngram_embedding_module, "is_uva_available", lambda: False)
|
||||||
|
config = SimpleNamespace(
|
||||||
|
ngram_size=2,
|
||||||
|
heads_per_ngram=2,
|
||||||
|
eos_token_id=0,
|
||||||
|
vocab_size=64,
|
||||||
|
ngram_vocab_size_base=8,
|
||||||
|
make_ngram_vocab_size_divisible_by=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="CPU offload requires UVA"):
|
||||||
|
Qwen4ExpNGramEmbedding(config, 4, 0, 4, data_parallel_rank=0, prefix="t")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("read_shards", "error"),
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
lambda path: [("ngram_embedding.shard_0.weight", torch.zeros(6, 2))],
|
||||||
|
"dtype torch.float32 must match the embedding dtype torch.bfloat16",
|
||||||
|
id="wrong_dtype",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
lambda path: [*load_file(path).items()] * 2,
|
||||||
|
"Duplicate PLE embedding shard 0",
|
||||||
|
id="duplicate_shard",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
lambda path: [s for s in load_file(path).items() if "_2." not in s[0]],
|
||||||
|
"PLE shards cover 15 of 21 rows",
|
||||||
|
id="missing_shard",
|
||||||
|
),
|
||||||
|
pytest.param(lambda path: [], "PLE shards cover 0 of 21 rows", id="no_shards"),
|
||||||
|
pytest.param(
|
||||||
|
lambda path: [(n, t.clone()) for n, t in load_file(path).items()],
|
||||||
|
"not a file-backed view",
|
||||||
|
id="anonymous",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
lambda path: load(path.read_bytes()).items(),
|
||||||
|
"not a file-backed view",
|
||||||
|
id="heap",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
lambda path: _dev_zero_shards(), "not a file-backed view", id="dev_zero"
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_file_gather_rejects_shards_that_cannot_serve_every_row_from_a_file(
|
||||||
|
monkeypatch, tmp_path, read_shards, error
|
||||||
|
) -> None:
|
||||||
|
"""A real load must fail instead of serving zeros or a copied table."""
|
||||||
|
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match=error):
|
||||||
|
module.load_weights(read_shards(tmp_path / "model.safetensors"))
|
||||||
|
module.ngram_embedding.bind_file_shards()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fp8", [False, True])
|
||||||
|
def test_file_gather_binds_zero_copy_shards_through_loader_hook(
|
||||||
|
monkeypatch, tmp_path, fp8
|
||||||
|
) -> None:
|
||||||
|
"""Fails if safetensors stops returning zero-copy views of the file."""
|
||||||
|
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, fp8=fp8)
|
||||||
|
embedding = module.ngram_embedding
|
||||||
|
|
||||||
|
loaded = _load_file_shards(module, tmp_path / "model.safetensors")
|
||||||
|
embedding.quant_method.process_weights_after_loading(embedding)
|
||||||
|
embedding.quant_method.process_weights_after_loading(embedding)
|
||||||
|
|
||||||
|
assert loaded == {"ngram_embedding.weight"} | (
|
||||||
|
{"ngram_embedding.weight_scale"} if fp8 else set()
|
||||||
|
)
|
||||||
|
assert embedding._bound
|
||||||
|
stored = torch.cat([embedding._shards[index] for index in range(4)])
|
||||||
|
assert torch.equal(stored, table.view(torch.uint8))
|
||||||
|
|
||||||
|
|
||||||
|
def test_file_gather_binds_shards_opened_through_a_symlink(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
"""HF cache snapshots link to suffix-less blobs, which is the mapped path."""
|
||||||
|
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||||
|
blob = (tmp_path / "model.safetensors").rename(tmp_path / "blob")
|
||||||
|
snapshot = tmp_path / "snapshot.safetensors"
|
||||||
|
snapshot.symlink_to(blob)
|
||||||
|
_load_file_shards(module, snapshot)
|
||||||
|
|
||||||
|
module.ngram_embedding.bind_file_shards()
|
||||||
|
|
||||||
|
assert module.ngram_embedding._bound
|
||||||
|
|
||||||
|
|
||||||
|
def test_file_gather_rejects_reload_and_stays_poisoned(monkeypatch, tmp_path) -> None:
|
||||||
|
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="does not support weight reload"):
|
||||||
|
module.load_weights([("ngram_embedding.shard_0.weight", table[:6])])
|
||||||
|
with pytest.raises(RuntimeError, match="unbound or was reloaded"):
|
||||||
|
module.ngram_embedding.stage_rows(1)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("bad_id", "error"),
|
||||||
|
[(21, IndexError), (24, IndexError), (-1, RuntimeError)],
|
||||||
|
ids=["past_last_shard_rows", "past_last_shard", "negative"],
|
||||||
|
)
|
||||||
|
def test_file_gather_out_of_range_id_keeps_previous_staged_rows(
|
||||||
|
monkeypatch, tmp_path, bad_id, error
|
||||||
|
) -> None:
|
||||||
|
"""Out-of-range ids fail before the H2D copy, so no wrong row is served."""
|
||||||
|
module, table = _make_file_gather_embedding(monkeypatch, tmp_path, bind=True)
|
||||||
|
embedding = module.ngram_embedding
|
||||||
|
ids = torch.tensor(
|
||||||
|
[
|
||||||
|
[20, 0],
|
||||||
|
[19, 1],
|
||||||
|
[18, 2],
|
||||||
|
[17, 3],
|
||||||
|
[16, 4],
|
||||||
|
[15, 5],
|
||||||
|
[14, 6],
|
||||||
|
[13, 7],
|
||||||
|
[12, 8],
|
||||||
|
[11, 9],
|
||||||
|
[10, 10],
|
||||||
|
]
|
||||||
|
)
|
||||||
|
embedding._host_ids[:11] = ids
|
||||||
|
embedding.stage_rows(11)
|
||||||
|
embedding._host_ids[5, 1] = bad_id
|
||||||
|
|
||||||
|
with pytest.raises(error):
|
||||||
|
embedding.stage_rows(11)
|
||||||
|
|
||||||
|
assert torch.equal(embedding._staging[:11], table[ids])
|
||||||
|
|
||||||
|
|
||||||
|
def test_file_gather_reads_ahead_the_coalesced_pages_of_staged_rows(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
"""Rows start 34 bytes before page 2 of a file mapped from page 1.
|
||||||
|
|
||||||
|
Rows 0-7 are in page 1, row 8 straddles pages 1 and 2, and the rows of shard 1
|
||||||
|
arrive unsorted.
|
||||||
|
"""
|
||||||
|
module, table = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||||
|
embedding, page = module.ngram_embedding, mmap.PAGESIZE
|
||||||
|
start = 2 * page - 34
|
||||||
|
path = tmp_path / "table.bin"
|
||||||
|
path.write_bytes(bytes(start) + table.view(torch.uint8).numpy().tobytes())
|
||||||
|
with open(path, "rb") as file:
|
||||||
|
mapping = mmap.mmap(file.fileno(), 0, offset=page, access=mmap.ACCESS_COPY)
|
||||||
|
rows = torch.frombuffer(mapping, dtype=torch.uint8)[start - page :]
|
||||||
|
module.load_weights(
|
||||||
|
(f"ngram_embedding.shard_{index}.weight", shard)
|
||||||
|
for index, shard in enumerate(rows.view(table.dtype).reshape(21, 2).split(6))
|
||||||
|
)
|
||||||
|
embedding.bind_file_shards()
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
nvidia_ngram_embedding_module.os,
|
||||||
|
"posix_fadvise",
|
||||||
|
lambda fd, *args: calls.append((os.fstat(fd).st_ino, *args)),
|
||||||
|
)
|
||||||
|
ids = torch.tensor([[0, 1], [8, 7]])
|
||||||
|
embedding._host_ids[:2] = ids
|
||||||
|
|
||||||
|
embedding.stage_rows(2)
|
||||||
|
|
||||||
|
inode, willneed = path.stat().st_ino, os.POSIX_FADV_WILLNEED
|
||||||
|
assert calls == [(inode, page, page, willneed), (inode, page, 2 * page, willneed)]
|
||||||
|
assert torch.equal(embedding._staging[:2], table[ids])
|
||||||
|
|
||||||
|
|
||||||
|
def test_file_gather_dummy_load_stages_zeros(monkeypatch, tmp_path) -> None:
|
||||||
|
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path, load_format="dummy")
|
||||||
|
embedding = module.ngram_embedding
|
||||||
|
embedding.quant_method.process_weights_after_loading(embedding)
|
||||||
|
embedding._host_rows.view(torch.uint8).fill_(0xFF)
|
||||||
|
embedding._staging.view(torch.uint8).fill_(0xFF)
|
||||||
|
|
||||||
|
embedding.stage_rows(4)
|
||||||
|
|
||||||
|
staged = embedding._staging.view(torch.uint8)
|
||||||
|
assert torch.all(staged[:4] == 0)
|
||||||
|
assert torch.all(staged[4:] == 0xFF)
|
||||||
|
|
||||||
|
|
||||||
|
def test_file_gather_forward_reads_staging_without_host_work(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
"""The forward must be a fixed-address read so FULL CUDA graphs replay it."""
|
||||||
|
module, _ = _make_file_gather_embedding(monkeypatch, tmp_path)
|
||||||
|
staging = module.ngram_embedding._staging
|
||||||
|
staging.view(torch.uint8).fill_(0xFF)
|
||||||
|
step = (torch.arange(6), torch.tensor([0, 6]), torch.tensor([[1]]))
|
||||||
|
hidden_states = torch.zeros(6, 8, dtype=torch.bfloat16)
|
||||||
|
|
||||||
|
assert module.start_prefetch(hidden_states, *step) is None
|
||||||
|
output = module(hidden_states, *step)
|
||||||
|
|
||||||
|
assert torch.all(staging.view(torch.uint8) == 0xFF)
|
||||||
|
assert output.data_ptr() == staging.data_ptr()
|
||||||
|
assert output.shape == (6, 4)
|
||||||
|
|
||||||
|
|
||||||
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
def test_dilated_ple_spec_state_rolls_back_before_next_forward() -> None:
|
||||||
conv_state_len = 6
|
conv_state_len = 6
|
||||||
dilation = 2
|
dilation = 2
|
||||||
|
|||||||
@@ -488,6 +488,67 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
|||||||
|
|
||||||
|
|
||||||
@requires_qsa_kernels
|
@requires_qsa_kernels
|
||||||
|
@requires_qsa_kernels
|
||||||
|
@pytest.mark.parametrize("layout", ["plain", "compressed", "circular"])
|
||||||
|
def test_qsa_draft_decode_metadata_update_matches_rebuild(layout: str) -> None:
|
||||||
|
"""A fused draft step's in-place update equals a fresh build at seq_lens + 1."""
|
||||||
|
device = torch.device("cuda")
|
||||||
|
compress_ratio = 4 if layout != "plain" else 1
|
||||||
|
|
||||||
|
def make_builder() -> QSAMetadataBuilder:
|
||||||
|
builder = QSAMetadataBuilder.__new__(QSAMetadataBuilder)
|
||||||
|
builder.compress_ratio = compress_ratio
|
||||||
|
builder.reorder_batch_threshold = 1
|
||||||
|
builder.is_circular_buffer = layout == "circular"
|
||||||
|
builder.kv_cache_spec = SimpleNamespace(block_size=4)
|
||||||
|
builder.storage_block_size = 16 if layout == "compressed" else 4
|
||||||
|
builder.token_to_req_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||||
|
builder.slot_mapping_buffer = torch.empty(8, dtype=torch.int64, device=device)
|
||||||
|
builder.logical_positions_buffer = torch.empty(
|
||||||
|
8, dtype=torch.int64, device=device
|
||||||
|
)
|
||||||
|
builder.visible_blocks_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||||
|
builder.request_capacity = 8
|
||||||
|
builder.k_work_metadata_buffer = torch.empty(
|
||||||
|
8 if layout == "compressed" else 0, 2, dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
return builder
|
||||||
|
|
||||||
|
query_start_loc = torch.tensor([0, 1, 2, 3], dtype=torch.int32, device=device)
|
||||||
|
token_to_req = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
|
||||||
|
|
||||||
|
def make_common(seq_lens: torch.Tensor) -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
num_actual_tokens=3,
|
||||||
|
num_reqs=3,
|
||||||
|
max_query_len=1,
|
||||||
|
max_seq_len=int(seq_lens.max()),
|
||||||
|
query_start_loc=query_start_loc,
|
||||||
|
query_start_loc_cpu=query_start_loc.cpu(),
|
||||||
|
seq_lens=seq_lens,
|
||||||
|
slot_mapping=torch.tensor([5, 9, 20], dtype=torch.int64, device=device),
|
||||||
|
block_table_tensor=torch.tensor(
|
||||||
|
[[3, 1], [0, 4], [2, 5]], dtype=torch.int32, device=device
|
||||||
|
),
|
||||||
|
token_to_req_indices=lambda buffer: buffer.copy_(token_to_req),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 7 -> 8 crosses a compression boundary, so visible_blocks must change.
|
||||||
|
seq_lens = torch.tensor([7, 11, 16], dtype=torch.int32, device=device)
|
||||||
|
builder = make_builder()
|
||||||
|
metadata = builder.build(0, make_common(seq_lens))
|
||||||
|
positions_before = metadata.logical_positions.clone()
|
||||||
|
|
||||||
|
seq_lens.add_(1)
|
||||||
|
builder.update_draft_decode_metadata(metadata)
|
||||||
|
expected = make_builder().build(0, make_common(seq_lens.clone()))
|
||||||
|
|
||||||
|
assert not torch.equal(metadata.logical_positions, positions_before)
|
||||||
|
for name in ("token_to_req", "logical_positions", "visible_blocks", "slot_mapping"):
|
||||||
|
torch.testing.assert_close(getattr(metadata, name), getattr(expected, name))
|
||||||
|
torch.testing.assert_close(metadata.k_work_metadata, expected.k_work_metadata)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.usefixtures("default_vllm_config")
|
@pytest.mark.usefixtures("default_vllm_config")
|
||||||
def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||||
"""Padded projected Q/K rows must not affect either side cache."""
|
"""Padded projected Q/K rows must not affect either side cache."""
|
||||||
|
|||||||
+41
-4
@@ -242,7 +242,7 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
|||||||
config = SimpleNamespace(
|
config = SimpleNamespace(
|
||||||
cache_config=SimpleNamespace(
|
cache_config=SimpleNamespace(
|
||||||
use_replayssm=True,
|
use_replayssm=True,
|
||||||
use_kda_recoverssm=False,
|
use_recoverssm=False,
|
||||||
mamba_cache_mode="none",
|
mamba_cache_mode="none",
|
||||||
replayssm_buffer_len=16,
|
replayssm_buffer_len=16,
|
||||||
),
|
),
|
||||||
@@ -262,7 +262,7 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
|||||||
|
|
||||||
VllmConfig.validate_mamba_cached_kernel(config)
|
VllmConfig.validate_mamba_cached_kernel(config)
|
||||||
assert config.cache_config.use_replayssm
|
assert config.cache_config.use_replayssm
|
||||||
assert config.cache_config.use_kda_recoverssm
|
assert config.cache_config.use_recoverssm
|
||||||
|
|
||||||
config.cache_config.mamba_cache_mode = "align"
|
config.cache_config.mamba_cache_mode = "align"
|
||||||
VllmConfig.validate_mamba_cached_kernel(config)
|
VllmConfig.validate_mamba_cached_kernel(config)
|
||||||
@@ -275,7 +275,7 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
|||||||
config.model_config.architecture = "NemotronHForCausalLM"
|
config.model_config.architecture = "NemotronHForCausalLM"
|
||||||
config.mamba_config.backend = MambaBackendEnum.FLASHINFER
|
config.mamba_config.backend = MambaBackendEnum.FLASHINFER
|
||||||
VllmConfig.validate_mamba_cached_kernel(config)
|
VllmConfig.validate_mamba_cached_kernel(config)
|
||||||
assert not config.cache_config.use_kda_recoverssm
|
assert not config.cache_config.use_recoverssm
|
||||||
|
|
||||||
config.model_config.architecture = "KimiLinearForCausalLM"
|
config.model_config.architecture = "KimiLinearForCausalLM"
|
||||||
config.parallel_config.pipeline_parallel_size = 2
|
config.parallel_config.pipeline_parallel_size = 2
|
||||||
@@ -1602,7 +1602,9 @@ def test_engram_tensor_parallel_size(dp_size: int, across_dp: bool, expected: in
|
|||||||
assert config.get_parallel_size(parallel) == expected
|
assert config.get_parallel_size(parallel) == expected
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("option", ["embedding_across_dp", "dp_shared_memory"])
|
@pytest.mark.parametrize(
|
||||||
|
"option", ["embedding_across_dp", "dp_shared_memory", "host_file_gather"]
|
||||||
|
)
|
||||||
def test_engram_rejects_elastic_cross_dp(option):
|
def test_engram_rejects_elastic_cross_dp(option):
|
||||||
parallel = ParallelConfig(
|
parallel = ParallelConfig(
|
||||||
tensor_parallel_size=4,
|
tensor_parallel_size=4,
|
||||||
@@ -1826,12 +1828,47 @@ def test_engram_draft_config_validates_target(monkeypatch, target_has_ple, expli
|
|||||||
assert config.engram_config.cpu_offload is True
|
assert config.engram_config.cpu_offload is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("architecture", "layer_field", "rocm", "expect_error"),
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
"DeepseekV41ForCausalLM", "engram_layer_ids", False, True, id="deepseek"
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
"Qwen4ExpForCausalLM", "ple_layer_ids", True, True, id="qwen4exp_rocm"
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
"Qwen4ExpForCausalLM", "ple_layer_ids", False, False, id="qwen4exp_cuda"
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_engram_host_file_gather_requires_cuda_qwen4exp(
|
||||||
|
monkeypatch, architecture, layer_field, rocm, expect_error
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
|
||||||
|
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
|
||||||
|
model = cast(
|
||||||
|
ModelConfig,
|
||||||
|
SimpleNamespace(
|
||||||
|
architecture=architecture,
|
||||||
|
hf_text_config=SimpleNamespace(**{layer_field: [1]}),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
config = EngramConfig(host_file_gather=True)
|
||||||
|
if expect_error:
|
||||||
|
with pytest.raises(ValueError, match="host_file_gather"):
|
||||||
|
config.verify_model_config(model)
|
||||||
|
else:
|
||||||
|
config.verify_model_config(model)
|
||||||
|
|
||||||
|
|
||||||
def test_engram_hash_tracks_execution_options():
|
def test_engram_hash_tracks_execution_options():
|
||||||
configs = [
|
configs = [
|
||||||
EngramConfig(cpu_offload=True),
|
EngramConfig(cpu_offload=True),
|
||||||
EngramConfig(cpu_offload=False),
|
EngramConfig(cpu_offload=False),
|
||||||
EngramConfig(cpu_offload=True, embedding_across_dp=True),
|
EngramConfig(cpu_offload=True, embedding_across_dp=True),
|
||||||
EngramConfig(cpu_offload=True, dp_shared_memory=True),
|
EngramConfig(cpu_offload=True, dp_shared_memory=True),
|
||||||
|
EngramConfig(host_file_gather=True),
|
||||||
]
|
]
|
||||||
assert len({config.compute_hash() for config in configs}) == len(configs)
|
assert len({config.compute_hash() for config in configs}) == len(configs)
|
||||||
|
|
||||||
|
|||||||
@@ -947,3 +947,73 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
|
|||||||
f"but allocator pulled {len(new_blocks)}"
|
f"but allocator pulled {len(new_blocks)}"
|
||||||
)
|
)
|
||||||
total_computed = num_tokens
|
total_computed = num_tokens
|
||||||
|
|
||||||
|
def test_mamba_honors_drop_eagle_block():
|
||||||
|
"""MambaManager must drop the EAGLE lookahead block, like FullAttentionManager.
|
||||||
|
|
||||||
|
EAGLE/MTP requires the final matched block of a cache hit be dropped and
|
||||||
|
recomputed: it can hold state written over draft positions that verification
|
||||||
|
later rejects. FullAttentionManager pops it. Before this fix MambaManager
|
||||||
|
accepted `drop_eagle_block` and ignored it, so its hit came back one block
|
||||||
|
longer, and the retained block's recurrent-state snapshot -- taken over the
|
||||||
|
unverified positions -- stayed hash-reachable for every later request sharing
|
||||||
|
the prefix (#43559).
|
||||||
|
|
||||||
|
block_size == alignment_tokens here, so the alignment loop is a no-op in both
|
||||||
|
managers and the eagle drop is the only behavior under test.
|
||||||
|
"""
|
||||||
|
block_size = 16
|
||||||
|
num_blocks = 5
|
||||||
|
full_gid, mamba_gid = 0, 1
|
||||||
|
|
||||||
|
block_pool = BlockPool(
|
||||||
|
num_gpu_blocks=64, enable_caching=True, hash_block_size=block_size
|
||||||
|
)
|
||||||
|
block_hashes = [BlockHash(f"blk{i:04d}".encode()) for i in range(num_blocks)]
|
||||||
|
|
||||||
|
# Cache the same prefix chain under both groups, each on real blocks.
|
||||||
|
for gid in (full_gid, mamba_gid):
|
||||||
|
for i, (block_hash, block) in enumerate(
|
||||||
|
zip(block_hashes, block_pool.get_new_blocks(num_blocks))
|
||||||
|
):
|
||||||
|
block_pool._insert_block_hash(
|
||||||
|
make_block_hash_with_group_id(block_hash, gid),
|
||||||
|
block,
|
||||||
|
num_tokens=(i + 1) * block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
full_spec = FullAttentionSpec(
|
||||||
|
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float16
|
||||||
|
)
|
||||||
|
mamba_spec = MambaSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
shapes=((1,),),
|
||||||
|
dtypes=(torch.float16,),
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
)
|
||||||
|
|
||||||
|
def hit_blocks(manager, gid, spec, drop_eagle_block):
|
||||||
|
return len(
|
||||||
|
manager.find_longest_cache_hit(
|
||||||
|
block_hashes=block_hashes,
|
||||||
|
max_length=num_blocks * block_size,
|
||||||
|
kv_cache_group_ids=[gid],
|
||||||
|
block_pool=block_pool,
|
||||||
|
kv_cache_spec=spec,
|
||||||
|
drop_eagle_block=drop_eagle_block,
|
||||||
|
alignment_tokens=block_size,
|
||||||
|
)[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Control: with no eagle drop, both groups see the whole cached prefix.
|
||||||
|
assert hit_blocks(FullAttentionManager, full_gid, full_spec, False) == num_blocks
|
||||||
|
assert hit_blocks(MambaManager, mamba_gid, mamba_spec, False) == num_blocks
|
||||||
|
|
||||||
|
# With the eagle drop requested, both must give up the final block.
|
||||||
|
full_hit = hit_blocks(FullAttentionManager, full_gid, full_spec, True)
|
||||||
|
mamba_hit = hit_blocks(MambaManager, mamba_gid, mamba_spec, True)
|
||||||
|
assert full_hit == num_blocks - 1
|
||||||
|
assert mamba_hit == full_hit, (
|
||||||
|
f"MambaManager ignored drop_eagle_block: full-attention dropped the "
|
||||||
|
f"lookahead block (hit={full_hit}), mamba kept it (hit={mamba_hit})"
|
||||||
|
)
|
||||||
|
|||||||
@@ -263,7 +263,7 @@ def _create_decode_vllm_config(
|
|||||||
num_speculative_tokens: int = 0,
|
num_speculative_tokens: int = 0,
|
||||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||||
max_num_seqs: int = 8,
|
max_num_seqs: int = 8,
|
||||||
use_kda_recoverssm: bool = False,
|
use_recoverssm: bool = False,
|
||||||
) -> MagicMock:
|
) -> MagicMock:
|
||||||
compilation_config = CompilationConfig(
|
compilation_config = CompilationConfig(
|
||||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||||
@@ -278,7 +278,7 @@ def _create_decode_vllm_config(
|
|||||||
max_num_seqs=max_num_seqs
|
max_num_seqs=max_num_seqs
|
||||||
)
|
)
|
||||||
vllm_config.parallel_config = ParallelConfig()
|
vllm_config.parallel_config = ParallelConfig()
|
||||||
vllm_config.cache_config = SimpleNamespace(use_kda_recoverssm=use_kda_recoverssm)
|
vllm_config.cache_config = SimpleNamespace(use_recoverssm=use_recoverssm)
|
||||||
vllm_config.num_speculative_tokens = num_speculative_tokens
|
vllm_config.num_speculative_tokens = num_speculative_tokens
|
||||||
if dynamic_spec_schedule is None:
|
if dynamic_spec_schedule is None:
|
||||||
vllm_config.speculative_config = None
|
vllm_config.speculative_config = None
|
||||||
@@ -297,7 +297,7 @@ def _make_spec_decode_manager(
|
|||||||
num_speculative_tokens: int = 0,
|
num_speculative_tokens: int = 0,
|
||||||
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
dynamic_spec_schedule: list[tuple[int, int, int]] | None = None,
|
||||||
max_num_seqs: int = 8,
|
max_num_seqs: int = 8,
|
||||||
use_kda_recoverssm: bool = False,
|
use_recoverssm: bool = False,
|
||||||
varlen_decode: bool = False,
|
varlen_decode: bool = False,
|
||||||
) -> gpu_cudagraph_utils.CudaGraphManager:
|
) -> gpu_cudagraph_utils.CudaGraphManager:
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
@@ -316,7 +316,7 @@ def _make_spec_decode_manager(
|
|||||||
num_speculative_tokens=num_speculative_tokens,
|
num_speculative_tokens=num_speculative_tokens,
|
||||||
dynamic_spec_schedule=dynamic_spec_schedule,
|
dynamic_spec_schedule=dynamic_spec_schedule,
|
||||||
max_num_seqs=max_num_seqs,
|
max_num_seqs=max_num_seqs,
|
||||||
use_kda_recoverssm=use_kda_recoverssm,
|
use_recoverssm=use_recoverssm,
|
||||||
),
|
),
|
||||||
device=torch.device("cpu"),
|
device=torch.device("cpu"),
|
||||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||||
|
|||||||
@@ -943,6 +943,102 @@ def _run_mamba_prefix_cache_mrv1(
|
|||||||
cleanup_dist_env_and_memory()
|
cleanup_dist_env_and_memory()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_mamba_eagle_block_reuse(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||||
|
):
|
||||||
|
"""Recreate the #43559 failure conditions: a request finishing right after a
|
||||||
|
block boundary it crossed with every draft rejected, then a second request
|
||||||
|
whose cache hit lands on that final (eagle) block.
|
||||||
|
|
||||||
|
Request A ends a few tokens past the 2nd aligned boundary; with
|
||||||
|
num_accepted_tokens=1 every boundary crossing happens with rejected drafts
|
||||||
|
in flight, and A finishes before any later step could repair its cached
|
||||||
|
snapshot. Request B shares the prefix: pre-fix MambaManager returns the
|
||||||
|
boundary block as part of the hit (B resumes on the possibly-poisoned
|
||||||
|
snapshot), post-fix the hit stops one block earlier and B recomputes it.
|
||||||
|
|
||||||
|
Assertions, all deterministic:
|
||||||
|
1. B's num_cached_tokens == BLOCK_SIZE (the eagle block was dropped).
|
||||||
|
2. A's cached snapshot at the boundary matches the no-spec reference.
|
||||||
|
3. B's state at its next boundary matches the reference -- corruption
|
||||||
|
resumed from the reused block would surface here (the user-visible
|
||||||
|
failure, at state level).
|
||||||
|
"""
|
||||||
|
global async_scheduling_mode
|
||||||
|
async_scheduling_mode = async_scheduling
|
||||||
|
run_ref_mamba_state_in_subprocess()
|
||||||
|
apply_patch(monkeypatch)
|
||||||
|
full_prompt = datasets.load_dataset("heheda/a_long_article")["train"][0]["text"]
|
||||||
|
|
||||||
|
engine = LLM(
|
||||||
|
model=MODEL,
|
||||||
|
load_format="dummy",
|
||||||
|
enable_prefix_caching=True,
|
||||||
|
block_size=BLOCK_SIZE,
|
||||||
|
mamba_cache_mode="align",
|
||||||
|
speculative_config={
|
||||||
|
"method": "qwen3_next_mtp",
|
||||||
|
"num_speculative_tokens": num_speculative_tokens,
|
||||||
|
},
|
||||||
|
max_num_batched_tokens=3072,
|
||||||
|
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||||
|
async_scheduling=async_scheduling,
|
||||||
|
seed=42,
|
||||||
|
)
|
||||||
|
global prompt_token_ids
|
||||||
|
prompt_token_ids = engine.get_tokenizer().encode(full_prompt)
|
||||||
|
global num_accepted_tokens
|
||||||
|
num_accepted_tokens = 1
|
||||||
|
global cur_step_action_idx, step_actions
|
||||||
|
cur_step_action_idx = 0
|
||||||
|
step_actions = []
|
||||||
|
|
||||||
|
boundary = BLOCK_SIZE * 2
|
||||||
|
# A decodes across BOTH boundaries (align mode only snapshots state at
|
||||||
|
# decode/chunk crossings, and B needs a fallback state block once the
|
||||||
|
# eagle block is dropped), rejecting 3 drafts every step, and finishes
|
||||||
|
# a few tokens past the second boundary -- no later step can repair the
|
||||||
|
# cached snapshot.
|
||||||
|
engine.generate(
|
||||||
|
[TokensPrompt(prompt_token_ids=prompt_token_ids[: BLOCK_SIZE - 2])],
|
||||||
|
SamplingParams(temperature=0.0, max_tokens=BLOCK_SIZE + 10),
|
||||||
|
)
|
||||||
|
a_boundary_state = mamba_kv_cache_dict.get(boundary)
|
||||||
|
assert a_boundary_state is not None, "A never cached the boundary block"
|
||||||
|
mamba_kv_cache_dict.clear()
|
||||||
|
|
||||||
|
# B: prefix through the boundary, prompt ending shy of the next one so a
|
||||||
|
# few decode steps cross it and its state gets captured for comparison.
|
||||||
|
next_boundary = BLOCK_SIZE * 3
|
||||||
|
(out_b,) = engine.generate(
|
||||||
|
[TokensPrompt(prompt_token_ids=prompt_token_ids[: next_boundary - 10])],
|
||||||
|
SamplingParams(temperature=0.0, max_tokens=15),
|
||||||
|
)
|
||||||
|
assert out_b.num_cached_tokens == BLOCK_SIZE, (
|
||||||
|
f"cache hit must drop the eagle block: expected {BLOCK_SIZE} cached "
|
||||||
|
f"tokens, got {out_b.num_cached_tokens}"
|
||||||
|
)
|
||||||
|
|
||||||
|
mamba_state_ref = torch.load("mamba_kv_cache_dict_ref.pth")
|
||||||
|
check_mamba_state_equal(mamba_state_ref, {boundary: a_boundary_state}, [boundary])
|
||||||
|
check_mamba_state_equal(mamba_state_ref, mamba_kv_cache_dict, [next_boundary])
|
||||||
|
|
||||||
|
mamba_kv_cache_dict.clear()
|
||||||
|
del engine
|
||||||
|
torch.accelerator.empty_cache()
|
||||||
|
cleanup_dist_env_and_memory()
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("spawn")
|
||||||
|
def test_mamba_eagle_block_reuse_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||||
|
_run_mamba_eagle_block_reuse(monkeypatch, async_scheduling=False)
|
||||||
|
|
||||||
|
|
||||||
|
@create_new_process_for_each_test("spawn")
|
||||||
|
def test_mamba_eagle_block_reuse_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||||
|
_run_mamba_eagle_block_reuse(monkeypatch, async_scheduling=True)
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("spawn")
|
@create_new_process_for_each_test("spawn")
|
||||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
|
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
"""Dense-MTP-head detection for pack-quantized checkpoints (#58807).
|
||||||
|
|
||||||
|
The whole fix hinges on reading the checkpoint's actual tensor names, so the
|
||||||
|
tests pin the naming contract: dense means plain `mtp.*.weight` keys with no
|
||||||
|
quantization artifact suffix anywhere under `mtp.`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.model_executor.models.qwen3_5_mtp import (
|
||||||
|
_checkpoint_ships_dense_mtp,
|
||||||
|
_mtp_weight_names,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
|
|
||||||
|
def _model_config_with_index(monkeypatch, names):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.model_executor.models.qwen3_5_mtp._mtp_weight_names",
|
||||||
|
lambda _config: names,
|
||||||
|
)
|
||||||
|
return SimpleNamespace()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"names,expected",
|
||||||
|
[
|
||||||
|
# The reporter's checkpoint: every mtp.* tensor dense.
|
||||||
|
(["mtp.fc.weight", "mtp.layers.0.mlp.down_proj.weight"], True),
|
||||||
|
# Biases are dense too and must not count as artifacts.
|
||||||
|
(["mtp.fc.weight", "mtp.fc.bias"], True),
|
||||||
|
# A properly pack-quantized MTP head keeps the target quantization.
|
||||||
|
(["mtp.fc.weight_packed", "mtp.fc.weight_scale"], False),
|
||||||
|
# Channel-wise quantized MTP head: dense name plus scale companion.
|
||||||
|
(["mtp.fc.weight", "mtp.fc.weight_scale"], False),
|
||||||
|
# No MTP head in the checkpoint at all.
|
||||||
|
(["model.layers.0.self_attn.q_proj.weight"], False),
|
||||||
|
# Layout unknown (no index): keep current behavior.
|
||||||
|
(None, False),
|
||||||
|
([], False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_dense_mtp_detection(monkeypatch, names, expected):
|
||||||
|
config = _model_config_with_index(monkeypatch, names)
|
||||||
|
assert _checkpoint_ships_dense_mtp(config) is expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_weight_names_read_from_local_index(tmp_path, monkeypatch):
|
||||||
|
index = {
|
||||||
|
"weight_map": {
|
||||||
|
"model.layers.0.self_attn.q_proj.weight_packed": "model-00001.safetensors",
|
||||||
|
"mtp.fc.weight": "model-mtp-bf16.safetensors",
|
||||||
|
"mtp.layers.0.mlp.down_proj.weight": "model-mtp-bf16.safetensors",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(tmp_path / "model.safetensors.index.json").write_text(json.dumps(index))
|
||||||
|
config = SimpleNamespace(model=str(tmp_path), revision=None)
|
||||||
|
|
||||||
|
names = _mtp_weight_names(config)
|
||||||
|
|
||||||
|
assert names == ["mtp.fc.weight", "mtp.layers.0.mlp.down_proj.weight"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_weight_names_none_without_index(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.model_executor.models.qwen3_5_mtp.huggingface_hub.hf_hub_download",
|
||||||
|
lambda *args, **kwargs: (_ for _ in ()).throw(FileNotFoundError()),
|
||||||
|
)
|
||||||
|
config = SimpleNamespace(model=str(tmp_path), revision=None)
|
||||||
|
|
||||||
|
assert _mtp_weight_names(config) is None
|
||||||
@@ -211,8 +211,8 @@ class CacheConfig:
|
|||||||
and mamba_cache_mode 'none'. In align mode flushes are most efficient when
|
and mamba_cache_mode 'none'. In align mode flushes are most efficient when
|
||||||
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
|
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
|
||||||
required."""
|
required."""
|
||||||
use_kda_recoverssm: bool = field(default=False, init=False)
|
use_recoverssm: bool = field(default=False, init=False)
|
||||||
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
|
"""Whether RecoverSSM speculative decode is used (Kimi-K3 KDA; Qwen4Exp GDN and PLE)."""
|
||||||
|
|
||||||
# Will be set after profiling.
|
# Will be set after profiling.
|
||||||
num_gpu_blocks: int | None = field(default=None, init=False)
|
num_gpu_blocks: int | None = field(default=None, init=False)
|
||||||
|
|||||||
@@ -767,9 +767,20 @@ class CompilationConfig:
|
|||||||
"vllm::mamba_mixer2",
|
"vllm::mamba_mixer2",
|
||||||
"vllm::mamba_mixer",
|
"vllm::mamba_mixer",
|
||||||
"vllm::short_conv",
|
"vllm::short_conv",
|
||||||
# Qwen4Exp's AMD backend still uses these splitting ops.
|
# Qwen4Exp: the AMD backend and (fork, compile-profile guard) the compile-restored
|
||||||
|
# NVIDIA backend. #55272 dropped the NVIDIA-only ngram-ids split; the
|
||||||
|
# prefetch pair is the UVA PLE side-stream fork/join (#54371).
|
||||||
|
"vllm::qwen4_exp_compute_ple_ngram_ids",
|
||||||
|
"vllm::qwen4_exp_ple_start_prefetch",
|
||||||
|
"vllm::qwen4_exp_ple_finalize_prefetch",
|
||||||
"vllm::qwen4_exp_ple_short_conv",
|
"vllm::qwen4_exp_ple_short_conv",
|
||||||
"vllm::qwen4_exp_qsa_with_output",
|
"vllm::qwen4_exp_qsa_with_output",
|
||||||
|
# Fork (pull transport with the compile-profile guard, found on the reference host): the PLE pull transport's wait+DMA
|
||||||
|
# mutates a module-owned buffer; inside an Inductor graph it failed
|
||||||
|
# post-grad with "auto_functionalized was not removed" on the stage-0
|
||||||
|
# ranks. As a split point it runs between pieces, like attention.
|
||||||
|
"vllm::ple_offload_wait_copy",
|
||||||
|
"vllm::ple_offload_wait",
|
||||||
"vllm::linear_attention",
|
"vllm::linear_attention",
|
||||||
"vllm::qwen_gdn_attention_core",
|
"vllm::qwen_gdn_attention_core",
|
||||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||||
@@ -1379,6 +1390,7 @@ class CompilationConfig:
|
|||||||
max_num_reqs: int | None = None,
|
max_num_reqs: int | None = None,
|
||||||
is_profiling: bool = False,
|
is_profiling: bool = False,
|
||||||
piecewise_capture_available: bool = True,
|
piecewise_capture_available: bool = True,
|
||||||
|
use_replayssm: bool = False,
|
||||||
) -> CUDAGraphMode:
|
) -> CUDAGraphMode:
|
||||||
from vllm.v1.attention.backend import AttentionCGSupport
|
from vllm.v1.attention.backend import AttentionCGSupport
|
||||||
|
|
||||||
@@ -1424,10 +1436,19 @@ class CompilationConfig:
|
|||||||
f"with {min_cg_attn_backend} backend (support: "
|
f"with {min_cg_attn_backend} backend (support: "
|
||||||
f"{min_cg_support})"
|
f"{min_cg_support})"
|
||||||
)
|
)
|
||||||
if self.mode == CompilationMode.VLLM_COMPILE and (
|
if (
|
||||||
self.splitting_ops_contain_attention()
|
self.mode == CompilationMode.VLLM_COMPILE
|
||||||
or self.use_inductor_graph_partition
|
and (
|
||||||
|
self.splitting_ops_contain_attention()
|
||||||
|
or self.use_inductor_graph_partition
|
||||||
|
)
|
||||||
|
) or (
|
||||||
|
envs.VLLM_USE_BREAKABLE_CUDAGRAPH
|
||||||
|
# CompilationConfig has no cache_config; the runner passes the flag.
|
||||||
|
and use_replayssm
|
||||||
):
|
):
|
||||||
|
# Breakable CUDA graphs break at the attention kernels at runtime,
|
||||||
|
# with or without torch.compile.
|
||||||
msg += (
|
msg += (
|
||||||
"; setting cudagraph_mode=PIECEWISE because "
|
"; setting cudagraph_mode=PIECEWISE because "
|
||||||
"attention is compiled piecewise"
|
"attention is compiled piecewise"
|
||||||
|
|||||||
+30
-3
@@ -3,9 +3,10 @@
|
|||||||
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from pydantic import model_validator
|
from pydantic import Field, model_validator
|
||||||
from typing_extensions import Self
|
from typing_extensions import Self
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.config.utils import config, get_hash_factors, hash_factors
|
from vllm.config.utils import config, get_hash_factors, hash_factors
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -21,6 +22,17 @@ _NGRAM_LAYER_FIELDS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _default_cpu_offload() -> bool:
|
||||||
|
# Fork: with VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a
|
||||||
|
# VLLM_PLE_CPU_OFFLOAD value that would enable UVA fails the boot. With the
|
||||||
|
# pull transport off, the upstream default below is returned unchanged.
|
||||||
|
from vllm.v1.e2_guards import pull_transport_cpu_offload_default
|
||||||
|
|
||||||
|
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
return envs.VLLM_PLE_CPU_OFFLOAD
|
||||||
|
return pull_transport_cpu_offload_default(False)
|
||||||
|
|
||||||
|
|
||||||
def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
||||||
"""Whether the model carries n-gram embedding layers."""
|
"""Whether the model carries n-gram embedding layers."""
|
||||||
if model_config is None:
|
if model_config is None:
|
||||||
@@ -35,8 +47,17 @@ def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
|||||||
class EngramConfig:
|
class EngramConfig:
|
||||||
"""Configuration for Engram embedding storage and sharding."""
|
"""Configuration for Engram embedding storage and sharding."""
|
||||||
|
|
||||||
cpu_offload: bool = True
|
cpu_offload: bool = Field(default_factory=_default_cpu_offload)
|
||||||
"""Store embedding weights in pinned CPU memory for UVA lookup."""
|
"""Store embedding weights in pinned CPU memory for UVA lookup.
|
||||||
|
Defaults to VLLM_PLE_CPU_OFFLOAD, which is enabled by default. An explicit
|
||||||
|
value takes precedence over the environment variable. With
|
||||||
|
VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a value that would
|
||||||
|
enable UVA fails the boot (vllm/v1/e2_guards.py)."""
|
||||||
|
|
||||||
|
host_file_gather: bool = False
|
||||||
|
"""Serve Qwen4Exp PLE rows by gathering them on the CPU from the loader's
|
||||||
|
file-backed checkpoint views and staging them on the device each step,
|
||||||
|
instead of allocating the table; takes precedence over `cpu_offload`."""
|
||||||
|
|
||||||
embedding_across_dp: bool = False
|
embedding_across_dp: bool = False
|
||||||
"""Shard embeddings across TP and all DP ranks when enabled.
|
"""Shard embeddings across TP and all DP ranks when enabled.
|
||||||
@@ -81,6 +102,10 @@ class EngramConfig:
|
|||||||
"EngramConfig requires a model with supported Engram "
|
"EngramConfig requires a model with supported Engram "
|
||||||
"embeddings and non-empty n-gram layer ids."
|
"embeddings and non-empty n-gram layer ids."
|
||||||
)
|
)
|
||||||
|
if self.host_file_gather and (
|
||||||
|
field != "ple_layer_ids" or current_platform.is_rocm()
|
||||||
|
):
|
||||||
|
raise ValueError("host_file_gather requires a Qwen4Exp model on CUDA")
|
||||||
|
|
||||||
def resolve_dp_shared_memory(self, parallel_config: "ParallelConfig") -> None:
|
def resolve_dp_shared_memory(self, parallel_config: "ParallelConfig") -> None:
|
||||||
"""Share host tables by default wherever the configuration permits."""
|
"""Share host tables by default wherever the configuration permits."""
|
||||||
@@ -107,6 +132,8 @@ class EngramConfig:
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Engram embedding_across_dp is not supported with elastic EP yet."
|
"Engram embedding_across_dp is not supported with elastic EP yet."
|
||||||
)
|
)
|
||||||
|
if self.host_file_gather and parallel_config.enable_elastic_ep:
|
||||||
|
raise ValueError("host_file_gather is not supported with elastic EP.")
|
||||||
|
|
||||||
def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
|
def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
|
||||||
"""Derive the embedding group size from the parallel configuration."""
|
"""Derive the embedding group size from the parallel configuration."""
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from vllm.config.fault_tolerance import FaultToleranceConfig
|
|||||||
from vllm.config.utils import config
|
from vllm.config.utils import config
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.network_utils import get_open_ports_list
|
from vllm.utils.network_utils import get_open_ports_list, get_open_zmq_ipc_path
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ray.runtime_env import RuntimeEnv
|
from ray.runtime_env import RuntimeEnv
|
||||||
@@ -369,6 +369,10 @@ class ParallelConfig:
|
|||||||
"""Port of the coordination TCPStore. Can be set by the API server; workers
|
"""Port of the coordination TCPStore. Can be set by the API server; workers
|
||||||
connect as clients to exchange self-picked group ports at runtime."""
|
connect as clients to exchange self-picked group ports at runtime."""
|
||||||
|
|
||||||
|
_ple_offload_ipc_path: str = ""
|
||||||
|
"""Node-local ZMQ IPC address for the fork's PLE pull-transport offload
|
||||||
|
process. Set only when VLLM_E2_PLE_PULL_TRANSPORT=1."""
|
||||||
|
|
||||||
decode_context_parallel_size: int = Field(default=1, ge=1)
|
decode_context_parallel_size: int = Field(default=1, ge=1)
|
||||||
"""Number of ranks that shard the decode KV cache. DCP does not expand
|
"""Number of ranks that shard the decode KV cache. DCP does not expand
|
||||||
the process world size. Without PCP, DCP reuses TP ranks. With PCP, DCP
|
the process world size. Without PCP, DCP reuses TP ranks. With PCP, DCP
|
||||||
@@ -517,6 +521,9 @@ class ParallelConfig:
|
|||||||
"The FT system assumes one AsyncMPClient manages all engines."
|
"The FT system assumes one AsyncMPClient manages all engines."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not self._ple_offload_ipc_path:
|
||||||
|
self._ple_offload_ipc_path = get_open_zmq_ipc_path()
|
||||||
|
|
||||||
if self.all2all_backend in ["pplx", "naive"]:
|
if self.all2all_backend in ["pplx", "naive"]:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"The '%s' all2all backend has been removed. "
|
"The '%s' all2all backend has been removed. "
|
||||||
@@ -910,6 +917,7 @@ class ParallelConfig:
|
|||||||
"worker_extension_cls",
|
"worker_extension_cls",
|
||||||
"_api_process_count",
|
"_api_process_count",
|
||||||
"_api_process_rank",
|
"_api_process_rank",
|
||||||
|
"_ple_offload_ipc_path",
|
||||||
# NUMA binding is per-rank host-side memory locality; it does
|
# NUMA binding is per-rank host-side memory locality; it does
|
||||||
# not affect collective-communication semantics. When numa_bind
|
# not affect collective-communication semantics. When numa_bind
|
||||||
# is enabled with auto-detection, each DP rank stores its own
|
# is enabled with auto-detection, each DP rank stores its own
|
||||||
|
|||||||
+72
-12
@@ -70,6 +70,13 @@ else:
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
# Architectures whose GDN (and PLE) layers run RecoverSSM under --use-replayssm.
|
||||||
|
_QWEN4EXP_RECOVERSSM_ARCHS = (
|
||||||
|
"Qwen4ExpForCausalLM",
|
||||||
|
"Qwen4ExpForConditionalGeneration",
|
||||||
|
"Qwen4ExpMTP",
|
||||||
|
)
|
||||||
|
|
||||||
# TODO(rocm): These models are either unsupported by MRV2 or slower with
|
# TODO(rocm): These models are either unsupported by MRV2 or slower with
|
||||||
# MRV2 on AMD GPUs.
|
# MRV2 on AMD GPUs.
|
||||||
ROCM_DEFAULT_MRV1_ARCHITECTURES = frozenset(
|
ROCM_DEFAULT_MRV1_ARCHITECTURES = frozenset(
|
||||||
@@ -99,9 +106,12 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
|||||||
"KimiLinearForCausalLM",
|
"KimiLinearForCausalLM",
|
||||||
"MiniMaxM3SparseForCausalLM",
|
"MiniMaxM3SparseForCausalLM",
|
||||||
"MiniMaxM3SparseForConditionalGeneration",
|
"MiniMaxM3SparseForConditionalGeneration",
|
||||||
"Qwen4ExpForCausalLM",
|
# Fork (compile-profile guard): Qwen4ExpForCausalLM,
|
||||||
"Qwen4ExpForConditionalGeneration",
|
# Qwen4ExpForConditionalGeneration and Qwen4ExpMTP are deliberately
|
||||||
"Qwen4ExpMTP",
|
# NOT here. #55272 added them when it removed torch.compile from the
|
||||||
|
# NVIDIA path; this tree restores compile, so the resolved profile for
|
||||||
|
# them is VLLM_COMPILE + FULL_AND_PIECEWISE with breakable capture off
|
||||||
|
# (vllm/v1/e2_guards.py asserts it at model construction).
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1242,6 +1252,11 @@ class VllmConfig:
|
|||||||
# PyTorch's expandable_segments allocator uses CUDA VMM, which can
|
# PyTorch's expandable_segments allocator uses CUDA VMM, which can
|
||||||
# remap a virtual address range to different physical pages over the
|
# remap a virtual address range to different physical pages over the
|
||||||
# engine's lifetime. KV connectors that pin KV cache memory (e.g.
|
# engine's lifetime. KV connectors that pin KV cache memory (e.g.
|
||||||
|
# Fork: an explicit cpu_offload=True beside the pull transport is
|
||||||
|
# a double offload; refused here, before any worker starts.
|
||||||
|
from vllm.v1.e2_guards import verify_engram_against_pull
|
||||||
|
|
||||||
|
verify_engram_against_pull(self.engram_config)
|
||||||
# NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their
|
# NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their
|
||||||
# registrations pointing at stale physical pages after any remap,
|
# registrations pointing at stale physical pages after any remap,
|
||||||
# producing RDMA failures like IBV_WC_REM_ACCESS_ERR /
|
# producing RDMA failures like IBV_WC_REM_ACCESS_ERR /
|
||||||
@@ -3482,9 +3497,8 @@ class VllmConfig:
|
|||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def validate_mamba_cached_kernel(self) -> "VllmConfig":
|
def validate_mamba_cached_kernel(self) -> "VllmConfig":
|
||||||
if not self.cache_config.use_replayssm:
|
if not self.cache_config.use_replayssm:
|
||||||
self.cache_config.use_kda_recoverssm = False
|
self.cache_config.use_recoverssm = False
|
||||||
return self
|
return self
|
||||||
|
|
||||||
kda_architectures = (
|
kda_architectures = (
|
||||||
"KimiLinearForCausalLM",
|
"KimiLinearForCausalLM",
|
||||||
"KimiK3ForConditionalGeneration",
|
"KimiK3ForConditionalGeneration",
|
||||||
@@ -3493,11 +3507,16 @@ class VllmConfig:
|
|||||||
self.model_config is not None
|
self.model_config is not None
|
||||||
and self.model_config.architecture in kda_architectures
|
and self.model_config.architecture in kda_architectures
|
||||||
)
|
)
|
||||||
self.cache_config.use_kda_recoverssm = (
|
is_qwen4exp_recoverssm_model = (
|
||||||
self.num_speculative_tokens > 0 and is_kda_model
|
self.model_config is not None
|
||||||
|
and self.model_config.architecture in _QWEN4EXP_RECOVERSSM_ARCHS
|
||||||
|
)
|
||||||
|
self.cache_config.use_recoverssm = (
|
||||||
|
self.num_speculative_tokens > 0
|
||||||
|
and (is_kda_model or is_qwen4exp_recoverssm_model)
|
||||||
)
|
)
|
||||||
use_mamba_replayssm_spec = (
|
use_mamba_replayssm_spec = (
|
||||||
self.num_speculative_tokens > 0 and not self.cache_config.use_kda_recoverssm
|
self.num_speculative_tokens > 0 and not self.cache_config.use_recoverssm
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.model_config is not None and not self.model_config.supports_replayssm:
|
if self.model_config is not None and not self.model_config.supports_replayssm:
|
||||||
@@ -3505,6 +3524,23 @@ class VllmConfig:
|
|||||||
"--use-replayssm is not supported for architecture "
|
"--use-replayssm is not supported for architecture "
|
||||||
f"{self.model_config.architecture!r}"
|
f"{self.model_config.architecture!r}"
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
self.model_config is not None
|
||||||
|
and self.model_config.architecture in _QWEN4EXP_RECOVERSSM_ARCHS
|
||||||
|
and not self.cache_config.use_recoverssm
|
||||||
|
):
|
||||||
|
# Qwen4Exp's GDN and PLE layers have no plain ReplaySSM path; with them the
|
||||||
|
# flag selects RecoverSSM, which needs speculative decoding. Fork (v2.5):
|
||||||
|
# without speculative decoding there are no drafts to recover from, so the
|
||||||
|
# flag falls back to the stock path (as if it were not passed) instead of
|
||||||
|
# refusing to start; one launch line then serves with and without MTP.
|
||||||
|
logger.warning(
|
||||||
|
"--use-replayssm on Qwen4Exp selects GDN RecoverSSM, which needs "
|
||||||
|
"speculative decoding; num_speculative_tokens=0, so the stock GDN "
|
||||||
|
"path is used (as without --use-replayssm)"
|
||||||
|
)
|
||||||
|
self.cache_config.use_replayssm = False
|
||||||
|
return self
|
||||||
if (
|
if (
|
||||||
self.mamba_config.backend == MambaBackendEnum.FLASHINFER
|
self.mamba_config.backend == MambaBackendEnum.FLASHINFER
|
||||||
and self.cache_config.replayssm_buffer_len > 16
|
and self.cache_config.replayssm_buffer_len > 16
|
||||||
@@ -3512,7 +3548,15 @@ class VllmConfig:
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
"FlashInfer ReplaySSM requires --replayssm-buffer-len <= 16"
|
"FlashInfer ReplaySSM requires --replayssm-buffer-len <= 16"
|
||||||
)
|
)
|
||||||
if self.cache_config.use_kda_recoverssm:
|
if self.cache_config.use_recoverssm:
|
||||||
|
if self.model_config is not None and self.model_config.architecture not in (
|
||||||
|
"KimiLinearForCausalLM",
|
||||||
|
"KimiK3ForConditionalGeneration",
|
||||||
|
*_QWEN4EXP_RECOVERSSM_ARCHS,
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"RecoverSSM is only supported for Kimi-K3 KDA and Qwen4Exp GDN"
|
||||||
|
)
|
||||||
if self.mamba_config.enable_stochastic_rounding:
|
if self.mamba_config.enable_stochastic_rounding:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"RecoverSSM supports bfloat16/float32 "
|
"RecoverSSM supports bfloat16/float32 "
|
||||||
@@ -3527,9 +3571,25 @@ class VllmConfig:
|
|||||||
"RecoverSSM with align mode requires VLLM_USE_V2_MODEL_RUNNER=1"
|
"RecoverSSM with align mode requires VLLM_USE_V2_MODEL_RUNNER=1"
|
||||||
)
|
)
|
||||||
if self.parallel_config.pipeline_parallel_size > 1:
|
if self.parallel_config.pipeline_parallel_size > 1:
|
||||||
raise ValueError(
|
# Non-last PP ranks get the accepted counts pp_size steps after
|
||||||
"RecoverSSM currently requires pipeline_parallel_size=1"
|
# their verify. Model Runner V2 defers the commit with a
|
||||||
)
|
# snapshot of its inputs (RecoverSSMState.detach_step) and
|
||||||
|
# masks freed rows; only the Qwen4Exp GDN/PLE metadata
|
||||||
|
# implement that snapshot (Kimi-K3 KDA does not).
|
||||||
|
if (
|
||||||
|
self.model_config is None
|
||||||
|
or self.model_config.architecture
|
||||||
|
not in _QWEN4EXP_RECOVERSSM_ARCHS
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"RecoverSSM with pipeline_parallel_size > 1 is supported "
|
||||||
|
"only for Qwen4Exp GDN/PLE"
|
||||||
|
)
|
||||||
|
if not self.use_v2_model_runner:
|
||||||
|
raise ValueError(
|
||||||
|
"RecoverSSM with pipeline_parallel_size > 1 requires "
|
||||||
|
"VLLM_USE_V2_MODEL_RUNNER=1"
|
||||||
|
)
|
||||||
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||||
raise ValueError("RecoverSSM requires --mamba-backend triton")
|
raise ValueError("RecoverSSM requires --mamba-backend triton")
|
||||||
elif use_mamba_replayssm_spec:
|
elif use_mamba_replayssm_spec:
|
||||||
|
|||||||
@@ -182,6 +182,22 @@ os.environ.setdefault("TRITON_CACHE_AUTOTUNING", "1")
|
|||||||
# debug layout (see https://github.com/vllm-project/vllm/issues/41410).
|
# debug layout (see https://github.com/vllm-project/vllm/issues/41410).
|
||||||
os.environ.setdefault("TILELANG_CLEANUP_TEMP_FILES", "1")
|
os.environ.setdefault("TILELANG_CLEANUP_TEMP_FILES", "1")
|
||||||
|
|
||||||
|
# Fork (v2.5, knob-absent boot): the MERLIN_* kernel toggles are read by
|
||||||
|
# getenv in C++ (FULL_K, TIE_DET, TIE_RECENT, TOPK_SORTED_EMIT) and by
|
||||||
|
# vllm.envs (QSA_SORT) with OFF as the code default, while every shipped
|
||||||
|
# launcher sets them to 1. Unset now means the shipped value; an explicit "0"
|
||||||
|
# still turns a toggle off. Set here because this module runs on `import vllm`,
|
||||||
|
# before any reader, and worker processes inherit the environment.
|
||||||
|
_SHIPPED_KERNEL_TOGGLE_DEFAULTS = (
|
||||||
|
"MERLIN_FULL_K",
|
||||||
|
"MERLIN_QSA_SORT",
|
||||||
|
"MERLIN_TIE_DET",
|
||||||
|
"MERLIN_TIE_RECENT",
|
||||||
|
"MERLIN_TOPK_SORTED_EMIT",
|
||||||
|
)
|
||||||
|
for _name in _SHIPPED_KERNEL_TOGGLE_DEFAULTS:
|
||||||
|
os.environ.setdefault(_name, "1")
|
||||||
|
|
||||||
# ===================================================
|
# ===================================================
|
||||||
# torch 2.9 Inductor PythonWrapperCodegen monkeypatch
|
# torch 2.9 Inductor PythonWrapperCodegen monkeypatch
|
||||||
# ===================================================
|
# ===================================================
|
||||||
|
|||||||
+583
@@ -304,6 +304,195 @@ if TYPE_CHECKING:
|
|||||||
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
||||||
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
|
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
|
||||||
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
|
VLLM_USE_V2_MODEL_RUNNER: bool | None = None
|
||||||
|
# Compile-cache factor (v2 host-embedding work): the pinned-host UVA embedding
|
||||||
|
# table changes the traced MTP-drafter graph (reads _uva_weight instead of
|
||||||
|
# embed_tokens.weight). Registered here so envs.compile_factors() keys the
|
||||||
|
# AOT cache on it; the layer itself still reads os.environ directly.
|
||||||
|
VLLM_HOST_EMBED_TABLE: bool = False
|
||||||
|
VLLM_PLE_OFFLOAD_READY_TIMEOUT: float = 600.0
|
||||||
|
# --- Flash-Next fork knobs (v2.2) --------------------------------
|
||||||
|
# Every name below was read straight from os.environ at its use site, so
|
||||||
|
# envs.validate_environ() logged it as unknown on every boot that set one
|
||||||
|
# and compile_factors() could not see it. Registering makes the hash
|
||||||
|
# complete; the read sites are unchanged (they still parse and fail closed
|
||||||
|
# themselves, which registration must not weaken). Runtime-only entries are
|
||||||
|
# listed in compile_factors().ignored_factors with their reason.
|
||||||
|
#
|
||||||
|
# TWO IDENTITIES, NOT ONE. "Read once at startup"
|
||||||
|
# settles neither of them, so every entry below is labelled with both:
|
||||||
|
#
|
||||||
|
# AOT identity -- does the value change the torch.compile / AOT
|
||||||
|
# artifact? This is the ONLY identity compile_factors()
|
||||||
|
# keys. It protects against reusing another process's
|
||||||
|
# cached artifact.
|
||||||
|
# CAPTURE identity -- does the value change what a cudagraph capture bakes
|
||||||
|
# in (ops present, launch constants, shapes)? Nothing
|
||||||
|
# in this tree keys anything on capture identity: the
|
||||||
|
# env is read before capture, in-process, so capture
|
||||||
|
# identity is protected only by "do not change the
|
||||||
|
# value between capture and replay in one process".
|
||||||
|
#
|
||||||
|
# So a knob that is a capture factor but NOT an AOT factor gets no
|
||||||
|
# protection from being registered here. Registering it is a contract
|
||||||
|
# statement and a cross-process guard, not a mechanism. Where the two
|
||||||
|
# identities are not both established by reading the code, the knob is
|
||||||
|
# hashed anyway: a spurious cache miss costs a recompile, a missed one costs
|
||||||
|
# a wrong graph.
|
||||||
|
#
|
||||||
|
# *** INERT ON THE 0.30 BASE. ***
|
||||||
|
# NO READ SITE, and therefore NOT HASHED -- moved out of the compile
|
||||||
|
# factors into ignored_factors below. Its only reader was the 21-line
|
||||||
|
# catch-all branch in _get_ple_embedding_quant_method, which lived in the
|
||||||
|
# half of models/qwen4_exp/nvidia/ple_layer.py that #54371 deleted. With no
|
||||||
|
# reader it cannot change an artifact, so hashing it would only invalidate
|
||||||
|
# caches for nothing. (The AOT artifact is real on this tree: #55272
|
||||||
|
# removed torch.compile from the NVIDIA Qwen4Exp path, the compile-profile guard restored it,
|
||||||
|
# and the vLLM 0.30 rebase ran AOT cache-load boots -- so the reason is "no
|
||||||
|
# reader", not "no artifact".) It gives NO capture protection either.
|
||||||
|
# v0.30.0 selects the FP8 PLE method from the CONFIG KEY
|
||||||
|
# text_config.ple_embedding_dtype == "float8_e4m3fn"
|
||||||
|
# (ngram_embedding.py:174-175) -- see
|
||||||
|
# tests/models/qwen4_exp/test_ple_fp8_selector_checkpoint.py.
|
||||||
|
# Setting this on the 0.30 base does NOTHING, and NOTHING WARNS: validate_environ
|
||||||
|
# (envs.py, below) warns only about names that are NOT registered, and this
|
||||||
|
# comment lives in a TYPE_CHECKING block that never runs. It is kept
|
||||||
|
# registered so a later PLE transport change can reland its reader and
|
||||||
|
# reclassify it, or delete it. DO NOT assume it works.
|
||||||
|
# When its reader returns: AOT yes, CAPTURE yes -- the FP8-with-global-scale
|
||||||
|
# table selects Qwen4ExpPLEFp8EmbeddingMethod over the checkpoint's method,
|
||||||
|
# a different weight dtype and a different embedding forward.
|
||||||
|
VLLM_PLE_FP8_GLOBAL_SCALE: bool = False
|
||||||
|
#
|
||||||
|
# COMPILE/CAPTURE FACTORS (hashed):
|
||||||
|
# CAPTURE: yes, established -- clip counting allocates persistent per-layer
|
||||||
|
# int64 counters and its increment ops are CAPTURED into the cudagraph (see
|
||||||
|
# the graph-safety note in models/qwen4_exp/nvidia/qsa.py); unset, those ops
|
||||||
|
# are absent. AOT: not established (the QSA forward is a custom op); hashed
|
||||||
|
# conservatively.
|
||||||
|
VLLM_QSA_KV_CLIP_COUNT: str = ""
|
||||||
|
# CAPTURE: n/a -- collection requires --enforce-eager, so an enabled run
|
||||||
|
# does not capture at all. AOT: adds absmax reduction ops to the attention
|
||||||
|
# forward; hashed conservatively.
|
||||||
|
VLLM_QSA_KV_COLLECT: str = ""
|
||||||
|
# CAPTURE: yes -- the resolved value is a launch constant baked into the
|
||||||
|
# graph at capture (csrc/libtorch_stable/persistent_topk.cuh:143-148).
|
||||||
|
# AOT: not established. Hashed conservatively.
|
||||||
|
# NOT A PYTHON READ SITE: the value is resolved once per process by
|
||||||
|
# exact_medium_enabled() in csrc/libtorch_stable/topk.cu:30-36, which is the
|
||||||
|
# single source of truth. This entry is a raw passthrough so nothing can
|
||||||
|
# mistake it for a second parser; do not use envs.MERLIN_TOPK_EXACT_MEDIUM
|
||||||
|
# to decide behaviour. Registered with the medium-path exact restart
|
||||||
|
# so the cache-factor contract is complete.
|
||||||
|
MERLIN_TOPK_EXACT_MEDIUM: str | None = None
|
||||||
|
# The fork's PLE pull transport -- the host-mapped,
|
||||||
|
# fail-closed offload process in vllm/v1/ple_offload/. Renamed from the E2
|
||||||
|
# base's VLLM_PLE_CPU_OFFLOAD, which vLLM v0.30.0 reuses for a DIFFERENT
|
||||||
|
# subsystem with the OPPOSITE default (upstream: pinned-host UVA lookup,
|
||||||
|
# default ON). The two subsystems no longer share a switch.
|
||||||
|
# Strict: unset or "0" = off, "1" = on, anything else is a boot error.
|
||||||
|
# On: the PLE table lives in the offload process, the GPU placeholder waits
|
||||||
|
# on a host flag and DMAs rows inside the captured forward, and upstream's
|
||||||
|
# UVA backend is refused (vllm/v1/e2_guards.py). Off: the tree is upstream.
|
||||||
|
# AOT: yes (different module class and forward). CAPTURE: yes (the captured
|
||||||
|
# graph contains the wait/copy ops instead of a table lookup). Hashed.
|
||||||
|
# UNSET = ON (since v2.5), the shipped value: upstream's UVA backend (the off
|
||||||
|
# path) compiles an fp8e4nv Triton kernel that sm_86 cannot build, so a launcher
|
||||||
|
# without this switch failed at profile_run. "0" still selects upstream's path.
|
||||||
|
VLLM_E2_PLE_PULL_TRANSPORT: bool = True
|
||||||
|
#
|
||||||
|
# RUNTIME-ONLY (registered for validate_environ + a single cached read):
|
||||||
|
# Static K/V scale sidecar: fill_()ed into existing fp32 device buffers
|
||||||
|
# after load and before capture, and the kernel does tl.load(k_scale_ptr),
|
||||||
|
# so the value is never a constexpr and the graph is identical.
|
||||||
|
VLLM_QSA_KV_SCALES: str = ""
|
||||||
|
# Guard telemetry: mode raise|warn|off and the reader poll interval in
|
||||||
|
# seconds. counter() returns a real tensor in every mode, so the Triton
|
||||||
|
# arguments and the captured graph do not depend on either.
|
||||||
|
VLLM_E2_GUARD_MODE: str = "warn"
|
||||||
|
VLLM_E2_GUARD_INTERVAL: str = "0.05"
|
||||||
|
# Debug allocator telemetry (poll seconds + output directory) and the
|
||||||
|
# scheduler allocation log path.
|
||||||
|
VLLM_E2_MEMSTATS: str = "0"
|
||||||
|
VLLM_E2_MEMSTATS_DIR: str = ""
|
||||||
|
VLLM_E2_SCHED_LOG: str = ""
|
||||||
|
# O(live blocks) host-side ownership assertion in the scheduler, which is
|
||||||
|
# plain Python and never compiled.
|
||||||
|
VLLM_E2_ZERO_OWNERSHIP_CHECK: bool = False
|
||||||
|
# PLE offload child placement: rewrites CUDA_VISIBLE_DEVICES for the child
|
||||||
|
# only (CUDA_VISIBLE_DEVICES is itself an ignored factor).
|
||||||
|
VLLM_PLE_OFFLOAD_HOME_DEVICE: str = ""
|
||||||
|
# PLE timeouts / deadlines, in seconds.
|
||||||
|
VLLM_PLE_OFFLOAD_WAIT_TIMEOUT: float = 120.0
|
||||||
|
VLLM_PLE_PROGRESS_DEADLINE_S: float = 300.0
|
||||||
|
VLLM_PLE_STARTUP_DEADLINE_S: float = 1800.0
|
||||||
|
# Test-only PLE fault injection: cut spec, marker log, bootstrap gate.
|
||||||
|
# AOT: no. CAPTURE: no. Read once at import by the vllm_ple_test_fault
|
||||||
|
# singleton, but the process that reads them is a real engine, so
|
||||||
|
# validate_environ sees them.
|
||||||
|
VLLM_PLE_TEST_FAULT: str | None = None
|
||||||
|
VLLM_PLE_TEST_FAULT_GATE: str | None = None
|
||||||
|
VLLM_PLE_TEST_FAULT_LOG: str | None = None
|
||||||
|
#
|
||||||
|
# --- Knobs registered ahead of their read sites (v2.2) -----------
|
||||||
|
# Registered ahead of their code so the MERGED tree is complete and
|
||||||
|
# validate_environ is quiet from the first boot (a
|
||||||
|
# cross-change knob table). Each mirrors its read site's
|
||||||
|
# parse; if a read site changes, the entry here must be reconciled.
|
||||||
|
# (Both read sites below are now live.)
|
||||||
|
# LIVE: read once at
|
||||||
|
# import as envs.VLLM_E2_PLE_FLAG_CHECK by vllm/v1/ple_offload/connector.py
|
||||||
|
# (the PLE pull-transport flag check, rank-gated, off by default). On
|
||||||
|
# the 0.30 base it had NO read site: the PLE-diagnostics half
|
||||||
|
# had not been relanded after the PLE pull-transport port restored connector.py. The
|
||||||
|
# census in tests/test_envs.py now asserts the read site exists.
|
||||||
|
# AOT: no. CAPTURE: no -- strictly host-side and outside capture.
|
||||||
|
# off|warn|raise (1 = warn), normalised by _e2_diagnostic_mode like the
|
||||||
|
# rest of the family. The mamba ownership check moved this gate off a plain
|
||||||
|
# boolean; reconciled here.
|
||||||
|
VLLM_E2_PLE_FLAG_CHECK: str = "off"
|
||||||
|
# AOT: no. CAPTURE: no -- strictly host-side. off|warn|raise (1 = warn).
|
||||||
|
# the mamba slot-ownership check, v1/worker/mamba_utils.py:74.
|
||||||
|
VLLM_E2_MAMBA_SLOT_OWNERSHIP: str = "off"
|
||||||
|
# v2.5: keep each running request's newest hashed align-mode Mamba
|
||||||
|
# state (its prefix-cache checkpoint) referenced until the request ends,
|
||||||
|
# instead of returning it to the free queue the moment decode passes it
|
||||||
|
# (single_type_kv_cache_manager.MambaManager._retire_state_blocks). Strict
|
||||||
|
# "0"/"1", default ON; "0" restores the old release-on-retire behaviour
|
||||||
|
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
|
||||||
|
# the worker sees the same block tables either way.
|
||||||
|
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
|
||||||
|
# v2.5 side-cache layout (Flash-Next PLE + QSA ring): stop charging each
|
||||||
|
# tiny side cache a whole packed pool block per request. The TP-replicated
|
||||||
|
# PLE short-conv state group is folded into a GDN group that has a spare
|
||||||
|
# page on every PP stage, and the one-block-per-request QSA ring group packs
|
||||||
|
# several requests' rings into one pool block. Strict "0"/"1", default ON;
|
||||||
|
# "0" restores the old one-group-each layout for A/B. Read in
|
||||||
|
# vllm/v1/core/kv_cache_utils.py (get_kv_cache_configs). AOT: yes, hashed --
|
||||||
|
# it changes the ring views' block stride and which block table the PLE
|
||||||
|
# state is read through, so ON and OFF must not share a compile cache.
|
||||||
|
VLLM_E2_SIDE_CACHE_LAYOUT: bool = True
|
||||||
|
# Profiling-teardown leak policy (vllm/v1/worker/gpu/cudagraph_utils.py): a profiling
|
||||||
|
# KV cache still allocated after teardown fails the boot by default; "0", "false" or
|
||||||
|
# "no" downgrade it to a warning while diagnosing. AOT: no. CAPTURE: no -- it only
|
||||||
|
# decides whether a host-side check raises.
|
||||||
|
VLLM_FAIL_ON_PROFILING_KV_LEAK: bool = True
|
||||||
|
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
||||||
|
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
||||||
|
# nothing itself; it only permits a boot whose identity IS different
|
||||||
|
# (VLLM_USE_BREAKABLE_CUDAGRAPH, which upstream already registers and
|
||||||
|
# hashes). Ignored factor for exactly that reason.
|
||||||
|
VLLM_E2_ALLOW_BREAKABLE: bool = False
|
||||||
|
# Expected RESOLVED cudagraph mode by name, asserted by the same guard when
|
||||||
|
# set (e.g. "FULL", "FULL_DECODE_ONLY"). AOT: no. CAPTURE: no -- it asserts
|
||||||
|
# an identity, it does not choose one. Ignored factor.
|
||||||
|
VLLM_E2_EXPECTED_CUDAGRAPH_MODE: str = ""
|
||||||
|
#
|
||||||
|
# The unaudited-knob fence that stood here listed the
|
||||||
|
# tie-determinism toggles as not-yet-in-tree. They landed with the
|
||||||
|
# v2.2 kernel port: the four are registered and
|
||||||
|
# classified in the "determinism toggles" block below, and the
|
||||||
|
# one knob still unclassified sits in that block's own fence, which
|
||||||
|
# tests/test_envs.py reads as a set of identifiers.
|
||||||
VLLM_LOG_MODEL_INSPECTION: bool = False
|
VLLM_LOG_MODEL_INSPECTION: bool = False
|
||||||
VLLM_DEBUG_MFU_METRICS: bool = False
|
VLLM_DEBUG_MFU_METRICS: bool = False
|
||||||
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
||||||
@@ -328,6 +517,167 @@ if TYPE_CHECKING:
|
|||||||
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
||||||
VLLM_NIC_SELECTION_VARS: str = ""
|
VLLM_NIC_SELECTION_VARS: str = ""
|
||||||
VLLM_ENABLE_HPC_OPS: bool = False
|
VLLM_ENABLE_HPC_OPS: bool = False
|
||||||
|
MERLIN_TOPK_EXACT_COUNTERS_LOG: int = 0
|
||||||
|
# Pre-tie-determinism fallback for the counter reader's slot count; runtime-only
|
||||||
|
# (ignored factor), never consulted on this build. See the dict entry.
|
||||||
|
MERLIN_TOPK_COUNTER_SLOTS: str | None = None
|
||||||
|
#
|
||||||
|
# --- determinism toggles -----------------------------------------
|
||||||
|
# The three nondeterminism mechanisms (FULL_K, QSA_SORT, TIE_DET/TIE_RECENT).
|
||||||
|
# Every one of them is
|
||||||
|
# OFF by default in this file; vllm/env_override.py sets each to 1 (the shipped value) when unset.
|
||||||
|
#
|
||||||
|
# Why these are registered at all: validate_environ() only inspects names
|
||||||
|
# beginning with VLLM_, so a MERLIN_ name never trips the unknown-variable
|
||||||
|
# warning. Registration here exists for ONE reason -- compile_factors() can
|
||||||
|
# only hash names it knows, and compile_factors() is a deny-list, so
|
||||||
|
# registering a name hashes it unless ignored_factors says otherwise. All
|
||||||
|
# four are deliberately absent from ignored_factors.
|
||||||
|
#
|
||||||
|
# CAPTURE: yes, all four. Each resolves once per process and the resolved
|
||||||
|
# value then reaches the GPU as a kernel launch argument, a launch
|
||||||
|
# configuration, or an extra op in the traced sequence -- so a cudagraph
|
||||||
|
# bakes in whatever was live at capture time. Flipping one between boots
|
||||||
|
# changes the captured graph, which is exactly what a cache key is for.
|
||||||
|
# AOT: not established for the three CUDA toggles (the Marlin and topk
|
||||||
|
# kernels are compiled into the extension at build time, not by the AOT
|
||||||
|
# compiler); established for MERLIN_QSA_SORT, which adds torch ops to the
|
||||||
|
# traced graph. Hashed conservatively throughout, the same rule
|
||||||
|
# already applied to MERLIN_TOPK_EXACT_MEDIUM.
|
||||||
|
#
|
||||||
|
# NOT A PYTHON READ SITE (FULL_K / TIE_DET / TIE_RECENT): each is resolved
|
||||||
|
# once by a std::getenv in C++ -- merlin_full_k() in
|
||||||
|
# csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu and merlin_det_ties() /
|
||||||
|
# merlin_tie_recent() in csrc/libtorch_stable/topk.cu -- and that is the
|
||||||
|
# single source of truth. The entries below are raw passthroughs so nothing
|
||||||
|
# can mistake them for a second parser; do not use them to decide
|
||||||
|
# behaviour. A raw passthrough hashes "0" and "00" differently even though
|
||||||
|
# the C++ treats them alike: over-invalidation, which is the safe direction.
|
||||||
|
#
|
||||||
|
# Marlin MoE split-K reduction order; the CUDA half of unmerged upstream
|
||||||
|
# PR #46639 (`use_full_k`).
|
||||||
|
MERLIN_FULL_K: str | None = None
|
||||||
|
# persistent_topk tie-break at the k-th cutoff, and its direction.
|
||||||
|
MERLIN_TIE_DET: str | None = None
|
||||||
|
MERLIN_TIE_RECENT: str | None = None
|
||||||
|
#
|
||||||
|
# A REAL Python read site, unlike the three above: parsed here and read
|
||||||
|
# once at import by models/qwen4_exp/nvidia/ops/qsa_indexer.py, which adds
|
||||||
|
# a torch.sort + two torch.where ops to the QSA selection path.
|
||||||
|
MERLIN_QSA_SORT: bool = False
|
||||||
|
#
|
||||||
|
# --- persistent_topk kernel switches ------------------------------
|
||||||
|
# Same classification as the three raw passthroughs above and registered
|
||||||
|
# for the same single reason: compile_factors() is a deny-list, so a name
|
||||||
|
# it does not know cannot be hashed, and a Ray deployment that propagates
|
||||||
|
# registered variables can otherwise drop an explicitly-disabled switch
|
||||||
|
# and leave a worker on the default. Each is resolved ONCE per process by
|
||||||
|
# a std::getenv in csrc/libtorch_stable/topk.cu -- that is the single
|
||||||
|
# source of truth -- and the resolved value reaches the GPU as a kernel
|
||||||
|
# launch argument, so a cudagraph bakes in whatever was live at capture.
|
||||||
|
# CAPTURE: yes, all three. AOT: not established (these kernels are built
|
||||||
|
# into the extension, not by the AOT compiler); hashed conservatively
|
||||||
|
# anyway, same rule as above. None is in ignored_factors.
|
||||||
|
#
|
||||||
|
# Raw passthroughs, so "unset" reads as None here. That is NOT the same as
|
||||||
|
# "off" for the C++: MERLIN_TOPK_EXACT_MEDIUM defaults ON. Read the getter
|
||||||
|
# in topk.cu, not this default, for the shipped behaviour.
|
||||||
|
#
|
||||||
|
# Take the exact restart on an overflowing MEDIUM row:
|
||||||
|
# MERLIN_TOPK_EXACT_MEDIUM. Declared ONCE, in the cross-change knob
|
||||||
|
# block above (same raw passthrough, same hashing); the 0.30 tree already
|
||||||
|
# carried that entry, so it is not repeated here.
|
||||||
|
# The same restart on an overflowing DECODE row.
|
||||||
|
MERLIN_TOPK_EXACT_DECODE: str | None = None
|
||||||
|
# Deterministic placement and tie membership on the LARGE path.
|
||||||
|
# LIVE on this tree since the v2.2 kernel port: the
|
||||||
|
# read site is det_large_enabled() in csrc/libtorch_stable/topk.cu, a
|
||||||
|
# std::getenv resolved once per process, and the resolved value reaches
|
||||||
|
# the large-path radix_topk as a launch argument, so a cudagraph bakes it
|
||||||
|
# in. CAPTURE: yes. AOT: not established (built into the extension, not by
|
||||||
|
# the AOT compiler); hashed conservatively like the switches above. A raw
|
||||||
|
# passthrough because the C++ getter is the single source of truth.
|
||||||
|
MERLIN_TOPK_DET_LARGE: str | None = None
|
||||||
|
#
|
||||||
|
# The large-path FAULT INJECTORS. TEST
|
||||||
|
# ONLY, and registered for exactly the same reason as the switches above,
|
||||||
|
# not because anything should set them in production:
|
||||||
|
#
|
||||||
|
# * both are resolved once per process by a std::getenv in topk.cu
|
||||||
|
# (large_fault_inject_cta(), large_fault_inject_emit()) and the
|
||||||
|
# resolved values reach the GPU as kernel launch arguments, so a
|
||||||
|
# cudagraph BAKES THEM IN. An injected build and a clean build differ
|
||||||
|
# in the captured graph while sharing a cache key -- which is the
|
||||||
|
# failure mode: a graph captured under injection returns an all -1 row
|
||||||
|
# for every deterministic large row, and on replay nothing re-reads the
|
||||||
|
# environment to notice the variable has since been unset. FAULT_INJECT
|
||||||
|
# was once "unregistered and capture-baked"; hashing
|
||||||
|
# them is what stops a cached artifact crossing between the arms.
|
||||||
|
# * compile_factors() is a deny-list, so a name it does not know cannot
|
||||||
|
# be hashed at all. Neither is in ignored_factors.
|
||||||
|
#
|
||||||
|
# Raw passthroughs, like the switches: the C++ getters are the single
|
||||||
|
# source of truth (INJECT parses with atoi and treats <0 and unset alike;
|
||||||
|
# KIND compares against the literal "emit"). Do not parse them here.
|
||||||
|
#
|
||||||
|
# Which CTA of the group has its fault predicate forced; unset / negative
|
||||||
|
# is off, and setting it also suppresses __trap().
|
||||||
|
MERLIN_TOPK_LARGE_FAULT_INJECT: str | None = None
|
||||||
|
# Which fault: unset / "pre" forces the pre-emission predicate caught at
|
||||||
|
# X3, "emit" forces the emission-bound slot check caught at X4. Ignored
|
||||||
|
# unless MERLIN_TOPK_LARGE_FAULT_INJECT names a CTA.
|
||||||
|
MERLIN_TOPK_LARGE_FAULT_KIND: str | None = None
|
||||||
|
#
|
||||||
|
# MERLIN_TOPK_SORTED_EMIT -- persistent_topk
|
||||||
|
# emits every row in the canonical order MERLIN_QSA_SORT imposes
|
||||||
|
# (ascending index, padding last as -1), and qsa_indexer.py then skips its
|
||||||
|
# Python sort. Same classification as the kernel switches above: resolved
|
||||||
|
# ONCE per process by a std::getenv in csrc/libtorch_stable/topk.cu
|
||||||
|
# (sorted_emit_enabled) -- the single source of truth -- and the resolved
|
||||||
|
# value reaches the GPU as a kernel launch argument AND removes three torch
|
||||||
|
# ops from the captured sequence, so a cudagraph bakes it in. CAPTURE: yes.
|
||||||
|
# AOT: not established for the kernel half; hashed conservatively (a fork
|
||||||
|
# env knob must be a compile factor: an os.environ knob that is not
|
||||||
|
# registered is not in the cache key). Raw passthrough: Python never parses
|
||||||
|
# it to decide anything -- qsa_indexer.py reads the C++ cache through
|
||||||
|
# persistent_topk_toggle_state() field 6. Not in ignored_factors.
|
||||||
|
MERLIN_TOPK_SORTED_EMIT: str | None = None
|
||||||
|
# Debug companion, a REAL Python read site (qsa_indexer._SORTED_EMIT_CHECK,
|
||||||
|
# read once at import): when the kernel sorted, assert on device that the
|
||||||
|
# row equals its canonical form instead of trusting it. Adds ops to the
|
||||||
|
# captured sequence, so it is hashed too. Default off.
|
||||||
|
MERLIN_TOPK_SORTED_EMIT_CHECK: bool = False
|
||||||
|
#
|
||||||
|
# NOT REGISTERED, deliberately: MERLIN_TOPK_EXACT_TRAP. It is a
|
||||||
|
# preprocessor define consumed by `#ifndef` in persistent_topk.cuh at
|
||||||
|
# BUILD time (-D), not a process environment variable -- nothing in the
|
||||||
|
# tree calls getenv on it. Registering it would advertise a runtime knob
|
||||||
|
# that does not exist. Its default (trap ON) is an open maintainer decision,
|
||||||
|
# not something to settle by
|
||||||
|
# quietly making it settable.
|
||||||
|
#
|
||||||
|
# --- UNAUDITED: reference-host knobs with no read site in this tree ----------
|
||||||
|
# A knob named inside the fence below is one this repository cannot
|
||||||
|
# classify, because the code that reads it is not in it. Registering one
|
||||||
|
# without first deleting it from the fence -- which means writing down its
|
||||||
|
# AOT and capture identity against its real read site -- is the exact
|
||||||
|
# failure the fork's env rule exists to prevent, and
|
||||||
|
# test_unaudited_host_side_knob_stays_listed_until_it_is_classified (in
|
||||||
|
# tests/test_envs.py) fails on it.
|
||||||
|
#
|
||||||
|
# The fence holds NOTHING BUT the names, one per line. Prose belongs out
|
||||||
|
# here: the gate reads the fence as a set of identifiers, and a knob name
|
||||||
|
# mentioned in passing inside it would otherwise read as a listing.
|
||||||
|
#
|
||||||
|
# The one entry, MERLIN_PIN_BSM, pins Marlin's block_size_m to 64 (Python,
|
||||||
|
# no rebuild). It is an ALTERNATIVE to the split-K toggle above, not a
|
||||||
|
# complement: on its own it fixed only the 74/102-token cells and left
|
||||||
|
# 1050 upward broken, while use_full_k fixed all of them with block_size_m
|
||||||
|
# still varying. The tie-determinism change promotes use_full_k and keeps this one off, so its
|
||||||
|
# read site was never brought across and it stays unclassified.
|
||||||
|
# --8<-- [start:unaudited-host-side-knobs]
|
||||||
|
# MERLIN_PIN_BSM
|
||||||
|
# --8<-- [end:unaudited-host-side-knobs]
|
||||||
|
|
||||||
|
|
||||||
def get_default_cache_root():
|
def get_default_cache_root():
|
||||||
@@ -609,6 +959,47 @@ def _resolve_rust_cli_path() -> str | None:
|
|||||||
return raw
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def _e2_strict_switch(name: str, default: bool = False) -> bool:
|
||||||
|
"""Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``.
|
||||||
|
|
||||||
|
Unset means ``default`` (off unless the knob says otherwise). Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
|
||||||
|
``"01"`` -- is a boot error: a switch that selects a different transport
|
||||||
|
must not be satisfiable by a spelling nobody meant.
|
||||||
|
"""
|
||||||
|
raw = os.environ.get(name)
|
||||||
|
if raw is None:
|
||||||
|
return default
|
||||||
|
if raw == "0":
|
||||||
|
return False
|
||||||
|
if raw == "1":
|
||||||
|
return True
|
||||||
|
raise ValueError(f'{name} must be exactly "0" or "1" (or unset), got {raw!r}')
|
||||||
|
|
||||||
|
|
||||||
|
def _e2_diagnostic_mode(name: str) -> str:
|
||||||
|
"""Normalize an E2 diagnostic gate to off|warn|raise.
|
||||||
|
|
||||||
|
One normalizer for every knob in the family, so a registry entry and the
|
||||||
|
module that acts on it cannot disagree about what the environment said.
|
||||||
|
``1`` is accepted as ``warn`` for continuity with the earlier boolean
|
||||||
|
gates. A set-but-unrecognised value is a boot error, not a silent ``off``:
|
||||||
|
a broken gate that disarms its own instrument is the failure mode these
|
||||||
|
diagnostics exist to avoid.
|
||||||
|
"""
|
||||||
|
# An EMPTY value means off, not a boot error. Shell scripts and pytest
|
||||||
|
# fixtures both use `VAR=` to mean "unset" (see tests/v1/worker/
|
||||||
|
# test_e2_memstats.py), and rejecting it would turn a routine way of
|
||||||
|
# clearing the variable into a crash.
|
||||||
|
value = os.environ.get(name, "off").strip().lower() or "off"
|
||||||
|
if value == "1":
|
||||||
|
return "warn"
|
||||||
|
if value not in ("off", "warn", "raise"):
|
||||||
|
raise ValueError(
|
||||||
|
f"{name} must be off|warn|raise (or 1 for warn), got {value!r}"
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
environment_variables: dict[str, Callable[[], Any]] = {
|
environment_variables: dict[str, Callable[[], Any]] = {
|
||||||
# ================== Installation Time Env Vars ==================
|
# ================== Installation Time Env Vars ==================
|
||||||
# Target device of vLLM, supporting [cuda (by default),
|
# Target device of vLLM, supporting [cuda (by default),
|
||||||
@@ -2118,6 +2509,56 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool(
|
"VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool(
|
||||||
os.getenv("VLLM_USE_V2_MODEL_RUNNER", None)
|
os.getenv("VLLM_USE_V2_MODEL_RUNNER", None)
|
||||||
),
|
),
|
||||||
|
"VLLM_HOST_EMBED_TABLE": lambda: os.getenv("VLLM_HOST_EMBED_TABLE", "") == "1",
|
||||||
|
# Timeout for PLE weight loading and TP worker registration.
|
||||||
|
"VLLM_PLE_OFFLOAD_READY_TIMEOUT": lambda: float(
|
||||||
|
os.getenv("VLLM_PLE_OFFLOAD_READY_TIMEOUT", "600")
|
||||||
|
),
|
||||||
|
# --- Flash-Next fork knobs (v2.2); see the TYPE_CHECKING block for
|
||||||
|
# which of these are compile/capture factors and which are ignored.
|
||||||
|
# Compile/capture factors.
|
||||||
|
"VLLM_PLE_FP8_GLOBAL_SCALE": lambda: (
|
||||||
|
os.getenv("VLLM_PLE_FP8_GLOBAL_SCALE", "") == "1"
|
||||||
|
),
|
||||||
|
"VLLM_QSA_KV_CLIP_COUNT": lambda: os.getenv("VLLM_QSA_KV_CLIP_COUNT", "").strip(),
|
||||||
|
"VLLM_QSA_KV_COLLECT": lambda: os.getenv("VLLM_QSA_KV_COLLECT", "").strip(),
|
||||||
|
# Capture-configuration factor, read in C++; raw passthrough on purpose.
|
||||||
|
"MERLIN_TOPK_EXACT_MEDIUM": lambda: os.getenv("MERLIN_TOPK_EXACT_MEDIUM", None),
|
||||||
|
# PLE pull transport switch; strict "0"/"1" (see the TYPE_CHECKING block).
|
||||||
|
"VLLM_E2_PLE_PULL_TRANSPORT": lambda: _e2_strict_switch(
|
||||||
|
"VLLM_E2_PLE_PULL_TRANSPORT", default=True
|
||||||
|
),
|
||||||
|
# Runtime only; all of the following are in compile_factors.ignored_factors.
|
||||||
|
"VLLM_E2_ALLOW_BREAKABLE": lambda: os.getenv("VLLM_E2_ALLOW_BREAKABLE", "") == "1",
|
||||||
|
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE": lambda: os.getenv(
|
||||||
|
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE", ""
|
||||||
|
).strip(),
|
||||||
|
"VLLM_QSA_KV_SCALES": lambda: os.getenv("VLLM_QSA_KV_SCALES", "").strip(),
|
||||||
|
"VLLM_E2_GUARD_MODE": lambda: (
|
||||||
|
os.getenv("VLLM_E2_GUARD_MODE", "warn").strip().lower()
|
||||||
|
),
|
||||||
|
"VLLM_E2_GUARD_INTERVAL": lambda: os.getenv("VLLM_E2_GUARD_INTERVAL", "0.05"),
|
||||||
|
"VLLM_E2_MEMSTATS": lambda: os.getenv("VLLM_E2_MEMSTATS", "0"),
|
||||||
|
"VLLM_E2_MEMSTATS_DIR": lambda: os.getenv("VLLM_E2_MEMSTATS_DIR", ""),
|
||||||
|
"VLLM_E2_SCHED_LOG": lambda: os.getenv("VLLM_E2_SCHED_LOG", ""),
|
||||||
|
"VLLM_E2_ZERO_OWNERSHIP_CHECK": lambda: (
|
||||||
|
os.getenv("VLLM_E2_ZERO_OWNERSHIP_CHECK", "") == "1"
|
||||||
|
),
|
||||||
|
"VLLM_PLE_OFFLOAD_HOME_DEVICE": lambda: os.getenv(
|
||||||
|
"VLLM_PLE_OFFLOAD_HOME_DEVICE", ""
|
||||||
|
).strip(),
|
||||||
|
"VLLM_PLE_OFFLOAD_WAIT_TIMEOUT": lambda: float(
|
||||||
|
os.getenv("VLLM_PLE_OFFLOAD_WAIT_TIMEOUT", "120")
|
||||||
|
),
|
||||||
|
"VLLM_PLE_PROGRESS_DEADLINE_S": lambda: float(
|
||||||
|
os.getenv("VLLM_PLE_PROGRESS_DEADLINE_S", "300")
|
||||||
|
),
|
||||||
|
"VLLM_PLE_STARTUP_DEADLINE_S": lambda: float(
|
||||||
|
os.getenv("VLLM_PLE_STARTUP_DEADLINE_S", "1800")
|
||||||
|
),
|
||||||
|
"VLLM_PLE_TEST_FAULT": lambda: os.getenv("VLLM_PLE_TEST_FAULT", None),
|
||||||
|
"VLLM_PLE_TEST_FAULT_GATE": lambda: os.getenv("VLLM_PLE_TEST_FAULT_GATE", None),
|
||||||
|
"VLLM_PLE_TEST_FAULT_LOG": lambda: os.getenv("VLLM_PLE_TEST_FAULT_LOG", None),
|
||||||
# Log model inspection after loading.
|
# Log model inspection after loading.
|
||||||
# If enabled, logs a transformers-style hierarchical view of the model
|
# If enabled, logs a transformers-style hierarchical view of the model
|
||||||
# with quantization methods and attention backends.
|
# with quantization methods and attention backends.
|
||||||
@@ -2182,6 +2623,23 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_ELASTIC_EP_DRAIN_REQUESTS": lambda: bool(
|
"VLLM_ELASTIC_EP_DRAIN_REQUESTS": lambda: bool(
|
||||||
int(os.getenv("VLLM_ELASTIC_EP_DRAIN_REQUESTS", "0"))
|
int(os.getenv("VLLM_ELASTIC_EP_DRAIN_REQUESTS", "0"))
|
||||||
),
|
),
|
||||||
|
# PLE diagnostics. Both are off|warn|raise (1 is accepted as warn) and both
|
||||||
|
# default to off. They are registered here so the registry is the single
|
||||||
|
# source of truth for their normalization -- the modules that use them read
|
||||||
|
# these getters rather than calling os.getenv a second time, so the value
|
||||||
|
# the registry reports cannot drift from the value the check applies.
|
||||||
|
# Both are excluded from compile factors in `compile_factors` below: they
|
||||||
|
# gate HOST-side Python outside any capture and add no node to a compiled
|
||||||
|
# graph. If either ever grows a device-side component, remove it from
|
||||||
|
# `ignored_factors` in the same change.
|
||||||
|
#
|
||||||
|
# Read the pull-transport PLE semaphore back before a real launch, on the
|
||||||
|
# publishing rank only (vllm#53899 one-step-lag class).
|
||||||
|
"VLLM_E2_PLE_FLAG_CHECK": lambda: _e2_diagnostic_mode("VLLM_E2_PLE_FLAG_CHECK"),
|
||||||
|
# Host-side align row-rule and state-block collision check.
|
||||||
|
"VLLM_E2_MAMBA_SLOT_OWNERSHIP": lambda: _e2_diagnostic_mode(
|
||||||
|
"VLLM_E2_MAMBA_SLOT_OWNERSHIP"
|
||||||
|
),
|
||||||
# If set to 1, enable CUDA graph memory estimation during memory profiling.
|
# If set to 1, enable CUDA graph memory estimation during memory profiling.
|
||||||
# This profiles CUDA graph memory usage to provide more accurate KV cache
|
# This profiles CUDA graph memory usage to provide more accurate KV cache
|
||||||
# memory allocation. Enabled by default as of v0.21.0
|
# memory allocation. Enabled by default as of v0.21.0
|
||||||
@@ -2249,6 +2707,60 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
|
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
|
||||||
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
|
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
|
||||||
),
|
),
|
||||||
|
# Mamba prefix-checkpoint retention (v2.5; default ON, "0" turns it off).
|
||||||
|
# Strict "0"/"1"; read when each MambaManager is constructed.
|
||||||
|
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
|
||||||
|
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
|
||||||
|
),
|
||||||
|
# Side-cache layout (v2.5; PLE state folded into a GDN group + QSA ring slot
|
||||||
|
# packing), default ON. Strict "0"/"1"; read once per get_kv_cache_configs call.
|
||||||
|
"VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch(
|
||||||
|
"VLLM_E2_SIDE_CACHE_LAYOUT", default=True
|
||||||
|
),
|
||||||
|
# Profiling-teardown leak policy: fail the boot unless "0"/"false"/"no".
|
||||||
|
"VLLM_FAIL_ON_PROFILING_KV_LEAK": lambda: os.getenv(
|
||||||
|
"VLLM_FAIL_ON_PROFILING_KV_LEAK", "1"
|
||||||
|
).strip().lower()
|
||||||
|
not in ("0", "false", "no"),
|
||||||
|
# Seconds between `topk-exact-counters` lines from each GPU worker
|
||||||
|
# (0, the default, disables the reader entirely). The line reports the
|
||||||
|
# medium-path exact-restart counters; see
|
||||||
|
# vllm/models/qwen4_exp/nvidia/ops/topk_counters.py.
|
||||||
|
"MERLIN_TOPK_EXACT_COUNTERS_LOG": lambda: int(
|
||||||
|
os.getenv("MERLIN_TOPK_EXACT_COUNTERS_LOG", "0")
|
||||||
|
),
|
||||||
|
# Fallback slot count for the counter reader on a build WITHOUT the
|
||||||
|
# persistent_topk_exact_counters_slots op (pre-tie-determinism). This tree registers
|
||||||
|
# that op, so topk_counters.slot_count() never consults the variable here.
|
||||||
|
# Raw passthrough: the reader's int(os.getenv(..., "5")) is the parser.
|
||||||
|
"MERLIN_TOPK_COUNTER_SLOTS": lambda: os.getenv("MERLIN_TOPK_COUNTER_SLOTS", None),
|
||||||
|
# Determinism toggles. See the declaration block above for the AOT /
|
||||||
|
# capture classification and for why three of these are raw passthroughs.
|
||||||
|
# All four are absent from ignored_factors on purpose, so all four are
|
||||||
|
# hashed into the compile cache key.
|
||||||
|
"MERLIN_FULL_K": lambda: os.getenv("MERLIN_FULL_K", None),
|
||||||
|
"MERLIN_TIE_DET": lambda: os.getenv("MERLIN_TIE_DET", None),
|
||||||
|
"MERLIN_TIE_RECENT": lambda: os.getenv("MERLIN_TIE_RECENT", None),
|
||||||
|
"MERLIN_QSA_SORT": lambda: bool(int(os.getenv("MERLIN_QSA_SORT", "0"))),
|
||||||
|
# persistent_topk kernel switches. Raw passthroughs
|
||||||
|
# of values C++ parses; see the declaration block above.
|
||||||
|
# MERLIN_TOPK_EXACT_MEDIUM is registered once, earlier in this dict.
|
||||||
|
"MERLIN_TOPK_EXACT_DECODE": lambda: os.getenv("MERLIN_TOPK_EXACT_DECODE", None),
|
||||||
|
"MERLIN_TOPK_DET_LARGE": lambda: os.getenv("MERLIN_TOPK_DET_LARGE", None),
|
||||||
|
# Large-path fault injectors. TEST ONLY, hashed for
|
||||||
|
# the reason in the declaration block: both are capture-baked.
|
||||||
|
"MERLIN_TOPK_LARGE_FAULT_INJECT": lambda: os.getenv(
|
||||||
|
"MERLIN_TOPK_LARGE_FAULT_INJECT", None
|
||||||
|
),
|
||||||
|
"MERLIN_TOPK_LARGE_FAULT_KIND": lambda: os.getenv(
|
||||||
|
"MERLIN_TOPK_LARGE_FAULT_KIND", None
|
||||||
|
),
|
||||||
|
# In-kernel canonical emission and its debug check; see the
|
||||||
|
# declaration block. Both hashed: both change the captured sequence.
|
||||||
|
"MERLIN_TOPK_SORTED_EMIT": lambda: os.getenv("MERLIN_TOPK_SORTED_EMIT", None),
|
||||||
|
"MERLIN_TOPK_SORTED_EMIT_CHECK": lambda: bool(
|
||||||
|
int(os.getenv("MERLIN_TOPK_SORTED_EMIT_CHECK", "0"))
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -2405,6 +2917,77 @@ def compile_factors() -> dict[str, object]:
|
|||||||
"CUDA_VISIBLE_DEVICES",
|
"CUDA_VISIBLE_DEVICES",
|
||||||
"NO_COLOR",
|
"NO_COLOR",
|
||||||
"VLLM_SKIP_VERSION_SUFFIX",
|
"VLLM_SKIP_VERSION_SUFFIX",
|
||||||
|
# --- Flash-Next fork knobs that are runtime-only (v2.2) ------
|
||||||
|
# Registering a name in environment_variables makes it a compile
|
||||||
|
# factor by default (this function is a deny-list), so every fork knob
|
||||||
|
# that cannot change a compiled or captured artifact is listed here
|
||||||
|
# with its reason. Hashing them would invalidate the whole AOT cache on
|
||||||
|
# a log path, a timeout or a guard-mode flip.
|
||||||
|
#
|
||||||
|
# Static QSA K/V scales: fill_()ed into existing fp32 device buffers
|
||||||
|
# after load and before capture; the kernel reads them with
|
||||||
|
# tl.load(k_scale_ptr), never as a constexpr.
|
||||||
|
"VLLM_QSA_KV_SCALES",
|
||||||
|
# Guard telemetry: counter() returns a real [3] int64 view in every
|
||||||
|
# mode, so the Triton arguments and the captured graph are identical;
|
||||||
|
# the mode only gates the side reader thread and the check_pending
|
||||||
|
# raise, and the interval is that thread's poll cadence.
|
||||||
|
"VLLM_E2_GUARD_MODE",
|
||||||
|
"VLLM_E2_GUARD_INTERVAL",
|
||||||
|
# Debug telemetry cadence and output locations.
|
||||||
|
"VLLM_E2_MEMSTATS",
|
||||||
|
"VLLM_E2_MEMSTATS_DIR",
|
||||||
|
"VLLM_E2_SCHED_LOG",
|
||||||
|
# Host-side scheduler assertion; the scheduler is never compiled.
|
||||||
|
"VLLM_E2_ZERO_OWNERSHIP_CHECK",
|
||||||
|
# Placement only: rewrites CUDA_VISIBLE_DEVICES for the PLE offload
|
||||||
|
# child, and CUDA_VISIBLE_DEVICES is already ignored above.
|
||||||
|
"VLLM_PLE_OFFLOAD_HOME_DEVICE",
|
||||||
|
# Timeouts and deadlines, in seconds. VLLM_PLE_OFFLOAD_READY_TIMEOUT
|
||||||
|
# was registered in the v2 host-embed commit and left hashed by
|
||||||
|
# oversight: serve-v2.sh sets it to 1800, so every boot that changed it
|
||||||
|
# missed the AOT cache for no reason.
|
||||||
|
"VLLM_PLE_OFFLOAD_READY_TIMEOUT",
|
||||||
|
"VLLM_PLE_OFFLOAD_WAIT_TIMEOUT",
|
||||||
|
"VLLM_PLE_PROGRESS_DEADLINE_S",
|
||||||
|
"VLLM_PLE_STARTUP_DEADLINE_S",
|
||||||
|
# Test-only fault injection: a cut spec, a marker log path and a
|
||||||
|
# bootstrap gate file. None of them reaches a kernel or a graph.
|
||||||
|
"VLLM_PLE_TEST_FAULT",
|
||||||
|
"VLLM_PLE_TEST_FAULT_GATE",
|
||||||
|
"VLLM_PLE_TEST_FAULT_LOG",
|
||||||
|
# Fork startup guard: the opt-out permits a different identity and the
|
||||||
|
# expected-mode pin asserts one; neither IS one.
|
||||||
|
"VLLM_E2_ALLOW_BREAKABLE",
|
||||||
|
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE",
|
||||||
|
# INERT on the 0.30 base: no read site, so it cannot change any artifact.
|
||||||
|
# Hashing it would invalidate caches for no behavioural change. Restore
|
||||||
|
# it to the compile factors in the same commit that relands its reader.
|
||||||
|
"VLLM_PLE_FP8_GLOBAL_SCALE",
|
||||||
|
# Diagnostic knobs, both strictly host-side and outside capture, so
|
||||||
|
# neither AOT nor capture identity depends on them. Hashing
|
||||||
|
# them would invalidate every compile cache the moment a diagnostic
|
||||||
|
# boot is taken -- exactly when the cache is wanted. Remove from this
|
||||||
|
# set if either ever grows a device-side component.
|
||||||
|
# MERLIN_TOPK_EXACT_MEDIUM is deliberately NOT here: it is a capture
|
||||||
|
# configuration factor and is hashed conservatively.
|
||||||
|
"VLLM_E2_PLE_FLAG_CHECK",
|
||||||
|
"VLLM_E2_MAMBA_SLOT_OWNERSHIP",
|
||||||
|
# Scheduler-side block retention (v2.5). The scheduler is never
|
||||||
|
# compiled and the worker's tables are identical in both worlds, so an
|
||||||
|
# A/B flip must reuse the same AOT cache.
|
||||||
|
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT",
|
||||||
|
# Boot-failure policy of a host-side profiling-teardown check; read by no
|
||||||
|
# compiled code.
|
||||||
|
"VLLM_FAIL_ON_PROFILING_KV_LEAK",
|
||||||
|
# Logging cadence for an already-compiled kernel's counters: it gates a
|
||||||
|
# host-side timer and a snapshot op, and is read by no compiled code, so
|
||||||
|
# hashing it would invalidate every compile cache for a log interval.
|
||||||
|
"MERLIN_TOPK_EXACT_COUNTERS_LOG",
|
||||||
|
# Host-side reader fallback (buffer length for a snapshot op outside
|
||||||
|
# capture); dead on a build that answers the _slots query.
|
||||||
|
"MERLIN_TOPK_COUNTER_SLOTS",
|
||||||
|
# ------------------------------------------------------------------
|
||||||
"FORCE_COLOR",
|
"FORCE_COLOR",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -74,11 +74,11 @@ class MambaBase(AttentionLayerBase):
|
|||||||
mamba_type=self.mamba_type,
|
mamba_type=self.mamba_type,
|
||||||
tp_replicated=self.is_kv_cache_tp_replicated,
|
tp_replicated=self.is_kv_cache_tp_replicated,
|
||||||
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
|
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
|
||||||
# ReplaySSM verifies the whole window off one checkpoint, so it
|
# RecoverSSM verifies the whole window off one checkpoint, so it
|
||||||
# never writes the baseline's per-draft-token state slots.
|
# never writes the baseline's per-draft-token state slots.
|
||||||
num_speculative_blocks=(
|
num_speculative_blocks=(
|
||||||
0
|
0
|
||||||
if vllm_config.cache_config.use_replayssm
|
if vllm_config.cache_config.use_recoverssm
|
||||||
else vllm_config.num_speculative_tokens
|
else vllm_config.num_speculative_tokens
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -370,10 +370,15 @@ class ChunkGatedDeltaRule(CustomOp):
|
|||||||
|
|
||||||
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
@PluggableLayer.register("qwen_gated_delta_net_attention")
|
||||||
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||||
def get_state_shape(
|
def _uses_gdn_recoverssm(self) -> bool:
|
||||||
self,
|
"""One predicate for the replay state's shape, dtype and the attention
|
||||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
backend."""
|
||||||
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
from vllm.model_executor.layers.mamba.recoverssm_utils import uses_recoverssm
|
||||||
|
|
||||||
|
return uses_recoverssm(self.cache_config, self.num_spec)
|
||||||
|
|
||||||
|
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
|
||||||
|
shapes = MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||||
self.tp_size,
|
self.tp_size,
|
||||||
self.num_k_heads,
|
self.num_k_heads,
|
||||||
self.num_v_heads,
|
self.num_v_heads,
|
||||||
@@ -382,6 +387,32 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
self.conv_kernel_size,
|
self.conv_kernel_size,
|
||||||
self.num_spec,
|
self.num_spec,
|
||||||
)
|
)
|
||||||
|
if self._uses_gdn_recoverssm():
|
||||||
|
# per-token replay record [HV, spec_query_len, V + K + 1] fp32
|
||||||
|
shapes = (
|
||||||
|
*shapes,
|
||||||
|
(
|
||||||
|
self.num_v_heads // self.tp_size,
|
||||||
|
self.num_spec + 1,
|
||||||
|
self.head_v_dim + self.head_k_dim + 1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return shapes
|
||||||
|
|
||||||
|
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||||
|
dtypes = super().get_state_dtype()
|
||||||
|
if self._uses_gdn_recoverssm():
|
||||||
|
dtypes = (*dtypes, torch.float32)
|
||||||
|
return dtypes
|
||||||
|
|
||||||
|
def get_attn_backend(self):
|
||||||
|
if self._uses_gdn_recoverssm():
|
||||||
|
from vllm.v1.attention.backends.gdn_recoverssm import (
|
||||||
|
GDNRecoverSSMAttentionBackend,
|
||||||
|
)
|
||||||
|
|
||||||
|
return GDNRecoverSSMAttentionBackend
|
||||||
|
return super().get_attn_backend()
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -531,9 +562,22 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
elif current_platform.is_xpu():
|
elif current_platform.is_xpu():
|
||||||
self.gdn_decode_kernel = "XPU"
|
self.gdn_decode_kernel = "XPU"
|
||||||
|
|
||||||
self.enable_fused_gdn_spec_decode = (
|
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
|
||||||
self.gdn_decode_kernel == "cuda" and self.speculative_config is not None
|
(
|
||||||
|
self.gdn_decode_kernel,
|
||||||
|
self.enable_fused_gdn_spec_decode,
|
||||||
|
self.enable_packed_recurrent_decode,
|
||||||
|
) = self._decode_paths(
|
||||||
|
self.gdn_decode_kernel,
|
||||||
|
self.use_gdn_recoverssm,
|
||||||
|
self.enable_packed_recurrent_decode,
|
||||||
|
self.speculative_config is not None,
|
||||||
)
|
)
|
||||||
|
if self.use_gdn_recoverssm:
|
||||||
|
logger.info_once(
|
||||||
|
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
|
||||||
|
self.num_spec + 1,
|
||||||
|
)
|
||||||
if self.speculative_config is not None:
|
if self.speculative_config is not None:
|
||||||
logger.info_once("GDN spec decode kernel: %s", self.gdn_decode_kernel)
|
logger.info_once("GDN spec decode kernel: %s", self.gdn_decode_kernel)
|
||||||
|
|
||||||
@@ -542,10 +586,33 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||||
compilation_config.static_forward_context[prefix] = self
|
compilation_config.static_forward_context[prefix] = self
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode_paths(
|
||||||
|
gdn_decode_kernel: str,
|
||||||
|
use_recoverssm: bool,
|
||||||
|
packed_recurrent_decode: bool,
|
||||||
|
has_speculative_config: bool,
|
||||||
|
) -> tuple[str, bool, bool]:
|
||||||
|
"""(decode kernel, fused spec-decode forward, packed recurrent decode).
|
||||||
|
|
||||||
|
The fused forward is on whenever the CUDA decode kernel is eligible and
|
||||||
|
the engine runs speculation (#59735 keeps non-speculative decode on the
|
||||||
|
standard path). RecoverSSM's verify replaces the per-draft snapshot path
|
||||||
|
and needs the Triton decode kernel for its spec rows, so with RecoverSSM
|
||||||
|
the decode kernel is Triton and the packed recurrent decode is off. The
|
||||||
|
fused forward stays on: it calls _forward_core for prefill and verify
|
||||||
|
rows alike and only fuses the input split and the gated norm. Turning it
|
||||||
|
off sends every prefill chunk down the unfused path.
|
||||||
|
"""
|
||||||
|
enable_fused_forward = gdn_decode_kernel == "cuda" and has_speculative_config
|
||||||
|
if use_recoverssm:
|
||||||
|
return "triton", enable_fused_forward, False
|
||||||
|
return gdn_decode_kernel, enable_fused_forward, packed_recurrent_decode
|
||||||
|
|
||||||
def _fused_gdn_decode_unsupported_reason(
|
def _fused_gdn_decode_unsupported_reason(
|
||||||
self, vllm_config: VllmConfig
|
self, vllm_config: VllmConfig
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
|
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2]
|
||||||
if (
|
if (
|
||||||
self.gqa_interleaved_layout
|
self.gqa_interleaved_layout
|
||||||
or self.head_k_dim != 128
|
or self.head_k_dim != 128
|
||||||
@@ -805,6 +872,25 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
|
|
||||||
return mixed_qkv_out, z_out, b_out, a_out
|
return mixed_qkv_out, z_out, b_out, a_out
|
||||||
|
|
||||||
|
def _recoverssm_verify_qkv(self, mixed_qkv):
|
||||||
|
"""Q/K/V for the RecoverSSM verify.
|
||||||
|
|
||||||
|
The verify kernel reads token-strided heads, so for a 2-D [tokens, q|k|v] conv
|
||||||
|
output with a contiguous last dimension it gets views instead of copies. Any
|
||||||
|
other layout falls back to the contiguous copies of rearrange_mixed_qkv().
|
||||||
|
Returns (query, key, value, is_view)."""
|
||||||
|
if mixed_qkv.ndim != 2 or mixed_qkv.stride(-1) != 1:
|
||||||
|
return (*self.rearrange_mixed_qkv(mixed_qkv), False)
|
||||||
|
q_dim = self.key_dim // self.tp_size
|
||||||
|
v_dim = self.value_dim // self.tp_size
|
||||||
|
num_tokens = mixed_qkv.shape[0]
|
||||||
|
query = mixed_qkv[:, :q_dim].view(1, num_tokens, -1, self.head_k_dim)
|
||||||
|
key = mixed_qkv[:, q_dim : 2 * q_dim].view(1, num_tokens, -1, self.head_k_dim)
|
||||||
|
value = mixed_qkv[:, 2 * q_dim : 2 * q_dim + v_dim].view(
|
||||||
|
1, num_tokens, -1, self.head_v_dim
|
||||||
|
)
|
||||||
|
return query, key, value, True
|
||||||
|
|
||||||
def rearrange_mixed_qkv(self, mixed_qkv):
|
def rearrange_mixed_qkv(self, mixed_qkv):
|
||||||
"""Split packed qkv into contiguous (1, seq, heads, dim) tensors.
|
"""Split packed qkv into contiguous (1, seq, heads, dim) tensors.
|
||||||
|
|
||||||
@@ -1090,7 +1176,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
dtype = qkv_or_qkvz.dtype
|
dtype = qkv_or_qkvz.dtype
|
||||||
num_k_heads = self.num_k_heads // self.tp_size
|
num_k_heads = self.num_k_heads // self.tp_size
|
||||||
num_v_heads = self.num_v_heads // self.tp_size
|
num_v_heads = self.num_v_heads // self.tp_size
|
||||||
_, state_dtype = self.get_state_dtype()
|
state_dtype = self.get_state_dtype()[1] # 3 dtypes with the replay record
|
||||||
|
|
||||||
# All kernels use BT = chunk_size, so a single pass with T = chunk_size
|
# All kernels use BT = chunk_size, so a single pass with T = chunk_size
|
||||||
# is sufficient to populate every autotuner cache. Mirror the real
|
# is sufficient to populate every autotuner cache. Mirror the real
|
||||||
@@ -1348,7 +1434,11 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
],
|
],
|
||||||
num_accepted_tokens=num_accepted_tokens,
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
query_start_loc=spec_query_start_loc,
|
query_start_loc=spec_query_start_loc,
|
||||||
max_query_len=spec_state_indices_tensor.size(-1),
|
max_query_len=(
|
||||||
|
self.num_spec + 1
|
||||||
|
if getattr(self, "use_gdn_recoverssm", False)
|
||||||
|
else spec_state_indices_tensor.size(-1)
|
||||||
|
),
|
||||||
validate_data=False,
|
validate_data=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1385,7 +1475,17 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
else:
|
else:
|
||||||
mixed_qkv_non_spec = None
|
mixed_qkv_non_spec = None
|
||||||
|
|
||||||
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
if (
|
||||||
|
spec_sequence_masks is not None
|
||||||
|
and mixed_qkv_spec is not None
|
||||||
|
and getattr(self, "use_gdn_recoverssm", False)
|
||||||
|
):
|
||||||
|
query_spec, key_spec, value_spec, verify_views = (
|
||||||
|
self._recoverssm_verify_qkv(mixed_qkv_spec)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
verify_views = False
|
||||||
|
query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec)
|
||||||
|
|
||||||
# Split mixed non-spec-decode+prefill to process independently
|
# Split mixed non-spec-decode+prefill to process independently
|
||||||
split_non_spec = (
|
split_non_spec = (
|
||||||
@@ -1448,7 +1548,43 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
# 2. Recurrent attention
|
# 2. Recurrent attention
|
||||||
|
|
||||||
# 2.1: Process the multi-query part
|
# 2.1: Process the multi-query part
|
||||||
if spec_sequence_masks is not None:
|
if spec_sequence_masks is not None and getattr(
|
||||||
|
self, "use_gdn_recoverssm", False
|
||||||
|
):
|
||||||
|
# verify the window off the checkpoint; the commit after sampling
|
||||||
|
# writes the state once
|
||||||
|
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||||
|
gdn_recoverssm_verify,
|
||||||
|
)
|
||||||
|
|
||||||
|
_n = attn_metadata.num_spec_decodes
|
||||||
|
assert spec_query_start_loc is not None
|
||||||
|
assert spec_state_indices_tensor is not None
|
||||||
|
core_attn_out_spec = gdn_recoverssm_verify(
|
||||||
|
self.A_log,
|
||||||
|
a_spec,
|
||||||
|
b_spec,
|
||||||
|
self.dt_bias,
|
||||||
|
query_spec,
|
||||||
|
key_spec,
|
||||||
|
value_spec,
|
||||||
|
checkpoint_state=ssm_state,
|
||||||
|
replay_cache=self_kv_cache[2],
|
||||||
|
query_start_loc=spec_query_start_loc[: _n + 1],
|
||||||
|
state_indices=spec_state_indices_tensor[:_n, 0],
|
||||||
|
spec_query_len=self.num_spec + 1,
|
||||||
|
use_qk_l2norm_in_kernel=True,
|
||||||
|
# spec-only batch: write straight into the layer output
|
||||||
|
out=(
|
||||||
|
core_attn_out[:num_actual_tokens].unsqueeze(0)
|
||||||
|
if verify_views
|
||||||
|
and mixed_qkv_non_spec is None
|
||||||
|
and query_spec.shape[1] == num_actual_tokens
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
last_recurrent_state = None
|
||||||
|
elif spec_sequence_masks is not None:
|
||||||
core_attn_out_spec, last_recurrent_state = (
|
core_attn_out_spec, last_recurrent_state = (
|
||||||
fused_sigmoid_gating_delta_rule_update(
|
fused_sigmoid_gating_delta_rule_update(
|
||||||
A_log=self.A_log,
|
A_log=self.A_log,
|
||||||
@@ -1563,7 +1699,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
0, non_spec_token_indx, core_attn_out_non_spec.squeeze(0)
|
0, non_spec_token_indx, core_attn_out_non_spec.squeeze(0)
|
||||||
)
|
)
|
||||||
elif spec_sequence_masks is not None:
|
elif spec_sequence_masks is not None:
|
||||||
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
if core_attn_out_spec.data_ptr() != core_attn_out.data_ptr():
|
||||||
|
core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0)
|
||||||
else:
|
else:
|
||||||
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,701 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""RecoverSSM for Gated DeltaNet (Qwen-style GDN): speculative verify from one
|
||||||
|
checkpoint without per-draft state snapshots, and accepted-state recovery after
|
||||||
|
sampling.
|
||||||
|
|
||||||
|
GDN port of the Kimi-K3 KDA RecoverSSM path
|
||||||
|
(vllm/models/kimi_k3/nvidia/ops/recoverssm.py). Differences from KDA: the gate is a
|
||||||
|
scalar per value head (g = -exp(A_log) * softplus(a + dt_bias)), q/k heads are grouped
|
||||||
|
over value heads (HV // H), and q/k are L2-normalised inside the kernel when requested.
|
||||||
|
The verify step caches, per token, the delta-rule correction c_t = beta_t * (v_t - S'_t
|
||||||
|
k_t) (V floats), the normalised key (K floats) and the decay exp(g_t) (1 float) in one
|
||||||
|
fp32 "replay" record. Recovery folds the accepted prefix: S_n = exp(sum_t g_t) S_0 +
|
||||||
|
sum_t exp(sum_{s>t} g_s) c_t k_t^T and writes the checkpoint once (plus the
|
||||||
|
block-boundary state in align mode)."""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.mamba.ops.recoverssm import (
|
||||||
|
compact_conv_state_kernel,
|
||||||
|
prepare_commit_plan_kernel,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.recoverssm_utils import recoverssm_require
|
||||||
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||||
|
|
||||||
|
|
||||||
|
def _require(cond: bool, msg: str) -> None:
|
||||||
|
recoverssm_require(cond, msg, "GDN RecoverSSM")
|
||||||
|
|
||||||
|
|
||||||
|
@triton.jit
|
||||||
|
def _gdn_recoverssm_verify_kernel(
|
||||||
|
q_ptr,
|
||||||
|
k_ptr,
|
||||||
|
v_ptr,
|
||||||
|
a_ptr,
|
||||||
|
b_ptr,
|
||||||
|
A_log_ptr,
|
||||||
|
dt_bias_ptr,
|
||||||
|
state_ptr,
|
||||||
|
replay_ptr,
|
||||||
|
out_ptr,
|
||||||
|
query_start_loc_ptr,
|
||||||
|
state_indices_ptr,
|
||||||
|
scale,
|
||||||
|
softplus_beta,
|
||||||
|
softplus_threshold,
|
||||||
|
null_block_id,
|
||||||
|
stride_q_token,
|
||||||
|
stride_k_token,
|
||||||
|
stride_v_token,
|
||||||
|
stride_a_token,
|
||||||
|
stride_b_token,
|
||||||
|
stride_state_block,
|
||||||
|
stride_state_head,
|
||||||
|
stride_state_v,
|
||||||
|
stride_state_k,
|
||||||
|
stride_replay_block,
|
||||||
|
stride_replay_head,
|
||||||
|
stride_replay_pos,
|
||||||
|
stride_replay_dim,
|
||||||
|
stride_out_token,
|
||||||
|
stride_qsl,
|
||||||
|
stride_si,
|
||||||
|
H: tl.constexpr,
|
||||||
|
HV: tl.constexpr,
|
||||||
|
K: tl.constexpr,
|
||||||
|
V: tl.constexpr,
|
||||||
|
BK: tl.constexpr,
|
||||||
|
BV: tl.constexpr,
|
||||||
|
SPEC_QUERY_LEN: tl.constexpr,
|
||||||
|
USE_QK_L2NORM: tl.constexpr,
|
||||||
|
):
|
||||||
|
pid_v = tl.program_id(0)
|
||||||
|
pid_b = tl.program_id(1)
|
||||||
|
pid_hv = tl.program_id(2)
|
||||||
|
pid_h = pid_hv // (HV // H)
|
||||||
|
bos = tl.load(query_start_loc_ptr + pid_b * stride_qsl).to(tl.int64)
|
||||||
|
eos = tl.load(query_start_loc_ptr + (pid_b + 1) * stride_qsl).to(tl.int64)
|
||||||
|
query_len = eos - bos
|
||||||
|
state_idx = tl.load(state_indices_ptr + pid_b * stride_si).to(tl.int64)
|
||||||
|
offs_k = tl.arange(0, BK)
|
||||||
|
offs_v = pid_v * BV + tl.arange(0, BV)
|
||||||
|
mask_k = offs_k < K
|
||||||
|
mask_v = offs_v < V
|
||||||
|
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||||
|
if state_idx <= null_block_id:
|
||||||
|
for t in tl.static_range(SPEC_QUERY_LEN):
|
||||||
|
tl.store(
|
||||||
|
out_ptr + (bos + t) * stride_out_token + pid_hv * V + offs_v,
|
||||||
|
tl.zeros([BV], dtype=tl.float32),
|
||||||
|
mask=(t < query_len) & mask_v,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
state = tl.load(
|
||||||
|
state_ptr
|
||||||
|
+ state_idx * stride_state_block
|
||||||
|
+ pid_hv * stride_state_head
|
||||||
|
+ offs_v[:, None] * stride_state_v
|
||||||
|
+ offs_k[None, :] * stride_state_k,
|
||||||
|
mask=mask_state,
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.float32)
|
||||||
|
neg_A = -tl.exp(tl.load(A_log_ptr + pid_hv).to(tl.float32))
|
||||||
|
dt_bias = tl.load(dt_bias_ptr + pid_hv).to(tl.float32)
|
||||||
|
rec_base = (
|
||||||
|
replay_ptr + state_idx * stride_replay_block + pid_hv * stride_replay_head
|
||||||
|
)
|
||||||
|
for t in tl.static_range(SPEC_QUERY_LEN):
|
||||||
|
valid = t < query_len
|
||||||
|
tok = bos + t
|
||||||
|
q = tl.load(
|
||||||
|
q_ptr + tok * stride_q_token + pid_h * K + offs_k,
|
||||||
|
mask=valid & mask_k,
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.float32)
|
||||||
|
k = tl.load(
|
||||||
|
k_ptr + tok * stride_k_token + pid_h * K + offs_k,
|
||||||
|
mask=valid & mask_k,
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.float32)
|
||||||
|
v = tl.load(
|
||||||
|
v_ptr + tok * stride_v_token + pid_hv * V + offs_v,
|
||||||
|
mask=valid & mask_v,
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.float32)
|
||||||
|
x = (
|
||||||
|
tl.load(a_ptr + tok * stride_a_token + pid_hv, mask=valid, other=0.0).to(
|
||||||
|
tl.float32
|
||||||
|
)
|
||||||
|
+ dt_bias
|
||||||
|
)
|
||||||
|
sp = tl.where(
|
||||||
|
softplus_beta * x <= softplus_threshold,
|
||||||
|
(1 / softplus_beta) * tl.log(1 + tl.exp(softplus_beta * x)),
|
||||||
|
x,
|
||||||
|
)
|
||||||
|
g = neg_A * sp
|
||||||
|
beta = tl.sigmoid(
|
||||||
|
tl.load(b_ptr + tok * stride_b_token + pid_hv, mask=valid, other=0.0).to(
|
||||||
|
tl.float32
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if USE_QK_L2NORM:
|
||||||
|
q = q * tl.rsqrt(tl.sum(q * q) + 1e-6)
|
||||||
|
k = k * tl.rsqrt(tl.sum(k * k) + 1e-6)
|
||||||
|
q = q * scale
|
||||||
|
decay = tl.exp(g)
|
||||||
|
new_state = state * decay
|
||||||
|
corr = (v - tl.sum(new_state * k[None, :], 1)) * beta
|
||||||
|
new_state = new_state + corr[:, None] * k[None, :]
|
||||||
|
state = tl.where(valid, new_state, state)
|
||||||
|
o = tl.sum(state * q[None, :], 1)
|
||||||
|
tl.store(
|
||||||
|
out_ptr + tok * stride_out_token + pid_hv * V + offs_v,
|
||||||
|
o.to(out_ptr.dtype.element_ty),
|
||||||
|
mask=valid & mask_v,
|
||||||
|
)
|
||||||
|
rec = rec_base + t * stride_replay_pos
|
||||||
|
tl.store(rec + offs_v * stride_replay_dim, corr, mask=valid & mask_v)
|
||||||
|
if pid_v == 0:
|
||||||
|
tl.store(rec + (V + offs_k) * stride_replay_dim, k, mask=valid & mask_k)
|
||||||
|
# the decay as multiplied here, so the commit replays this exact float
|
||||||
|
tl.store(rec + (V + K) * stride_replay_dim, decay, mask=valid)
|
||||||
|
|
||||||
|
|
||||||
|
def gdn_recoverssm_verify(
|
||||||
|
A_log: torch.Tensor,
|
||||||
|
a: torch.Tensor,
|
||||||
|
b: torch.Tensor,
|
||||||
|
dt_bias: torch.Tensor,
|
||||||
|
q: torch.Tensor,
|
||||||
|
k: torch.Tensor,
|
||||||
|
v: torch.Tensor,
|
||||||
|
*,
|
||||||
|
checkpoint_state: torch.Tensor,
|
||||||
|
replay_cache: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
state_indices: torch.Tensor,
|
||||||
|
spec_query_len: int,
|
||||||
|
scale: float | None = None,
|
||||||
|
softplus_beta: float = 1.0,
|
||||||
|
softplus_threshold: float = 20.0,
|
||||||
|
use_qk_l2norm_in_kernel: bool = True,
|
||||||
|
out: torch.Tensor | None = None,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Verify a GDN speculative window against its checkpoint without modifying the
|
||||||
|
checkpoint. q, k: [1, T, H, K]; v: [1, T, HV, V]; a, b: [T, HV] (or [1, T, HV]);
|
||||||
|
checkpoint_state: [blocks, HV, V, K]; replay_cache: [blocks, HV, spec_query_len, V +
|
||||||
|
K + 1] fp32. Returns out [1, T, HV, V]."""
|
||||||
|
_require(
|
||||||
|
q.ndim == 4 and q.shape[0] == 1, "q must have shape [1, tokens, heads, dim]"
|
||||||
|
)
|
||||||
|
_, total, H, K = q.shape
|
||||||
|
_require(k.shape == q.shape, "q and k shapes differ")
|
||||||
|
_require(
|
||||||
|
v.ndim == 4 and tuple(v.shape[:2]) == (1, total),
|
||||||
|
"v must have shape [1, tokens, value heads, dim]",
|
||||||
|
)
|
||||||
|
HV, V = v.shape[-2], v.shape[-1]
|
||||||
|
_require(
|
||||||
|
H > 0 and HV >= H and HV % H == 0,
|
||||||
|
"value heads must be a positive multiple of the q/k heads",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
a.numel() == total * HV and b.numel() == total * HV,
|
||||||
|
"gate or beta shape is incompatible",
|
||||||
|
)
|
||||||
|
a2 = a.reshape(total, HV)
|
||||||
|
b2 = b.reshape(total, HV)
|
||||||
|
_require(
|
||||||
|
a2.stride(1) == 1 and b2.stride(1) == 1,
|
||||||
|
"gate and beta heads must be contiguous",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
tuple(A_log.shape) == (HV,) and tuple(dt_bias.shape) == (HV,),
|
||||||
|
"A_log or dt_bias shape is incompatible",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
all(t_.stride(-1) == 1 and t_.stride(-2) == K for t_ in (q, k)),
|
||||||
|
"q and k heads must be contiguous",
|
||||||
|
)
|
||||||
|
_require(v.stride(-1) == 1 and v.stride(-2) == V, "v heads must be contiguous")
|
||||||
|
_require(checkpoint_state.ndim == 4, "checkpoint must be four-dimensional")
|
||||||
|
nb = checkpoint_state.shape[0]
|
||||||
|
_require(
|
||||||
|
tuple(checkpoint_state.shape[1:]) == (HV, V, K),
|
||||||
|
"checkpoint shape is incompatible",
|
||||||
|
)
|
||||||
|
_rec = (nb, HV, spec_query_len, V + K + 1)
|
||||||
|
_require(tuple(replay_cache.shape) == _rec, f"replay buffer needs shape {_rec}")
|
||||||
|
_require(replay_cache.dtype == torch.float32, "replay buffer must use float32")
|
||||||
|
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||||
|
_require(
|
||||||
|
query_start_loc.ndim == 1
|
||||||
|
and query_start_loc.shape[0] == state_indices.shape[0] + 1,
|
||||||
|
"query metadata is incompatible",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
total <= state_indices.shape[0] * spec_query_len,
|
||||||
|
"speculative decode input exceeds its activation capacity",
|
||||||
|
)
|
||||||
|
_dev = q.device
|
||||||
|
_require(
|
||||||
|
all(
|
||||||
|
t_.device == _dev
|
||||||
|
for t_ in (
|
||||||
|
k,
|
||||||
|
v,
|
||||||
|
a,
|
||||||
|
b,
|
||||||
|
A_log,
|
||||||
|
dt_bias,
|
||||||
|
checkpoint_state,
|
||||||
|
replay_cache,
|
||||||
|
query_start_loc,
|
||||||
|
state_indices,
|
||||||
|
)
|
||||||
|
if t_ is not None
|
||||||
|
)
|
||||||
|
and (out is None or out.device == _dev),
|
||||||
|
"inputs must be on the same device",
|
||||||
|
)
|
||||||
|
if scale is None:
|
||||||
|
scale = K**-0.5
|
||||||
|
if out is None:
|
||||||
|
out = torch.empty(1, total, HV, V, dtype=v.dtype, device=v.device)
|
||||||
|
_require(
|
||||||
|
tuple(out.shape) == (1, total, HV, V) and out.stride()[2:] == (V, 1),
|
||||||
|
"output shape or layout is incompatible",
|
||||||
|
)
|
||||||
|
batch = state_indices.shape[0]
|
||||||
|
if total == 0 or batch == 0:
|
||||||
|
return out
|
||||||
|
BK = triton.next_power_of_2(K)
|
||||||
|
BV = min(triton.next_power_of_2(V), 32)
|
||||||
|
grid = (triton.cdiv(V, BV), batch, HV)
|
||||||
|
_gdn_recoverssm_verify_kernel[grid](
|
||||||
|
q,
|
||||||
|
k,
|
||||||
|
v,
|
||||||
|
a2,
|
||||||
|
b2,
|
||||||
|
A_log,
|
||||||
|
dt_bias,
|
||||||
|
checkpoint_state,
|
||||||
|
replay_cache,
|
||||||
|
out,
|
||||||
|
query_start_loc,
|
||||||
|
state_indices,
|
||||||
|
scale,
|
||||||
|
softplus_beta,
|
||||||
|
softplus_threshold,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
q.stride(1),
|
||||||
|
k.stride(1),
|
||||||
|
v.stride(1),
|
||||||
|
a2.stride(0),
|
||||||
|
b2.stride(0),
|
||||||
|
checkpoint_state.stride(0),
|
||||||
|
checkpoint_state.stride(1),
|
||||||
|
checkpoint_state.stride(2),
|
||||||
|
checkpoint_state.stride(3),
|
||||||
|
replay_cache.stride(0),
|
||||||
|
replay_cache.stride(1),
|
||||||
|
replay_cache.stride(2),
|
||||||
|
replay_cache.stride(3),
|
||||||
|
out.stride(1),
|
||||||
|
query_start_loc.stride(0),
|
||||||
|
state_indices.stride(0),
|
||||||
|
H=H,
|
||||||
|
HV=HV,
|
||||||
|
K=K,
|
||||||
|
V=V,
|
||||||
|
BK=BK,
|
||||||
|
BV=BV,
|
||||||
|
SPEC_QUERY_LEN=spec_query_len,
|
||||||
|
USE_QK_L2NORM=use_qk_l2norm_in_kernel,
|
||||||
|
num_warps=4,
|
||||||
|
num_stages=2,
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@triton.jit
|
||||||
|
def _commit_gdn_state_kernel(
|
||||||
|
state_ref_ptr,
|
||||||
|
state_base_addrs_ptr,
|
||||||
|
state_block_strides_ptr,
|
||||||
|
replay_ref_ptr,
|
||||||
|
replay_base_addrs_ptr,
|
||||||
|
replay_block_strides_ptr,
|
||||||
|
state_indices_ptr,
|
||||||
|
commit_lens_ptr,
|
||||||
|
final_state_indices_ptr,
|
||||||
|
boundary_state_indices_ptr,
|
||||||
|
boundary_recovery_lens_ptr,
|
||||||
|
null_block_id,
|
||||||
|
stride_state_head,
|
||||||
|
stride_state_v,
|
||||||
|
stride_state_k,
|
||||||
|
stride_replay_head,
|
||||||
|
stride_replay_pos,
|
||||||
|
stride_replay_dim,
|
||||||
|
stride_si,
|
||||||
|
K: tl.constexpr,
|
||||||
|
V: tl.constexpr,
|
||||||
|
BK: tl.constexpr,
|
||||||
|
BV: tl.constexpr,
|
||||||
|
NUM_HEADS: tl.constexpr,
|
||||||
|
BOUNDARY: tl.constexpr,
|
||||||
|
):
|
||||||
|
# Every program runs one fold and keeps one tile live. In align mode the
|
||||||
|
# block-boundary state (the window's first boundary_len tokens) is folded by a
|
||||||
|
# second launch (BOUNDARY): a crossing window's boundary state goes into its
|
||||||
|
# source block, which the first launch's final-state fold must still read.
|
||||||
|
pid_v = tl.program_id(0)
|
||||||
|
pid_b = tl.program_id(1)
|
||||||
|
pid_lh = tl.program_id(2)
|
||||||
|
pid_l = pid_lh // NUM_HEADS
|
||||||
|
pid_h = pid_lh % NUM_HEADS
|
||||||
|
src = tl.load(state_indices_ptr + pid_b * stride_si).to(tl.int64)
|
||||||
|
if src <= null_block_id:
|
||||||
|
return
|
||||||
|
n = tl.load(commit_lens_ptr + pid_b)
|
||||||
|
if n == 0:
|
||||||
|
return
|
||||||
|
final_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||||
|
if final_idx <= null_block_id:
|
||||||
|
return
|
||||||
|
dst_idx = final_idx
|
||||||
|
if BOUNDARY:
|
||||||
|
dst_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||||
|
# A window ending exactly on the boundary keeps its final state in that
|
||||||
|
# block: the first launch already wrote it (and may have overwritten src).
|
||||||
|
if dst_idx <= null_block_id or dst_idx == final_idx:
|
||||||
|
return
|
||||||
|
n = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||||
|
state_ptr = tl.load(state_base_addrs_ptr + pid_l).to(
|
||||||
|
tl.pointer_type(state_ref_ptr.dtype.element_ty)
|
||||||
|
)
|
||||||
|
sbs = tl.load(state_block_strides_ptr + pid_l)
|
||||||
|
replay_ptr = tl.load(replay_base_addrs_ptr + pid_l).to(
|
||||||
|
tl.pointer_type(replay_ref_ptr.dtype.element_ty)
|
||||||
|
)
|
||||||
|
rbs = tl.load(replay_block_strides_ptr + pid_l)
|
||||||
|
rec_base = replay_ptr + src * rbs + pid_h * stride_replay_head
|
||||||
|
offs_k = tl.arange(0, BK)
|
||||||
|
offs_v = pid_v * BV + tl.arange(0, BV)
|
||||||
|
mask_k = offs_k < K
|
||||||
|
mask_v = offs_v < V
|
||||||
|
mask_state = mask_v[:, None] & mask_k[None, :]
|
||||||
|
sp = offs_v[:, None] * stride_state_v + offs_k[None, :] * stride_state_k
|
||||||
|
s0 = tl.load(
|
||||||
|
state_ptr + src * sbs + pid_h * stride_state_head + sp,
|
||||||
|
mask=mask_state,
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.float32)
|
||||||
|
# Forward over the accepted tokens with the verify step's own update, so the committed
|
||||||
|
# state is the per-token state bit for bit (the decay and corr stored are the floats
|
||||||
|
# the verify kernel used).
|
||||||
|
h = s0
|
||||||
|
for t in range(n):
|
||||||
|
rec = rec_base + t * stride_replay_pos
|
||||||
|
c = tl.load(rec + offs_v * stride_replay_dim, mask=mask_v, other=0.0)
|
||||||
|
kk = tl.load(rec + (V + offs_k) * stride_replay_dim, mask=mask_k, other=0.0)
|
||||||
|
decay = tl.load(rec + (V + K) * stride_replay_dim)
|
||||||
|
h = tl.fma(c[:, None], kk[None, :], h * decay)
|
||||||
|
tl.store(
|
||||||
|
state_ptr + dst_idx * sbs + pid_h * stride_state_head + sp,
|
||||||
|
h.to(state_ref_ptr.dtype.element_ty),
|
||||||
|
mask=mask_state,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GDNRecoverSSMCommitContext:
|
||||||
|
conv_states: tuple[torch.Tensor, ...]
|
||||||
|
conv_state_base_addrs: torch.Tensor
|
||||||
|
conv_state_block_strides: torch.Tensor
|
||||||
|
conv_state_dim_strides: torch.Tensor
|
||||||
|
conv_state_token_strides: torch.Tensor
|
||||||
|
conv_history_len: int
|
||||||
|
checkpoints: tuple[torch.Tensor, ...]
|
||||||
|
state_base_addrs: torch.Tensor
|
||||||
|
state_block_strides: torch.Tensor
|
||||||
|
replays: tuple[torch.Tensor, ...]
|
||||||
|
replay_base_addrs: torch.Tensor
|
||||||
|
replay_block_strides: torch.Tensor
|
||||||
|
commit_lens: torch.Tensor
|
||||||
|
final_state_indices: torch.Tensor
|
||||||
|
boundary_state_indices: torch.Tensor
|
||||||
|
boundary_recovery_lens: torch.Tensor
|
||||||
|
spec_query_len: int
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_tensors(
|
||||||
|
cls,
|
||||||
|
conv_states: Sequence[torch.Tensor],
|
||||||
|
checkpoints: Sequence[torch.Tensor],
|
||||||
|
replays: Sequence[torch.Tensor],
|
||||||
|
*,
|
||||||
|
spec_query_len: int,
|
||||||
|
max_num_reqs: int,
|
||||||
|
conv_dim_first: bool = True,
|
||||||
|
) -> "GDNRecoverSSMCommitContext":
|
||||||
|
_require(len(checkpoints) > 0, "commit requires at least one layer")
|
||||||
|
_require(
|
||||||
|
len(conv_states) == len(checkpoints) == len(replays),
|
||||||
|
"conv, state and replay lists differ",
|
||||||
|
)
|
||||||
|
if not conv_dim_first:
|
||||||
|
conv_states = [s.transpose(-1, -2) for s in conv_states]
|
||||||
|
ref = checkpoints[0]
|
||||||
|
_require(ref.ndim == 4, "checkpoint must be four-dimensional")
|
||||||
|
nb, HV, V, K = ref.shape
|
||||||
|
_dev = ref.device
|
||||||
|
for s in checkpoints:
|
||||||
|
_require(
|
||||||
|
s.shape == ref.shape
|
||||||
|
and s.dtype == ref.dtype
|
||||||
|
and s.stride()[1:] == ref.stride()[1:]
|
||||||
|
and s.device == _dev,
|
||||||
|
"layers need matching checkpoints",
|
||||||
|
)
|
||||||
|
_rec = (nb, HV, spec_query_len, V + K + 1)
|
||||||
|
for r in replays:
|
||||||
|
_require(
|
||||||
|
tuple(r.shape) == _rec
|
||||||
|
and r.dtype == torch.float32
|
||||||
|
and r.stride()[1:] == replays[0].stride()[1:]
|
||||||
|
and r.device == _dev,
|
||||||
|
f"layers need matching float32 replay buffers of shape {_rec}",
|
||||||
|
)
|
||||||
|
conv_ref = conv_states[0]
|
||||||
|
_require(
|
||||||
|
conv_ref.ndim == 3 and conv_ref.shape[0] == nb,
|
||||||
|
"conv state must be [blocks, dim, window] with the checkpoint's "
|
||||||
|
"block count",
|
||||||
|
)
|
||||||
|
for c in conv_states:
|
||||||
|
_require(
|
||||||
|
c.shape == conv_ref.shape
|
||||||
|
and c.dtype == conv_ref.dtype
|
||||||
|
and c.stride()[1:] == conv_ref.stride()[1:]
|
||||||
|
and c.device == _dev,
|
||||||
|
"layers need matching conv states",
|
||||||
|
)
|
||||||
|
conv_dim, conv_len = conv_ref.shape[1:]
|
||||||
|
hist = conv_len - spec_query_len + 1
|
||||||
|
_require(hist > 0, "conv state is shorter than its window")
|
||||||
|
dev = ref.device
|
||||||
|
addr = lambda ts: torch.tensor(
|
||||||
|
[t.data_ptr() for t in ts], dtype=torch.int64, device=dev
|
||||||
|
)
|
||||||
|
bstr = lambda ts: torch.tensor(
|
||||||
|
[t.stride(0) for t in ts], dtype=torch.int64, device=dev
|
||||||
|
)
|
||||||
|
z = lambda: torch.empty(max_num_reqs, dtype=torch.int32, device=dev)
|
||||||
|
return cls(
|
||||||
|
tuple(conv_states),
|
||||||
|
addr(conv_states),
|
||||||
|
bstr(conv_states),
|
||||||
|
torch.tensor(
|
||||||
|
[s.stride(1) for s in conv_states], dtype=torch.int64, device=dev
|
||||||
|
),
|
||||||
|
torch.tensor(
|
||||||
|
[s.stride(2) for s in conv_states], dtype=torch.int64, device=dev
|
||||||
|
),
|
||||||
|
hist,
|
||||||
|
tuple(checkpoints),
|
||||||
|
addr(checkpoints),
|
||||||
|
bstr(checkpoints),
|
||||||
|
tuple(replays),
|
||||||
|
addr(replays),
|
||||||
|
bstr(replays),
|
||||||
|
z(),
|
||||||
|
z(),
|
||||||
|
z(),
|
||||||
|
z(),
|
||||||
|
spec_query_len,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create(
|
||||||
|
cls, layers: Sequence[Any], *, spec_query_len: int, max_num_reqs: int
|
||||||
|
) -> "GDNRecoverSSMCommitContext":
|
||||||
|
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||||
|
|
||||||
|
if any(len(layer.kv_cache) != 3 for layer in layers):
|
||||||
|
raise ValueError("GDN RecoverSSM pages must contain conv, state and replay")
|
||||||
|
return cls.from_tensors(
|
||||||
|
[layer.kv_cache[0] for layer in layers],
|
||||||
|
[layer.kv_cache[1] for layer in layers],
|
||||||
|
[layer.kv_cache[2] for layer in layers],
|
||||||
|
spec_query_len=spec_query_len,
|
||||||
|
max_num_reqs=max_num_reqs,
|
||||||
|
conv_dim_first=is_conv_state_dim_first(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def commit(
|
||||||
|
self,
|
||||||
|
num_accepted_tokens: torch.Tensor,
|
||||||
|
state_indices: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
request_indices: torch.Tensor | None = None,
|
||||||
|
block_table: torch.Tensor | None = None,
|
||||||
|
num_computed_tokens: torch.Tensor | None = None,
|
||||||
|
mamba_block_size: int | None = None,
|
||||||
|
commit_conv: bool = True,
|
||||||
|
) -> None:
|
||||||
|
"""Fold accepted GDN and convolution inputs into every layer's checkpoint."""
|
||||||
|
batch = state_indices.shape[0]
|
||||||
|
if batch == 0:
|
||||||
|
return
|
||||||
|
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||||
|
_require(
|
||||||
|
batch <= self.commit_lens.shape[0], "commit batch exceeds its plan capacity"
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
query_start_loc.ndim == 1 and query_start_loc.shape[0] == batch + 1,
|
||||||
|
"commit metadata is incompatible",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
request_indices is None or request_indices.shape[0] >= batch,
|
||||||
|
"request mapping is too short",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
num_accepted_tokens.ndim == 1
|
||||||
|
and (request_indices is not None or num_accepted_tokens.shape[0] >= batch),
|
||||||
|
"accepted-token counts are too short",
|
||||||
|
)
|
||||||
|
_align = (block_table, num_computed_tokens, mamba_block_size)
|
||||||
|
_require(
|
||||||
|
all(x is None for x in _align) or all(x is not None for x in _align),
|
||||||
|
"align metadata is incomplete",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
mamba_block_size is None or mamba_block_size >= self.spec_query_len,
|
||||||
|
"align block size must cover one speculative window",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
block_table is None or block_table.ndim == 2,
|
||||||
|
"block table must be two-dimensional",
|
||||||
|
)
|
||||||
|
_dev = self.checkpoints[0].device
|
||||||
|
_require(
|
||||||
|
all(
|
||||||
|
t_ is None or t_.device == _dev
|
||||||
|
for t_ in (
|
||||||
|
num_accepted_tokens,
|
||||||
|
state_indices,
|
||||||
|
query_start_loc,
|
||||||
|
request_indices,
|
||||||
|
block_table,
|
||||||
|
num_computed_tokens,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"commit inputs must be on the same device",
|
||||||
|
)
|
||||||
|
bt_stride = (0, 0) if block_table is None else block_table.stride()
|
||||||
|
prepare_commit_plan_kernel[(batch,)](
|
||||||
|
num_accepted_tokens,
|
||||||
|
request_indices,
|
||||||
|
state_indices,
|
||||||
|
query_start_loc,
|
||||||
|
block_table,
|
||||||
|
num_computed_tokens,
|
||||||
|
self.commit_lens,
|
||||||
|
self.final_state_indices,
|
||||||
|
self.boundary_state_indices,
|
||||||
|
self.boundary_recovery_lens,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
mamba_block_size or 1,
|
||||||
|
block_table.shape[1] if block_table is not None else 1,
|
||||||
|
num_accepted_tokens.stride(0),
|
||||||
|
request_indices.stride(0) if request_indices is not None else 0,
|
||||||
|
state_indices.stride(0),
|
||||||
|
query_start_loc.stride(0),
|
||||||
|
bt_stride[0],
|
||||||
|
bt_stride[1],
|
||||||
|
0 if num_computed_tokens is None else num_computed_tokens.stride(0),
|
||||||
|
SPEC_QUERY_LEN=self.spec_query_len,
|
||||||
|
num_warps=1,
|
||||||
|
)
|
||||||
|
num_layers = len(self.checkpoints)
|
||||||
|
if commit_conv:
|
||||||
|
conv_ref = self.conv_states[0]
|
||||||
|
conv_dim = conv_ref.shape[1]
|
||||||
|
compact_conv_state_kernel[(triton.cdiv(conv_dim, 256), batch, num_layers)](
|
||||||
|
conv_ref,
|
||||||
|
self.conv_state_base_addrs,
|
||||||
|
self.conv_state_block_strides,
|
||||||
|
self.conv_state_dim_strides,
|
||||||
|
self.conv_state_token_strides,
|
||||||
|
state_indices,
|
||||||
|
self.commit_lens,
|
||||||
|
self.final_state_indices,
|
||||||
|
self.boundary_state_indices,
|
||||||
|
self.boundary_recovery_lens,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
conv_dim,
|
||||||
|
self.conv_history_len,
|
||||||
|
state_indices.stride(0),
|
||||||
|
BLOCK_D=256,
|
||||||
|
BLOCK_HISTORY=triton.next_power_of_2(self.conv_history_len),
|
||||||
|
ALIGN_MODE=block_table is not None,
|
||||||
|
num_warps=4,
|
||||||
|
)
|
||||||
|
ref = self.checkpoints[0]
|
||||||
|
_, HV, V, K = ref.shape
|
||||||
|
BK = triton.next_power_of_2(K)
|
||||||
|
# 16-row tiles: the commit streams each state once, and smaller tiles
|
||||||
|
# keep more of them in flight (about 13% more bandwidth than 32 on H200).
|
||||||
|
BV = min(triton.next_power_of_2(V), 16)
|
||||||
|
rr = self.replays[0]
|
||||||
|
align = block_table is not None
|
||||||
|
args = (
|
||||||
|
ref,
|
||||||
|
self.state_base_addrs,
|
||||||
|
self.state_block_strides,
|
||||||
|
rr,
|
||||||
|
self.replay_base_addrs,
|
||||||
|
self.replay_block_strides,
|
||||||
|
state_indices,
|
||||||
|
self.commit_lens,
|
||||||
|
self.final_state_indices,
|
||||||
|
self.boundary_state_indices,
|
||||||
|
self.boundary_recovery_lens,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
ref.stride(1),
|
||||||
|
ref.stride(2),
|
||||||
|
ref.stride(3),
|
||||||
|
rr.stride(1),
|
||||||
|
rr.stride(2),
|
||||||
|
rr.stride(3),
|
||||||
|
state_indices.stride(0),
|
||||||
|
)
|
||||||
|
grid = (triton.cdiv(V, BV), batch, num_layers * HV)
|
||||||
|
# Final states first: a boundary state can land in the window's source
|
||||||
|
# slot, which the final-state fold reads (stream order separates them).
|
||||||
|
for boundary in (False, True) if align else (False,):
|
||||||
|
_commit_gdn_state_kernel[grid](
|
||||||
|
*args,
|
||||||
|
K=K,
|
||||||
|
V=V,
|
||||||
|
BK=BK,
|
||||||
|
BV=BV,
|
||||||
|
NUM_HEADS=HV,
|
||||||
|
BOUNDARY=boundary,
|
||||||
|
num_warps=4,
|
||||||
|
num_stages=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["GDNRecoverSSMCommitContext", "gdn_recoverssm_verify"]
|
||||||
@@ -0,0 +1,200 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Model-independent RecoverSSM Triton kernels shared by every recurrent-state
|
||||||
|
implementation.
|
||||||
|
|
||||||
|
RecoverSSM verifies a speculative window from a read-only checkpoint and, after
|
||||||
|
sampling, commits the accepted prefix once. The per-model parts (the KDA and GDN verify
|
||||||
|
kernels and their state reconstruction) live with their models; the kernels they share
|
||||||
|
are here:
|
||||||
|
|
||||||
|
- ``prepare_commit_plan_kernel``: per request, the commit length and the destination
|
||||||
|
state blocks (final and, in align mode, the block-boundary block) from the accepted
|
||||||
|
counts and the block table;
|
||||||
|
- ``compact_conv_state_kernel``: moves the accepted suffix of an extended short-conv
|
||||||
|
window to the front of the destination block (and writes the boundary window in align
|
||||||
|
mode)."""
|
||||||
|
|
||||||
|
from vllm.triton_utils import tl, triton
|
||||||
|
|
||||||
|
|
||||||
|
@triton.heuristics(
|
||||||
|
{
|
||||||
|
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
|
||||||
|
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
@triton.jit
|
||||||
|
def prepare_commit_plan_kernel(
|
||||||
|
num_accepted_ptr,
|
||||||
|
request_indices_ptr,
|
||||||
|
state_indices_ptr,
|
||||||
|
query_start_loc_ptr,
|
||||||
|
block_table_ptr,
|
||||||
|
num_computed_ptr,
|
||||||
|
commit_lens_ptr,
|
||||||
|
final_state_indices_ptr,
|
||||||
|
boundary_state_indices_ptr,
|
||||||
|
boundary_recovery_lens_ptr,
|
||||||
|
null_block_id,
|
||||||
|
mamba_block_size,
|
||||||
|
block_table_width,
|
||||||
|
stride_num_accepted,
|
||||||
|
stride_request_indices,
|
||||||
|
stride_state_indices,
|
||||||
|
stride_query_start_loc,
|
||||||
|
stride_block_table_row,
|
||||||
|
stride_block_table_col,
|
||||||
|
stride_num_computed,
|
||||||
|
SPEC_QUERY_LEN: tl.constexpr,
|
||||||
|
HAS_REQUEST_INDICES: tl.constexpr,
|
||||||
|
ALIGN_MODE: tl.constexpr,
|
||||||
|
):
|
||||||
|
spec_idx = tl.program_id(0)
|
||||||
|
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
|
||||||
|
tl.int64
|
||||||
|
)
|
||||||
|
request_idx = spec_idx
|
||||||
|
if HAS_REQUEST_INDICES:
|
||||||
|
request_idx = tl.load(
|
||||||
|
request_indices_ptr + spec_idx * stride_request_indices
|
||||||
|
).to(tl.int64)
|
||||||
|
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
|
||||||
|
tl.int32
|
||||||
|
)
|
||||||
|
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
|
||||||
|
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
|
||||||
|
tl.int64
|
||||||
|
)
|
||||||
|
query_len = (eos - bos).to(tl.int32)
|
||||||
|
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
|
||||||
|
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
|
||||||
|
|
||||||
|
final_state_idx = source_state_idx
|
||||||
|
boundary_state_idx = null_block_id
|
||||||
|
boundary_recovery_len = 0
|
||||||
|
if ALIGN_MODE:
|
||||||
|
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
|
||||||
|
tl.int32
|
||||||
|
)
|
||||||
|
final_num_computed = num_computed + commit_len
|
||||||
|
# The committed state belongs to the block holding the last committed
|
||||||
|
# token. A window ending exactly on a boundary stays in that block; the
|
||||||
|
# next block may still be unallocated.
|
||||||
|
final_state_col = tl.minimum(
|
||||||
|
tl.maximum(final_num_computed - 1, 0) // mamba_block_size,
|
||||||
|
block_table_width - 1,
|
||||||
|
)
|
||||||
|
final_state_idx = tl.load(
|
||||||
|
block_table_ptr
|
||||||
|
+ request_idx * stride_block_table_row
|
||||||
|
+ final_state_col * stride_block_table_col
|
||||||
|
).to(tl.int64)
|
||||||
|
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
|
||||||
|
crosses_boundary = final_num_computed >= next_boundary
|
||||||
|
boundary_recovery_len = next_boundary - num_computed
|
||||||
|
boundary_state_idx = tl.load(
|
||||||
|
block_table_ptr
|
||||||
|
+ request_idx * stride_block_table_row
|
||||||
|
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
|
||||||
|
mask=crosses_boundary,
|
||||||
|
other=null_block_id,
|
||||||
|
).to(tl.int64)
|
||||||
|
valid = (source_state_idx > null_block_id) & (commit_len > 0)
|
||||||
|
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
|
||||||
|
tl.store(
|
||||||
|
final_state_indices_ptr + spec_idx,
|
||||||
|
tl.where(valid, final_state_idx, null_block_id),
|
||||||
|
)
|
||||||
|
tl.store(
|
||||||
|
boundary_state_indices_ptr + spec_idx,
|
||||||
|
tl.where(valid, boundary_state_idx, null_block_id),
|
||||||
|
)
|
||||||
|
tl.store(
|
||||||
|
boundary_recovery_lens_ptr + spec_idx,
|
||||||
|
tl.where(valid, boundary_recovery_len, 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@triton.jit
|
||||||
|
def compact_conv_state_kernel(
|
||||||
|
conv_state_ref_ptr,
|
||||||
|
conv_state_base_addrs_ptr,
|
||||||
|
conv_state_block_strides_ptr,
|
||||||
|
conv_state_dim_strides_ptr,
|
||||||
|
conv_state_token_strides_ptr,
|
||||||
|
state_indices_ptr,
|
||||||
|
commit_lens_ptr,
|
||||||
|
final_state_indices_ptr,
|
||||||
|
boundary_state_indices_ptr,
|
||||||
|
boundary_recovery_lens_ptr,
|
||||||
|
null_block_id,
|
||||||
|
conv_dim,
|
||||||
|
conv_history_len,
|
||||||
|
stride_state_indices,
|
||||||
|
BLOCK_D: tl.constexpr,
|
||||||
|
BLOCK_HISTORY: tl.constexpr,
|
||||||
|
ALIGN_MODE: tl.constexpr,
|
||||||
|
):
|
||||||
|
pid_d = tl.program_id(0)
|
||||||
|
pid_b = tl.program_id(1)
|
||||||
|
pid_l = tl.program_id(2)
|
||||||
|
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
|
||||||
|
tl.int64
|
||||||
|
)
|
||||||
|
if source_state_idx <= null_block_id:
|
||||||
|
return
|
||||||
|
|
||||||
|
commit_len = tl.load(commit_lens_ptr + pid_b)
|
||||||
|
if commit_len == 0:
|
||||||
|
return
|
||||||
|
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
||||||
|
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
||||||
|
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
||||||
|
|
||||||
|
if final_state_idx <= null_block_id:
|
||||||
|
return
|
||||||
|
|
||||||
|
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
|
||||||
|
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
|
||||||
|
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
|
||||||
|
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
|
||||||
|
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
|
||||||
|
source_ptr = conv_state_ptr + source_state_idx * block_stride
|
||||||
|
final_ptr = conv_state_ptr + final_state_idx * block_stride
|
||||||
|
|
||||||
|
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
|
||||||
|
offs_h = tl.arange(0, BLOCK_HISTORY)
|
||||||
|
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
|
||||||
|
final_values = tl.load(
|
||||||
|
source_ptr
|
||||||
|
+ offs_d[:, None] * dim_stride
|
||||||
|
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
|
||||||
|
mask=mask,
|
||||||
|
)
|
||||||
|
if ALIGN_MODE:
|
||||||
|
boundary_values = tl.load(
|
||||||
|
source_ptr
|
||||||
|
+ offs_d[:, None] * dim_stride
|
||||||
|
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
|
||||||
|
mask=mask & (boundary_state_idx > null_block_id),
|
||||||
|
)
|
||||||
|
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
|
||||||
|
tl.store(
|
||||||
|
boundary_ptr
|
||||||
|
+ offs_d[:, None] * dim_stride
|
||||||
|
+ offs_h[None, :] * token_stride,
|
||||||
|
boundary_values,
|
||||||
|
mask=mask & (boundary_state_idx > null_block_id),
|
||||||
|
)
|
||||||
|
tl.store(
|
||||||
|
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
|
||||||
|
final_values,
|
||||||
|
mask=mask,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"compact_conv_state_kernel",
|
||||||
|
"prepare_commit_plan_kernel",
|
||||||
|
]
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Model-independent RecoverSSM Python helpers: the activation predicate shared by
|
||||||
|
layers, models and backends, and argument validation that raises ``ValueError``
|
||||||
|
before any raw-pointer kernel launch."""
|
||||||
|
|
||||||
|
|
||||||
|
def recoverssm_require(cond: bool, msg: str, component: str = "RecoverSSM") -> None:
|
||||||
|
if not cond:
|
||||||
|
raise ValueError(f"{component}: {msg}")
|
||||||
|
|
||||||
|
|
||||||
|
def uses_recoverssm(cache_config, num_speculative_tokens: int) -> bool:
|
||||||
|
"""True when this model instance runs RecoverSSM for its recurrent states.
|
||||||
|
|
||||||
|
VllmConfig sets ``cache_config.use_recoverssm`` for every model whose recurrent
|
||||||
|
states support it (Kimi-K3 KDA, Qwen GDN/PLE). Without speculative tokens there is
|
||||||
|
nothing to recover, so the stock path runs."""
|
||||||
|
return (
|
||||||
|
bool(getattr(cache_config, "use_recoverssm", False))
|
||||||
|
and num_speculative_tokens > 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["recoverssm_require", "uses_recoverssm"]
|
||||||
@@ -0,0 +1,563 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Base class for PLE layers that support cross-process CPU offload.
|
||||||
|
|
||||||
|
Synchronization between the offload process and the GPU main stream is
|
||||||
|
handled by :class:`CpuGpuSemaphore`, which uses CUDA's
|
||||||
|
``cuStreamWaitValue32`` / ``cuStreamWriteValue32`` stream memory operations.
|
||||||
|
|
||||||
|
Decode-step timeline, host-mapped PULL design
|
||||||
|
--------------------------------------------------------
|
||||||
|
Offload process (CPU, no CUDA) GPU main stream (forward)
|
||||||
|
------------------------------ ------------------------------
|
||||||
|
forward_impl() on CPU ... other GPU ops ...
|
||||||
|
wait host flag == 0 WaitValue32(host flag == 1) <- in graph
|
||||||
|
rows -> shared pinned host slot cuMemcpyHtoDAsync(slot -> GPU) <- in graph
|
||||||
|
publish fence; host flag = 1 return _gpu_output_buffer[:n]
|
||||||
|
... model consumes output ...
|
||||||
|
WriteValue32(host flag = 0) <- release_outputs
|
||||||
|
|
||||||
|
``release_outputs`` performs the sole reset. Safety requires serialized local
|
||||||
|
forwards on one ordered model stream: a new transaction cannot begin until the
|
||||||
|
prior release has been enqueued. If any prepared forward is abandoned or its
|
||||||
|
correspondence becomes uncertain, the worker must terminate; resetting,
|
||||||
|
retrying, or continuing could reuse stale rows or overwrite live input.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import functools
|
||||||
|
import threading
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from collections.abc import Callable, Iterator
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from cuda.bindings import driver as cuda_driver
|
||||||
|
from cuda.bindings.driver import CUstreamWaitValue_flags
|
||||||
|
from torch import nn
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
|
|
||||||
|
# Module-level flag set to True inside the offload subprocess.
|
||||||
|
# Because the offload process and GPU worker processes are separate OS
|
||||||
|
# processes (spawned via multiprocessing), each has its own memory space.
|
||||||
|
# A plain module-level bool is sufficient -- no thread-local storage needed.
|
||||||
|
_offload_worker_flag = False
|
||||||
|
|
||||||
|
# Native FULL capture runs on one Python thread. The custom-op implementation
|
||||||
|
# appends the exact arguments from every wait-copy invocation while that thread
|
||||||
|
# is inside the graph manager's capture scope.
|
||||||
|
_ple_copy_capture = threading.local()
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def observe_ple_copy_geometry() -> Iterator[list[tuple[str, int, int]]]:
|
||||||
|
"""Collect ``(layer, rows, bytes)`` from actual wait-copy executions."""
|
||||||
|
if getattr(_ple_copy_capture, "records", None) is not None:
|
||||||
|
raise RuntimeError("nested PLE copy-geometry observation is unsupported")
|
||||||
|
records: list[tuple[str, int, int]] = []
|
||||||
|
_ple_copy_capture.records = records
|
||||||
|
try:
|
||||||
|
yield records
|
||||||
|
finally:
|
||||||
|
del _ple_copy_capture.records
|
||||||
|
|
||||||
|
|
||||||
|
def _record_ple_copy_geometry(
|
||||||
|
layer_name: str,
|
||||||
|
num_tokens: int,
|
||||||
|
row_bytes: int,
|
||||||
|
) -> None:
|
||||||
|
records = getattr(_ple_copy_capture, "records", None)
|
||||||
|
if records is not None:
|
||||||
|
rows = int(num_tokens)
|
||||||
|
records.append((str(layer_name), rows, rows * int(row_bytes)))
|
||||||
|
|
||||||
|
|
||||||
|
def is_offload_process() -> bool:
|
||||||
|
"""Return True inside the dedicated PLE CPU-offload subprocess."""
|
||||||
|
return _offload_worker_flag
|
||||||
|
|
||||||
|
|
||||||
|
def mark_as_offload_worker() -> None:
|
||||||
|
"""Mark the current process as the dedicated PLE offload worker."""
|
||||||
|
global _offload_worker_flag
|
||||||
|
_offload_worker_flag = True
|
||||||
|
|
||||||
|
|
||||||
|
def _cuda_check(result: Any, operation: str) -> Any:
|
||||||
|
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
|
||||||
|
error = result[0] if isinstance(result, tuple) else result
|
||||||
|
if error.value != 0:
|
||||||
|
raise RuntimeError(f"{operation} failed: {error}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class CpuGpuSemaphore:
|
||||||
|
"""Cross-process semaphore backed by a one-element int32 CUDA tensor.
|
||||||
|
|
||||||
|
The flag is stored in a regular GPU tensor that is shared through
|
||||||
|
PyTorch's CUDA IPC mechanism. Both processes obtain device pointers that
|
||||||
|
map to the same physical GPU memory, so a stream-memory wait in the GPU
|
||||||
|
worker observes the write issued by the offload process.
|
||||||
|
"""
|
||||||
|
|
||||||
|
RESET_VALUE = 0
|
||||||
|
DONE_VALUE = 1
|
||||||
|
|
||||||
|
def __init__(self, device: torch.device) -> None:
|
||||||
|
self._flag_tensor = torch.zeros(1, dtype=torch.int32, device=device)
|
||||||
|
self._flag_ptr: int | None = None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_ipc_tensor(cls, flag_tensor: torch.Tensor) -> "CpuGpuSemaphore":
|
||||||
|
"""Construct a semaphore from a CUDA tensor received through IPC."""
|
||||||
|
semaphore = cls.__new__(cls)
|
||||||
|
semaphore._flag_tensor = flag_tensor
|
||||||
|
semaphore._flag_ptr = None
|
||||||
|
return semaphore
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_host_mapped(
|
||||||
|
cls, host_flag_tensor: torch.Tensor, device_ptr: int
|
||||||
|
) -> "CpuGpuSemaphore":
|
||||||
|
"""Semaphore on a page-locked, device-mapped shared host int32.
|
||||||
|
|
||||||
|
``host_flag_tensor`` keeps the shared storage alive; ``device_ptr`` is
|
||||||
|
its device address (``cuMemHostGetDevicePointer``). Stream memory ops
|
||||||
|
on it are executed by the GPU against host memory, so a CPU process
|
||||||
|
with no CUDA context can signal it with a plain store.
|
||||||
|
"""
|
||||||
|
semaphore = cls.__new__(cls)
|
||||||
|
semaphore._flag_tensor = host_flag_tensor
|
||||||
|
semaphore._flag_ptr = int(device_ptr)
|
||||||
|
return semaphore
|
||||||
|
|
||||||
|
@property
|
||||||
|
def flag_tensor(self) -> torch.Tensor:
|
||||||
|
"""Return the tensor that backs the semaphore flag."""
|
||||||
|
return self._flag_tensor
|
||||||
|
|
||||||
|
@property
|
||||||
|
def flag_ptr(self) -> int:
|
||||||
|
"""Device address used by the stream memory operations."""
|
||||||
|
if self._flag_ptr is not None:
|
||||||
|
return self._flag_ptr
|
||||||
|
return self._flag_tensor.data_ptr()
|
||||||
|
|
||||||
|
def reset(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||||
|
"""Enqueue ``WriteValue32(flag=0)`` on ``stream``."""
|
||||||
|
if stream is None:
|
||||||
|
stream = torch.cuda.current_stream()
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuStreamWriteValue32(
|
||||||
|
cuda_driver.CUstream(stream.cuda_stream),
|
||||||
|
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||||
|
self.RESET_VALUE,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
"CpuGpuSemaphore.reset",
|
||||||
|
)
|
||||||
|
|
||||||
|
def signal(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||||
|
"""Enqueue ``WriteValue32(flag=1)`` on ``stream``."""
|
||||||
|
if stream is None:
|
||||||
|
stream = torch.cuda.current_stream()
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuStreamWriteValue32(
|
||||||
|
cuda_driver.CUstream(stream.cuda_stream),
|
||||||
|
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||||
|
self.DONE_VALUE,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
"CpuGpuSemaphore.signal",
|
||||||
|
)
|
||||||
|
|
||||||
|
def wait_reset(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||||
|
"""Enqueue ``WaitValue32(flag==0)`` on ``stream``."""
|
||||||
|
if stream is None:
|
||||||
|
stream = torch.cuda.current_stream()
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuStreamWaitValue32(
|
||||||
|
cuda_driver.CUstream(stream.cuda_stream),
|
||||||
|
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||||
|
self.RESET_VALUE,
|
||||||
|
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||||
|
),
|
||||||
|
"CpuGpuSemaphore.wait_reset",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Custom op: vllm::ple_offload_wait
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Wraps cuStreamWaitValue32 as a torch custom op so that
|
||||||
|
# torch.compile(fullgraph=True) treats the CUDA Driver call as an opaque node.
|
||||||
|
# The hidden_states argument creates a dynamo data-dependency edge that prevents
|
||||||
|
# the wait from being reordered before preceding GPU work.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_offload_wait_impl(
|
||||||
|
sem_flag_tensor: torch.Tensor,
|
||||||
|
gpu_output_buffer: torch.Tensor,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
"""Wait for the CPU result without releasing its output buffer."""
|
||||||
|
stream = torch.cuda.current_stream()
|
||||||
|
cuda_stream = cuda_driver.CUstream(stream.cuda_stream)
|
||||||
|
dev_ptr = cuda_driver.CUdeviceptr(sem_flag_tensor.data_ptr())
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuStreamWaitValue32(
|
||||||
|
cuda_stream,
|
||||||
|
dev_ptr,
|
||||||
|
CpuGpuSemaphore.DONE_VALUE,
|
||||||
|
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||||
|
),
|
||||||
|
"cuStreamWaitValue32(done)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_offload_wait_fake(
|
||||||
|
sem_flag_tensor: torch.Tensor,
|
||||||
|
gpu_output_buffer: torch.Tensor,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
"""Represent the side-effect-only wait during dynamo tracing."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="ple_offload_wait",
|
||||||
|
op_func=_ple_offload_wait_impl,
|
||||||
|
mutates_args=["gpu_output_buffer"],
|
||||||
|
fake_impl=_ple_offload_wait_fake,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Custom op: vllm::ple_offload_wait_copy (host-mapped pull design)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# The offload process writes the result into a page-locked shared host buffer
|
||||||
|
# and stores flag=1 in a page-locked shared host int32; it holds no CUDA
|
||||||
|
# context. On the CURRENT stream (captured into the graph like the wait above):
|
||||||
|
# cuStreamWaitValue32(host flag == 1) -> GPU polls host memory
|
||||||
|
# cuMemcpyAsync(gpu buffer <- host buffer, num_tokens * row_bytes)
|
||||||
|
# The copy engine reads host memory coherently after the flag was observed,
|
||||||
|
# and the consumers read device memory as before. The flag is NOT reset here:
|
||||||
|
# ``release_outputs`` resets it after the forward (unchanged semantics), so
|
||||||
|
# capture warm-ups and dummy runs behave exactly as with ``ple_offload_wait``.
|
||||||
|
# No CPU tensor and no raw address enters the compiled graph: the op resolves
|
||||||
|
# the live pointers by layer name (see _PULL_LAYERS).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
# Fork (found on the reference host): the host flag / host output pointers and
|
||||||
|
# the row size are NOT op arguments any more. As Python ints they were
|
||||||
|
# specialised into the compiled graph as constants, so a boot that LOADED the
|
||||||
|
# AOT compile cache (no Dynamo pass) waited on the previous process's addresses:
|
||||||
|
# cuStreamWaitValue32 -> CUDA_ERROR_INVALID_VALUE on the first forward. The op
|
||||||
|
# now carries only process-stable values (buffer, token count, layer name) and
|
||||||
|
# resolves the live pointers from this registry on every call.
|
||||||
|
_PULL_LAYERS: dict[str, "PleOffloadLayer"] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_offload_wait_copy_impl(
|
||||||
|
gpu_output_buffer: torch.Tensor,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
num_tokens: int,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Resolve this process's live pointers for ``layer_name``, then wait+DMA."""
|
||||||
|
layer = _PULL_LAYERS.get(layer_name)
|
||||||
|
if layer is None or layer._host_output_ptr is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE wait_copy: no armed pull layer named {layer_name!r} in this "
|
||||||
|
f"process (armed: {sorted(_PULL_LAYERS)})"
|
||||||
|
)
|
||||||
|
_ple_offload_wait_copy_raw(
|
||||||
|
gpu_output_buffer,
|
||||||
|
hidden_states,
|
||||||
|
layer._sem.flag_ptr,
|
||||||
|
layer._host_output_ptr,
|
||||||
|
layer._host_row_bytes,
|
||||||
|
num_tokens,
|
||||||
|
layer_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_offload_wait_copy_raw(
|
||||||
|
gpu_output_buffer: torch.Tensor,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
flag_ptr: int,
|
||||||
|
host_ptr: int,
|
||||||
|
row_bytes: int,
|
||||||
|
num_tokens: int,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Wait for the host-side result, then DMA it into the GPU buffer."""
|
||||||
|
if not 0 <= int(num_tokens) <= gpu_output_buffer.shape[0] or int(row_bytes) != (
|
||||||
|
gpu_output_buffer.shape[1] * gpu_output_buffer.element_size()
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE wait_copy out of bounds: {num_tokens} rows x {row_bytes} B vs "
|
||||||
|
f"GPU buffer {tuple(gpu_output_buffer.shape)} {gpu_output_buffer.dtype}"
|
||||||
|
)
|
||||||
|
nbytes = int(num_tokens) * int(row_bytes)
|
||||||
|
_record_ple_copy_geometry(layer_name, num_tokens, row_bytes)
|
||||||
|
stream = cuda_driver.CUstream(torch.cuda.current_stream().cuda_stream)
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuStreamWaitValue32(
|
||||||
|
stream,
|
||||||
|
cuda_driver.CUdeviceptr(flag_ptr),
|
||||||
|
CpuGpuSemaphore.DONE_VALUE,
|
||||||
|
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||||
|
),
|
||||||
|
"cuStreamWaitValue32(host flag done)",
|
||||||
|
)
|
||||||
|
if nbytes > 0:
|
||||||
|
# host_ptr is this worker's host address of page-locked (registered)
|
||||||
|
# memory, so the copy is a true async DMA and is capturable.
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuMemcpyHtoDAsync(
|
||||||
|
cuda_driver.CUdeviceptr(gpu_output_buffer.data_ptr()),
|
||||||
|
host_ptr,
|
||||||
|
nbytes,
|
||||||
|
stream,
|
||||||
|
),
|
||||||
|
"cuMemcpyHtoDAsync(PLE host -> GPU)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_offload_wait_copy_fake(
|
||||||
|
gpu_output_buffer: torch.Tensor,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
num_tokens: int,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Represent the side-effect-only wait + copy during dynamo tracing."""
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="ple_offload_wait_copy",
|
||||||
|
op_func=_ple_offload_wait_copy_impl,
|
||||||
|
mutates_args=["gpu_output_buffer"],
|
||||||
|
fake_impl=_ple_offload_wait_copy_fake,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PleOffloadLayer(nn.Module, ABC):
|
||||||
|
"""Base class for embedding-like PLE layers that can run on CPU.
|
||||||
|
|
||||||
|
Subclasses implement :meth:`forward_impl`, which contains the regular
|
||||||
|
on-device computation. In a GPU worker with PLE offload enabled, subclass
|
||||||
|
constructors are skipped so their large weights are never allocated. The
|
||||||
|
CPU process constructs a complete copy and owns those weights instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_is_cpu_offloaded = False
|
||||||
|
_gpu_output_buffer: torch.Tensor
|
||||||
|
_sem: CpuGpuSemaphore
|
||||||
|
# Host-mapped pull design: host address of the shared output buffer and
|
||||||
|
# the bytes per output row. ``None`` keeps the IPC push behaviour.
|
||||||
|
_host_output_ptr: int | None = None
|
||||||
|
_host_row_bytes: int = 0
|
||||||
|
_ple_offload_name: str = "<unbound>"
|
||||||
|
|
||||||
|
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||||
|
"""Skip subclass initialization in cross-process GPU-worker mode."""
|
||||||
|
super().__init_subclass__(**kwargs)
|
||||||
|
original_init_obj = cls.__dict__.get("__init__")
|
||||||
|
if original_init_obj is None or not callable(original_init_obj):
|
||||||
|
return
|
||||||
|
original_init = cast(Callable[..., None], original_init_obj)
|
||||||
|
|
||||||
|
@functools.wraps(original_init)
|
||||||
|
def guarded_init(
|
||||||
|
self: "PleOffloadLayer", *args: object, **kwargs: object
|
||||||
|
) -> None:
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not is_offload_process():
|
||||||
|
nn.Module.__init__(self)
|
||||||
|
return
|
||||||
|
original_init(self, *args, **kwargs)
|
||||||
|
|
||||||
|
cls.__init__ = guarded_init # type: ignore[method-assign, assignment]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_target_device(cls) -> torch.device:
|
||||||
|
"""Return CPU for the offload process and the active GPU otherwise."""
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
return torch.device("cpu")
|
||||||
|
return torch.device("cuda", torch.accelerator.current_device_index())
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def forward_impl(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
*args: object,
|
||||||
|
**kwargs: object,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Execute the actual embedding computation on the owning device."""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
|
||||||
|
"""Return the dtype used by cross-process output buffers."""
|
||||||
|
return default_dtype
|
||||||
|
|
||||||
|
def get_offload_output_dim(self, default_dim: int) -> int:
|
||||||
|
"""Return the last dimension used by cross-process output buffers."""
|
||||||
|
return default_dim
|
||||||
|
|
||||||
|
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
|
||||||
|
"""Initialize metadata needed by dummy offload execution."""
|
||||||
|
|
||||||
|
def setup_cross_process_offload(
|
||||||
|
self,
|
||||||
|
gpu_output_buffer: torch.Tensor,
|
||||||
|
semaphore: CpuGpuSemaphore,
|
||||||
|
host_output_ptr: int | None = None,
|
||||||
|
host_row_bytes: int = 0,
|
||||||
|
) -> None:
|
||||||
|
"""Configure the GPU-worker placeholder with its offload resources."""
|
||||||
|
self._is_cpu_offloaded = True
|
||||||
|
self._gpu_output_buffer = gpu_output_buffer
|
||||||
|
self._sem = semaphore
|
||||||
|
self._host_output_ptr = host_output_ptr
|
||||||
|
self._host_row_bytes = int(host_row_bytes)
|
||||||
|
if host_output_ptr is not None:
|
||||||
|
# Hardening: the registry key is the name the
|
||||||
|
# traced graph carries, so it must be a real, unique layer name.
|
||||||
|
name = self._ple_offload_name
|
||||||
|
if name == "<unbound>":
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE pull: setup_cross_process_offload on a layer with no "
|
||||||
|
"_ple_offload_name; the op would resolve nothing by name"
|
||||||
|
)
|
||||||
|
prior = _PULL_LAYERS.get(name)
|
||||||
|
if prior is not None and prior is not self:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE pull: two layers armed under the name {name!r}; the op "
|
||||||
|
"would resolve the wrong layer's pointers"
|
||||||
|
)
|
||||||
|
_PULL_LAYERS[name] = self
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
*args: object,
|
||||||
|
**kwargs: object,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Wait for an offloaded result or delegate to ``forward_impl``."""
|
||||||
|
if self._is_cpu_offloaded:
|
||||||
|
if self._host_output_ptr is not None:
|
||||||
|
torch.ops.vllm.ple_offload_wait_copy(
|
||||||
|
self._gpu_output_buffer,
|
||||||
|
hidden_states,
|
||||||
|
input_ids.shape[0],
|
||||||
|
self._ple_offload_name,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
torch.ops.vllm.ple_offload_wait(
|
||||||
|
self._sem.flag_tensor,
|
||||||
|
self._gpu_output_buffer,
|
||||||
|
hidden_states,
|
||||||
|
)
|
||||||
|
return self._gpu_output_buffer[: input_ids.shape[0]]
|
||||||
|
return self.forward_impl(hidden_states, input_ids, *args, **kwargs)
|
||||||
|
|
||||||
|
def release_offloaded_output(
|
||||||
|
self,
|
||||||
|
stream: torch.cuda.Stream | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Mark the cross-process output buffer reusable on ``stream``."""
|
||||||
|
if self._is_cpu_offloaded:
|
||||||
|
self._sem.reset(stream)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Pre-launch staleness check for the pull transport (#53899 class)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# WHAT IT TESTS. The transport's correctness rests on one precondition: when a
|
||||||
|
# real request is published, every layer's semaphore must be CLEAR, so the
|
||||||
|
# ``cuStreamWaitValue32(flag == DONE_VALUE)`` inside the forward (line 279 for
|
||||||
|
# the pull path, line 215 for the IPC push path) can only be satisfied by the
|
||||||
|
# copy THIS step publishes. If a leftover ``DONE_VALUE`` survives into a real
|
||||||
|
# step, that wait is satisfied immediately by the PREVIOUS step's signal and
|
||||||
|
# the forward consumes the previous step's PLE rows. That is vllm#53899 on
|
||||||
|
# GB10/sm_121 exactly: "every forward consumes the previous step's PLE
|
||||||
|
# outputs", for the life of the server, with ``cudagraph_mode=NONE`` correct.
|
||||||
|
#
|
||||||
|
# In this tree the precondition is held structurally -- the dummy signal is
|
||||||
|
# consumed by the dummy forward, ``_release_outputs_impl`` resets every layer
|
||||||
|
# and synchronizes the model stream while ``_after_dummy`` is set, and the
|
||||||
|
# dummy->real transition takes a TP barrier -- but nothing ever READ the flag
|
||||||
|
# back, so the structural argument had no instrument. This is that instrument.
|
||||||
|
#
|
||||||
|
# WHY NOT AN EPOCH WORD. The stronger fix is a monotone epoch: the launch
|
||||||
|
# writes ``epoch = step``, the worker signals ``epoch``, and the placeholder
|
||||||
|
# waits ``EQ(epoch)``, so a stale signal carries the previous epoch and cannot
|
||||||
|
# satisfy the wait. It is not implementable on this transport as it stands.
|
||||||
|
# The compare value is an IMMEDIATE OPERAND of ``cuStreamWaitValue32``
|
||||||
|
# (``_ple_offload_wait_copy_impl``, this file, line 279), and that call is
|
||||||
|
# issued inside the model forward, which is captured: PIECEWISE and
|
||||||
|
# FULL_AND_PIECEWISE both bake the wait into the graph with the value frozen at
|
||||||
|
# capture time. Making the compare value change per replay needs the captured
|
||||||
|
# node's parameters rewritten before each launch
|
||||||
|
# (``cuGraphExecBatchMemOpNodeSetParams``); nothing in this tree's capture path
|
||||||
|
# exposes the graph exec handle to the connector, and the wait is reached
|
||||||
|
# through a torch custom op inside a compiled region. An epoch word would
|
||||||
|
# therefore work only with ``cudagraph_mode=NONE``, i.e. only in the
|
||||||
|
# configuration where #53899 does not reproduce. So: binary flag, read back.
|
||||||
|
#
|
||||||
|
# The check is HOST-SIDE and needs a stream synchronize to be ordered against
|
||||||
|
# a pending reset, so it is never run inside a capture and is off by default.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def flag_check_is_authoritative(tp_rank: int) -> bool:
|
||||||
|
"""Whether ``flag == RESET_VALUE`` before a launch is an INVARIANT here.
|
||||||
|
|
||||||
|
Only on the publishing rank (TP0). Why: the offload worker fans a
|
||||||
|
result out to every registered TP target, each gated only on that target's
|
||||||
|
own flag having been reset, so a non-publishing rank's flag can legitimately
|
||||||
|
carry THIS step's signal before that rank reaches its own pre-launch check.
|
||||||
|
Reading a set flag there and calling it stale terminates a healthy run.
|
||||||
|
|
||||||
|
On TP0 the invariant holds by construction: the worker learns of a request
|
||||||
|
only from ``_launch``, which runs after the check, and the progress monitor
|
||||||
|
refuses overlapping serving submissions, so the previous transaction has
|
||||||
|
been released and (after a stream synchronize) its reset has retired.
|
||||||
|
|
||||||
|
A non-publishing rank cannot tell "this step's fan-out arrived early" from
|
||||||
|
"last step's signal was never consumed", because the flag is BINARY and
|
||||||
|
carries no transaction identity -- the same representational limit that
|
||||||
|
rules out an epoch word. So those ranks are skipped, not checked weakly.
|
||||||
|
"""
|
||||||
|
return tp_rank == 0
|
||||||
|
|
||||||
|
|
||||||
|
def stale_offload_flags(
|
||||||
|
layers: "dict[str, PleOffloadLayer]",
|
||||||
|
) -> list[tuple[str, int]]:
|
||||||
|
"""Return ``(layer_name, flag_value)`` for every layer whose flag is set.
|
||||||
|
|
||||||
|
An empty list is the precondition holding. A non-empty list means at least
|
||||||
|
one semaphore still carries a signal that no forward consumed, so the next
|
||||||
|
``wait`` would be satisfied by the previous step's copy.
|
||||||
|
|
||||||
|
Pure and side-effect free: the caller owns the ordering (synchronize the
|
||||||
|
model stream first, and never call this inside a stream capture). Layers
|
||||||
|
that are not cross-process offloaded have no semaphore and are skipped.
|
||||||
|
"""
|
||||||
|
stale: list[tuple[str, int]] = []
|
||||||
|
for name, layer in layers.items():
|
||||||
|
if not getattr(layer, "_is_cpu_offloaded", False):
|
||||||
|
continue
|
||||||
|
value = int(layer._sem.flag_tensor.reshape(-1)[0].item())
|
||||||
|
if value != CpuGpuSemaphore.RESET_VALUE:
|
||||||
|
stale.append((name, value))
|
||||||
|
return stale
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import os
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
@@ -31,8 +32,26 @@ from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
|
|||||||
from vllm.model_executor.parameter import BasevLLMParameter
|
from vllm.model_executor.parameter import BasevLLMParameter
|
||||||
from vllm.model_executor.utils import set_weight_attrs
|
from vllm.model_executor.utils import set_weight_attrs
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.torch_utils import (
|
||||||
|
PIN_MEMORY,
|
||||||
|
get_accelerator_view_from_cpu_tensor,
|
||||||
|
)
|
||||||
|
|
||||||
DEFAULT_VOCAB_PADDING_SIZE = 64
|
DEFAULT_VOCAB_PADDING_SIZE = 64
|
||||||
|
HOST_EMBED_TABLE_ENV = "VLLM_HOST_EMBED_TABLE"
|
||||||
|
|
||||||
|
|
||||||
|
def host_embed_table_enabled() -> bool:
|
||||||
|
"""Return whether the exact host-embedding opt-in is set."""
|
||||||
|
return os.environ.get(HOST_EMBED_TABLE_ENV, "") == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def should_use_host_embed_table() -> bool:
|
||||||
|
"""Keep the PLE offload worker's model construction on the meta device."""
|
||||||
|
return (
|
||||||
|
host_embed_table_enabled()
|
||||||
|
and torch.empty(0).device.type != "meta"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||||
@@ -91,6 +110,46 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
|||||||
return layer
|
return layer
|
||||||
|
|
||||||
|
|
||||||
|
class _PinnedHostUnquantizedEmbeddingMethod(UnquantizedEmbeddingMethod):
|
||||||
|
"""Allocate an unquantized embedding shard without a CUDA transient."""
|
||||||
|
|
||||||
|
requires_device_loading = False
|
||||||
|
|
||||||
|
def __init__(self, *, device: torch.device, pin_memory: bool) -> None:
|
||||||
|
self.device = device
|
||||||
|
self.pin_memory = pin_memory
|
||||||
|
|
||||||
|
def create_weights(
|
||||||
|
self,
|
||||||
|
layer: torch.nn.Module,
|
||||||
|
input_size_per_partition: int,
|
||||||
|
output_partition_sizes: list[int],
|
||||||
|
input_size: int,
|
||||||
|
output_size: int,
|
||||||
|
params_dtype: torch.dtype,
|
||||||
|
**extra_weight_attrs,
|
||||||
|
) -> None:
|
||||||
|
del input_size, output_size
|
||||||
|
weight = Parameter(
|
||||||
|
torch.empty(
|
||||||
|
sum(output_partition_sizes),
|
||||||
|
input_size_per_partition,
|
||||||
|
dtype=params_dtype,
|
||||||
|
device=self.device,
|
||||||
|
pin_memory=self.pin_memory,
|
||||||
|
),
|
||||||
|
requires_grad=False,
|
||||||
|
)
|
||||||
|
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
|
||||||
|
layer.register_parameter("weight", weight)
|
||||||
|
set_weight_attrs(weight, extra_weight_attrs)
|
||||||
|
|
||||||
|
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||||
|
# The serving layer must retain the same pinned allocation whose mapped
|
||||||
|
# address was captured. CPU-only reference tests need no GEMM dispatch.
|
||||||
|
del layer
|
||||||
|
|
||||||
|
|
||||||
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
|
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
|
||||||
"""Pad the vocab size to the given value."""
|
"""Pad the vocab size to the given value."""
|
||||||
return ((vocab_size + pad_to - 1) // pad_to) * pad_to
|
return ((vocab_size + pad_to - 1) // pad_to) * pad_to
|
||||||
@@ -572,6 +631,124 @@ class VocabParallelEmbedding(PluggableLayer):
|
|||||||
return s
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
class PinnedHostVocabParallelEmbedding(VocabParallelEmbedding):
|
||||||
|
"""TP-sharded embedding read on CUDA through mapped pinned host memory.
|
||||||
|
|
||||||
|
Host provisioning must use measured pinned allocation, not tensor payload.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
num_embeddings: int,
|
||||||
|
embedding_dim: int,
|
||||||
|
params_dtype: torch.dtype | None = None,
|
||||||
|
org_num_embeddings: int | None = None,
|
||||||
|
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
|
||||||
|
prefix: str = "",
|
||||||
|
*,
|
||||||
|
disable_tp: bool = False,
|
||||||
|
create_accelerator_view: bool = True,
|
||||||
|
) -> None:
|
||||||
|
construction_device = torch.empty(0).device
|
||||||
|
is_meta = construction_device.type == "meta"
|
||||||
|
if (
|
||||||
|
create_accelerator_view
|
||||||
|
and not is_meta
|
||||||
|
and current_platform.is_cuda()
|
||||||
|
and not PIN_MEMORY
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"VLLM_HOST_EMBED_TABLE requires available pinned host memory"
|
||||||
|
)
|
||||||
|
|
||||||
|
storage_device = construction_device if is_meta else torch.device("cpu")
|
||||||
|
pin_memory = (
|
||||||
|
not is_meta
|
||||||
|
and current_platform.is_cuda()
|
||||||
|
and PIN_MEMORY
|
||||||
|
)
|
||||||
|
quant_method = _PinnedHostUnquantizedEmbeddingMethod(
|
||||||
|
device=storage_device,
|
||||||
|
pin_memory=pin_memory,
|
||||||
|
)
|
||||||
|
super().__init__(
|
||||||
|
num_embeddings,
|
||||||
|
embedding_dim,
|
||||||
|
params_dtype=params_dtype,
|
||||||
|
org_num_embeddings=org_num_embeddings,
|
||||||
|
padding_size=padding_size,
|
||||||
|
prefix=prefix,
|
||||||
|
disable_tp=disable_tp,
|
||||||
|
quant_method=quant_method,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The custom fused kernel is specified for ordinary device-resident
|
||||||
|
# shards, not device-mapped host allocations. Use the regular masked
|
||||||
|
# embedding path whose CUDA tensor argument is the mapped view.
|
||||||
|
self.use_fused_embedding = False
|
||||||
|
self._uva_weight: torch.Tensor | None = None
|
||||||
|
if (
|
||||||
|
create_accelerator_view
|
||||||
|
and self.weight.device.type == "cpu"
|
||||||
|
and current_platform.is_cuda()
|
||||||
|
):
|
||||||
|
if not self.weight.is_pinned():
|
||||||
|
raise RuntimeError(
|
||||||
|
"VLLM_HOST_EMBED_TABLE weight was not allocated pinned"
|
||||||
|
)
|
||||||
|
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def accelerator_weight_view(self) -> torch.Tensor | None:
|
||||||
|
return self._uva_weight
|
||||||
|
|
||||||
|
def _apply(self, fn, recurse: bool = True):
|
||||||
|
# Probe the transformation without applying it to the pinned shard.
|
||||||
|
# Device-only parent moves remain no-ops, but silently retaining this
|
||||||
|
# layer's dtype during a parent dtype conversion would create a
|
||||||
|
# mixed-dtype model.
|
||||||
|
del recurse
|
||||||
|
probe = torch.empty(0, dtype=self.weight.dtype, device=self.weight.device)
|
||||||
|
if fn(probe).dtype != self.weight.dtype:
|
||||||
|
raise RuntimeError("Pinned host embedding has a fixed dtype")
|
||||||
|
return self
|
||||||
|
|
||||||
|
def _lookup_weight(self, input_: torch.Tensor) -> torch.Tensor:
|
||||||
|
if input_.device.type == "cpu":
|
||||||
|
return self.weight
|
||||||
|
if self._uva_weight is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Host embedding received accelerator IDs without a UVA view"
|
||||||
|
)
|
||||||
|
if self._uva_weight.device != input_.device:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Host embedding UVA view and input IDs are on different devices"
|
||||||
|
)
|
||||||
|
return self._uva_weight
|
||||||
|
|
||||||
|
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||||
|
if self.tp_size == 1:
|
||||||
|
return F.embedding(
|
||||||
|
input_.long(),
|
||||||
|
self._lookup_weight(input_),
|
||||||
|
)
|
||||||
|
|
||||||
|
masked_input, input_mask = get_masked_input_and_mask(
|
||||||
|
input_,
|
||||||
|
self.shard_indices.org_vocab_start_index,
|
||||||
|
self.shard_indices.org_vocab_end_index,
|
||||||
|
self.shard_indices.num_org_vocab_padding,
|
||||||
|
self.shard_indices.added_vocab_start_index,
|
||||||
|
self.shard_indices.added_vocab_end_index,
|
||||||
|
)
|
||||||
|
output_parallel = F.embedding(
|
||||||
|
masked_input.long(),
|
||||||
|
self._lookup_weight(input_),
|
||||||
|
)
|
||||||
|
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||||
|
return tensor_model_parallel_all_reduce(output_parallel)
|
||||||
|
|
||||||
|
|
||||||
# --8<-- [start:parallel_lm_head]
|
# --8<-- [start:parallel_lm_head]
|
||||||
@PluggableLayer.register("parallel_lm_head")
|
@PluggableLayer.register("parallel_lm_head")
|
||||||
class ParallelLMHead(VocabParallelEmbedding):
|
class ParallelLMHead(VocabParallelEmbedding):
|
||||||
|
|||||||
@@ -883,12 +883,25 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
|
|||||||
)
|
)
|
||||||
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
|
# Checked again in Qwen4ExpModelState; rejecting it here keeps the
|
||||||
# engine from loading weights first.
|
# engine from loading weights first.
|
||||||
|
# vllm#54709: PLE only needs raw input_ids on the rank that owns a PLE
|
||||||
|
# layer. Allow PP>1 as long as every PLE layer lives on pipeline rank 0.
|
||||||
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
|
if text_config.ple_layer_ids and parallel_config.pipeline_parallel_size > 1:
|
||||||
raise NotImplementedError(
|
from vllm.distributed.utils import get_pp_indices
|
||||||
"Qwen4Exp N-gram PLE embedding requires pipeline_parallel_size=1 "
|
|
||||||
"because non-first pipeline ranks do not receive the raw input_ids "
|
rank0_start, rank0_end = get_pp_indices(
|
||||||
"it needs. Please run with PP=1."
|
int(text_config.num_hidden_layers),
|
||||||
|
0,
|
||||||
|
parallel_config.pipeline_parallel_size,
|
||||||
)
|
)
|
||||||
|
ple_idx = [int(i) - 1 for i in text_config.ple_layer_ids]
|
||||||
|
if min(ple_idx) < rank0_start or max(ple_idx) >= rank0_end:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"Qwen4Exp N-gram PLE embedding requires every PLE layer to "
|
||||||
|
f"live on pipeline rank 0 (layers {rank0_start}.."
|
||||||
|
f"{rank0_end - 1}), but PLE layers are at {ple_idx}. Give "
|
||||||
|
"rank 0 more layers via VLLM_PP_LAYER_PARTITION, or run "
|
||||||
|
"with PP=1."
|
||||||
|
)
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.multimodal_config
|
||||||
if multimodal_config is not None and multimodal_config.language_model_only:
|
if multimodal_config is not None and multimodal_config.language_model_only:
|
||||||
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
_strip_qwen4_exp_mrope(vllm_config.model_config)
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ MambaStateShapes: TypeAlias = (
|
|||||||
| tuple[tuple[int, int, int]]
|
| tuple[tuple[int, int, int]]
|
||||||
| tuple[tuple[int, int], tuple[int, int]]
|
| tuple[tuple[int, int], tuple[int, int]]
|
||||||
| tuple[tuple[int, int], tuple[int, int, int]]
|
| tuple[tuple[int, int], tuple[int, int, int]]
|
||||||
|
| tuple[tuple[int, int], tuple[int, int, int], tuple[int, int, int]]
|
||||||
| tuple[
|
| tuple[
|
||||||
tuple[int, int],
|
tuple[int, int],
|
||||||
tuple[int, int, int],
|
tuple[int, int, int],
|
||||||
|
|||||||
@@ -248,6 +248,8 @@ class Qwen3_5Model(Qwen3NextModel):
|
|||||||
self.embed_tokens = VocabParallelEmbedding(
|
self.embed_tokens = VocabParallelEmbedding(
|
||||||
self.vocab_size,
|
self.vocab_size,
|
||||||
config.hidden_size,
|
config.hidden_size,
|
||||||
|
quant_config=self.quant_config,
|
||||||
|
prefix=f"{prefix}.embed_tokens",
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_layer(prefix: str):
|
def get_layer(prefix: str):
|
||||||
|
|||||||
@@ -2,13 +2,15 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Inference-only Qwen3_5 MTP model."""
|
"""Inference-only Qwen3_5 MTP model."""
|
||||||
|
|
||||||
|
import json
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
import huggingface_hub
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
|
|
||||||
from vllm.compilation.decorators import support_torch_compile
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import ModelConfig, VllmConfig
|
||||||
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
|
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe.utils import (
|
from vllm.model_executor.layers.fused_moe.utils import (
|
||||||
@@ -34,6 +36,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk
|
|||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
|
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
|
||||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
||||||
|
from vllm.transformers_utils.repo_utils import try_get_local_file
|
||||||
|
|
||||||
from .interfaces import (
|
from .interfaces import (
|
||||||
MultiModalEmbeddings,
|
MultiModalEmbeddings,
|
||||||
@@ -52,6 +55,64 @@ from .utils import (
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
# Parameter suffixes a quantized MTP head would carry in the checkpoint.
|
||||||
|
_MTP_QUANT_ARTIFACT_SUFFIXES = (
|
||||||
|
"weight_packed",
|
||||||
|
"weight_scale",
|
||||||
|
"weight_zero_point",
|
||||||
|
"input_scale",
|
||||||
|
"input_zero_point",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _mtp_weight_names(model_config: ModelConfig) -> list[str] | None:
|
||||||
|
"""Return the checkpoint's mtp.* tensor names, or None when unknown.
|
||||||
|
|
||||||
|
Reads the safetensors index from the local cache or the Hub, the same way
|
||||||
|
config-time helpers read recipe.yaml. A single-file checkpoint has no
|
||||||
|
index, so its layout stays unknown (callers keep the default behavior).
|
||||||
|
"""
|
||||||
|
file_path = try_get_local_file(
|
||||||
|
model=model_config.model,
|
||||||
|
file_name="model.safetensors.index.json",
|
||||||
|
revision=model_config.revision,
|
||||||
|
)
|
||||||
|
if file_path is None:
|
||||||
|
try:
|
||||||
|
file_path = huggingface_hub.hf_hub_download(
|
||||||
|
model_config.model,
|
||||||
|
"model.safetensors.index.json",
|
||||||
|
revision=model_config.revision,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
with open(file_path) as f:
|
||||||
|
weight_map = json.load(f).get("weight_map", {})
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return None
|
||||||
|
return [name for name in weight_map if name.startswith("mtp.")]
|
||||||
|
|
||||||
|
|
||||||
|
def _checkpoint_ships_dense_mtp(model_config: ModelConfig) -> bool:
|
||||||
|
"""Whether the checkpoint ships the MTP head as dense weights.
|
||||||
|
|
||||||
|
Quantizers sometimes leave the MTP head unquantized without declaring it
|
||||||
|
in the quant config's ignore list; the tensors then arrive as plain
|
||||||
|
`mtp.*.weight` with no packed or scale companions (#58807). Detection is
|
||||||
|
name-based, so a properly quantized MTP head (any artifact suffix on any
|
||||||
|
mtp.* tensor) keeps the target's quantization.
|
||||||
|
"""
|
||||||
|
names = _mtp_weight_names(model_config)
|
||||||
|
if not names:
|
||||||
|
return False
|
||||||
|
mtp_names = [name for name in names if name.startswith("mtp.")]
|
||||||
|
if not mtp_names:
|
||||||
|
return False
|
||||||
|
return not any(
|
||||||
|
name.rsplit(".", 1)[-1] in _MTP_QUANT_ARTIFACT_SUFFIXES for name in mtp_names
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@support_torch_compile(
|
@support_torch_compile(
|
||||||
dynamic_arg_dims={
|
dynamic_arg_dims={
|
||||||
@@ -91,9 +152,19 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
|||||||
# missing from hf_quant_config.json exclude_modules. Force unquantized.
|
# missing from hf_quant_config.json exclude_modules. Force unquantized.
|
||||||
# Ref: https://github.com/vllm-project/vllm/pull/38650
|
# Ref: https://github.com/vllm-project/vllm/pull/38650
|
||||||
# Ref: https://github.com/NVIDIA/Model-Optimizer/pull/1124
|
# Ref: https://github.com/NVIDIA/Model-Optimizer/pull/1124
|
||||||
|
# compressed-tensors pack-quantized checkpoints may likewise ship the
|
||||||
|
# whole MTP head dense without declaring it in the ignore list (#58807).
|
||||||
|
mtp_dense = (
|
||||||
|
quant_config is not None
|
||||||
|
and quant_config.get_name() == "compressed-tensors"
|
||||||
|
and _checkpoint_ships_dense_mtp(model_config)
|
||||||
|
)
|
||||||
fc_quant = (
|
fc_quant = (
|
||||||
None
|
None
|
||||||
if (quant_config and quant_config.get_name() == "modelopt_fp4")
|
if (
|
||||||
|
quant_config
|
||||||
|
and (quant_config.get_name() == "modelopt_fp4" or mtp_dense)
|
||||||
|
)
|
||||||
else quant_config
|
else quant_config
|
||||||
)
|
)
|
||||||
self.fc = ColumnParallelLinear(
|
self.fc = ColumnParallelLinear(
|
||||||
@@ -109,13 +180,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
|
|||||||
# GPTQ: quantized checkpoints may exclude MTP from quantization via
|
# GPTQ: quantized checkpoints may exclude MTP from quantization via
|
||||||
# quantization_config.dynamic with "-:pattern" entries. When detected,
|
# quantization_config.dynamic with "-:pattern" entries. When detected,
|
||||||
# disable quantization for MTP layers so they use unquantized params.
|
# disable quantization for MTP layers so they use unquantized params.
|
||||||
|
# The dense-MTP detection above covers the same need for
|
||||||
|
# compressed-tensors checkpoints that declare nothing at all.
|
||||||
original_quant = vllm_config.quant_config
|
original_quant = vllm_config.quant_config
|
||||||
if quant_config and quant_config.get_name() not in ("modelopt_fp4",):
|
if quant_config and quant_config.get_name() not in ("modelopt_fp4",):
|
||||||
hf_qc = getattr(model_config.hf_config, "quantization_config", None)
|
hf_qc = getattr(model_config.hf_config, "quantization_config", None)
|
||||||
if isinstance(hf_qc, dict):
|
if mtp_dense or (
|
||||||
dynamic = hf_qc.get("dynamic", {})
|
isinstance(hf_qc, dict)
|
||||||
if any(k.startswith("-:") and "mtp" in k for k in dynamic):
|
and any(
|
||||||
vllm_config.quant_config = None
|
k.startswith("-:") and "mtp" in k for k in hf_qc.get("dynamic", {})
|
||||||
|
)
|
||||||
|
):
|
||||||
|
vllm_config.quant_config = None
|
||||||
self.layers = torch.nn.ModuleList(
|
self.layers = torch.nn.ModuleList(
|
||||||
Qwen3_5DecoderLayer(
|
Qwen3_5DecoderLayer(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
|
|||||||
@@ -22,6 +22,11 @@ _QWEN_MODEL_TYPES = frozenset(
|
|||||||
"qwen3_5_text",
|
"qwen3_5_text",
|
||||||
"qwen3_5_moe",
|
"qwen3_5_moe",
|
||||||
"qwen3_5_moe_text",
|
"qwen3_5_moe_text",
|
||||||
|
# Qwen4Exp runs its GDN prefill and QSA on other backends, but the
|
||||||
|
# kernels this file warms are the Triton ones around them -- the gated
|
||||||
|
# RMS norm, the causal conv, the post-conv preparation, the decode
|
||||||
|
# state update -- and those stay whichever prefill kernel is chosen.
|
||||||
|
# Left out of this set they compile on the first prompt instead.
|
||||||
"qwen4_exp",
|
"qwen4_exp",
|
||||||
"qwen4_exp_text",
|
"qwen4_exp_text",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -555,7 +555,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
|||||||
self.cache_config.mamba_cache_dtype,
|
self.cache_config.mamba_cache_dtype,
|
||||||
self.cache_config.mamba_ssm_cache_dtype,
|
self.cache_config.mamba_ssm_cache_dtype,
|
||||||
)
|
)
|
||||||
if self.cache_config.use_kda_recoverssm:
|
if self.cache_config.use_recoverssm:
|
||||||
return MambaStateDtypeCalculator.append_kda_recoverssm_record(
|
return MambaStateDtypeCalculator.append_kda_recoverssm_record(
|
||||||
base_dtypes, self.model_config.dtype
|
base_dtypes, self.model_config.dtype
|
||||||
)
|
)
|
||||||
@@ -571,7 +571,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
|||||||
conv_kernel_size=self.conv_size,
|
conv_kernel_size=self.conv_size,
|
||||||
num_spec=self.num_spec,
|
num_spec=self.num_spec,
|
||||||
)
|
)
|
||||||
if self.cache_config.use_kda_recoverssm:
|
if self.cache_config.use_recoverssm:
|
||||||
return MambaStateShapeCalculator.append_kda_recoverssm_record(
|
return MambaStateShapeCalculator.append_kda_recoverssm_record(
|
||||||
base_shapes,
|
base_shapes,
|
||||||
self.num_heads,
|
self.num_heads,
|
||||||
@@ -590,7 +590,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
|||||||
run_gemm_rs_ar: bool = False,
|
run_gemm_rs_ar: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(config, vllm_config, prefix)
|
super().__init__(config, vllm_config, prefix)
|
||||||
self.use_recoverssm = self.cache_config.use_kda_recoverssm
|
self.use_recoverssm = self.cache_config.use_recoverssm
|
||||||
if self.cache_config.use_replayssm and not self.use_recoverssm:
|
if self.cache_config.use_replayssm and not self.use_recoverssm:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Kimi-K3 supports --use-replayssm only with speculative decoding"
|
"Kimi-K3 supports --use-replayssm only with speculative decoding"
|
||||||
|
|||||||
@@ -319,7 +319,7 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
|||||||
isinstance(additional_config, dict)
|
isinstance(additional_config, dict)
|
||||||
and additional_config.get("kda_prefill_backend") == "flashinfer"
|
and additional_config.get("kda_prefill_backend") == "flashinfer"
|
||||||
)
|
)
|
||||||
self.use_recoverssm = vllm_config.cache_config.use_kda_recoverssm
|
self.use_recoverssm = vllm_config.cache_config.use_recoverssm
|
||||||
self.spec_state_slots = 1 if self.use_recoverssm else self.num_spec + 1
|
self.spec_state_slots = 1 if self.use_recoverssm else self.num_spec + 1
|
||||||
self.recoverssm_num_accepted_tokens: torch.Tensor | None = None
|
self.recoverssm_num_accepted_tokens: torch.Tensor | None = None
|
||||||
self.recoverssm_context: KDARecoverSSMCommitContext | None = None
|
self.recoverssm_context: KDARecoverSSMCommitContext | None = None
|
||||||
|
|||||||
@@ -1637,7 +1637,7 @@ class KimiLinearForCausalLM(
|
|||||||
vllm_config.cache_config.mamba_cache_dtype,
|
vllm_config.cache_config.mamba_cache_dtype,
|
||||||
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
||||||
)
|
)
|
||||||
if vllm_config.cache_config.use_kda_recoverssm:
|
if vllm_config.cache_config.use_recoverssm:
|
||||||
dtypes = MambaStateDtypeCalculator.append_kda_recoverssm_record(
|
dtypes = MambaStateDtypeCalculator.append_kda_recoverssm_record(
|
||||||
dtypes, vllm_config.model_config.dtype
|
dtypes, vllm_config.model_config.dtype
|
||||||
)
|
)
|
||||||
@@ -1662,7 +1662,7 @@ class KimiLinearForCausalLM(
|
|||||||
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
|
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
|
||||||
num_spec=num_spec,
|
num_spec=num_spec,
|
||||||
)
|
)
|
||||||
if vllm_config.cache_config.use_kda_recoverssm:
|
if vllm_config.cache_config.use_recoverssm:
|
||||||
return MambaStateShapeCalculator.append_kda_recoverssm_record(
|
return MambaStateShapeCalculator.append_kda_recoverssm_record(
|
||||||
shapes,
|
shapes,
|
||||||
hf_config.linear_attn_config["num_heads"],
|
hf_config.linear_attn_config["num_heads"],
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ from typing import Any
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||||
|
from vllm.model_executor.layers.mamba.ops.recoverssm import (
|
||||||
|
compact_conv_state_kernel as _compact_conv_state_kernel,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.ops.recoverssm import (
|
||||||
|
prepare_commit_plan_kernel as _prepare_commit_plan_kernel,
|
||||||
|
)
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||||
|
|
||||||
@@ -223,178 +229,6 @@ def _kda_recoverssm_verify_kernel(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@triton.heuristics(
|
|
||||||
{
|
|
||||||
"HAS_REQUEST_INDICES": lambda args: args["request_indices_ptr"] is not None,
|
|
||||||
"ALIGN_MODE": lambda args: args["block_table_ptr"] is not None,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
@triton.jit
|
|
||||||
def _prepare_commit_plan_kernel(
|
|
||||||
num_accepted_ptr,
|
|
||||||
request_indices_ptr,
|
|
||||||
state_indices_ptr,
|
|
||||||
query_start_loc_ptr,
|
|
||||||
block_table_ptr,
|
|
||||||
num_computed_ptr,
|
|
||||||
commit_lens_ptr,
|
|
||||||
final_state_indices_ptr,
|
|
||||||
boundary_state_indices_ptr,
|
|
||||||
boundary_recovery_lens_ptr,
|
|
||||||
null_block_id,
|
|
||||||
mamba_block_size,
|
|
||||||
block_table_width,
|
|
||||||
stride_num_accepted,
|
|
||||||
stride_request_indices,
|
|
||||||
stride_state_indices,
|
|
||||||
stride_query_start_loc,
|
|
||||||
stride_block_table_row,
|
|
||||||
stride_block_table_col,
|
|
||||||
stride_num_computed,
|
|
||||||
SPEC_QUERY_LEN: tl.constexpr,
|
|
||||||
HAS_REQUEST_INDICES: tl.constexpr,
|
|
||||||
ALIGN_MODE: tl.constexpr,
|
|
||||||
):
|
|
||||||
spec_idx = tl.program_id(0)
|
|
||||||
source_state_idx = tl.load(state_indices_ptr + spec_idx * stride_state_indices).to(
|
|
||||||
tl.int64
|
|
||||||
)
|
|
||||||
request_idx = spec_idx
|
|
||||||
if HAS_REQUEST_INDICES:
|
|
||||||
request_idx = tl.load(
|
|
||||||
request_indices_ptr + spec_idx * stride_request_indices
|
|
||||||
).to(tl.int64)
|
|
||||||
num_accepted = tl.load(num_accepted_ptr + request_idx * stride_num_accepted).to(
|
|
||||||
tl.int32
|
|
||||||
)
|
|
||||||
bos = tl.load(query_start_loc_ptr + spec_idx * stride_query_start_loc).to(tl.int64)
|
|
||||||
eos = tl.load(query_start_loc_ptr + (spec_idx + 1) * stride_query_start_loc).to(
|
|
||||||
tl.int64
|
|
||||||
)
|
|
||||||
query_len = (eos - bos).to(tl.int32)
|
|
||||||
commit_len = tl.minimum(tl.maximum(num_accepted, 0), query_len)
|
|
||||||
commit_len = tl.minimum(commit_len, SPEC_QUERY_LEN)
|
|
||||||
|
|
||||||
final_state_idx = source_state_idx
|
|
||||||
boundary_state_idx = null_block_id
|
|
||||||
boundary_recovery_len = 0
|
|
||||||
if ALIGN_MODE:
|
|
||||||
num_computed = tl.load(num_computed_ptr + request_idx * stride_num_computed).to(
|
|
||||||
tl.int32
|
|
||||||
)
|
|
||||||
final_num_computed = num_computed + commit_len
|
|
||||||
final_state_col = tl.minimum(
|
|
||||||
final_num_computed // mamba_block_size, block_table_width - 1
|
|
||||||
)
|
|
||||||
final_state_idx = tl.load(
|
|
||||||
block_table_ptr
|
|
||||||
+ request_idx * stride_block_table_row
|
|
||||||
+ final_state_col * stride_block_table_col
|
|
||||||
).to(tl.int64)
|
|
||||||
next_boundary = (num_computed // mamba_block_size + 1) * mamba_block_size
|
|
||||||
crosses_boundary = final_num_computed >= next_boundary
|
|
||||||
boundary_recovery_len = next_boundary - num_computed
|
|
||||||
boundary_state_idx = tl.load(
|
|
||||||
block_table_ptr
|
|
||||||
+ request_idx * stride_block_table_row
|
|
||||||
+ (next_boundary // mamba_block_size - 1) * stride_block_table_col,
|
|
||||||
mask=crosses_boundary,
|
|
||||||
other=null_block_id,
|
|
||||||
).to(tl.int64)
|
|
||||||
valid = (source_state_idx > null_block_id) & (commit_len > 0)
|
|
||||||
tl.store(commit_lens_ptr + spec_idx, tl.where(valid, commit_len, 0))
|
|
||||||
tl.store(
|
|
||||||
final_state_indices_ptr + spec_idx,
|
|
||||||
tl.where(valid, final_state_idx, null_block_id),
|
|
||||||
)
|
|
||||||
tl.store(
|
|
||||||
boundary_state_indices_ptr + spec_idx,
|
|
||||||
tl.where(valid, boundary_state_idx, null_block_id),
|
|
||||||
)
|
|
||||||
tl.store(
|
|
||||||
boundary_recovery_lens_ptr + spec_idx,
|
|
||||||
tl.where(valid, boundary_recovery_len, 0),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@triton.jit
|
|
||||||
def _compact_conv_state_kernel(
|
|
||||||
conv_state_ref_ptr,
|
|
||||||
conv_state_base_addrs_ptr,
|
|
||||||
conv_state_block_strides_ptr,
|
|
||||||
conv_state_dim_strides_ptr,
|
|
||||||
conv_state_token_strides_ptr,
|
|
||||||
state_indices_ptr,
|
|
||||||
commit_lens_ptr,
|
|
||||||
final_state_indices_ptr,
|
|
||||||
boundary_state_indices_ptr,
|
|
||||||
boundary_recovery_lens_ptr,
|
|
||||||
null_block_id,
|
|
||||||
conv_dim,
|
|
||||||
conv_history_len,
|
|
||||||
stride_state_indices,
|
|
||||||
BLOCK_D: tl.constexpr,
|
|
||||||
BLOCK_HISTORY: tl.constexpr,
|
|
||||||
ALIGN_MODE: tl.constexpr,
|
|
||||||
):
|
|
||||||
pid_d = tl.program_id(0)
|
|
||||||
pid_b = tl.program_id(1)
|
|
||||||
pid_l = tl.program_id(2)
|
|
||||||
source_state_idx = tl.load(state_indices_ptr + pid_b * stride_state_indices).to(
|
|
||||||
tl.int64
|
|
||||||
)
|
|
||||||
if source_state_idx <= null_block_id:
|
|
||||||
return
|
|
||||||
|
|
||||||
commit_len = tl.load(commit_lens_ptr + pid_b)
|
|
||||||
if commit_len == 0:
|
|
||||||
return
|
|
||||||
final_state_idx = tl.load(final_state_indices_ptr + pid_b).to(tl.int64)
|
|
||||||
boundary_state_idx = tl.load(boundary_state_indices_ptr + pid_b).to(tl.int64)
|
|
||||||
boundary_recovery_len = tl.load(boundary_recovery_lens_ptr + pid_b)
|
|
||||||
|
|
||||||
if final_state_idx <= null_block_id:
|
|
||||||
return
|
|
||||||
|
|
||||||
base_addr = tl.load(conv_state_base_addrs_ptr + pid_l)
|
|
||||||
block_stride = tl.load(conv_state_block_strides_ptr + pid_l)
|
|
||||||
dim_stride = tl.load(conv_state_dim_strides_ptr + pid_l)
|
|
||||||
token_stride = tl.load(conv_state_token_strides_ptr + pid_l)
|
|
||||||
conv_state_ptr = base_addr.to(tl.pointer_type(conv_state_ref_ptr.dtype.element_ty))
|
|
||||||
source_ptr = conv_state_ptr + source_state_idx * block_stride
|
|
||||||
final_ptr = conv_state_ptr + final_state_idx * block_stride
|
|
||||||
|
|
||||||
offs_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D)
|
|
||||||
offs_h = tl.arange(0, BLOCK_HISTORY)
|
|
||||||
mask = (offs_d[:, None] < conv_dim) & (offs_h[None, :] < conv_history_len)
|
|
||||||
final_values = tl.load(
|
|
||||||
source_ptr
|
|
||||||
+ offs_d[:, None] * dim_stride
|
|
||||||
+ (commit_len - 1 + offs_h[None, :]) * token_stride,
|
|
||||||
mask=mask,
|
|
||||||
)
|
|
||||||
if ALIGN_MODE:
|
|
||||||
boundary_values = tl.load(
|
|
||||||
source_ptr
|
|
||||||
+ offs_d[:, None] * dim_stride
|
|
||||||
+ (boundary_recovery_len - 1 + offs_h[None, :]) * token_stride,
|
|
||||||
mask=mask & (boundary_state_idx > null_block_id),
|
|
||||||
)
|
|
||||||
boundary_ptr = conv_state_ptr + boundary_state_idx * block_stride
|
|
||||||
tl.store(
|
|
||||||
boundary_ptr
|
|
||||||
+ offs_d[:, None] * dim_stride
|
|
||||||
+ offs_h[None, :] * token_stride,
|
|
||||||
boundary_values,
|
|
||||||
mask=mask & (boundary_state_idx > null_block_id),
|
|
||||||
)
|
|
||||||
tl.store(
|
|
||||||
final_ptr + offs_d[:, None] * dim_stride + offs_h[None, :] * token_stride,
|
|
||||||
final_values,
|
|
||||||
mask=mask,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@triton.jit
|
@triton.jit
|
||||||
def _commit_kda_state_kernel(
|
def _commit_kda_state_kernel(
|
||||||
state_ref_ptr,
|
state_ref_ptr,
|
||||||
|
|||||||
@@ -119,6 +119,15 @@ class QSAIndexer(nn.Module):
|
|||||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# QSAKeyStateCache rows now carry
|
||||||
|
# a tag word the NVIDIA pre-indexer writes and validates; this path
|
||||||
|
# neither writes nor checks it, so it would run the widened layout
|
||||||
|
# with no ring validation at all. Refuse it rather than share it.
|
||||||
|
raise NotImplementedError(
|
||||||
|
"Qwen4Exp QSA on the AMD path is not supported on this fork: the "
|
||||||
|
"ring-row tag contract is implemented for the NVIDIA "
|
||||||
|
"pre-indexer only (vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py)."
|
||||||
|
)
|
||||||
cache_config = vllm_config.cache_config
|
cache_config = vllm_config.cache_config
|
||||||
cache_prefix = f"{prefix}." if prefix else ""
|
cache_prefix = f"{prefix}." if prefix else ""
|
||||||
self.raw_key_cache = QSAKeyStateCache(
|
self.raw_key_cache = QSAKeyStateCache(
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config
|
|||||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||||
from vllm.v1.kv_cache_interface import MambaSpec
|
from vllm.v1.kv_cache_interface import MambaSpec
|
||||||
|
|
||||||
|
from ..common.mtp import make_mtp_hidden_buffer
|
||||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||||
from .ple_layer import Qwen4ExpPLELayer
|
from .ple_layer import Qwen4ExpPLELayer
|
||||||
@@ -428,25 +429,11 @@ class Qwen4ExpModel(nn.Module):
|
|||||||
else:
|
else:
|
||||||
self.hyper_connection_mixer = None
|
self.hyper_connection_mixer = None
|
||||||
|
|
||||||
spec_config = vllm_config.speculative_config
|
self._mtp_hidden_buffer = make_mtp_hidden_buffer(
|
||||||
# MTP HC multi-stream outputs: when speculative method=="mtp" and the
|
vllm_config,
|
||||||
# model uses HC with hc_count>1, retain the pre-final-mixer multi-stream
|
config,
|
||||||
# hidden state [T, hc_count*H] so the MTP drafter can feed a real
|
is_last_rank=get_pp_group().is_last_rank,
|
||||||
# multi-stream backbone hidden on its first step (scheme A). Derived
|
|
||||||
# purely from config (NOT node identity) so P/D nodes stay consistent.
|
|
||||||
needs_mtp_hidden = (
|
|
||||||
spec_config is not None
|
|
||||||
and getattr(spec_config, "method", None) == "mtp"
|
|
||||||
and get_pp_group().is_last_rank
|
|
||||||
)
|
)
|
||||||
if needs_mtp_hidden:
|
|
||||||
self._mtp_hidden_buffer = torch.empty(
|
|
||||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
||||||
config.hc_count * config.hidden_size,
|
|
||||||
dtype=vllm_config.model_config.dtype,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self._mtp_hidden_buffer = None
|
|
||||||
|
|
||||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||||
return self.embed_tokens(input_ids)
|
return self.embed_tokens(input_ids)
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
|||||||
VocabParallelEmbedding,
|
VocabParallelEmbedding,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||||
from vllm.model_executor.models.interfaces import SupportsPP
|
from vllm.model_executor.models.interfaces import LocalArgmaxMixin, SupportsPP
|
||||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||||
from vllm.model_executor.models.utils import (
|
from vllm.model_executor.models.utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
@@ -377,7 +377,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
|||||||
"hidden_states": 0,
|
"hidden_states": 0,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
class Qwen4ExpMTP(LocalArgmaxMixin, nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Shared MTP drafter state for Qwen4Exp."""
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from transformers import Qwen4ExpTextConfig
|
||||||
|
|
||||||
|
|
||||||
|
def make_mtp_hidden_buffer(
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
config: Qwen4ExpTextConfig,
|
||||||
|
*,
|
||||||
|
is_last_rank: bool,
|
||||||
|
) -> torch.Tensor | None:
|
||||||
|
"""Buffer holding the pre-final-mixer multi-stream hidden state, or None.
|
||||||
|
|
||||||
|
When speculative ``method == "mtp"`` the drafter feeds a real multi-stream
|
||||||
|
backbone hidden on its first step, so the last stage retains
|
||||||
|
``[T, hc_count * H]``. Whether it is needed is derived from the config
|
||||||
|
alone, not from node identity, so prefill and decode nodes agree.
|
||||||
|
"""
|
||||||
|
spec_config = vllm_config.speculative_config
|
||||||
|
needs_mtp_hidden = (
|
||||||
|
spec_config is not None
|
||||||
|
and getattr(spec_config, "method", None) == "mtp"
|
||||||
|
and is_last_rank
|
||||||
|
)
|
||||||
|
if not needs_mtp_hidden:
|
||||||
|
return None
|
||||||
|
return torch.empty(
|
||||||
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||||
|
config.hc_count * config.hidden_size,
|
||||||
|
dtype=vllm_config.model_config.dtype,
|
||||||
|
device=vllm_config.device_config.device,
|
||||||
|
)
|
||||||
@@ -15,6 +15,7 @@ import torch
|
|||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from torch import nn
|
from torch import nn
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||||
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
|
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
|
||||||
from vllm.forward_context import DPMetadata, get_forward_context
|
from vllm.forward_context import DPMetadata, get_forward_context
|
||||||
@@ -53,7 +54,7 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
||||||
"""ETP-sharded PLE table shared by device and pinned-host backends."""
|
"""PLE table base shared by the device, pinned-host, and file-gather backends."""
|
||||||
|
|
||||||
supports_prefetch: ClassVar[bool] = False
|
supports_prefetch: ClassVar[bool] = False
|
||||||
|
|
||||||
@@ -162,9 +163,9 @@ class Qwen4ExpPLEEmbedding(PLEVocabParallelEmbedding, ABC):
|
|||||||
|
|
||||||
|
|
||||||
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||||
"""Quantization interface shared by resident and pinned PLE tables."""
|
"""Quantization interface shared by device, pinned, and file-gather PLE tables."""
|
||||||
|
|
||||||
# PLE post-load processing only validates scales in their current storage.
|
# PLE post-load processing validates scales in place and binds host file shards.
|
||||||
requires_device_loading: bool = False
|
requires_device_loading: bool = False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -231,6 +232,12 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
|||||||
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
def embedding(self, layer: nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
||||||
return F.embedding(input_, layer.weight)
|
return F.embedding(input_, layer.weight)
|
||||||
|
|
||||||
|
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||||
|
"""Bind host-file-gather tables once every checkpoint shard is loaded."""
|
||||||
|
bind = getattr(layer, "bind_file_shards", None)
|
||||||
|
if bind is not None:
|
||||||
|
bind()
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def dequantize(
|
def dequantize(
|
||||||
self,
|
self,
|
||||||
@@ -320,6 +327,7 @@ class Qwen4ExpPLEFp8EmbeddingMethod(Qwen4ExpPLEEmbeddingMethod):
|
|||||||
sentinel = torch.finfo(torch.float32).min
|
sentinel = torch.finfo(torch.float32).min
|
||||||
if torch.any(layer.weight_scale == sentinel):
|
if torch.any(layer.weight_scale == sentinel):
|
||||||
raise ValueError("FP8 PLE checkpoint is missing its global scale")
|
raise ValueError("FP8 PLE checkpoint is missing its global scale")
|
||||||
|
super().process_weights_after_loading(layer)
|
||||||
|
|
||||||
def dequantize(
|
def dequantize(
|
||||||
self,
|
self,
|
||||||
@@ -415,6 +423,13 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
|||||||
max_total_tokens: int = 0,
|
max_total_tokens: int = 0,
|
||||||
data_parallel_rank: int = 0,
|
data_parallel_rank: int = 0,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
# Fork: never build the UVA table beside the E2 pull transport --
|
||||||
|
# that would be a second offload of the same PLE table.
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
|
||||||
|
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
|
||||||
|
)
|
||||||
if not is_uva_available():
|
if not is_uva_available():
|
||||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||||
super().__init__(
|
super().__init__(
|
||||||
@@ -477,10 +492,17 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
|||||||
|
|
||||||
flat_ids = input_ids.reshape(-1).long()
|
flat_ids = input_ids.reshape(-1).long()
|
||||||
if flat_ids.numel():
|
if flat_ids.numel():
|
||||||
|
if self.weight.dtype == torch.float8_e4m3fn:
|
||||||
|
uva_weight_view = self._uva_weight.view(torch.int8)
|
||||||
|
output_view = output.view(torch.int8)
|
||||||
|
else:
|
||||||
|
uva_weight_view = self._uva_weight
|
||||||
|
output_view = output
|
||||||
|
|
||||||
_lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)](
|
_lookup_ple_embedding_from_pinned_kernel[(flat_ids.numel(),)](
|
||||||
self._uva_weight,
|
uva_weight_view,
|
||||||
flat_ids,
|
flat_ids,
|
||||||
output,
|
output_view,
|
||||||
self.embedding_dim,
|
self.embedding_dim,
|
||||||
self.shard_indices.org_vocab_start_index,
|
self.shard_indices.org_vocab_start_index,
|
||||||
self.shard_indices.org_vocab_end_index,
|
self.shard_indices.org_vocab_end_index,
|
||||||
|
|||||||
@@ -44,6 +44,42 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
MLAAttentionSpec,
|
MLAAttentionSpec,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Ring-row tag contract. Every raw-key ring row is the key (bf16), padded to a
|
||||||
|
# whole int64, followed by a tail of four int64 words:
|
||||||
|
# (pos_t, pos_h, pos_w, tag), tag = logical_position + 1.
|
||||||
|
# The tail exists on every ring, with or without cached RoPE positions: the tag
|
||||||
|
# is what lets the reader prove the row it is about to pool was committed by
|
||||||
|
# this request for exactly that logical position. 0 is never a valid tag (so a
|
||||||
|
# cleared row can never validate) and the manager clears the tag word of every
|
||||||
|
# newly claimed ring block before its new owner's first step.
|
||||||
|
QSA_RING_TAIL_WORDS = 4
|
||||||
|
QSA_RING_TAG_WORD = 3
|
||||||
|
QSA_RING_BF16_PER_INT64 = 4
|
||||||
|
QSA_RING_TAIL_ELEMENTS = QSA_RING_TAIL_WORDS * QSA_RING_BF16_PER_INT64
|
||||||
|
|
||||||
|
|
||||||
|
def qsa_ring_row_tags(logical_positions: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""The tag the writer stores for each committed row (int64)."""
|
||||||
|
return logical_positions.to(torch.int64) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def qsa_ring_group_rows_valid(
|
||||||
|
found_tags: torch.Tensor,
|
||||||
|
source_positions: torch.Tensor,
|
||||||
|
historical: torch.Tensor,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Reference for the pre-indexer's per-group check (torch, any device).
|
||||||
|
|
||||||
|
``found_tags``/``source_positions``/``historical`` are ``[groups, ratio]``:
|
||||||
|
the tag word read from each source row's ring slot, the row's logical
|
||||||
|
position, and whether the row must come from the ring (it precedes the
|
||||||
|
chunk start). Returns a ``[groups]`` bool: True when EVERY historical row
|
||||||
|
carries ``source_position + 1``. In-chunk rows are never checked.
|
||||||
|
"""
|
||||||
|
expected = qsa_ring_row_tags(source_positions)
|
||||||
|
bad = historical & (found_tags.to(torch.int64) != expected)
|
||||||
|
return ~bad.any(dim=-1)
|
||||||
|
|
||||||
|
|
||||||
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
|
def canonical_qsa_rope_positions(positions: torch.Tensor) -> torch.Tensor:
|
||||||
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
|
"""Return exact per-token positions as ``[tokens, 1, 3]`` int64 rows."""
|
||||||
@@ -398,12 +434,53 @@ def build_qsa_metadata_triton(
|
|||||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||||
"""Build QSA side-cache and optional pre-indexer work metadata."""
|
"""Build QSA side-cache and optional pre-indexer work metadata."""
|
||||||
num_tokens = common_attn_metadata.num_actual_tokens
|
num_tokens = common_attn_metadata.num_actual_tokens
|
||||||
num_mapped_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
|
|
||||||
token_to_req = token_to_req_buffer[:num_tokens]
|
token_to_req = token_to_req_buffer[:num_tokens]
|
||||||
logical_positions = logical_positions_buffer[:num_tokens]
|
logical_positions = logical_positions_buffer[:num_tokens]
|
||||||
visible_blocks = visible_blocks_buffer[:num_tokens]
|
visible_blocks = visible_blocks_buffer[:num_tokens]
|
||||||
slot_mapping = slot_mapping_buffer[:num_tokens]
|
slot_mapping = slot_mapping_buffer[:num_tokens]
|
||||||
num_reqs = common_attn_metadata.query_start_loc.shape[0] - 1
|
if num_tokens == 0 and k_work_metadata_buffer is None:
|
||||||
|
return token_to_req, logical_positions, visible_blocks, slot_mapping
|
||||||
|
_launch_qsa_metadata_kernel(
|
||||||
|
common_attn_metadata.query_start_loc,
|
||||||
|
common_attn_metadata.seq_lens,
|
||||||
|
common_attn_metadata.slot_mapping,
|
||||||
|
common_attn_metadata.block_table_tensor,
|
||||||
|
token_to_req,
|
||||||
|
logical_positions,
|
||||||
|
visible_blocks,
|
||||||
|
slot_mapping,
|
||||||
|
k_work_metadata_buffer,
|
||||||
|
num_mapped_tokens=int(common_attn_metadata.query_start_loc_cpu[-1]),
|
||||||
|
storage_block_size=storage_block_size,
|
||||||
|
compress_ratio=compress_ratio,
|
||||||
|
circular_buffer_size=circular_buffer_size,
|
||||||
|
request_capacity=request_capacity,
|
||||||
|
)
|
||||||
|
if circular_buffer_size == 0 and compress_ratio == 1:
|
||||||
|
slot_mapping = common_attn_metadata.slot_mapping[:num_tokens]
|
||||||
|
return token_to_req, logical_positions, visible_blocks, slot_mapping
|
||||||
|
|
||||||
|
|
||||||
|
def _launch_qsa_metadata_kernel(
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
seq_lens: torch.Tensor,
|
||||||
|
common_slot_mapping: torch.Tensor,
|
||||||
|
block_table: torch.Tensor,
|
||||||
|
token_to_req: torch.Tensor,
|
||||||
|
logical_positions: torch.Tensor,
|
||||||
|
visible_blocks: torch.Tensor,
|
||||||
|
slot_mapping: torch.Tensor,
|
||||||
|
k_work_metadata_buffer: torch.Tensor | None,
|
||||||
|
*,
|
||||||
|
num_mapped_tokens: int,
|
||||||
|
storage_block_size: int,
|
||||||
|
compress_ratio: int,
|
||||||
|
circular_buffer_size: int,
|
||||||
|
request_capacity: int | None,
|
||||||
|
) -> None:
|
||||||
|
"""Fill QSA metadata in place; capture-safe given fixed shapes and scalars."""
|
||||||
|
num_tokens = token_to_req.shape[0]
|
||||||
|
num_reqs = query_start_loc.shape[0] - 1
|
||||||
assert num_reqs > 0
|
assert num_reqs > 0
|
||||||
|
|
||||||
if k_work_metadata_buffer is not None:
|
if k_work_metadata_buffer is not None:
|
||||||
@@ -417,10 +494,6 @@ def build_qsa_metadata_triton(
|
|||||||
request_scan_size = 1
|
request_scan_size = 1
|
||||||
max_num_work = 0
|
max_num_work = 0
|
||||||
|
|
||||||
if num_tokens == 0 and k_work_metadata_buffer is None:
|
|
||||||
return token_to_req, logical_positions, visible_blocks, slot_mapping
|
|
||||||
|
|
||||||
block_table = common_attn_metadata.block_table_tensor
|
|
||||||
num_search_steps = int(math.ceil(math.log2(num_reqs)))
|
num_search_steps = int(math.ceil(math.log2(num_reqs)))
|
||||||
work_search_steps = int(math.ceil(math.log2(num_reqs)))
|
work_search_steps = int(math.ceil(math.log2(num_reqs)))
|
||||||
# The same grid covers token tiles and, for the compressed cache, work tiles.
|
# The same grid covers token tiles and, for the compressed cache, work tiles.
|
||||||
@@ -429,9 +502,9 @@ def build_qsa_metadata_triton(
|
|||||||
cdiv(max_num_work, 256) if k_work_metadata_buffer is not None else 0
|
cdiv(max_num_work, 256) if k_work_metadata_buffer is not None else 0
|
||||||
)
|
)
|
||||||
_build_qsa_metadata_kernel[(max(num_token_blocks, num_work_blocks, 1),)](
|
_build_qsa_metadata_kernel[(max(num_token_blocks, num_work_blocks, 1),)](
|
||||||
common_attn_metadata.query_start_loc,
|
query_start_loc,
|
||||||
common_attn_metadata.seq_lens,
|
seq_lens,
|
||||||
common_attn_metadata.slot_mapping,
|
common_slot_mapping,
|
||||||
block_table,
|
block_table,
|
||||||
token_to_req,
|
token_to_req,
|
||||||
logical_positions,
|
logical_positions,
|
||||||
@@ -456,9 +529,6 @@ def build_qsa_metadata_triton(
|
|||||||
WORK_BLOCK_SIZE=256,
|
WORK_BLOCK_SIZE=256,
|
||||||
num_warps=4,
|
num_warps=4,
|
||||||
)
|
)
|
||||||
if circular_buffer_size == 0 and compress_ratio == 1:
|
|
||||||
slot_mapping = common_attn_metadata.slot_mapping[:num_tokens]
|
|
||||||
return token_to_req, logical_positions, visible_blocks, slot_mapping
|
|
||||||
|
|
||||||
|
|
||||||
def _build_qsa_metadata_torch(
|
def _build_qsa_metadata_torch(
|
||||||
@@ -581,6 +651,8 @@ class QSAForwardMetadata(AttentionMetadata):
|
|||||||
logical_positions: torch.Tensor
|
logical_positions: torch.Tensor
|
||||||
visible_blocks: torch.Tensor
|
visible_blocks: torch.Tensor
|
||||||
k_work_metadata: torch.Tensor
|
k_work_metadata: torch.Tensor
|
||||||
|
common_slot_mapping: torch.Tensor
|
||||||
|
num_mapped_tokens: int
|
||||||
num_actual_tokens: int
|
num_actual_tokens: int
|
||||||
num_decodes: int
|
num_decodes: int
|
||||||
num_decode_tokens: int
|
num_decode_tokens: int
|
||||||
@@ -597,6 +669,7 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
|
|||||||
"""Build QSA metadata from vLLM's cache-group-specific common metadata."""
|
"""Build QSA metadata from vLLM's cache-group-specific common metadata."""
|
||||||
|
|
||||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
supports_draft_decode_metadata_update = HAS_TRITON
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -708,6 +781,8 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
|
|||||||
logical_positions=logical_positions,
|
logical_positions=logical_positions,
|
||||||
visible_blocks=visible_blocks,
|
visible_blocks=visible_blocks,
|
||||||
k_work_metadata=k_work_metadata,
|
k_work_metadata=k_work_metadata,
|
||||||
|
common_slot_mapping=common_attn_metadata.slot_mapping,
|
||||||
|
num_mapped_tokens=int(common_attn_metadata.query_start_loc_cpu[-1]),
|
||||||
num_actual_tokens=num_tokens,
|
num_actual_tokens=num_tokens,
|
||||||
num_decodes=num_decodes,
|
num_decodes=num_decodes,
|
||||||
num_decode_tokens=num_decode_tokens,
|
num_decode_tokens=num_decode_tokens,
|
||||||
@@ -720,6 +795,29 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
|
|||||||
compress_ratio=self.compress_ratio,
|
compress_ratio=self.compress_ratio,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def update_draft_decode_metadata(self, metadata: QSAForwardMetadata) -> None:
|
||||||
|
if metadata.num_actual_tokens == 0:
|
||||||
|
return
|
||||||
|
build_k_work = not self.is_circular_buffer and self.compress_ratio != 1
|
||||||
|
_launch_qsa_metadata_kernel(
|
||||||
|
metadata.query_start_loc,
|
||||||
|
metadata.seq_lens,
|
||||||
|
metadata.common_slot_mapping,
|
||||||
|
metadata.block_table,
|
||||||
|
metadata.token_to_req,
|
||||||
|
metadata.logical_positions,
|
||||||
|
metadata.visible_blocks,
|
||||||
|
metadata.slot_mapping,
|
||||||
|
metadata.k_work_metadata if build_k_work else None,
|
||||||
|
num_mapped_tokens=metadata.num_mapped_tokens,
|
||||||
|
storage_block_size=self.storage_block_size,
|
||||||
|
compress_ratio=self.compress_ratio,
|
||||||
|
circular_buffer_size=(
|
||||||
|
self.kv_cache_spec.block_size if self.is_circular_buffer else 0
|
||||||
|
),
|
||||||
|
request_capacity=self.request_capacity if build_k_work else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class QSAStateBackend(AttentionBackend):
|
class QSAStateBackend(AttentionBackend):
|
||||||
"""Key-only dummy backend for out-of-band QSA side-cache operations."""
|
"""Key-only dummy backend for out-of-band QSA side-cache operations."""
|
||||||
@@ -806,37 +904,65 @@ class _QSAStateCache(nn.Module, AttentionLayerBase):
|
|||||||
|
|
||||||
|
|
||||||
class QSAKeyStateCache(_QSAStateCache):
|
class QSAKeyStateCache(_QSAStateCache):
|
||||||
"""Raw BF16 key, optionally followed by exact int64 MRoPE positions."""
|
"""Raw BF16 key followed by the int64 ring tail (pos_t, pos_h, pos_w, tag).
|
||||||
|
|
||||||
_BF16_PER_INT64 = 4
|
The position words are consumed only when ``cache_rope_positions``; the tag
|
||||||
|
word is always written and always validated."""
|
||||||
|
|
||||||
|
_BF16_PER_INT64 = QSA_RING_BF16_PER_INT64
|
||||||
_NUM_ROPE_AXES = 3
|
_NUM_ROPE_AXES = 3
|
||||||
|
|
||||||
def __init__(self, *, cache_rope_positions: bool = False, **kwargs) -> None:
|
def __init__(self, *, cache_rope_positions: bool = False, **kwargs) -> None:
|
||||||
key_head_size = int(kwargs.pop("head_size"))
|
key_head_size = int(kwargs.pop("head_size"))
|
||||||
self.key_head_size = key_head_size
|
self.key_head_size = key_head_size
|
||||||
self.cache_rope_positions = bool(cache_rope_positions)
|
self.cache_rope_positions = bool(cache_rope_positions)
|
||||||
|
# Element offset of the int64 tail (the name predates the tag word).
|
||||||
self.rope_position_offset = (
|
self.rope_position_offset = (
|
||||||
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
|
(key_head_size + self._BF16_PER_INT64 - 1) // self._BF16_PER_INT64
|
||||||
) * self._BF16_PER_INT64
|
) * self._BF16_PER_INT64
|
||||||
storage_head_size = key_head_size
|
storage_head_size = self.rope_position_offset + QSA_RING_TAIL_ELEMENTS
|
||||||
if self.cache_rope_positions:
|
|
||||||
storage_head_size = self.rope_position_offset + (
|
|
||||||
self._NUM_ROPE_AXES * self._BF16_PER_INT64
|
|
||||||
)
|
|
||||||
super().__init__(head_size=storage_head_size, **kwargs)
|
super().__init__(head_size=storage_head_size, **kwargs)
|
||||||
|
self._ring_tag_cache: torch.Tensor | None = None
|
||||||
|
self._ring_tail: torch.Tensor | None = None
|
||||||
|
|
||||||
|
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
|
||||||
|
super().bind_kv_cache(kv_cache)
|
||||||
|
qsa_cache = self.kv_cache
|
||||||
|
tail_end = self.rope_position_offset + QSA_RING_TAIL_ELEMENTS
|
||||||
|
tail = qsa_cache[..., self.rope_position_offset : tail_end].view(torch.int64)
|
||||||
|
assert tail.shape[-1] == QSA_RING_TAIL_WORDS
|
||||||
|
# Derived on access so `kv_cache` stays the only reference to the bound
|
||||||
|
# storage: clearing it (e.g. after CUDA graph memory profiling) must free
|
||||||
|
# the cache, or the freed block stays pinned in the allocator. Only the
|
||||||
|
# tag word is read outside a bind, so it is the one cached attribute.
|
||||||
|
self._ring_tail = tail
|
||||||
|
self._ring_tag_cache = tail[..., QSA_RING_TAG_WORD : QSA_RING_TAG_WORD + 1]
|
||||||
|
|
||||||
# Derived on access so `kv_cache` stays the only reference to the bound
|
|
||||||
# storage: clearing it (e.g. after CUDA graph memory profiling) must free
|
|
||||||
# the cache, or the freed block stays pinned in the allocator.
|
|
||||||
@property
|
@property
|
||||||
def key_cache(self) -> torch.Tensor:
|
def key_cache(self) -> torch.Tensor:
|
||||||
return self.kv_cache[..., : self.key_head_size]
|
return self.kv_cache[..., : self.key_head_size]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def rope_position_cache(self) -> torch.Tensor | None:
|
def rope_position_cache(self) -> torch.Tensor | None:
|
||||||
if not self.cache_rope_positions:
|
if not self.cache_rope_positions or self._ring_tail is None:
|
||||||
return None
|
return None
|
||||||
return self.kv_cache[..., self.rope_position_offset :].view(torch.int64)
|
return self._ring_tail[..., : self._NUM_ROPE_AXES]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ring_tag_cache(self) -> torch.Tensor | None:
|
||||||
|
return self._ring_tag_cache
|
||||||
|
|
||||||
|
def clear_ring_row_tags(self, block_ids: torch.Tensor) -> None:
|
||||||
|
"""Zero the tag word of every row of ``block_ids`` (ring-row tag claim clear).
|
||||||
|
|
||||||
|
Only the tag word: a claimed ring block's keys and positions stay as
|
||||||
|
the previous occupant left them, and the zero tag is what makes every
|
||||||
|
such row fail validation until this request commits it."""
|
||||||
|
if self._ring_tag_cache is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"{self.prefix}: ring tag clear before the KV cache was bound"
|
||||||
|
)
|
||||||
|
self._ring_tag_cache.index_fill_(0, block_ids, 0)
|
||||||
|
|
||||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||||
# Hold the open group's committed keys plus every row a speculative
|
# Hold the open group's committed keys plus every row a speculative
|
||||||
@@ -874,6 +1000,9 @@ class QSACompressedKeyCache(_QSAStateCache):
|
|||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
"QSA_RING_TAG_WORD",
|
||||||
|
"QSA_RING_TAIL_ELEMENTS",
|
||||||
|
"QSA_RING_TAIL_WORDS",
|
||||||
"QSACompressedKeyCache",
|
"QSACompressedKeyCache",
|
||||||
"QSAForwardMetadata",
|
"QSAForwardMetadata",
|
||||||
"QSAKeyStateCache",
|
"QSAKeyStateCache",
|
||||||
@@ -882,4 +1011,6 @@ __all__ = [
|
|||||||
"canonical_qsa_rope_positions",
|
"canonical_qsa_rope_positions",
|
||||||
"circular_qsa_slot_mapping",
|
"circular_qsa_slot_mapping",
|
||||||
"compressed_qsa_slot_mapping",
|
"compressed_qsa_slot_mapping",
|
||||||
|
"qsa_ring_group_rows_valid",
|
||||||
|
"qsa_ring_row_tags",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from ..common.qsa_cache import (
|
|||||||
QSAForwardMetadata,
|
QSAForwardMetadata,
|
||||||
QSAKeyStateCache,
|
QSAKeyStateCache,
|
||||||
canonical_qsa_rope_positions,
|
canonical_qsa_rope_positions,
|
||||||
|
qsa_ring_row_tags,
|
||||||
)
|
)
|
||||||
from .ops.qsa_prepare import qsa_prepare
|
from .ops.qsa_prepare import qsa_prepare
|
||||||
|
|
||||||
@@ -339,6 +340,7 @@ class QSAIndexer(nn.Module):
|
|||||||
main_slot_mapping=slot_mapping[:num_tokens],
|
main_slot_mapping=slot_mapping[:num_tokens],
|
||||||
main_k_scale=attn._k_scale_float,
|
main_k_scale=attn._k_scale_float,
|
||||||
main_v_scale=attn._v_scale_float,
|
main_v_scale=attn._v_scale_float,
|
||||||
|
layer_id=self.layer_id,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# Unfused reference path
|
# Unfused reference path
|
||||||
@@ -405,6 +407,18 @@ class QSAIndexer(nn.Module):
|
|||||||
raw_metadata.slot_mapping,
|
raw_metadata.slot_mapping,
|
||||||
position_rows,
|
position_rows,
|
||||||
)
|
)
|
||||||
|
# The unfused writer commits the same tag word as the fused
|
||||||
|
# one, so both paths leave byte-identical rings. (This reference
|
||||||
|
# reader does not validate tags; the fused kernel does.)
|
||||||
|
ring_tag_cache = getattr(raw_key_state_cache, "ring_tag_cache", None)
|
||||||
|
if ring_tag_cache is not None:
|
||||||
|
qsa_store_cache_rows(
|
||||||
|
ring_tag_cache,
|
||||||
|
raw_metadata.slot_mapping,
|
||||||
|
qsa_ring_row_tags(raw_metadata.logical_positions[:num_tokens]).view(
|
||||||
|
-1, 1, 1
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
if self.skip_topk:
|
if self.skip_topk:
|
||||||
if out is None:
|
if out is None:
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Inference-only Qwen4Exp model."""
|
"""Inference-only Qwen4Exp model."""
|
||||||
|
|
||||||
|
import os
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from itertools import islice
|
from itertools import islice
|
||||||
|
|
||||||
@@ -9,8 +10,10 @@ import torch
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
|
||||||
|
|
||||||
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed import get_pp_group
|
from vllm.distributed import get_pp_group
|
||||||
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe.utils import (
|
from vllm.model_executor.layers.fused_moe.utils import (
|
||||||
is_model_fused_shared_expert_compatible,
|
is_model_fused_shared_expert_compatible,
|
||||||
)
|
)
|
||||||
@@ -29,16 +32,20 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
|||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
|
PinnedHostVocabParallelEmbedding,
|
||||||
VocabParallelEmbedding,
|
VocabParallelEmbedding,
|
||||||
|
should_use_host_embed_table,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.interfaces import (
|
from vllm.model_executor.models.interfaces import (
|
||||||
HasInnerState,
|
HasInnerState,
|
||||||
IsHybrid,
|
IsHybrid,
|
||||||
|
MambaStateShapes,
|
||||||
MixtureOfExperts,
|
MixtureOfExperts,
|
||||||
MultiModalEmbeddings,
|
MultiModalEmbeddings,
|
||||||
SupportsLoRA,
|
SupportsLoRA,
|
||||||
SupportsMRoPE,
|
SupportsMRoPE,
|
||||||
SupportsPP,
|
SupportsPP,
|
||||||
|
SupportsReplaySSM,
|
||||||
_require_is_multimodal,
|
_require_is_multimodal,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.qwen3_5 import (
|
from vllm.model_executor.models.qwen3_5 import (
|
||||||
@@ -57,6 +64,7 @@ from vllm.model_executor.models.qwen3_vl import (
|
|||||||
)
|
)
|
||||||
from vllm.model_executor.models.utils import (
|
from vllm.model_executor.models.utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
|
PPMissingLayer,
|
||||||
StageMissingLayer,
|
StageMissingLayer,
|
||||||
WeightsMapper,
|
WeightsMapper,
|
||||||
_merge_multimodal_embeddings,
|
_merge_multimodal_embeddings,
|
||||||
@@ -71,13 +79,21 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
|
|||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||||
|
from vllm.v1.e2_guards import (
|
||||||
|
assert_compiled_submodule,
|
||||||
|
assert_graph_profile_is_executable,
|
||||||
|
assert_ple_mode,
|
||||||
|
)
|
||||||
from vllm.v1.kv_cache_interface import MambaSpec
|
from vllm.v1.kv_cache_interface import MambaSpec
|
||||||
|
|
||||||
|
from ..common.mtp import make_mtp_hidden_buffer
|
||||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||||
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
||||||
from .ple_layer import Qwen4ExpPLELayer
|
from .ple_layer import Qwen4ExpPLELayer
|
||||||
from .qsa import Qwen4ExpQSAAttention
|
from .qsa import Qwen4ExpQSAAttention, load_qsa_static_kv_scales
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention`
|
# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention`
|
||||||
# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version.
|
# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version.
|
||||||
@@ -158,6 +174,104 @@ _EXTRA_WEIGHTS_MAPPER = WeightsMapper(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_QSA_KV_SCALES_ENV = "VLLM_QSA_KV_SCALES"
|
||||||
|
|
||||||
|
|
||||||
|
def _maybe_load_qsa_static_kv_scales(model: nn.Module) -> None:
|
||||||
|
"""Apply a static QSA K/V scale sidecar when VLLM_QSA_KV_SCALES names one.
|
||||||
|
|
||||||
|
Placed where the loader's own docstring requires: after checkpoint weights
|
||||||
|
are loaded, before profiling, warmup or cudagraph capture. Unset env var ->
|
||||||
|
no-op (the attention layer logs the default 1.0 per layer so the
|
||||||
|
uncalibrated state stays visible). Set but unusable -> the exception
|
||||||
|
propagates: a calibration that was requested and silently not applied is
|
||||||
|
worse than one never requested.
|
||||||
|
"""
|
||||||
|
path = os.environ.get(_QSA_KV_SCALES_ENV, "").strip()
|
||||||
|
if not path:
|
||||||
|
return
|
||||||
|
# The PLE offload worker builds the whole target on the meta device and loads
|
||||||
|
# only PLE weights through load_weights; a fill_ on its meta scale buffers
|
||||||
|
# would log success without calibrating any GPU worker.
|
||||||
|
local_qsa: list[Qwen4ExpQSAAttention] = []
|
||||||
|
inner: Qwen4ExpModel | None = None
|
||||||
|
for module in model.modules():
|
||||||
|
if isinstance(module, Qwen4ExpModel) and inner is None:
|
||||||
|
inner = module
|
||||||
|
if isinstance(module, Qwen4ExpQSAAttention):
|
||||||
|
local_qsa.append(module)
|
||||||
|
for layer in local_qsa:
|
||||||
|
k_scale = getattr(layer, "_k_scale", None)
|
||||||
|
if isinstance(k_scale, torch.Tensor) and k_scale.device.type == "meta":
|
||||||
|
logger.info(
|
||||||
|
"Qwen4Exp QSA: model is on meta (PLE offload worker); "
|
||||||
|
"static K/V scales are applied by the GPU workers only"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if inner is None:
|
||||||
|
raise ValueError("Qwen4Exp QSA: no Qwen4ExpModel found for the sidecar")
|
||||||
|
if not inner._qsa_layer_ids:
|
||||||
|
# A requested calibration for a model without QSA layers is a
|
||||||
|
# configuration error, never a silent no-op.
|
||||||
|
raise ValueError(
|
||||||
|
f"Qwen4Exp QSA: {_QSA_KV_SCALES_ENV} is set but the model has no "
|
||||||
|
"QSA (indexer) attention layers"
|
||||||
|
)
|
||||||
|
# Whole-model QSA layer set (all pipeline stages) = "<stack>.<idx>.self_attn.attn"
|
||||||
|
# for every configured QSA index. The stack prefix is read from a LOCAL QSA
|
||||||
|
# layer's registered layer_name whenever one exists: this hook also runs
|
||||||
|
# inside the nested Qwen4ExpForCausalLM.load_weights, where a module path
|
||||||
|
# relative to that submodule ("model.layers") would not match the names
|
||||||
|
# registered under the outer prefix ("language_model.model.layers") - the
|
||||||
|
# earlier boot failure. Only a stage with no QSA layer
|
||||||
|
# falls back to the module path (then no local name can disagree).
|
||||||
|
if local_qsa:
|
||||||
|
stack, sep, tail = local_qsa[0].layer_name.rpartition(".layers.")
|
||||||
|
if not sep or not tail.split(".")[0].isdigit():
|
||||||
|
raise ValueError(
|
||||||
|
f"Qwen4Exp QSA: unexpected layer name {local_qsa[0].layer_name!r}"
|
||||||
|
)
|
||||||
|
layers_path = f"{stack}.layers"
|
||||||
|
else:
|
||||||
|
layers_path = next(
|
||||||
|
(name for name, module in model.named_modules() if module is inner.layers),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if layers_path is None:
|
||||||
|
raise ValueError("Qwen4Exp QSA: cannot locate the decoder layer stack")
|
||||||
|
expected = {
|
||||||
|
f"{layers_path}.{idx}.self_attn.attn" for idx in inner._qsa_layer_ids
|
||||||
|
}
|
||||||
|
if not local_qsa:
|
||||||
|
# A pipeline stage without a full-attention layer (only for a very
|
||||||
|
# uneven split): validate the complete sidecar here too, apply nothing.
|
||||||
|
logger.warning(
|
||||||
|
"Qwen4Exp QSA: this pipeline stage holds no QSA layer "
|
||||||
|
"(pipeline layers %d-%d); sidecar %s validated, nothing to apply",
|
||||||
|
inner.start_layer,
|
||||||
|
inner.end_layer,
|
||||||
|
path,
|
||||||
|
)
|
||||||
|
# strict: every QSA layer of the whole model must be in the sidecar. An empty
|
||||||
|
# or partial file silently leaving layers at scale 1.0 is exactly the failure
|
||||||
|
# this call site exists to prevent.
|
||||||
|
applied = load_qsa_static_kv_scales(
|
||||||
|
model, path, strict=True, expected_layer_names=expected
|
||||||
|
)
|
||||||
|
shown = ", ".join(sorted(applied)[:4])
|
||||||
|
logger.info(
|
||||||
|
"Qwen4Exp QSA: applied static K/V scales from %s to %d of %d layer(s) "
|
||||||
|
"(pipeline layers %d-%d): %s%s",
|
||||||
|
path,
|
||||||
|
len(applied),
|
||||||
|
len(expected),
|
||||||
|
inner.start_layer,
|
||||||
|
inner.end_layer,
|
||||||
|
shown,
|
||||||
|
" ..." if len(applied) > 4 else "",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
|
class Qwen4ExpSparseMoeBlock(Qwen3NextSparseMoeBlock):
|
||||||
"""Qwen3Next MoE with Qwen4Exp HC validation."""
|
"""Qwen3Next MoE with Qwen4Exp HC validation."""
|
||||||
|
|
||||||
@@ -363,6 +477,25 @@ class Qwen4ExpMixtureOfExperts(MixtureOfExperts):
|
|||||||
moe.experts.update_expert_map()
|
moe.experts.update_expert_map()
|
||||||
|
|
||||||
|
|
||||||
|
# Compile-profile guard: torch.compile restored on the NVIDIA
|
||||||
|
# path, reverting #55272's removal. Every request-dependent region is a custom
|
||||||
|
# op listed in CompilationConfig._attention_ops (qsa_with_output,
|
||||||
|
# ple_short_conv, compute_ple_ngram_ids, ple_{start,finalize}_prefetch), so
|
||||||
|
# VLLM_COMPILE + FULL_AND_PIECEWISE is executable again with breakable capture
|
||||||
|
# off. query_start_loc / ngram_context are fixed-capacity since #55272
|
||||||
|
# (model_state.py); they stay marked dynamic as on v2.0.1 and only ever reach
|
||||||
|
# the opaque PLE ops.
|
||||||
|
@support_torch_compile(
|
||||||
|
dynamic_arg_dims={
|
||||||
|
"input_ids": 0,
|
||||||
|
"positions": -1,
|
||||||
|
"intermediate_tensors": 0,
|
||||||
|
"inputs_embeds": 0,
|
||||||
|
"query_start_loc": 0,
|
||||||
|
"ngram_context": 0,
|
||||||
|
"deepstack_input_embeds": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
class Qwen4ExpModel(nn.Module):
|
class Qwen4ExpModel(nn.Module):
|
||||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||||
|
|
||||||
@@ -380,7 +513,31 @@ class Qwen4ExpModel(nn.Module):
|
|||||||
if layer_type in _QSA_LAYER_TYPES
|
if layer_type in _QSA_LAYER_TYPES
|
||||||
and getattr(config, "indexer_n_heads", None) is not None
|
and getattr(config, "indexer_n_heads", None) is not None
|
||||||
)
|
)
|
||||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
self.uses_host_embed_table = (
|
||||||
|
get_pp_group().is_first_rank
|
||||||
|
and should_use_host_embed_table()
|
||||||
|
)
|
||||||
|
if self.uses_host_embed_table and getattr(
|
||||||
|
config, "tie_word_embeddings", False
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"VLLM_HOST_EMBED_TABLE=1 is incompatible with "
|
||||||
|
"tie_word_embeddings=True"
|
||||||
|
)
|
||||||
|
# PP ownership: the input embedding runs on
|
||||||
|
# the first stage only (see forward); later stages carried an unused
|
||||||
|
# 248320 x 2560 bf16 table (0.59 GiB per GPU at TP2) beside the MTP
|
||||||
|
# drafter's own copy. PP=1 ownership is unchanged, and the PLE offload
|
||||||
|
# worker retains meta construction.
|
||||||
|
if get_pp_group().is_first_rank:
|
||||||
|
embedding_cls = (
|
||||||
|
PinnedHostVocabParallelEmbedding
|
||||||
|
if self.uses_host_embed_table
|
||||||
|
else VocabParallelEmbedding
|
||||||
|
)
|
||||||
|
self.embed_tokens = embedding_cls(self.vocab_size, config.hidden_size)
|
||||||
|
else:
|
||||||
|
self.embed_tokens = PPMissingLayer()
|
||||||
|
|
||||||
def get_layer(prefix: str) -> Qwen4ExpDecoderLayer:
|
def get_layer(prefix: str) -> Qwen4ExpDecoderLayer:
|
||||||
layer_idx = extract_layer_index(prefix)
|
layer_idx = extract_layer_index(prefix)
|
||||||
@@ -421,25 +578,11 @@ class Qwen4ExpModel(nn.Module):
|
|||||||
else:
|
else:
|
||||||
self.hyper_connection_mixer = None
|
self.hyper_connection_mixer = None
|
||||||
|
|
||||||
spec_config = vllm_config.speculative_config
|
self._mtp_hidden_buffer = make_mtp_hidden_buffer(
|
||||||
# MTP HC multi-stream outputs: when speculative method=="mtp" and the
|
vllm_config,
|
||||||
# model uses HC with hc_count>1, retain the pre-final-mixer multi-stream
|
config,
|
||||||
# hidden state [T, hc_count*H] so the MTP drafter can feed a real
|
is_last_rank=get_pp_group().is_last_rank,
|
||||||
# multi-stream backbone hidden on its first step (scheme A). Derived
|
|
||||||
# purely from config (NOT node identity) so P/D nodes stay consistent.
|
|
||||||
needs_mtp_hidden = (
|
|
||||||
spec_config is not None
|
|
||||||
and getattr(spec_config, "method", None) == "mtp"
|
|
||||||
and get_pp_group().is_last_rank
|
|
||||||
)
|
)
|
||||||
if needs_mtp_hidden:
|
|
||||||
self._mtp_hidden_buffer = torch.empty(
|
|
||||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
||||||
config.hc_count * config.hidden_size,
|
|
||||||
dtype=vllm_config.model_config.dtype,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self._mtp_hidden_buffer = None
|
|
||||||
|
|
||||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||||
return self.embed_tokens(input_ids)
|
return self.embed_tokens(input_ids)
|
||||||
@@ -620,6 +763,7 @@ class Qwen4ExpForCausalLM(
|
|||||||
SupportsPP,
|
SupportsPP,
|
||||||
Qwen4ExpMixtureOfExperts,
|
Qwen4ExpMixtureOfExperts,
|
||||||
IsHybrid,
|
IsHybrid,
|
||||||
|
SupportsReplaySSM,
|
||||||
):
|
):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj", "indexer.index_qk_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj", "indexer.index_qk_proj"],
|
||||||
@@ -640,6 +784,13 @@ class Qwen4ExpForCausalLM(
|
|||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
# Compile-profile guard: the only qualified profile on this tree is VLLM_COMPILE +
|
||||||
|
# FULL_AND_PIECEWISE with breakable capture off (the v2.0.1 identity).
|
||||||
|
# Fail at construction, before profiling or capture, on anything else.
|
||||||
|
# Qwen4ExpForConditionalGeneration constructs this class; Qwen4ExpMTP
|
||||||
|
# does not, and guards itself.
|
||||||
|
assert_graph_profile_is_executable(vllm_config, role="target")
|
||||||
|
assert_ple_mode(vllm_config, role="target")
|
||||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||||
if vllm_config.lora_config is not None:
|
if vllm_config.lora_config is not None:
|
||||||
# LoRA does not support the merged QKV/indexer projection, so its
|
# LoRA does not support the merged QKV/indexer projection, so its
|
||||||
@@ -655,11 +806,19 @@ class Qwen4ExpForCausalLM(
|
|||||||
self.model = Qwen4ExpModel(
|
self.model = Qwen4ExpModel(
|
||||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||||
)
|
)
|
||||||
self.lm_head = ParallelLMHead(
|
# has_compiled_submodule() at gpu/cudagraph_utils.py:605 is what capture
|
||||||
config.vocab_size,
|
# actually tests; check the same property here.
|
||||||
config.hidden_size,
|
assert_compiled_submodule(self.model, vllm_config, role="target")
|
||||||
prefix=maybe_prefix(prefix, "lm_head"),
|
# PP ownership: logits are computed on the
|
||||||
)
|
# last stage only; earlier stages carried an unused 0.59 GiB head.
|
||||||
|
if get_pp_group().is_last_rank:
|
||||||
|
self.lm_head = ParallelLMHead(
|
||||||
|
config.vocab_size,
|
||||||
|
config.hidden_size,
|
||||||
|
prefix=maybe_prefix(prefix, "lm_head"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.lm_head = PPMissingLayer()
|
||||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
self.model.make_empty_intermediate_tensors
|
self.model.make_empty_intermediate_tensors
|
||||||
@@ -736,20 +895,38 @@ class Qwen4ExpForCausalLM(
|
|||||||
num_spec=num_spec,
|
num_spec=num_spec,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _uses_gdn_recoverssm(cls, vllm_config: VllmConfig) -> bool:
|
||||||
|
"""One predicate for the replay record's shape AND dtype (they must
|
||||||
|
agree for the MambaSpec)."""
|
||||||
|
num_spec = (
|
||||||
|
vllm_config.speculative_config.num_speculative_tokens
|
||||||
|
if vllm_config.speculative_config
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.recoverssm_utils import uses_recoverssm
|
||||||
|
|
||||||
|
return uses_recoverssm(vllm_config.cache_config, num_spec or 0)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_gdn_mamba_state_dtype_from_config(
|
def get_gdn_mamba_state_dtype_from_config(
|
||||||
cls, vllm_config: VllmConfig
|
cls, vllm_config: VllmConfig
|
||||||
) -> tuple[torch.dtype, torch.dtype]:
|
) -> tuple[torch.dtype, ...]:
|
||||||
return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
|
dtypes: tuple[torch.dtype, ...] = (
|
||||||
vllm_config.model_config.dtype,
|
MambaStateDtypeCalculator.gated_delta_net_state_dtype(
|
||||||
vllm_config.cache_config.mamba_cache_dtype,
|
vllm_config.model_config.dtype,
|
||||||
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
vllm_config.cache_config.mamba_cache_dtype,
|
||||||
|
vllm_config.cache_config.mamba_ssm_cache_dtype,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
if cls._uses_gdn_recoverssm(vllm_config): # replay record, fp32
|
||||||
|
dtypes = (*dtypes, torch.float32)
|
||||||
|
return dtypes
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_gdn_mamba_state_shape_from_config(
|
def get_gdn_mamba_state_shape_from_config(
|
||||||
cls, vllm_config: VllmConfig
|
cls, vllm_config: VllmConfig
|
||||||
) -> tuple[tuple[int, int], tuple[int, int]]:
|
) -> MambaStateShapes:
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
hf_config = vllm_config.model_config.hf_text_config
|
hf_config = vllm_config.model_config.hf_text_config
|
||||||
tp_size = parallel_config.tensor_parallel_size
|
tp_size = parallel_config.tensor_parallel_size
|
||||||
@@ -758,7 +935,7 @@ class Qwen4ExpForCausalLM(
|
|||||||
if vllm_config.speculative_config
|
if vllm_config.speculative_config
|
||||||
else 0
|
else 0
|
||||||
)
|
)
|
||||||
return MambaStateShapeCalculator.gated_delta_net_state_shape(
|
shapes = MambaStateShapeCalculator.gated_delta_net_state_shape(
|
||||||
tp_size,
|
tp_size,
|
||||||
hf_config.linear_num_key_heads,
|
hf_config.linear_num_key_heads,
|
||||||
hf_config.linear_num_value_heads,
|
hf_config.linear_num_value_heads,
|
||||||
@@ -767,18 +944,30 @@ class Qwen4ExpForCausalLM(
|
|||||||
hf_config.linear_conv_kernel_dim,
|
hf_config.linear_conv_kernel_dim,
|
||||||
num_spec,
|
num_spec,
|
||||||
)
|
)
|
||||||
|
if cls._uses_gdn_recoverssm(
|
||||||
|
vllm_config
|
||||||
|
): # replay record [HV, spec_query_len, V + K + 1]
|
||||||
|
shapes = (
|
||||||
|
*shapes,
|
||||||
|
(
|
||||||
|
hf_config.linear_num_value_heads // tp_size,
|
||||||
|
num_spec + 1,
|
||||||
|
hf_config.linear_value_head_dim + hf_config.linear_key_head_dim + 1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return shapes
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_mamba_state_dtype_from_config(
|
def get_mamba_state_dtype_from_config(
|
||||||
cls,
|
cls,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
) -> tuple[torch.dtype, torch.dtype]:
|
) -> tuple[torch.dtype, ...]:
|
||||||
return cls.get_gdn_mamba_state_dtype_from_config(vllm_config)
|
return cls.get_gdn_mamba_state_dtype_from_config(vllm_config)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_mamba_state_shape_from_config(
|
def get_mamba_state_shape_from_config(
|
||||||
cls, vllm_config: VllmConfig
|
cls, vllm_config: VllmConfig
|
||||||
) -> tuple[tuple[int, int], tuple[int, int]]:
|
) -> MambaStateShapes:
|
||||||
return cls.get_gdn_mamba_state_shape_from_config(vllm_config)
|
return cls.get_gdn_mamba_state_shape_from_config(vllm_config)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -849,7 +1038,9 @@ class Qwen4ExpForCausalLM(
|
|||||||
self,
|
self,
|
||||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||||
)
|
)
|
||||||
return loader.load_weights(weights, mapper=mapper)
|
loaded = loader.load_weights(weights, mapper=mapper)
|
||||||
|
_maybe_load_qsa_static_kv_scales(self)
|
||||||
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
|
class Qwen4ExpProcessingInfo(Qwen3VLProcessingInfo):
|
||||||
@@ -866,6 +1057,7 @@ class Qwen4ExpForConditionalGeneration(
|
|||||||
Qwen3_5ForConditionalGeneration,
|
Qwen3_5ForConditionalGeneration,
|
||||||
HasInnerState,
|
HasInnerState,
|
||||||
Qwen4ExpMixtureOfExperts,
|
Qwen4ExpMixtureOfExperts,
|
||||||
|
SupportsReplaySSM,
|
||||||
):
|
):
|
||||||
"""Qwen3-VL vision tower backed by the Qwen4Exp language model."""
|
"""Qwen3-VL vision tower backed by the Qwen4Exp language model."""
|
||||||
|
|
||||||
@@ -907,6 +1099,14 @@ class Qwen4ExpForConditionalGeneration(
|
|||||||
self.model_config = vllm_config.model_config
|
self.model_config = vllm_config.model_config
|
||||||
self.multimodal_config = multimodal_config
|
self.multimodal_config = multimodal_config
|
||||||
self.language_model_only = multimodal_config.language_model_only
|
self.language_model_only = multimodal_config.language_model_only
|
||||||
|
# PP ownership: the encoder cache and the image
|
||||||
|
# embeddings exist on the first pipeline stage only; later stages
|
||||||
|
# carried an unused vision tower (~0.42 GiB per GPU at TP2). Absence
|
||||||
|
# on a later stage is an ownership decision, not language_model_only:
|
||||||
|
# the multimodal configuration stays global.
|
||||||
|
self._owns_vision_tower = (
|
||||||
|
not self.language_model_only and get_pp_group().is_first_rank
|
||||||
|
)
|
||||||
if self.language_model_only:
|
if self.language_model_only:
|
||||||
self.use_data_parallel = False
|
self.use_data_parallel = False
|
||||||
self.is_multimodal_pruning_enabled = False
|
self.is_multimodal_pruning_enabled = False
|
||||||
@@ -915,6 +1115,12 @@ class Qwen4ExpForConditionalGeneration(
|
|||||||
self._tokenizer = None
|
self._tokenizer = None
|
||||||
self.visual = StageMissingLayer("vision_tower")
|
self.visual = StageMissingLayer("vision_tower")
|
||||||
self._tower_model_names = []
|
self._tower_model_names = []
|
||||||
|
elif not self._owns_vision_tower:
|
||||||
|
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||||
|
self._init_video_pruning(multimodal_config)
|
||||||
|
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||||
|
self.visual = StageMissingLayer("vision_tower")
|
||||||
|
self._tower_model_names = []
|
||||||
else:
|
else:
|
||||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||||
self._init_video_pruning(multimodal_config)
|
self._init_video_pruning(multimodal_config)
|
||||||
@@ -961,7 +1167,16 @@ class Qwen4ExpForConditionalGeneration(
|
|||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
self.language_model.make_empty_intermediate_tensors
|
self.language_model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
if not get_pp_group().is_first_rank and self.use_deepstack:
|
# Deepstack levels are injected into the first len(indexes) decoder
|
||||||
|
# layers on the first stage and are not forwarded through PP; keep the
|
||||||
|
# boundary assertion independent of use_deepstack (False on stages
|
||||||
|
# without a tower) so a future checkpoint with deepstack cannot be
|
||||||
|
# split across the boundary silently.
|
||||||
|
if (
|
||||||
|
not get_pp_group().is_first_rank
|
||||||
|
and not self.language_model_only
|
||||||
|
and bool(config.vision_config.deepstack_visual_indexes)
|
||||||
|
):
|
||||||
assert self.language_model.model.start_layer >= len(
|
assert self.language_model.model.start_layer >= len(
|
||||||
config.vision_config.deepstack_visual_indexes
|
config.vision_config.deepstack_visual_indexes
|
||||||
), (
|
), (
|
||||||
@@ -1052,20 +1267,23 @@ class Qwen4ExpForConditionalGeneration(
|
|||||||
self,
|
self,
|
||||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||||
)
|
)
|
||||||
return loader.load_weights(weights, mapper=mapper)
|
loaded = loader.load_weights(weights, mapper=mapper)
|
||||||
|
_maybe_load_qsa_static_kv_scales(self)
|
||||||
|
return loaded
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_mamba_state_dtype_from_config(
|
def get_mamba_state_dtype_from_config(
|
||||||
cls,
|
cls,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
) -> tuple[torch.dtype, torch.dtype]:
|
) -> tuple[torch.dtype, ...]:
|
||||||
return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config)
|
return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_mamba_state_shape_from_config(
|
def get_mamba_state_shape_from_config( # type: ignore[override]
|
||||||
cls,
|
cls,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
) -> tuple[tuple[int, int], tuple[int, int]]:
|
) -> MambaStateShapes:
|
||||||
|
# GDN RecoverSSM adds a third state (the replay record) to Qwen3.5's two.
|
||||||
return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config)
|
return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -13,6 +13,12 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
|||||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||||
from vllm.v1.worker.gpu.states import RequestState
|
from vllm.v1.worker.gpu.states import RequestState
|
||||||
|
|
||||||
|
from .ngram_embedding import (
|
||||||
|
Qwen4ExpNGramEmbedding,
|
||||||
|
Qwen4ExpPLEFileGatherEmbedding,
|
||||||
|
stage_checkpoint_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Qwen4ExpModelState(MambaHybridModelState):
|
class Qwen4ExpModelState(MambaHybridModelState):
|
||||||
"""Add rollback-safe PLE n-gram context to the model inputs."""
|
"""Add rollback-safe PLE n-gram context to the model inputs."""
|
||||||
@@ -32,13 +38,7 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
|||||||
self.ngram_eos_token_id = 0
|
self.ngram_eos_token_id = 0
|
||||||
return
|
return
|
||||||
|
|
||||||
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
# vllm#54709: PP gate moved to config.py (placement-based).
|
||||||
raise RuntimeError(
|
|
||||||
"N-gram PLE embedding currently requires "
|
|
||||||
"pipeline_parallel_size=1 because non-first pipeline ranks do "
|
|
||||||
"not receive the raw input_ids required by PLE. Please run "
|
|
||||||
"with PP=1."
|
|
||||||
)
|
|
||||||
|
|
||||||
self.ngram_context_len = int(config.ngram_size) - 1
|
self.ngram_context_len = int(config.ngram_size) - 1
|
||||||
if self.ngram_context_len <= 0:
|
if self.ngram_context_len <= 0:
|
||||||
@@ -63,6 +63,12 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
|||||||
dtype=torch.int32,
|
dtype=torch.int32,
|
||||||
device=self.device,
|
device=self.device,
|
||||||
)
|
)
|
||||||
|
self.file_gather_modules = [
|
||||||
|
module
|
||||||
|
for module in model.modules()
|
||||||
|
if isinstance(module, Qwen4ExpNGramEmbedding)
|
||||||
|
and isinstance(module.ngram_embedding, Qwen4ExpPLEFileGatherEmbedding)
|
||||||
|
]
|
||||||
|
|
||||||
def _prepare_ngram_context(
|
def _prepare_ngram_context(
|
||||||
self,
|
self,
|
||||||
@@ -106,9 +112,17 @@ class Qwen4ExpModelState(MambaHybridModelState):
|
|||||||
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
|
query_start_loc[: num_reqs_padded + 1].copy_(input_batch.query_start_loc)
|
||||||
# Represent unused capacity as trailing zero-length requests.
|
# Represent unused capacity as trailing zero-length requests.
|
||||||
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
|
query_start_loc[num_reqs_padded + 1 :].copy_(input_batch.query_start_loc[-1])
|
||||||
|
ngram_context = self._prepare_ngram_context(input_batch, req_states)
|
||||||
|
if self.file_gather_modules:
|
||||||
|
stage_checkpoint_rows(
|
||||||
|
self.file_gather_modules,
|
||||||
|
input_batch.input_ids,
|
||||||
|
query_start_loc,
|
||||||
|
ngram_context,
|
||||||
|
)
|
||||||
model_inputs.update(
|
model_inputs.update(
|
||||||
query_start_loc=query_start_loc,
|
query_start_loc=query_start_loc,
|
||||||
ngram_context=self._prepare_ngram_context(input_batch, req_states),
|
ngram_context=ngram_context,
|
||||||
)
|
)
|
||||||
return model_inputs
|
return model_inputs
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import torch
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import Qwen4ExpTextConfig
|
from transformers import Qwen4ExpTextConfig
|
||||||
|
|
||||||
|
from vllm.compilation.decorators import support_torch_compile
|
||||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||||
from vllm.distributed import get_pp_group
|
from vllm.distributed import get_pp_group
|
||||||
from vllm.model_executor.layers.fused_moe.utils import (
|
from vllm.model_executor.layers.fused_moe.utils import (
|
||||||
@@ -30,10 +31,16 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear
|
|||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
|
PinnedHostVocabParallelEmbedding,
|
||||||
VocabParallelEmbedding,
|
VocabParallelEmbedding,
|
||||||
|
should_use_host_embed_table,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||||
from vllm.model_executor.models.interfaces import SupportsPP
|
from vllm.model_executor.models.interfaces import (
|
||||||
|
LocalArgmaxMixin,
|
||||||
|
SupportsPP,
|
||||||
|
SupportsReplaySSM,
|
||||||
|
)
|
||||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||||
from vllm.model_executor.models.utils import (
|
from vllm.model_executor.models.utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
@@ -45,6 +52,11 @@ from vllm.model_executor.models.utils import (
|
|||||||
maybe_prefix,
|
maybe_prefix,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
|
from vllm.v1.e2_guards import (
|
||||||
|
assert_compiled_submodule,
|
||||||
|
assert_graph_profile_is_executable,
|
||||||
|
assert_ple_mode,
|
||||||
|
)
|
||||||
|
|
||||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||||
@@ -161,6 +173,18 @@ def _make_draft_vllm_config(
|
|||||||
return draft_vllm_config
|
return draft_vllm_config
|
||||||
|
|
||||||
|
|
||||||
|
# Compile restored on the drafter too (reverts #55272 here), as
|
||||||
|
# on v2.0.1: both classes wrapped; the outer one compiles, the inner is
|
||||||
|
# inlined while Dynamo traces.
|
||||||
|
@support_torch_compile(
|
||||||
|
dynamic_arg_dims={
|
||||||
|
"input_ids": 0,
|
||||||
|
"positions": -1,
|
||||||
|
"intermediate_tensors": 0,
|
||||||
|
"inputs_embeds": 0,
|
||||||
|
"hidden_states": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||||
|
|
||||||
@@ -179,7 +203,20 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
|||||||
self.hidden_size = config.hidden_size
|
self.hidden_size = config.hidden_size
|
||||||
self.hc_count = config.hc_count
|
self.hc_count = config.hc_count
|
||||||
|
|
||||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
|
self.uses_host_embed_table = should_use_host_embed_table()
|
||||||
|
if self.uses_host_embed_table and getattr(
|
||||||
|
config, "tie_word_embeddings", False
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"VLLM_HOST_EMBED_TABLE=1 is incompatible with "
|
||||||
|
"tie_word_embeddings=True"
|
||||||
|
)
|
||||||
|
embedding_cls = (
|
||||||
|
PinnedHostVocabParallelEmbedding
|
||||||
|
if self.uses_host_embed_table
|
||||||
|
else VocabParallelEmbedding
|
||||||
|
)
|
||||||
|
self.embed_tokens = embedding_cls(self.vocab_size, self.hidden_size)
|
||||||
draft_vllm_config = _make_draft_vllm_config(
|
draft_vllm_config = _make_draft_vllm_config(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
self.mtp_start_layer_idx,
|
self.mtp_start_layer_idx,
|
||||||
@@ -284,7 +321,11 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
|||||||
hidden_size = self.hidden_size
|
hidden_size = self.hidden_size
|
||||||
prev_block_output: torch.Tensor | None = None
|
prev_block_output: torch.Tensor | None = None
|
||||||
|
|
||||||
if get_pp_group().is_first_rank:
|
# Branch on the inputs, not the rank (vllm PR #46994's rule, applied
|
||||||
|
# there to Qwen3_5MTP / DeepSeek MTP): the drafter is built entirely on
|
||||||
|
# the LAST pipeline stage, where is_first_rank is False, and it always
|
||||||
|
# starts from the target's hidden state + token ids it embeds itself.
|
||||||
|
if intermediate_tensors is None:
|
||||||
assert hidden_states is not None
|
assert hidden_states is not None
|
||||||
if inputs_embeds is None:
|
if inputs_embeds is None:
|
||||||
assert input_ids is not None
|
assert input_ids is not None
|
||||||
@@ -306,7 +347,6 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
|||||||
hidden_states = hidden_states.flatten(-2)
|
hidden_states = hidden_states.flatten(-2)
|
||||||
prev_block_output = inputs_embeds
|
prev_block_output = inputs_embeds
|
||||||
else:
|
else:
|
||||||
assert intermediate_tensors is not None
|
|
||||||
hidden_states = intermediate_tensors["hidden_states"]
|
hidden_states = intermediate_tensors["hidden_states"]
|
||||||
|
|
||||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||||
@@ -357,7 +397,9 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
|||||||
return loader.load_weights(weights, mapper=mapper)
|
return loader.load_weights(weights, mapper=mapper)
|
||||||
|
|
||||||
|
|
||||||
class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
class Qwen4ExpMTP(
|
||||||
|
LocalArgmaxMixin, nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts, SupportsReplaySSM
|
||||||
|
):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
@@ -371,6 +413,11 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||||
|
# The drafter is NOT reached through Qwen4ExpForCausalLM --
|
||||||
|
# this class builds Qwen4ExpMultiTokenPredictor directly, from a config
|
||||||
|
# rebuilt at mtp.py:159 -- so its graph profile is checked here.
|
||||||
|
assert_graph_profile_is_executable(vllm_config, role="draft")
|
||||||
|
assert_ple_mode(vllm_config, role="draft")
|
||||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||||
self.vllm_config = vllm_config
|
self.vllm_config = vllm_config
|
||||||
self.quant_config = vllm_config.quant_config
|
self.quant_config = vllm_config.quant_config
|
||||||
@@ -381,6 +428,7 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
|||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
prefix=maybe_prefix(prefix, "mtp"),
|
prefix=maybe_prefix(prefix, "mtp"),
|
||||||
)
|
)
|
||||||
|
assert_compiled_submodule(self.model, vllm_config, role="draft")
|
||||||
|
|
||||||
if get_pp_group().is_last_rank:
|
if get_pp_group().is_last_rank:
|
||||||
if config.tie_word_embeddings:
|
if config.tie_word_embeddings:
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
"""Qwen4Exp n-gram embeddings with device and pinned-host storage."""
|
"""Qwen4Exp n-gram embeddings with device, pinned-host and host-file-gather storage."""
|
||||||
|
|
||||||
|
import mmap
|
||||||
|
import os
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import Qwen4ExpTextConfig
|
from transformers import Qwen4ExpTextConfig
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.config import get_current_vllm_config
|
from vllm.config import get_current_vllm_config
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.quantization.base_config import (
|
from vllm.model_executor.layers.quantization.base_config import (
|
||||||
QuantizationConfig,
|
QuantizationConfig,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||||
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
|
|
||||||
from ..common.ngram_embedding import (
|
from ..common.ngram_embedding import (
|
||||||
Qwen4ExpPLEDeviceEmbedding,
|
Qwen4ExpPLEDeviceEmbedding,
|
||||||
@@ -38,6 +42,164 @@ __all__ = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _maps_entry(ptr: int) -> tuple[str, int]:
|
||||||
|
"""Return the path and file offset of ``ptr`` from ``/proc/self/maps``."""
|
||||||
|
with open("/proc/self/maps") as maps:
|
||||||
|
for line in maps:
|
||||||
|
fields = line.split(maxsplit=5)
|
||||||
|
low, high = (int(bound, 16) for bound in fields[0].split("-"))
|
||||||
|
if low <= ptr < high:
|
||||||
|
path = fields[5].rstrip("\n") if len(fields) > 5 else ""
|
||||||
|
return path, ptr - low + int(fields[2], 16)
|
||||||
|
return "", 0
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen4ExpPLEFileGatherEmbedding(Qwen4ExpPLEEmbedding):
|
||||||
|
"""PLE rows gathered on the host from the loader's file-backed shard views."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
num_embeddings: int,
|
||||||
|
embedding_dim: int,
|
||||||
|
*,
|
||||||
|
params_dtype: torch.dtype,
|
||||||
|
padding_size: int,
|
||||||
|
prefix: str,
|
||||||
|
embedding_method: Qwen4ExpPLEEmbeddingMethod,
|
||||||
|
num_ngram_heads: int = 1,
|
||||||
|
max_total_tokens: int = 0,
|
||||||
|
data_parallel_rank: int = 0,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
num_embeddings,
|
||||||
|
embedding_dim,
|
||||||
|
params_dtype=params_dtype,
|
||||||
|
padding_size=padding_size,
|
||||||
|
prefix=prefix,
|
||||||
|
embedding_method=embedding_method,
|
||||||
|
data_parallel_rank=data_parallel_rank,
|
||||||
|
)
|
||||||
|
self._dummy = get_current_vllm_config().load_config.load_format == "dummy"
|
||||||
|
shape = (max_total_tokens, num_ngram_heads, embedding_dim)
|
||||||
|
dtype, pin = self.weight.dtype, self.weight.device.type != "cpu"
|
||||||
|
self._staging = torch.zeros(shape, dtype=dtype, device=self.weight.device)
|
||||||
|
self._host_ids = torch.empty(
|
||||||
|
shape[:2], dtype=torch.int64, device="cpu", pin_memory=pin
|
||||||
|
)
|
||||||
|
# Step N+1 writes these rows only after its stream sync, so after step N's H2D.
|
||||||
|
self._host_rows = torch.empty(shape, dtype=dtype, device="cpu", pin_memory=pin)
|
||||||
|
self._shards: dict[int, torch.Tensor] | None = {}
|
||||||
|
self._shard_size = 0
|
||||||
|
self._bound = False
|
||||||
|
self._fds: dict[str, int] = {}
|
||||||
|
self._fds_t = self._bases = torch.empty(0, dtype=torch.int64)
|
||||||
|
|
||||||
|
def allocate_embedding_weight(
|
||||||
|
self,
|
||||||
|
num_embeddings: int,
|
||||||
|
embedding_dim: int,
|
||||||
|
dtype: torch.dtype,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Allocate no rows; they stay in the checkpoint files."""
|
||||||
|
return torch.empty(0, embedding_dim, dtype=dtype)
|
||||||
|
|
||||||
|
def accept_checkpoint_shard(
|
||||||
|
self,
|
||||||
|
shard_index: int,
|
||||||
|
loaded_weight: torch.Tensor,
|
||||||
|
shard_size: int,
|
||||||
|
) -> None:
|
||||||
|
"""Keep a checkpoint shard as a zero-copy byte view of its storage."""
|
||||||
|
if self._bound or self._shards is None:
|
||||||
|
self._shards = None
|
||||||
|
raise RuntimeError("PLE host file gather does not support weight reload")
|
||||||
|
if loaded_weight.dtype != self.weight.dtype:
|
||||||
|
raise ValueError(
|
||||||
|
f"PLE shard dtype {loaded_weight.dtype} must match the embedding "
|
||||||
|
f"dtype {self.weight.dtype}; host file gather serves rows as stored"
|
||||||
|
)
|
||||||
|
if shard_index in self._shards:
|
||||||
|
raise ValueError(f"Duplicate PLE embedding shard {shard_index}")
|
||||||
|
self._shards[shard_index] = loaded_weight.view(torch.uint8)
|
||||||
|
self._shard_size = shard_size
|
||||||
|
|
||||||
|
def bind_file_shards(self) -> None:
|
||||||
|
"""Check that file-backed shards cover every row; dummy loads serve zeros."""
|
||||||
|
assert self._shards is not None
|
||||||
|
if not self._dummy:
|
||||||
|
rows = sum(shard.shape[0] for shard in self._shards.values())
|
||||||
|
if rows != self.org_vocab_size:
|
||||||
|
raise ValueError(
|
||||||
|
f"PLE shards cover {rows} of {self.org_vocab_size} rows"
|
||||||
|
)
|
||||||
|
self._fds_t, self._bases = torch.zeros(2, len(self._shards)).long()
|
||||||
|
for index, shard in self._shards.items():
|
||||||
|
path, offset = _maps_entry(shard.data_ptr())
|
||||||
|
if not path or not os.path.isfile(path):
|
||||||
|
raise ValueError(
|
||||||
|
f"PLE embedding shard {index} is not a file-backed view "
|
||||||
|
f"({path or 'anonymous'})"
|
||||||
|
)
|
||||||
|
if path not in self._fds:
|
||||||
|
self._fds[path] = os.open(path, os.O_RDONLY)
|
||||||
|
self._fds_t[index], self._bases[index] = self._fds[path], offset
|
||||||
|
self._bound = True
|
||||||
|
|
||||||
|
def stage_rows(self, num_tokens: int) -> None:
|
||||||
|
"""Gather the rows for ``_host_ids[:num_tokens]`` and copy them to staging."""
|
||||||
|
shards = self._shards
|
||||||
|
if shards is None or not self._bound:
|
||||||
|
raise RuntimeError("PLE host file gather is unbound or was reloaded")
|
||||||
|
ids = self._host_ids[:num_tokens].flatten()
|
||||||
|
out = self._host_rows[:num_tokens].flatten(0, 1).view(torch.uint8)
|
||||||
|
if not shards:
|
||||||
|
out.zero_()
|
||||||
|
else:
|
||||||
|
ids, order = ids.sort()
|
||||||
|
shard = ids // self._shard_size
|
||||||
|
local = ids - shard * self._shard_size
|
||||||
|
counts = torch.bincount(shard, minlength=len(shards)).tolist()
|
||||||
|
page, row_bytes = mmap.PAGESIZE, out.shape[1]
|
||||||
|
start = self._bases[shard] + local * row_bytes
|
||||||
|
first, last = start // page, (start + row_bytes - 1) // page
|
||||||
|
run = torch.ones_like(first, dtype=torch.bool)
|
||||||
|
run[1:] = (first[1:] > last[:-1] + 1) | (shard[1:] != shard[:-1])
|
||||||
|
fds, lows = self._fds_t[shard[run]].tolist(), first[run].tolist()
|
||||||
|
# Queue every page read before the gather's first serial fault.
|
||||||
|
for fd, lo, hi in zip(fds, lows, last[run.roll(-1)].tolist()):
|
||||||
|
os.posix_fadvise(
|
||||||
|
fd, lo * page, (hi - lo + 1) * page, os.POSIX_FADV_WILLNEED
|
||||||
|
)
|
||||||
|
groups = zip(order.split(counts), local.split(counts))
|
||||||
|
for index, (rows, rows_local) in enumerate(groups):
|
||||||
|
if rows.numel():
|
||||||
|
out.index_copy_(0, rows, shards[index].index_select(0, rows_local))
|
||||||
|
self._staging[:num_tokens].copy_(
|
||||||
|
self._host_rows[:num_tokens], non_blocking=True
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def prefetch_buffer(self) -> torch.Tensor:
|
||||||
|
"""Side-stream lookup destination (declared mutated by the start op)."""
|
||||||
|
return self._prefetch_buffer
|
||||||
|
|
||||||
|
def new_prefetch_output(self, num_tokens: int) -> torch.Tensor:
|
||||||
|
"""Graph-owned output storage for the finalize op."""
|
||||||
|
return self._prefetch_buffer.new_empty((num_tokens, self._output_dim))
|
||||||
|
|
||||||
|
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""Return the rows staged for this step."""
|
||||||
|
return self._staging[: hidden_states.shape[0]].flatten(-2)
|
||||||
|
|
||||||
|
def start_prefetch(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
ngram_ids: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
"""Rows are staged in ``prepare_inputs``, so prefetch is a no-op."""
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class Qwen4ExpNGramEmbedding(nn.Module):
|
class Qwen4ExpNGramEmbedding(nn.Module):
|
||||||
_MASK64 = (1 << 64) - 1
|
_MASK64 = (1 << 64) - 1
|
||||||
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
||||||
@@ -144,10 +306,16 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
*,
|
*,
|
||||||
data_parallel_rank: int,
|
data_parallel_rank: int,
|
||||||
prefix: str,
|
prefix: str,
|
||||||
|
layer_name: str | None = None,
|
||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
params_dtype: torch.dtype | None = None,
|
params_dtype: torch.dtype | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
# Compile-profile guard: the owning Qwen4ExpPLELayer's static_forward_context key.
|
||||||
|
# When set, request-dependent work runs through the custom ops at the
|
||||||
|
# bottom of this file so VLLM_COMPILE never traces it (#55272 removed
|
||||||
|
# them together with @support_torch_compile). None = direct calls.
|
||||||
|
self.layer_name = layer_name
|
||||||
self.embedding_dim = embedding_dim
|
self.embedding_dim = embedding_dim
|
||||||
self.ngram_size = int(config.ngram_size)
|
self.ngram_size = int(config.ngram_size)
|
||||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||||
@@ -206,9 +374,18 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
if params_dtype is None:
|
if params_dtype is None:
|
||||||
params_dtype = torch.get_default_dtype()
|
params_dtype = torch.get_default_dtype()
|
||||||
engram_config = get_current_vllm_config().engram_config
|
engram_config = get_current_vllm_config().engram_config
|
||||||
|
use_uva = engram_config is not None and engram_config.cpu_offload
|
||||||
|
if use_uva and envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
# Fork: no silent fallback between transports.
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE backend selection: cpu_offload=True (upstream UVA) with "
|
||||||
|
"VLLM_E2_PLE_PULL_TRANSPORT=1 -- refusing a double offload"
|
||||||
|
)
|
||||||
embedding_cls = (
|
embedding_cls = (
|
||||||
Qwen4ExpPLEPinnedHostEmbedding
|
Qwen4ExpPLEFileGatherEmbedding
|
||||||
if engram_config is not None and engram_config.cpu_offload
|
if engram_config is not None and engram_config.host_file_gather
|
||||||
|
else Qwen4ExpPLEPinnedHostEmbedding
|
||||||
|
if use_uva
|
||||||
else Qwen4ExpPLEDeviceEmbedding
|
else Qwen4ExpPLEDeviceEmbedding
|
||||||
)
|
)
|
||||||
self.ngram_embedding = embedding_cls(
|
self.ngram_embedding = embedding_cls(
|
||||||
@@ -229,14 +406,21 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
max_total_tokens, self.ngram_heads, dtype=torch.long
|
max_total_tokens, self.ngram_heads, dtype=torch.long
|
||||||
)
|
)
|
||||||
weight = self.ngram_embedding.weight
|
weight = self.ngram_embedding.weight
|
||||||
|
# Fork: backend= names the class (or, inside the pull transport's
|
||||||
|
# offload process, the table it serves), so a smoke need not derive
|
||||||
|
# the backend from pinned=. Fields before it are upstream's, unchanged.
|
||||||
|
backend = type(self.ngram_embedding).__name__
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
backend = f"E2PullTransport.offload_table:{backend}"
|
||||||
logger.info(
|
logger.info(
|
||||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
|
||||||
embedding_prefix,
|
embedding_prefix,
|
||||||
type(embedding_quant_method).__name__,
|
type(embedding_quant_method).__name__,
|
||||||
weight.dtype,
|
weight.dtype,
|
||||||
weight.device,
|
weight.device,
|
||||||
weight.is_pinned(),
|
weight.is_pinned(),
|
||||||
|
backend,
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -249,6 +433,13 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
|
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
|
||||||
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
|
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
|
||||||
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
|
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
|
||||||
|
# Fork: never build the UVA table beside the E2 pull transport --
|
||||||
|
# that would be a second offload of the same PLE table.
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
|
||||||
|
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
|
||||||
|
)
|
||||||
previous_eos = torch.cat(
|
previous_eos = torch.cat(
|
||||||
[
|
[
|
||||||
eos_positions.new_full((batch_size, 1), -1),
|
eos_positions.new_full((batch_size, 1), -1),
|
||||||
@@ -356,10 +547,43 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
ngram_context: torch.Tensor,
|
ngram_context: torch.Tensor,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
embedding = self.ngram_embedding
|
embedding = self.ngram_embedding
|
||||||
if embedding.supports_prefetch:
|
if isinstance(embedding, Qwen4ExpPLEFileGatherEmbedding):
|
||||||
|
# Fork: this base's third backend stages rows in prepare_inputs,
|
||||||
|
# so it has no graph-owned lookup op.
|
||||||
return embedding(hidden_states)
|
return embedding(hidden_states)
|
||||||
ngram_ids = self.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
if self.layer_name is None:
|
||||||
return self.ngram_embedding(ngram_ids).flatten(-2)
|
if embedding.supports_prefetch:
|
||||||
|
return embedding(hidden_states)
|
||||||
|
ngram_ids = self.compute_ngram_ids(
|
||||||
|
input_ids, query_start_loc, ngram_context
|
||||||
|
)
|
||||||
|
return self.ngram_embedding(ngram_ids).flatten(-2)
|
||||||
|
|
||||||
|
if embedding.supports_prefetch:
|
||||||
|
output = embedding.new_prefetch_output(hidden_states.shape[0])
|
||||||
|
torch.ops.vllm.qwen4_exp_ple_finalize_prefetch(output, self.layer_name)
|
||||||
|
return output
|
||||||
|
input_ids = input_ids.reshape(-1)
|
||||||
|
# Request-layout-dependent ID generation stays outside PIECEWISE graphs
|
||||||
|
# (a splitting op), as on v2.0.1; the op writes graph-owned storage.
|
||||||
|
ngram_ids = input_ids.new_empty(
|
||||||
|
(input_ids.numel(), self.ngram_heads), dtype=torch.long
|
||||||
|
)
|
||||||
|
torch.ops.vllm.qwen4_exp_compute_ple_ngram_ids(
|
||||||
|
input_ids,
|
||||||
|
query_start_loc,
|
||||||
|
ngram_context,
|
||||||
|
ngram_ids,
|
||||||
|
self.layer_name,
|
||||||
|
)
|
||||||
|
# The table lookup is opaque so Inductor never sees (or autotunes
|
||||||
|
# against) the full PLE table -- #55272's ~50 GB activation peak.
|
||||||
|
output = ngram_ids.new_empty(
|
||||||
|
(ngram_ids.shape[0], self.ngram_heads * self.head_dim),
|
||||||
|
dtype=embedding.weight.dtype,
|
||||||
|
)
|
||||||
|
torch.ops.vllm.qwen4_exp_ple_device_lookup(ngram_ids, output, self.layer_name)
|
||||||
|
return output
|
||||||
|
|
||||||
def start_prefetch(
|
def start_prefetch(
|
||||||
self,
|
self,
|
||||||
@@ -372,6 +596,32 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
embedding = self.ngram_embedding
|
embedding = self.ngram_embedding
|
||||||
if not embedding.supports_prefetch:
|
if not embedding.supports_prefetch:
|
||||||
return
|
return
|
||||||
|
if self.layer_name is not None:
|
||||||
|
# Forks onto the prefetch side stream; the join is in the finalize
|
||||||
|
# op. Both are splitting ops, so neither end of the fork is ever
|
||||||
|
# inside a piecewise capture.
|
||||||
|
torch.ops.vllm.qwen4_exp_ple_start_prefetch(
|
||||||
|
hidden_states,
|
||||||
|
input_ids,
|
||||||
|
query_start_loc,
|
||||||
|
ngram_context,
|
||||||
|
embedding.prefetch_buffer,
|
||||||
|
self.layer_name,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
self._start_prefetch_eager(
|
||||||
|
hidden_states, input_ids, query_start_loc, ngram_context
|
||||||
|
)
|
||||||
|
|
||||||
|
def _start_prefetch_eager(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
"""Compute IDs and launch the pinned lookup (runs outside compile)."""
|
||||||
|
embedding = self.ngram_embedding
|
||||||
ngram_ids = self.compute_ngram_ids(
|
ngram_ids = self.compute_ngram_ids(
|
||||||
input_ids,
|
input_ids,
|
||||||
query_start_loc,
|
query_start_loc,
|
||||||
@@ -432,6 +682,12 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
f"expected {expected_shape}, got "
|
f"expected {expected_shape}, got "
|
||||||
f"{tuple(loaded_weight.shape)}"
|
f"{tuple(loaded_weight.shape)}"
|
||||||
)
|
)
|
||||||
|
if isinstance(embedding, Qwen4ExpPLEFileGatherEmbedding):
|
||||||
|
embedding.accept_checkpoint_shard(
|
||||||
|
shard_index, loaded_weight, shard_size
|
||||||
|
)
|
||||||
|
loaded.add("ngram_embedding.weight")
|
||||||
|
continue
|
||||||
embedding.weight.weight_loader(
|
embedding.weight.weight_loader(
|
||||||
embedding.weight,
|
embedding.weight,
|
||||||
loaded_weight,
|
loaded_weight,
|
||||||
@@ -446,6 +702,155 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
|||||||
return loaded
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
|
def stage_checkpoint_rows(
|
||||||
|
modules: list[Qwen4ExpNGramEmbedding],
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
"""Stage host-file-gather PLE rows for one step behind a single stream sync."""
|
||||||
|
num_tokens = input_ids.shape[0]
|
||||||
|
for module in modules:
|
||||||
|
ids = module.compute_ngram_ids(input_ids, query_start_loc, ngram_context)
|
||||||
|
module.ngram_embedding._host_ids[:num_tokens].copy_(ids, non_blocking=True)
|
||||||
|
if input_ids.device.type != "cpu":
|
||||||
|
torch.accelerator.current_stream().synchronize()
|
||||||
|
for module in modules:
|
||||||
|
module.ngram_embedding.stage_rows(num_tokens)
|
||||||
|
|
||||||
|
|
||||||
|
# --- compile support ------------------------------------------------
|
||||||
|
# The four ops below are the NVIDIA PLE's boundary with torch.compile. Each
|
||||||
|
# resolves its Qwen4ExpPLELayer through the forward context by name, as
|
||||||
|
# v2.0.1's qwen4_exp_compute_ple_ngram_ids did before #55272 removed it.
|
||||||
|
# Splitting ops (vllm/config/compilation.py ``_attention_ops``):
|
||||||
|
# qwen4_exp_compute_ple_ngram_ids, qwen4_exp_ple_start_prefetch,
|
||||||
|
# qwen4_exp_ple_finalize_prefetch.
|
||||||
|
# qwen4_exp_ple_device_lookup is opaque but NOT a split: a gather plus the ETP
|
||||||
|
# all-reduce is capturable; it only has to be hidden from Inductor.
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_embedding_of(layer_name: str) -> Qwen4ExpNGramEmbedding:
|
||||||
|
layer = get_forward_context().no_compile_layers[layer_name]
|
||||||
|
embedding = getattr(layer, "ple_embedding", None)
|
||||||
|
if not isinstance(embedding, Qwen4ExpNGramEmbedding):
|
||||||
|
raise TypeError(f"{layer_name} does not own a Qwen4Exp PLE embedding")
|
||||||
|
if hasattr(type(embedding), "dequantize_pulled"):
|
||||||
|
# Fork (pull transport + compile profile): the pull transport's GPU placeholder owns no
|
||||||
|
# table and dispatches through PleOffloadLayer.forward; it must never
|
||||||
|
# reach these device/UVA ops (ple_layer.py builds it without a
|
||||||
|
# layer_name). Same marker ple_layer.py uses to select dequantisation.
|
||||||
|
raise TypeError(
|
||||||
|
f"{layer_name}: PLE pull transport module routed through an "
|
||||||
|
"upstream device/UVA PLE op"
|
||||||
|
)
|
||||||
|
return embedding
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_compute_ple_ngram_ids(
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Compute request-dependent PLE n-gram IDs outside piecewise graphs."""
|
||||||
|
_ple_embedding_of(layer_name).compute_ngram_ids(
|
||||||
|
input_ids, query_start_loc, ngram_context, output
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_ple_device_lookup(
|
||||||
|
ngram_ids: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Look up device-resident PLE rows; keeps the table out of Inductor."""
|
||||||
|
embedding = _ple_embedding_of(layer_name)
|
||||||
|
output.copy_(embedding.ngram_embedding(ngram_ids).flatten(-2))
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_ple_start_prefetch(
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
prefetch_buffer: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""Start the pinned-host (UVA) lookup on the side stream."""
|
||||||
|
embedding = _ple_embedding_of(layer_name)
|
||||||
|
owned = embedding.ngram_embedding.prefetch_buffer
|
||||||
|
if prefetch_buffer.data_ptr() != owned.data_ptr():
|
||||||
|
raise RuntimeError(f"{layer_name}: PLE prefetch buffer identity changed")
|
||||||
|
embedding._start_prefetch_eager(
|
||||||
|
hidden_states, input_ids, query_start_loc, ngram_context
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_ple_finalize_prefetch(output: torch.Tensor, layer_name: str) -> None:
|
||||||
|
"""Join the side stream and write the pinned lookup into ``output``."""
|
||||||
|
pinned = _ple_embedding_of(layer_name).ngram_embedding
|
||||||
|
pinned._finalize_prefetch(pinned.prefetch_buffer, output)
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_ple_ngram_ids_fake(
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_device_lookup_fake(
|
||||||
|
ngram_ids: torch.Tensor, output: torch.Tensor, layer_name: str
|
||||||
|
) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_start_prefetch_fake(
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
prefetch_buffer: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_finalize_prefetch_fake(output: torch.Tensor, layer_name: str) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="qwen4_exp_compute_ple_ngram_ids",
|
||||||
|
op_func=qwen4_exp_compute_ple_ngram_ids,
|
||||||
|
mutates_args=["output"],
|
||||||
|
fake_impl=_compute_ple_ngram_ids_fake,
|
||||||
|
)
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="qwen4_exp_ple_device_lookup",
|
||||||
|
op_func=qwen4_exp_ple_device_lookup,
|
||||||
|
mutates_args=["output"],
|
||||||
|
fake_impl=_ple_device_lookup_fake,
|
||||||
|
)
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="qwen4_exp_ple_start_prefetch",
|
||||||
|
op_func=qwen4_exp_ple_start_prefetch,
|
||||||
|
mutates_args=["prefetch_buffer"],
|
||||||
|
fake_impl=_ple_start_prefetch_fake,
|
||||||
|
)
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="qwen4_exp_ple_finalize_prefetch",
|
||||||
|
op_func=qwen4_exp_ple_finalize_prefetch,
|
||||||
|
mutates_args=["output"],
|
||||||
|
fake_impl=_ple_finalize_prefetch_fake,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Qwen4ExpNGramEmbedding",
|
"Qwen4ExpNGramEmbedding",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -26,13 +26,32 @@ def _is_sm120() -> bool:
|
|||||||
def _is_sm90() -> bool:
|
def _is_sm90() -> bool:
|
||||||
"""True on sm_90 (H100/H200/H20): selects the sm_90 tuning table."""
|
"""True on sm_90 (H100/H200/H20): selects the sm_90 tuning table."""
|
||||||
return current_platform.get_device_capability() == (9, 0)
|
return current_platform.get_device_capability() == (9, 0)
|
||||||
|
@triton.jit
|
||||||
|
def _decode_e4m3_to_bf16(b):
|
||||||
|
# Shifting the 7 magnitude bits left by 7 lands the E4M3 exponent in the
|
||||||
|
# FP16 exponent field and the 3 mantissa bits at the top of the FP16
|
||||||
|
# mantissa field, giving the right number under the wrong exponent bias
|
||||||
|
# (E4M3 7 vs FP16 15). That is a constant factor of 2^8. E4M3 subnormals
|
||||||
|
# land on FP16 subnormals and the same factor renormalizes them, which is
|
||||||
|
# only true because sm_86 does not flush FP16 subnormals -- verified on
|
||||||
|
# device, not assumed.
|
||||||
|
#
|
||||||
|
# Bit-exact against torch on all 254 finite codepoints: every E4M3 value
|
||||||
|
# fits in BF16's 7 mantissa bits, and x256 is exact in FP16.
|
||||||
|
#
|
||||||
|
# Deviation: 0x7F/0xFF are NaN in e4m3fn but decode to +-480 here. The CUDA
|
||||||
|
# writer clamps to +-448 and never emits them, so this is unreachable from
|
||||||
|
# a cache written by reshape_and_cache_flash.
|
||||||
|
b = b.to(tl.uint16)
|
||||||
|
bits = ((b & 0x80) << 8) | ((b & 0x7F) << 7)
|
||||||
|
return (bits.to(tl.uint16).to(tl.float16, bitcast=True) * 256.0).to(tl.bfloat16)
|
||||||
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
|
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
|
||||||
def _qsa_sparse_paged_gqa_splitk_kernel(
|
def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||||
q_ptr,
|
q_ptr,
|
||||||
k_cache_ptr,
|
k_cache_ptr,
|
||||||
v_cache_ptr,
|
v_cache_ptr,
|
||||||
|
k_scale_ptr,
|
||||||
|
v_scale_ptr,
|
||||||
indices_ptr,
|
indices_ptr,
|
||||||
block_table_ptr,
|
block_table_ptr,
|
||||||
token_to_req_ptr,
|
token_to_req_ptr,
|
||||||
@@ -67,6 +86,7 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
|||||||
NUM_QUERY_HEADS: tl.constexpr,
|
NUM_QUERY_HEADS: tl.constexpr,
|
||||||
NUM_SPLITS: tl.constexpr,
|
NUM_SPLITS: tl.constexpr,
|
||||||
NUM_TILES: tl.constexpr,
|
NUM_TILES: tl.constexpr,
|
||||||
|
KV_DTYPE: tl.constexpr,
|
||||||
BLOCK_M: tl.constexpr,
|
BLOCK_M: tl.constexpr,
|
||||||
BLOCK_N: tl.constexpr,
|
BLOCK_N: tl.constexpr,
|
||||||
IS_FP8: tl.constexpr,
|
IS_FP8: tl.constexpr,
|
||||||
@@ -104,6 +124,10 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
|||||||
# here for the exp2-based online softmax.
|
# here for the exp2-based online softmax.
|
||||||
score_scale = softmax_scale * 1.4426950408889634
|
score_scale = softmax_scale * 1.4426950408889634
|
||||||
|
|
||||||
|
if KV_DTYPE != 0:
|
||||||
|
k_scale_value = tl.load(k_scale_ptr)
|
||||||
|
v_scale_value = tl.load(v_scale_ptr)
|
||||||
|
|
||||||
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
|
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
|
||||||
|
|
||||||
for tile in range(split_id, tile_end, NUM_SPLITS):
|
for tile in range(split_id, tile_end, NUM_SPLITS):
|
||||||
@@ -132,44 +156,87 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
|||||||
valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)
|
valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)
|
||||||
# physical_page * block stride can overflow int32 for large caches.
|
# physical_page * block stride can overflow int32 for large caches.
|
||||||
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
||||||
keys = tl.load(
|
if KV_DTYPE == 0:
|
||||||
k_cache_ptr
|
keys = tl.load(
|
||||||
+ safe_page[None, :] * stride_k_block
|
k_cache_ptr
|
||||||
+ page_offset[None, :] * stride_k_token
|
+ safe_page[None, :] * stride_k_block
|
||||||
+ kv_head * stride_k_head
|
+ page_offset[None, :] * stride_k_token
|
||||||
+ dim_offsets[:, None],
|
+ kv_head * stride_k_head
|
||||||
mask=valid[None, :],
|
+ dim_offsets[:, None],
|
||||||
other=0.0,
|
mask=valid[None, :],
|
||||||
)
|
other=0.0,
|
||||||
values = tl.load(
|
)
|
||||||
v_cache_ptr
|
values = tl.load(
|
||||||
+ safe_page[:, None] * stride_v_block
|
v_cache_ptr
|
||||||
+ page_offset[:, None] * stride_v_token
|
+ safe_page[:, None] * stride_v_block
|
||||||
+ kv_head * stride_v_head
|
+ page_offset[:, None] * stride_v_token
|
||||||
+ dim_offsets[None, :],
|
+ kv_head * stride_v_head
|
||||||
mask=valid[:, None],
|
+ dim_offsets[None, :],
|
||||||
other=0.0,
|
mask=valid[:, None],
|
||||||
)
|
other=0.0,
|
||||||
if IS_FP8:
|
)
|
||||||
# e4m3 -> Q dtype is exact; keep the QK dot in Q's dtype (fp8 QK
|
elif KV_DTYPE == 1:
|
||||||
# measured slower here and less accurate).
|
key_bytes = tl.load(
|
||||||
keys = keys.to(query.dtype)
|
k_cache_ptr
|
||||||
|
+ safe_page[None, :] * stride_k_block
|
||||||
|
+ page_offset[None, :] * stride_k_token
|
||||||
|
+ kv_head * stride_k_head
|
||||||
|
+ dim_offsets[:, None],
|
||||||
|
mask=valid[None, :],
|
||||||
|
other=0,
|
||||||
|
)
|
||||||
|
# UNSCALED: k_scale is folded into the score scaling below.
|
||||||
|
keys = _decode_e4m3_to_bf16(key_bytes)
|
||||||
|
else:
|
||||||
|
keys = tl.load(
|
||||||
|
k_cache_ptr
|
||||||
|
+ safe_page[None, :] * stride_k_block
|
||||||
|
+ page_offset[None, :] * stride_k_token
|
||||||
|
+ kv_head * stride_k_head
|
||||||
|
+ dim_offsets[:, None],
|
||||||
|
mask=valid[None, :],
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.bfloat16)
|
||||||
scores = tl.dot(query, keys)
|
scores = tl.dot(query, keys)
|
||||||
# Scaling scores avoids re-quantizing a scaled query to BF16; for fp8
|
# Scaling scores avoids re-quantizing a scaled query to BF16.
|
||||||
# caches the K dequant scale is already folded into softmax_scale on the
|
if KV_DTYPE == 0:
|
||||||
# host.
|
scores *= softmax_scale_log2
|
||||||
scores *= score_scale
|
else:
|
||||||
|
# k_scale is a scalar, so scaling each SCORE is equivalent to
|
||||||
|
# scaling every key element -- but costs one multiply per score
|
||||||
|
# instead of one per element, and skips the fp32 round trip and
|
||||||
|
# the BF16 re-quantization of the scaled key (which for a
|
||||||
|
# non-power-of-two scale also lost mantissa bits). Strictly fewer
|
||||||
|
# roundings than the per-element form it replaces.
|
||||||
|
scores *= softmax_scale_log2 * k_scale_value
|
||||||
scores = tl.where(valid[None, :], scores, -1.0e20)
|
scores = tl.where(valid[None, :], scores, -1.0e20)
|
||||||
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
|
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
|
||||||
alpha = tl.math.exp2(max_value - next_max)
|
alpha = tl.math.exp2(max_value - next_max)
|
||||||
probabilities = tl.where(
|
probabilities = tl.where(
|
||||||
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
|
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
|
||||||
)
|
)
|
||||||
if IS_FP8:
|
if KV_DTYPE == 1:
|
||||||
# Dequant V to fp16 (not bf16) for the PV dot: P <= 1 (online
|
value_bytes = tl.load(
|
||||||
# softmax) so fp16 has the range, its wider mantissa is more
|
v_cache_ptr
|
||||||
# accurate, and the fp8->fp16 upcast with an fp16 PV dot is faster.
|
+ safe_page[:, None] * stride_v_block
|
||||||
values = values.to(tl.float16)
|
+ page_offset[:, None] * stride_v_token
|
||||||
|
+ kv_head * stride_v_head
|
||||||
|
+ dim_offsets[None, :],
|
||||||
|
mask=valid[:, None],
|
||||||
|
other=0,
|
||||||
|
)
|
||||||
|
# UNSCALED: v_scale is folded into the epilogue below.
|
||||||
|
values = _decode_e4m3_to_bf16(value_bytes)
|
||||||
|
elif KV_DTYPE == 2:
|
||||||
|
values = tl.load(
|
||||||
|
v_cache_ptr
|
||||||
|
+ safe_page[:, None] * stride_v_block
|
||||||
|
+ page_offset[:, None] * stride_v_token
|
||||||
|
+ kv_head * stride_v_head
|
||||||
|
+ dim_offsets[None, :],
|
||||||
|
mask=valid[:, None],
|
||||||
|
other=0.0,
|
||||||
|
).to(tl.bfloat16)
|
||||||
accumulator = tl.dot(
|
accumulator = tl.dot(
|
||||||
probabilities.to(values.dtype),
|
probabilities.to(values.dtype),
|
||||||
values,
|
values,
|
||||||
@@ -189,6 +256,13 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
|||||||
accumulator * inv_normalizer[:, None],
|
accumulator * inv_normalizer[:, None],
|
||||||
0.0,
|
0.0,
|
||||||
)
|
)
|
||||||
|
if KV_DTYPE != 0:
|
||||||
|
# v_scale folded out of the per-element V decode: the attention output
|
||||||
|
# is linear in V, so one multiply per output element replaces one per
|
||||||
|
# cache element per tile. Safe under split-K too -- every partial is
|
||||||
|
# scaled by the same constant before the LSE merge, which is itself
|
||||||
|
# linear in the partial outputs.
|
||||||
|
normalized_output *= v_scale_value
|
||||||
output_mask = head_offsets[:, None] < GROUP_SIZE
|
output_mask = head_offsets[:, None] < GROUP_SIZE
|
||||||
if NUM_SPLITS == 1:
|
if NUM_SPLITS == 1:
|
||||||
# Preserve the unfused path's BF16 attention-output rounding before
|
# Preserve the unfused path's BF16 attention-output rounding before
|
||||||
@@ -600,17 +674,18 @@ def qsa_sparse_paged_attention(
|
|||||||
token_to_req: torch.Tensor,
|
token_to_req: torch.Tensor,
|
||||||
use_prefill_config: bool,
|
use_prefill_config: bool,
|
||||||
out: torch.Tensor | None = None,
|
out: torch.Tensor | None = None,
|
||||||
k_scale: float | None = None,
|
k_scale: torch.Tensor | None = None,
|
||||||
v_scale: float | None = None,
|
v_scale: torch.Tensor | None = None,
|
||||||
|
_force_num_warps: int | None = None,
|
||||||
*,
|
*,
|
||||||
output_gate: torch.Tensor,
|
output_gate: torch.Tensor,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""Run sparse GQA directly over paged BF16 or FP8-e4m3 K/V caches.
|
"""Run sparse GQA over paged BF16 or FP8 (E4M3 via static scalar scales) K/V caches.
|
||||||
|
|
||||||
With fp8 caches, k_scale/v_scale are the layer's per-tensor dequant scales
|
``_force_num_warps`` overrides the tuned launch profile for the warp-count
|
||||||
as host floats (e.g. layer._k_scale_float/_v_scale_float): k_scale is
|
equivalence test only; production callers leave it None. Hardware policy
|
||||||
pre-multiplied into the softmax scale and v_scale becomes the kernel's
|
(E4M3 only on CUDA sm_86) is enforced once at Qwen4ExpQSAAttention /
|
||||||
output scale, so the kernel needs no device scale buffers.
|
Qwen4ExpQSAFlashAttentionImpl construction, not per call.
|
||||||
|
|
||||||
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
|
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
|
||||||
with the trailing column holding each row's valid-entry count (written by
|
with the trailing column holding each row's valid-entry count (written by
|
||||||
@@ -634,19 +709,26 @@ def qsa_sparse_paged_attention(
|
|||||||
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
||||||
head_dim = q.shape[2]
|
head_dim = q.shape[2]
|
||||||
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
|
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
|
||||||
assert q.dtype == torch.bfloat16
|
if q.dtype != torch.bfloat16:
|
||||||
assert k_cache.dtype == v_cache.dtype
|
raise ValueError("QSA sparse attention requires BF16 queries")
|
||||||
is_fp8 = k_cache.dtype == torch.float8_e4m3fn
|
if k_cache.dtype != v_cache.dtype:
|
||||||
if is_fp8:
|
raise ValueError("QSA sparse attention requires matching K/V dtypes")
|
||||||
assert k_scale is not None and v_scale is not None
|
fp8_dtype = current_platform.fp8_dtype()
|
||||||
# Host pre-multiply: fold the K dequant scale into the attention scale
|
if k_cache.dtype == torch.bfloat16:
|
||||||
# and pass V's dequant scale as the kernel's output scale.
|
kv_dtype = 0
|
||||||
softmax_scale = (head_dim**-0.5) * float(k_scale)
|
elif k_cache.dtype in (torch.uint8, fp8_dtype):
|
||||||
output_scale = float(v_scale)
|
kv_dtype = 1
|
||||||
|
# Present platform E4M3 storage as bytes so Triton never materializes
|
||||||
|
# the unsupported fp8e4nv type on SM86.
|
||||||
|
if k_cache.dtype == fp8_dtype:
|
||||||
|
k_cache = k_cache.view(torch.uint8)
|
||||||
|
v_cache = v_cache.view(torch.uint8)
|
||||||
|
elif k_cache.dtype == torch.float8_e5m2:
|
||||||
|
kv_dtype = 2
|
||||||
else:
|
else:
|
||||||
assert k_cache.dtype == torch.bfloat16
|
raise ValueError(
|
||||||
softmax_scale = head_dim**-0.5
|
"QSA supports only BF16, uint8/E4M3, and E5M2 K/V caches"
|
||||||
output_scale = 1.0
|
)
|
||||||
assert logical_indices.dtype == block_table.dtype == torch.int32
|
assert logical_indices.dtype == block_table.dtype == torch.int32
|
||||||
assert token_to_req.dtype == torch.int32
|
assert token_to_req.dtype == torch.int32
|
||||||
assert q.device == k_cache.device == v_cache.device
|
assert q.device == k_cache.device == v_cache.device
|
||||||
@@ -667,12 +749,37 @@ def qsa_sparse_paged_attention(
|
|||||||
if not q.shape[0]:
|
if not q.shape[0]:
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
if kv_dtype == 0:
|
||||||
|
# Scale pointers are compile-time dead in the unchanged BF16 kernel.
|
||||||
|
k_scale_ptr = q
|
||||||
|
v_scale_ptr = q
|
||||||
|
else:
|
||||||
|
if k_scale is None:
|
||||||
|
k_scale = torch.tensor(1.0, dtype=torch.float32, device=q.device)
|
||||||
|
if v_scale is None:
|
||||||
|
v_scale = torch.tensor(1.0, dtype=torch.float32, device=q.device)
|
||||||
|
if (
|
||||||
|
k_scale.numel() != 1
|
||||||
|
or v_scale.numel() != 1
|
||||||
|
or k_scale.dtype != torch.float32
|
||||||
|
or v_scale.dtype != torch.float32
|
||||||
|
or k_scale.device != q.device
|
||||||
|
or v_scale.device != q.device
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"QSA K/V scales must be scalar float32 tensors on the query device"
|
||||||
|
)
|
||||||
|
k_scale_ptr = k_scale
|
||||||
|
v_scale_ptr = v_scale
|
||||||
|
|
||||||
group_size = q.shape[1] // k_cache.shape[2]
|
group_size = q.shape[1] // k_cache.shape[2]
|
||||||
block_m = triton.next_power_of_2(group_size)
|
block_m = triton.next_power_of_2(group_size)
|
||||||
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
|
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
|
||||||
block_n, partial_warps, num_tiles, num_splits = _select_config(
|
block_n, partial_warps, num_tiles, num_splits = _select_config(
|
||||||
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width, is_fp8
|
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width, is_fp8
|
||||||
)
|
)
|
||||||
|
if _force_num_warps is not None:
|
||||||
|
partial_warps = _force_num_warps
|
||||||
|
|
||||||
# Split=1 writes output directly and compiles out all workspace accesses.
|
# Split=1 writes output directly and compiles out all workspace accesses.
|
||||||
if num_splits == 1:
|
if num_splits == 1:
|
||||||
@@ -695,6 +802,8 @@ def qsa_sparse_paged_attention(
|
|||||||
q,
|
q,
|
||||||
k_cache,
|
k_cache,
|
||||||
v_cache,
|
v_cache,
|
||||||
|
k_scale_ptr,
|
||||||
|
v_scale_ptr,
|
||||||
logical_indices,
|
logical_indices,
|
||||||
block_table,
|
block_table,
|
||||||
token_to_req,
|
token_to_req,
|
||||||
@@ -729,6 +838,7 @@ def qsa_sparse_paged_attention(
|
|||||||
NUM_QUERY_HEADS=q.shape[1],
|
NUM_QUERY_HEADS=q.shape[1],
|
||||||
NUM_SPLITS=num_splits,
|
NUM_SPLITS=num_splits,
|
||||||
NUM_TILES=num_tiles,
|
NUM_TILES=num_tiles,
|
||||||
|
KV_DTYPE=kv_dtype,
|
||||||
BLOCK_M=block_m,
|
BLOCK_M=block_m,
|
||||||
BLOCK_N=block_n,
|
BLOCK_N=block_n,
|
||||||
IS_FP8=is_fp8,
|
IS_FP8=is_fp8,
|
||||||
@@ -792,13 +902,32 @@ def warmup_qsa_sparse_paged_attention(
|
|||||||
q_ptr = TritonWarmupTensor(
|
q_ptr = TritonWarmupTensor(
|
||||||
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
||||||
)
|
)
|
||||||
|
# Mirror qsa_sparse_paged_attention's dtype dispatch: E4M3 storage is launched
|
||||||
|
# as a uint8 view with scalar float32 scale pointers.
|
||||||
|
if key_cache.dtype == torch.bfloat16:
|
||||||
|
warm_kv_dtype = 0
|
||||||
|
warm_cache_dtype = key_cache.dtype
|
||||||
|
elif key_cache.dtype in (torch.uint8, current_platform.fp8_dtype()):
|
||||||
|
warm_kv_dtype = 1
|
||||||
|
warm_cache_dtype = torch.uint8
|
||||||
|
elif key_cache.dtype == torch.float8_e5m2:
|
||||||
|
warm_kv_dtype = 2
|
||||||
|
warm_cache_dtype = key_cache.dtype
|
||||||
|
else:
|
||||||
|
raise ValueError("QSA warmup: unsupported K/V cache dtype")
|
||||||
|
if warm_kv_dtype == 0:
|
||||||
|
k_scale_ptr = q_ptr
|
||||||
|
v_scale_ptr = q_ptr
|
||||||
|
else:
|
||||||
|
k_scale_ptr = TritonWarmupTensor(torch.float32)
|
||||||
|
v_scale_ptr = TritonWarmupTensor(torch.float32)
|
||||||
k_cache_ptr = TritonWarmupTensor(
|
k_cache_ptr = TritonWarmupTensor(
|
||||||
cache_dtype,
|
warm_cache_dtype,
|
||||||
shape=tuple(key_cache.shape),
|
shape=tuple(key_cache.shape),
|
||||||
strides=tuple(key_cache.stride()),
|
strides=tuple(key_cache.stride()),
|
||||||
)
|
)
|
||||||
v_cache_ptr = TritonWarmupTensor(
|
v_cache_ptr = TritonWarmupTensor(
|
||||||
cache_dtype,
|
warm_cache_dtype,
|
||||||
shape=tuple(value_cache.shape),
|
shape=tuple(value_cache.shape),
|
||||||
strides=tuple(value_cache.stride()),
|
strides=tuple(value_cache.stride()),
|
||||||
)
|
)
|
||||||
@@ -838,6 +967,8 @@ def warmup_qsa_sparse_paged_attention(
|
|||||||
q_ptr,
|
q_ptr,
|
||||||
k_cache_ptr,
|
k_cache_ptr,
|
||||||
v_cache_ptr,
|
v_cache_ptr,
|
||||||
|
k_scale_ptr,
|
||||||
|
v_scale_ptr,
|
||||||
indices_ptr,
|
indices_ptr,
|
||||||
block_table_ptr,
|
block_table_ptr,
|
||||||
token_to_req_ptr,
|
token_to_req_ptr,
|
||||||
@@ -872,6 +1003,7 @@ def warmup_qsa_sparse_paged_attention(
|
|||||||
NUM_QUERY_HEADS=num_query_heads,
|
NUM_QUERY_HEADS=num_query_heads,
|
||||||
NUM_SPLITS=num_splits,
|
NUM_SPLITS=num_splits,
|
||||||
NUM_TILES=num_tiles,
|
NUM_TILES=num_tiles,
|
||||||
|
KV_DTYPE=warm_kv_dtype,
|
||||||
BLOCK_M=block_m,
|
BLOCK_M=block_m,
|
||||||
BLOCK_N=block_n,
|
BLOCK_N=block_n,
|
||||||
IS_FP8=is_fp8,
|
IS_FP8=is_fp8,
|
||||||
|
|||||||
@@ -10,11 +10,126 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
|||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.worker import guard_telemetry
|
||||||
|
|
||||||
|
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||||
|
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||||
|
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||||
|
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||||
|
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||||
|
|
||||||
|
|
||||||
_TOPK_WORKSPACE_BYTES = 1024 * 1024
|
_TOPK_WORKSPACE_BYTES = 1024 * 1024
|
||||||
_DECODE_BLOCK_N = 64
|
_DECODE_BLOCK_N = 64
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# MERLIN_QSA_SORT: canonicalise the ORDER of the selected block indices.
|
||||||
|
#
|
||||||
|
# persistent_topk assigns output slots by atomicAdd, so two identical requests
|
||||||
|
# get the same selected SET in a different ORDER (measured on the reference host: matching
|
||||||
|
# sorted hashes, differing raw hashes). Sparse attention then accumulates over
|
||||||
|
# those blocks in whatever order it was handed, and fp32 addition is not
|
||||||
|
# associative, so the arithmetic moves. Sorting is semantically a no-op -- the
|
||||||
|
# same blocks are attended either way -- and makes the accumulation order
|
||||||
|
# canonical.
|
||||||
|
#
|
||||||
|
# Padding (negative sentinels) is pushed to the END so the valid prefix is
|
||||||
|
# preserved: `_expand_qsa_indices_kernel` truncates by POSITION
|
||||||
|
# (`complete_blocks = min(visible_blocks, BLOCK_TOPK)`), and position is never
|
||||||
|
# compared against a score anywhere in the chain, so there is no relevance-rank
|
||||||
|
# contract for the sort to break.
|
||||||
|
#
|
||||||
|
# Branch-free and sync-free on purpose. `.any()` / `.item()` raise
|
||||||
|
# cudaErrorStreamCaptureUnsupported inside graph capture; that cost one boot
|
||||||
|
# on the reference host and is why this version has no Python-visible branch on data.
|
||||||
|
#
|
||||||
|
# Resolved once at import so it is a launch constant: a cudagraph captures the
|
||||||
|
# op sequence that was live at capture time. Code default OFF; vllm/env_override.py
|
||||||
|
# sets 1 (the shipped value) when unset.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
_QSA_SORT = envs.MERLIN_QSA_SORT
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# MERLIN_TOPK_SORTED_EMIT: persistent_topk emits
|
||||||
|
# every row in the canonical order below by itself (sorted_emit_row() in
|
||||||
|
# csrc/libtorch_stable/persistent_topk.cuh), so the Python sort is skipped.
|
||||||
|
#
|
||||||
|
# The decision is NOT a Python parse of the variable. The C++ getter
|
||||||
|
# (topk.cu sorted_emit_enabled) is the single source of truth and publishes
|
||||||
|
# what it cached into persistent_topk_toggle_state() field 6; that is read
|
||||||
|
# once, host-side, right after this process's first persistent_topk launch
|
||||||
|
# (the static is filled by then) and latched. So the sort is dropped only when
|
||||||
|
# the kernel that actually ran says it sorted; an older build (layout 1, no
|
||||||
|
# field 6) or an absent op keeps the Python sort. cooperative_topk (SM90+) has
|
||||||
|
# no sorted emission and always keeps the Python sort.
|
||||||
|
#
|
||||||
|
# MERLIN_TOPK_SORTED_EMIT_CHECK=1 (debug, default off): instead of trusting
|
||||||
|
# the kernel, assert on device that the row already equals its canonical form.
|
||||||
|
# torch._assert_async is capture-safe (no host sync); a violation is a device
|
||||||
|
# assert, i.e. a dead context -- which is the point of a debug check.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
_SORTED_EMIT_CHECK = envs.MERLIN_TOPK_SORTED_EMIT_CHECK
|
||||||
|
_kernel_sorted_emit_state: bool | None = None # None: not read yet
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(block_indices: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""Ascending, padding last as -1 -- THE definition of the canonical order,
|
||||||
|
shared by the Python sort and the debug check (and the reference the
|
||||||
|
kernel's sorted emission is tested against)."""
|
||||||
|
big = torch.iinfo(block_indices.dtype).max
|
||||||
|
tmp = torch.where(
|
||||||
|
block_indices < 0, torch.full_like(block_indices, big), block_indices
|
||||||
|
)
|
||||||
|
tmp = torch.sort(tmp, dim=-1).values
|
||||||
|
return torch.where(tmp == big, torch.full_like(tmp, -1), tmp)
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_canonicalise(block_indices: torch.Tensor) -> None:
|
||||||
|
"""Sort selected block indices ascending, padding last. In place, same shape."""
|
||||||
|
if not _QSA_SORT:
|
||||||
|
return
|
||||||
|
block_indices.copy_(_canonical(block_indices))
|
||||||
|
|
||||||
|
|
||||||
|
def kernel_sorted_emit() -> bool:
|
||||||
|
"""True iff this process's persistent_topk emits canonical order.
|
||||||
|
|
||||||
|
Host-only (a tensor-free query op; no CUDA call), so it is safe inside a
|
||||||
|
capture. Latches once the C++ static is known; an uninitialised static
|
||||||
|
(no launch yet) answers False without latching.
|
||||||
|
"""
|
||||||
|
global _kernel_sorted_emit_state
|
||||||
|
if _kernel_sorted_emit_state is not None:
|
||||||
|
return _kernel_sorted_emit_state
|
||||||
|
op = getattr(torch.ops._C, "persistent_topk_toggle_state", None)
|
||||||
|
if op is None:
|
||||||
|
_kernel_sorted_emit_state = False
|
||||||
|
return False
|
||||||
|
from vllm.v1.worker.e2_toggles import decode_topk_state
|
||||||
|
|
||||||
|
decoded = decode_topk_state(int(op()))
|
||||||
|
if isinstance(decoded, str) or "sorted_emit" not in decoded:
|
||||||
|
# Unknown layout, or layout 1 (no field 6): this build cannot sort.
|
||||||
|
_kernel_sorted_emit_state = False
|
||||||
|
return False
|
||||||
|
value = decoded["sorted_emit"]
|
||||||
|
if value == "uninit":
|
||||||
|
return False # not latched: ask again after a launch
|
||||||
|
_kernel_sorted_emit_state = value == "1"
|
||||||
|
return _kernel_sorted_emit_state
|
||||||
|
|
||||||
|
|
||||||
|
def _check_sorted_emission(block_indices: torch.Tensor) -> None:
|
||||||
|
"""Debug only: device-side assert that the kernel's row is canonical."""
|
||||||
|
if not _SORTED_EMIT_CHECK:
|
||||||
|
return
|
||||||
|
torch._assert_async(
|
||||||
|
torch.eq(block_indices, _canonical(block_indices)).all(),
|
||||||
|
"MERLIN_TOPK_SORTED_EMIT: persistent_topk row is not in canonical order",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@triton.jit
|
@triton.jit
|
||||||
def _qsa_mqa_paged_uniform_kernel(
|
def _qsa_mqa_paged_uniform_kernel(
|
||||||
q_ptr,
|
q_ptr,
|
||||||
@@ -28,6 +143,8 @@ def _qsa_mqa_paged_uniform_kernel(
|
|||||||
stride_cache_token,
|
stride_cache_token,
|
||||||
stride_table_req,
|
stride_table_req,
|
||||||
stride_logits_row,
|
stride_logits_row,
|
||||||
|
num_cache_blocks, # bounds-guard instrumentation
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
PAGE_SIZE: tl.constexpr,
|
PAGE_SIZE: tl.constexpr,
|
||||||
PAGE_TABLE_WIDTH: tl.constexpr,
|
PAGE_TABLE_WIDTH: tl.constexpr,
|
||||||
NUM_HEADS: tl.constexpr,
|
NUM_HEADS: tl.constexpr,
|
||||||
@@ -72,6 +189,11 @@ def _qsa_mqa_paged_uniform_kernel(
|
|||||||
other=0.0,
|
other=0.0,
|
||||||
)
|
)
|
||||||
column_offsets = tl.arange(0, BLOCK_N)
|
column_offsets = tl.arange(0, BLOCK_N)
|
||||||
|
# Bounds-guard instrumentation: accumulate in registers inside the pipelined loop
|
||||||
|
# and do the cross-lane reduction + atomics once after it, so an
|
||||||
|
# atomic-bearing conditional does not sit in the loop body.
|
||||||
|
guard_bad_n = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||||
|
guard_bad_col = tl.full([BLOCK_N], -1, tl.int32)
|
||||||
for tile in tl.range(tile_start, tile_end, num_stages=STAGES):
|
for tile in tl.range(tile_start, tile_end, num_stages=STAGES):
|
||||||
columns = tile * BLOCK_N + column_offsets
|
columns = tile * BLOCK_N + column_offsets
|
||||||
live = columns < max_visible
|
live = columns < max_visible
|
||||||
@@ -80,11 +202,22 @@ def _qsa_mqa_paged_uniform_kernel(
|
|||||||
physical_page = tl.load(
|
physical_page = tl.load(
|
||||||
page_table_ptr + request * stride_table_req + logical_page,
|
page_table_ptr + request * stride_table_req + logical_page,
|
||||||
mask=live,
|
mask=live,
|
||||||
other=0,
|
other=-1,
|
||||||
)
|
)
|
||||||
|
# Mirror of the QSA attention kernel (qsa.py:137): the VALUE read out of
|
||||||
|
# the page table is untrusted and must be bounded before it becomes an
|
||||||
|
# address. Folds into the existing live mask.
|
||||||
|
page_bad = live & ((physical_page < 0) | (physical_page >= num_cache_blocks))
|
||||||
|
guard_bad_n += page_bad.to(tl.int32)
|
||||||
|
guard_bad_col = tl.maximum(
|
||||||
|
guard_bad_col, tl.where(page_bad, logical_page, -1).to(tl.int32)
|
||||||
|
)
|
||||||
|
live = live & (physical_page >= 0) & (physical_page < num_cache_blocks)
|
||||||
|
# physical_page * block stride can overflow int32 for large caches.
|
||||||
|
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
||||||
keys = tl.load(
|
keys = tl.load(
|
||||||
k_cache_ptr
|
k_cache_ptr
|
||||||
+ physical_page[:, None].to(tl.int64) * stride_cache_block
|
+ safe_page[:, None] * stride_cache_block
|
||||||
+ page_offset[:, None] * stride_cache_token
|
+ page_offset[:, None] * stride_cache_token
|
||||||
+ dims[None, :],
|
+ dims[None, :],
|
||||||
mask=live[:, None] & (dims[None, :] < HEAD_DIM),
|
mask=live[:, None] & (dims[None, :] < HEAD_DIM),
|
||||||
@@ -106,6 +239,15 @@ def _qsa_mqa_paged_uniform_kernel(
|
|||||||
& (columns[:, None] < visible[None, :]),
|
& (columns[:, None] < visible[None, :]),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Bounds-guard instrumentation: one reduction and at most one atomic per program.
|
||||||
|
guard_n = tl.sum(guard_bad_n, axis=0)
|
||||||
|
if guard_n > 0:
|
||||||
|
tl.atomic_add(guard_ctr, guard_n.to(tl.int64))
|
||||||
|
tl.atomic_max(guard_ctr + 1, (request).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(
|
||||||
|
guard_ctr + 2, (tl.max(guard_bad_col, axis=0)).to(tl.int64) + _GUARD_BIAS
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@triton.jit(do_not_specialize=["num_rows", "query_offset"])
|
@triton.jit(do_not_specialize=["num_rows", "query_offset"])
|
||||||
def _qsa_mqa_paged_prefill_kernel(
|
def _qsa_mqa_paged_prefill_kernel(
|
||||||
@@ -124,6 +266,8 @@ def _qsa_mqa_paged_prefill_kernel(
|
|||||||
num_rows,
|
num_rows,
|
||||||
query_offset,
|
query_offset,
|
||||||
page_table_width,
|
page_table_width,
|
||||||
|
num_cache_blocks, # bounds-guard instrumentation
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
PAGE_SIZE: tl.constexpr,
|
PAGE_SIZE: tl.constexpr,
|
||||||
NUM_HEADS: tl.constexpr,
|
NUM_HEADS: tl.constexpr,
|
||||||
HEAD_DIM: tl.constexpr,
|
HEAD_DIM: tl.constexpr,
|
||||||
@@ -181,6 +325,11 @@ def _qsa_mqa_paged_prefill_kernel(
|
|||||||
other=0.0,
|
other=0.0,
|
||||||
)
|
)
|
||||||
column_offsets = tl.arange(0, BLOCK_N)
|
column_offsets = tl.arange(0, BLOCK_N)
|
||||||
|
# Bounds-guard instrumentation: accumulate in registers inside the pipelined loop
|
||||||
|
# and do the cross-lane reduction + atomics once after it, so an
|
||||||
|
# atomic-bearing conditional does not sit in the loop body.
|
||||||
|
guard_bad_n = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||||
|
guard_bad_col = tl.full([BLOCK_N], -1, tl.int32)
|
||||||
for tile in tl.range(k_tile_start, k_tile_end, num_stages=STAGES):
|
for tile in tl.range(k_tile_start, k_tile_end, num_stages=STAGES):
|
||||||
columns = tile * BLOCK_N + column_offsets
|
columns = tile * BLOCK_N + column_offsets
|
||||||
live = columns < max_visible
|
live = columns < max_visible
|
||||||
@@ -189,11 +338,22 @@ def _qsa_mqa_paged_prefill_kernel(
|
|||||||
physical_page = tl.load(
|
physical_page = tl.load(
|
||||||
page_table_ptr + request * stride_table_req + logical_page,
|
page_table_ptr + request * stride_table_req + logical_page,
|
||||||
mask=live,
|
mask=live,
|
||||||
other=0,
|
other=-1,
|
||||||
)
|
)
|
||||||
|
# Mirror of the QSA attention kernel (qsa.py:137): the VALUE read out of
|
||||||
|
# the page table is untrusted and must be bounded before it becomes an
|
||||||
|
# address. Folds into the existing live mask.
|
||||||
|
page_bad = live & ((physical_page < 0) | (physical_page >= num_cache_blocks))
|
||||||
|
guard_bad_n += page_bad.to(tl.int32)
|
||||||
|
guard_bad_col = tl.maximum(
|
||||||
|
guard_bad_col, tl.where(page_bad, logical_page, -1).to(tl.int32)
|
||||||
|
)
|
||||||
|
live = live & (physical_page >= 0) & (physical_page < num_cache_blocks)
|
||||||
|
# physical_page * block stride can overflow int32 for large caches.
|
||||||
|
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
||||||
keys = tl.load(
|
keys = tl.load(
|
||||||
k_cache_ptr
|
k_cache_ptr
|
||||||
+ physical_page[:, None].to(tl.int64) * stride_cache_block
|
+ safe_page[:, None] * stride_cache_block
|
||||||
+ page_offset[:, None] * stride_cache_token
|
+ page_offset[:, None] * stride_cache_token
|
||||||
+ dims[None, :],
|
+ dims[None, :],
|
||||||
mask=live[:, None] & (dims[None, :] < HEAD_DIM),
|
mask=live[:, None] & (dims[None, :] < HEAD_DIM),
|
||||||
@@ -217,6 +377,15 @@ def _qsa_mqa_paged_prefill_kernel(
|
|||||||
mask=store_mask,
|
mask=store_mask,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Bounds-guard instrumentation: one reduction and at most one atomic per program.
|
||||||
|
guard_n = tl.sum(guard_bad_n, axis=0)
|
||||||
|
if guard_n > 0:
|
||||||
|
tl.atomic_add(guard_ctr, guard_n.to(tl.int64))
|
||||||
|
tl.atomic_max(guard_ctr + 1, (request).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(
|
||||||
|
guard_ctr + 2, (tl.max(guard_bad_col, axis=0)).to(tl.int64) + _GUARD_BIAS
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@triton.jit
|
@triton.jit
|
||||||
def _expand_qsa_indices_kernel(
|
def _expand_qsa_indices_kernel(
|
||||||
@@ -359,6 +528,8 @@ def warmup_qsa_mqa_paged_decode(
|
|||||||
k_cache.stride(1),
|
k_cache.stride(1),
|
||||||
page_table.stride(0),
|
page_table.stride(0),
|
||||||
columns,
|
columns,
|
||||||
|
k_cache.shape[0],
|
||||||
|
TritonWarmupTensor(torch.int64),
|
||||||
PAGE_SIZE=page_size,
|
PAGE_SIZE=page_size,
|
||||||
PAGE_TABLE_WIDTH=page_table_width,
|
PAGE_TABLE_WIDTH=page_table_width,
|
||||||
NUM_HEADS=num_heads,
|
NUM_HEADS=num_heads,
|
||||||
@@ -427,6 +598,8 @@ def _prefill_logits(
|
|||||||
num_queries,
|
num_queries,
|
||||||
query_offset,
|
query_offset,
|
||||||
page_table.shape[1],
|
page_table.shape[1],
|
||||||
|
k_cache.shape[0],
|
||||||
|
guard_telemetry.counter(guard_telemetry.SITE_QSA_PREFILL, q.device),
|
||||||
PAGE_SIZE=k_cache.shape[1],
|
PAGE_SIZE=k_cache.shape[1],
|
||||||
NUM_HEADS=q.shape[1],
|
NUM_HEADS=q.shape[1],
|
||||||
HEAD_DIM=q.shape[2],
|
HEAD_DIM=q.shape[2],
|
||||||
@@ -502,6 +675,12 @@ def _topk(
|
|||||||
block_topk,
|
block_topk,
|
||||||
logits.shape[1],
|
logits.shape[1],
|
||||||
)
|
)
|
||||||
|
# MERLIN_TOPK_SORTED_EMIT: the kernel already emitted the canonical order,
|
||||||
|
# so the Python sort would be a no-op -- skip it (optionally check it).
|
||||||
|
if not use_cooperative_topk and kernel_sorted_emit():
|
||||||
|
_check_sorted_emission(block_indices)
|
||||||
|
return
|
||||||
|
_qsa_canonicalise(block_indices)
|
||||||
|
|
||||||
|
|
||||||
def qsa_select_paged_decode(
|
def qsa_select_paged_decode(
|
||||||
@@ -554,6 +733,8 @@ def qsa_select_paged_decode(
|
|||||||
*k_cache.stride()[:2],
|
*k_cache.stride()[:2],
|
||||||
*page_table.stride()[:-1],
|
*page_table.stride()[:-1],
|
||||||
*logits.stride()[:-1],
|
*logits.stride()[:-1],
|
||||||
|
k_cache.shape[0],
|
||||||
|
guard_telemetry.counter(guard_telemetry.SITE_QSA_DECODE, q.device),
|
||||||
PAGE_SIZE=k_cache.shape[1],
|
PAGE_SIZE=k_cache.shape[1],
|
||||||
PAGE_TABLE_WIDTH=page_table.shape[1],
|
PAGE_TABLE_WIDTH=page_table.shape[1],
|
||||||
NUM_HEADS=q.shape[1],
|
NUM_HEADS=q.shape[1],
|
||||||
|
|||||||
@@ -4,7 +4,27 @@
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||||
|
QSA_RING_TAG_WORD,
|
||||||
|
QSA_RING_TAIL_ELEMENTS,
|
||||||
|
)
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.worker import guard_telemetry
|
||||||
|
|
||||||
|
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||||
|
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||||
|
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||||
|
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||||
|
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||||
|
# Int64 word of the ring tail that carries `logical_position + 1`.
|
||||||
|
_RING_TAG_WORD = tl.constexpr(QSA_RING_TAG_WORD)
|
||||||
|
# Ring-row tag record packing and the encodable range of the found-tag slot.
|
||||||
|
_REC_POS_BITS = tl.constexpr(guard_telemetry.RING_RECORD_POS_BITS)
|
||||||
|
_REC_REQ_BITS = tl.constexpr(guard_telemetry.RING_RECORD_REQ_BITS)
|
||||||
|
_REC_LAYER_BITS = tl.constexpr(guard_telemetry.RING_RECORD_LAYER_BITS)
|
||||||
|
_GUARD_MAX_ENC = tl.constexpr(guard_telemetry.GUARD_MAX_ENCODABLE)
|
||||||
|
_GUARD_MIN_ENC = tl.constexpr(guard_telemetry.GUARD_MIN_ENCODABLE)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@triton.jit
|
@triton.jit
|
||||||
@@ -15,6 +35,8 @@ def _norm_rope(
|
|||||||
pos_w,
|
pos_w,
|
||||||
cos_sin_ptr,
|
cos_sin_ptr,
|
||||||
cos_sin_stride,
|
cos_sin_stride,
|
||||||
|
cos_sin_rows, # bounds-guard instrumentation
|
||||||
|
guard_ctr, # bounds-guard persistent int64[3], SITE_QSA_ROPE_POS
|
||||||
norm_weight_ptr,
|
norm_weight_ptr,
|
||||||
eps,
|
eps,
|
||||||
IS_MROPE: tl.constexpr,
|
IS_MROPE: tl.constexpr,
|
||||||
@@ -22,6 +44,72 @@ def _norm_rope(
|
|||||||
MROPE_W: tl.constexpr,
|
MROPE_W: tl.constexpr,
|
||||||
):
|
):
|
||||||
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
|
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
|
||||||
|
# Bounds-guard instrumentation (ring-residue crash replay): the position is a
|
||||||
|
# ROW INDEX into the cos/sin table and nothing bounds it. The masks below
|
||||||
|
# bound which rotary PAIR is read, never which row, so
|
||||||
|
# `cos_sin_ptr + pos * cos_sin_stride` is an address formed from an
|
||||||
|
# unchecked value.
|
||||||
|
#
|
||||||
|
# On the pooled cross-chunk path that value is read straight out of a
|
||||||
|
# raw-key RING row (:305-317). The ring is excluded from zeroing
|
||||||
|
# (single_type_kv_cache_manager.py:96-101), so a re-claimed block keeps the
|
||||||
|
# previous occupant's bytes -- and nothing ties the row the reader selects
|
||||||
|
# (`first_position % STATE_SIZE`) to a row any occupant ever committed, so
|
||||||
|
# it can be allocator residue rather than a position at all. A crash replay drove
|
||||||
|
# exactly this and compute-sanitizer reported 11 invalid reads at
|
||||||
|
# +4.6e18 bytes, the filler bytes decoded as an int64.
|
||||||
|
#
|
||||||
|
# This is the ONLY guard in the series that covers that chain: the request guard
|
||||||
|
# bounds the pre-indexer's `request`, which sits upstream of the ring read
|
||||||
|
# and cannot see a bad position.
|
||||||
|
# WHICH AXIS. Under the production mrope section (11, 11, 10)
|
||||||
|
# all three components are live row indices: the IS_MROPE branch masks
|
||||||
|
# COLUMNS per axis while the row comes unmasked from pos_rows[axis], so a
|
||||||
|
# bad pos_h or pos_w faults exactly as a bad pos_t does. Recording pos_t
|
||||||
|
# unconditionally would name the wrong axis and, for a t-clean/h-bad case,
|
||||||
|
# record an in-range value next to a non-zero count.
|
||||||
|
#
|
||||||
|
# The filler under tl.where must be strongly negative, NOT 0: an offender
|
||||||
|
# that is entirely negative (i_t == -1 is the shape that started this
|
||||||
|
# series) would otherwise max to 0 and read back as position 0 -- the
|
||||||
|
# defect re-entering at the one site with a reproduced fault.
|
||||||
|
_FILL: tl.constexpr = -(1 << 62)
|
||||||
|
bad_t = (pos_t < 0) | (pos_t >= cos_sin_rows)
|
||||||
|
if IS_MROPE:
|
||||||
|
bad_h = (pos_h < 0) | (pos_h >= cos_sin_rows)
|
||||||
|
bad_w = (pos_w < 0) | (pos_w >= cos_sin_rows)
|
||||||
|
else:
|
||||||
|
# the non-mrope branch reads only pos_t (:63-66), and with NO mask at
|
||||||
|
# all, which is why this guard clamps rather than masks.
|
||||||
|
# always-false, but correctly shaped and typed for the max below.
|
||||||
|
bad_h = pos_h < pos_h
|
||||||
|
bad_w = pos_w < pos_w
|
||||||
|
guard_bad = bad_t | bad_h | bad_w
|
||||||
|
guard_n = tl.sum(guard_bad.to(tl.int32), axis=0)
|
||||||
|
if guard_n > 0:
|
||||||
|
worst_t = tl.max(tl.where(bad_t, pos_t, _FILL), axis=0).to(tl.int64)
|
||||||
|
worst_h = tl.max(tl.where(bad_h, pos_h, _FILL), axis=0).to(tl.int64)
|
||||||
|
worst_w = tl.max(tl.where(bad_w, pos_w, _FILL), axis=0).to(tl.int64)
|
||||||
|
worst = tl.maximum(worst_t, tl.maximum(worst_h, worst_w))
|
||||||
|
# slot 2 names the axis: 0 = t, 1 = h, 2 = w. Derived from the MASKS
|
||||||
|
# (deliberately), not by comparing the winner's VALUE against each
|
||||||
|
# axis's max. Value comparison mis-attributes whenever two axes share
|
||||||
|
# the worst value -- which is the common case here, because pos_h and
|
||||||
|
# pos_w are frequently equal to pos_t (the non-2D path sets them so,
|
||||||
|
# and a stale ring tail writes the same int64 into all three). Asking
|
||||||
|
# "did this axis have any offender at all", in t/h/w order, cannot.
|
||||||
|
any_t = tl.max(bad_t.to(tl.int32), axis=0) > 0
|
||||||
|
any_h = tl.max(bad_h.to(tl.int32), axis=0) > 0
|
||||||
|
axis = tl.where(any_t, 0, tl.where(any_h, 1, 2))
|
||||||
|
tl.atomic_add(guard_ctr, guard_n.to(tl.int64))
|
||||||
|
tl.atomic_max(guard_ctr + 1, worst + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, axis.to(tl.int64) + _GUARD_BIAS)
|
||||||
|
# Clamp rather than skip: the rotation is then wrong for these rows, which
|
||||||
|
# is what a masked fault IS. The counter is the result, not the output.
|
||||||
|
pos_t = tl.minimum(tl.maximum(pos_t, 0), cos_sin_rows - 1)
|
||||||
|
pos_h = tl.minimum(tl.maximum(pos_h, 0), cos_sin_rows - 1)
|
||||||
|
pos_w = tl.minimum(tl.maximum(pos_w, 0), cos_sin_rows - 1)
|
||||||
|
|
||||||
TILE_T: tl.constexpr = x.shape[0]
|
TILE_T: tl.constexpr = x.shape[0]
|
||||||
TILE_H: tl.constexpr = x.shape[1]
|
TILE_H: tl.constexpr = x.shape[1]
|
||||||
D: tl.constexpr = x.shape[2]
|
D: tl.constexpr = x.shape[2]
|
||||||
@@ -107,6 +195,8 @@ def _store_rotated(dst, y, o1, o2, scale):
|
|||||||
"num_state_blocks",
|
"num_state_blocks",
|
||||||
"num_compressed_blocks",
|
"num_compressed_blocks",
|
||||||
"num_k_work",
|
"num_k_work",
|
||||||
|
"num_requests",
|
||||||
|
"layer_id",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
def _qsa_prepare_kernel(
|
def _qsa_prepare_kernel(
|
||||||
@@ -118,6 +208,8 @@ def _qsa_prepare_kernel(
|
|||||||
pos_stride_axis,
|
pos_stride_axis,
|
||||||
pos_stride_token,
|
pos_stride_token,
|
||||||
cos_sin_ptr,
|
cos_sin_ptr,
|
||||||
|
cos_sin_rows, # bounds-guard instrumentation
|
||||||
|
rope_guard_ctr, # bounds-guard persistent int64[3], SITE_QSA_ROPE_POS
|
||||||
q_norm_weight_ptr,
|
q_norm_weight_ptr,
|
||||||
k_norm_weight_ptr,
|
k_norm_weight_ptr,
|
||||||
eps,
|
eps,
|
||||||
@@ -141,6 +233,11 @@ def _qsa_prepare_kernel(
|
|||||||
num_state_blocks,
|
num_state_blocks,
|
||||||
num_compressed_blocks,
|
num_compressed_blocks,
|
||||||
num_k_work,
|
num_k_work,
|
||||||
|
num_requests, # bounds-guard instrumentation
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
|
ring_skip_ctr, # ring-skip telemetry, persistent int64[3], SITE_QSA_RING_SKIP
|
||||||
|
ring_uncommitted_ctr, # ring-row tag fix, persistent int64[3], SITE_QSA_RING_UNCOMMITTED
|
||||||
|
layer_id, # ring-row tag fix: named in the uncommitted-row record only
|
||||||
HQ: tl.constexpr,
|
HQ: tl.constexpr,
|
||||||
D: tl.constexpr,
|
D: tl.constexpr,
|
||||||
TILE_T_Q: tl.constexpr,
|
TILE_T_Q: tl.constexpr,
|
||||||
@@ -278,6 +375,8 @@ def _qsa_prepare_kernel(
|
|||||||
pos_w,
|
pos_w,
|
||||||
cos_sin_ptr,
|
cos_sin_ptr,
|
||||||
D // 2,
|
D // 2,
|
||||||
|
cos_sin_rows,
|
||||||
|
rope_guard_ctr,
|
||||||
q_norm_weight_ptr,
|
q_norm_weight_ptr,
|
||||||
eps,
|
eps,
|
||||||
IS_2D_POSITIONS,
|
IS_2D_POSITIONS,
|
||||||
@@ -300,6 +399,17 @@ def _qsa_prepare_kernel(
|
|||||||
request, work_in_request = tl.split(work_metadata)
|
request, work_in_request = tl.split(work_metadata)
|
||||||
if request < 0:
|
if request < 0:
|
||||||
return
|
return
|
||||||
|
# Second bounds guard: the early return above rejects only the -1
|
||||||
|
# sentinel the builder writes for inactive work items. Any other
|
||||||
|
# corrupted value was loaded straight out of query_start_loc, which has
|
||||||
|
# num_requests + 1 entries -- 8 bytes for a single-request batch. The
|
||||||
|
# chunk_end load below derives its index from query_end, so a bad row
|
||||||
|
# also walked logical_positions; returning here stops both.
|
||||||
|
if request >= num_requests:
|
||||||
|
tl.atomic_add(guard_ctr, 1)
|
||||||
|
tl.atomic_max(guard_ctr + 1, (request).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, (work_in_request).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
return
|
||||||
|
|
||||||
query_start = tl.load(query_start_loc_ptr + request)
|
query_start = tl.load(query_start_loc_ptr + request)
|
||||||
query_end = tl.load(query_start_loc_ptr + request + 1)
|
query_end = tl.load(query_start_loc_ptr + request + 1)
|
||||||
@@ -351,6 +461,64 @@ def _qsa_prepare_kernel(
|
|||||||
+ safe_state_block * state_cache_stride_block
|
+ safe_state_block * state_cache_stride_block
|
||||||
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
|
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
|
||||||
)
|
)
|
||||||
|
# Ring-row tag fix: EVERY historical source row must carry the tag the
|
||||||
|
# writer stores for exactly this logical position, before anything
|
||||||
|
# reads its key or its position tail. A missing ring block (state
|
||||||
|
# table entry out of range) reads tag 0 and fails the same way --
|
||||||
|
# before the ring-row tag fix it pooled zeros silently. In-chunk rows come from
|
||||||
|
# this launch's own k and are never checked.
|
||||||
|
historical = valid & ~source_in_chunk
|
||||||
|
source_tags = tl.load(
|
||||||
|
(
|
||||||
|
state_cache_ptr
|
||||||
|
+ safe_state_block * state_cache_stride_block
|
||||||
|
+ (source_positions % STATE_SIZE) * state_cache_stride_token
|
||||||
|
+ D
|
||||||
|
).to(tl.pointer_type(tl.int64))
|
||||||
|
+ _RING_TAG_WORD,
|
||||||
|
mask=historical & state_block_valid,
|
||||||
|
other=0,
|
||||||
|
)
|
||||||
|
row_bad = historical & (source_tags != source_positions.to(tl.int64) + 1)
|
||||||
|
num_bad_rows = tl.sum(row_bad.to(tl.int32), axis=0)
|
||||||
|
group_ok = num_bad_rows == 0
|
||||||
|
if num_bad_rows > 0:
|
||||||
|
# One coherent record per site, first offender wins: the CAS
|
||||||
|
# claims slot 1 and only its winner writes slot 2, so both
|
||||||
|
# fields always describe the same row.
|
||||||
|
bad_position = tl.min(
|
||||||
|
tl.where(row_bad, source_positions.to(tl.int64), 1 << 62),
|
||||||
|
axis=0,
|
||||||
|
)
|
||||||
|
bad_tag = tl.sum(
|
||||||
|
tl.where(
|
||||||
|
row_bad & (source_positions.to(tl.int64) == bad_position),
|
||||||
|
source_tags,
|
||||||
|
0,
|
||||||
|
),
|
||||||
|
axis=0,
|
||||||
|
)
|
||||||
|
record = (
|
||||||
|
(
|
||||||
|
((layer_id + 1).to(tl.int64) & ((1 << _REC_LAYER_BITS) - 1))
|
||||||
|
<< (_REC_REQ_BITS + _REC_POS_BITS)
|
||||||
|
)
|
||||||
|
| (
|
||||||
|
(request.to(tl.int64) & ((1 << _REC_REQ_BITS) - 1))
|
||||||
|
<< _REC_POS_BITS
|
||||||
|
)
|
||||||
|
| (bad_position & ((1 << _REC_POS_BITS) - 1))
|
||||||
|
)
|
||||||
|
tl.atomic_add(ring_uncommitted_ctr, num_bad_rows.to(tl.int64))
|
||||||
|
zero64 = bad_position * 0
|
||||||
|
prev = tl.atomic_cas(
|
||||||
|
ring_uncommitted_ctr + 1, zero64, record + _GUARD_BIAS
|
||||||
|
)
|
||||||
|
if prev == 0:
|
||||||
|
clamped_tag = tl.minimum(
|
||||||
|
tl.maximum(bad_tag, _GUARD_MIN_ENC), _GUARD_MAX_ENC
|
||||||
|
)
|
||||||
|
tl.atomic_xchg(ring_uncommitted_ctr + 2, clamped_tag + _GUARD_BIAS)
|
||||||
# Only the first completed group can cross the chunk boundary. Select
|
# Only the first completed group can cross the chunk boundary. Select
|
||||||
# historical rows from the ring without issuing two masked loads.
|
# historical rows from the ring without issuing two masked loads.
|
||||||
source_base = tl.where(source_in_chunk[:, None], current_base, cached_base)
|
source_base = tl.where(source_in_chunk[:, None], current_base, cached_base)
|
||||||
@@ -359,9 +527,11 @@ def _qsa_prepare_kernel(
|
|||||||
source_valid = tl.where(
|
source_valid = tl.where(
|
||||||
source_in_chunk, source_tokens_valid, state_block_valid
|
source_in_chunk, source_tokens_valid, state_block_valid
|
||||||
)
|
)
|
||||||
|
# A failed group loads nothing (the mask, not a tl.where on the
|
||||||
|
# result, is what suppresses the memory operation).
|
||||||
source = tl.load(
|
source = tl.load(
|
||||||
source_base + dims[None, :],
|
source_base + dims[None, :],
|
||||||
mask=valid & source_valid[:, None],
|
mask=valid & group_ok & source_valid[:, None],
|
||||||
other=0.0,
|
other=0.0,
|
||||||
).to(tl.float32)
|
).to(tl.float32)
|
||||||
# Match the unfused path's BF16 pooled tensor before RMSNorm.
|
# Match the unfused path's BF16 pooled tensor before RMSNorm.
|
||||||
@@ -409,7 +579,9 @@ def _qsa_prepare_kernel(
|
|||||||
+ (first_position % STATE_SIZE) * state_cache_stride_token
|
+ (first_position % STATE_SIZE) * state_cache_stride_token
|
||||||
+ D
|
+ D
|
||||||
).to(tl.pointer_type(tl.int64))
|
).to(tl.pointer_type(tl.int64))
|
||||||
load_cached_position = ~first_in_chunk & state_block_valid
|
# The first row's tag was validated above with the rest;
|
||||||
|
# a failed group never reads its tail (pos stays 0, in range).
|
||||||
|
load_cached_position = ~first_in_chunk & state_block_valid & group_ok
|
||||||
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
|
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
|
||||||
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
|
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
|
||||||
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
|
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
|
||||||
@@ -427,6 +599,8 @@ def _qsa_prepare_kernel(
|
|||||||
pos_w + tl.arange(0, 1),
|
pos_w + tl.arange(0, 1),
|
||||||
cos_sin_ptr,
|
cos_sin_ptr,
|
||||||
D // 2,
|
D // 2,
|
||||||
|
cos_sin_rows,
|
||||||
|
rope_guard_ctr,
|
||||||
k_norm_weight_ptr,
|
k_norm_weight_ptr,
|
||||||
eps,
|
eps,
|
||||||
IS_K_MROPE,
|
IS_K_MROPE,
|
||||||
@@ -435,13 +609,16 @@ def _qsa_prepare_kernel(
|
|||||||
)
|
)
|
||||||
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
|
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
|
||||||
compressed_row = compressed_slot % COMP_PAGE_SIZE
|
compressed_row = compressed_slot % COMP_PAGE_SIZE
|
||||||
|
# A group built from an uncommitted row must never become
|
||||||
|
# usable compressed state. The slot stays unwritten; the site is
|
||||||
|
# fatal because the selection kernels will still read it.
|
||||||
tl.store(
|
tl.store(
|
||||||
compressed_cache_ptr
|
compressed_cache_ptr
|
||||||
+ compressed_block * compressed_cache_stride_block
|
+ compressed_block * compressed_cache_stride_block
|
||||||
+ compressed_row * compressed_cache_stride_token
|
+ compressed_row * compressed_cache_stride_token
|
||||||
+ dims,
|
+ dims,
|
||||||
tl.reshape(y, (D,)),
|
tl.reshape(y, (D,)),
|
||||||
mask=valid,
|
mask=valid & group_ok,
|
||||||
)
|
)
|
||||||
|
|
||||||
if work_in_request == 0:
|
if work_in_request == 0:
|
||||||
@@ -459,6 +636,19 @@ def _qsa_prepare_kernel(
|
|||||||
valid_slot = (
|
valid_slot = (
|
||||||
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
|
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
|
||||||
)
|
)
|
||||||
|
# Ring-skip telemetry: a live token whose slot fails the range check
|
||||||
|
# has its ring row silently NOT written (every store below is
|
||||||
|
# masked by valid_slot). That token's raw key is then absent
|
||||||
|
# from the ring the next step reads (a candidate cause). Count
|
||||||
|
# it -- the write stays skipped, the reconciliation is the ring-row tag fix.
|
||||||
|
skipped = valid_token & ~valid_slot
|
||||||
|
tl.atomic_add(ring_skip_ctr, 1, mask=skipped)
|
||||||
|
tl.atomic_max(
|
||||||
|
ring_skip_ctr + 1, token.to(tl.int64) + _GUARD_BIAS, mask=skipped
|
||||||
|
)
|
||||||
|
tl.atomic_max(
|
||||||
|
ring_skip_ctr + 2, slot.to(tl.int64) + _GUARD_BIAS, mask=skipped
|
||||||
|
)
|
||||||
safe_slot = tl.maximum(slot, 0)
|
safe_slot = tl.maximum(slot, 0)
|
||||||
state_row = (
|
state_row = (
|
||||||
state_cache_ptr
|
state_cache_ptr
|
||||||
@@ -499,6 +689,20 @@ def _qsa_prepare_kernel(
|
|||||||
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
|
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
|
||||||
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
|
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
|
||||||
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
|
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
|
||||||
|
# Ring-row tag fix: the tag names the logical position this row now
|
||||||
|
# holds, taken from the authoritative per-token metadata (not
|
||||||
|
# re-derived from chunk arithmetic), stored after key + tail.
|
||||||
|
# Every committed row gets one, with or without RoPE positions.
|
||||||
|
token_logical = tl.load(
|
||||||
|
logical_positions_ptr + tl.maximum(token, 0),
|
||||||
|
mask=valid_slot,
|
||||||
|
other=-1,
|
||||||
|
)
|
||||||
|
tl.store(
|
||||||
|
(state_row + D).to(tl.pointer_type(tl.int64)) + _RING_TAG_WORD,
|
||||||
|
token_logical.to(tl.int64) + 1,
|
||||||
|
mask=valid_slot,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def qsa_prepare(
|
def qsa_prepare(
|
||||||
@@ -530,11 +734,16 @@ def qsa_prepare(
|
|||||||
main_slot_mapping: torch.Tensor,
|
main_slot_mapping: torch.Tensor,
|
||||||
main_k_scale: float,
|
main_k_scale: float,
|
||||||
main_v_scale: float,
|
main_v_scale: float,
|
||||||
|
layer_id: int = -1,
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
"""Normalize Q, compress K, then update the circular raw state.
|
"""Normalize Q, compress K, then update the circular raw state.
|
||||||
|
|
||||||
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
|
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
|
||||||
and returns its Q and gate.
|
and returns its Q and gate.
|
||||||
|
|
||||||
|
``state_cache`` rows are ``[key D bf16 | 4 int64: pos_t, pos_h, pos_w, tag]``
|
||||||
|
(``QSAKeyStateCache``, ring-row tag fix); the tail is present with or without
|
||||||
|
``rope_pos_offset``. ``layer_id`` only labels the uncommitted-row record.
|
||||||
"""
|
"""
|
||||||
num_tokens = q.shape[0]
|
num_tokens = q.shape[0]
|
||||||
main_head_dim = main_kv_cache.shape[-1] // 2
|
main_head_dim = main_kv_cache.shape[-1] // 2
|
||||||
@@ -559,6 +768,15 @@ def qsa_prepare(
|
|||||||
is_k_mrope = bool(mrope_section)
|
is_k_mrope = bool(mrope_section)
|
||||||
cache_has_rope_pos = rope_pos_offset is not None
|
cache_has_rope_pos = rope_pos_offset is not None
|
||||||
assert rope_pos_offset is None or rope_pos_offset == head_dim
|
assert rope_pos_offset is None or rope_pos_offset == head_dim
|
||||||
|
# The kernel addresses the tail at +head_dim and the tag at word 3 of
|
||||||
|
# it. A ring without the four-word tail (the older D or D+12 layouts)
|
||||||
|
# would put the tag store into the NEXT row, so refuse it outright.
|
||||||
|
assert state_cache.shape[-1] == head_dim + QSA_RING_TAIL_ELEMENTS, (
|
||||||
|
f"QSA ring row width {state_cache.shape[-1]} != head_dim + "
|
||||||
|
f"{QSA_RING_TAIL_ELEMENTS} (key + pos_t/h/w + tag)"
|
||||||
|
)
|
||||||
|
assert state_cache.dtype == torch.bfloat16
|
||||||
|
assert state_cache.stride(1) % 4 == 0 and state_cache.stride(0) % 4 == 0
|
||||||
if is_2d_positions:
|
if is_2d_positions:
|
||||||
assert positions.shape == (3, num_tokens)
|
assert positions.shape == (3, num_tokens)
|
||||||
assert is_k_mrope
|
assert is_k_mrope
|
||||||
@@ -589,6 +807,8 @@ def qsa_prepare(
|
|||||||
pos_stride_axis,
|
pos_stride_axis,
|
||||||
pos_stride_token,
|
pos_stride_token,
|
||||||
cos_sin_cache,
|
cos_sin_cache,
|
||||||
|
cos_sin_cache.shape[0],
|
||||||
|
guard_telemetry.counter(guard_telemetry.SITE_QSA_ROPE_POS, q.device),
|
||||||
q_norm_weight,
|
q_norm_weight,
|
||||||
k_norm_weight,
|
k_norm_weight,
|
||||||
eps,
|
eps,
|
||||||
@@ -612,6 +832,11 @@ def qsa_prepare(
|
|||||||
state_cache.shape[0],
|
state_cache.shape[0],
|
||||||
compressed_cache.shape[0],
|
compressed_cache.shape[0],
|
||||||
num_k_work,
|
num_k_work,
|
||||||
|
min(query_start_loc.numel() - 1, state_block_table.shape[0]),
|
||||||
|
guard_telemetry.counter(guard_telemetry.SITE_QSA_PRE_INDEXER, q.device),
|
||||||
|
guard_telemetry.counter(guard_telemetry.SITE_QSA_RING_SKIP, q.device),
|
||||||
|
guard_telemetry.counter(guard_telemetry.SITE_QSA_RING_UNCOMMITTED, q.device),
|
||||||
|
int(layer_id),
|
||||||
HQ=num_q_heads,
|
HQ=num_q_heads,
|
||||||
D=head_dim,
|
D=head_dim,
|
||||||
TILE_T_Q=TILE_T_Q,
|
TILE_T_Q=TILE_T_Q,
|
||||||
|
|||||||
@@ -0,0 +1,208 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Python reader for the medium-path exact-restart top-k counters.
|
||||||
|
|
||||||
|
The counters themselves are device-global uint64s owned by
|
||||||
|
`csrc/libtorch_stable/topk.cu`; `torch.ops._C.persistent_topk_exact_counters`
|
||||||
|
snapshots them into an **int64** CUDA tensor. Without this module nothing in a
|
||||||
|
running engine ever calls that op, so the instrument records into a buffer no
|
||||||
|
one reads -- which measures nothing.
|
||||||
|
|
||||||
|
DTYPE IS PART OF THE PROTOCOL (large-path determinism change). Version 1 was u32
|
||||||
|
device counters read into a signed int32 tensor, which goes negative after
|
||||||
|
~33k all-equal rows at k=512 and wraps after ~66k, so a delta taken across the
|
||||||
|
boundary is a wrong number that looks like a measurement. Version 2 widens
|
||||||
|
both sides, and the op REJECTS an int32 buffer rather than filling it from half
|
||||||
|
of each counter. An older build whose op still demands int32 will therefore
|
||||||
|
raise here; that is the intended signal, not a regression to paper over.
|
||||||
|
|
||||||
|
SLOT LAYOUT is build-dependent, so nothing here hard-codes five. `slot_count()`
|
||||||
|
asks the build first -- the tie-determinism change added the `..._slots` op this was written to look
|
||||||
|
for, so on a build that has it the env variable is never consulted -- and
|
||||||
|
otherwise reads `MERLIN_TOPK_COUNTER_SLOTS` (default 5). The op demands
|
||||||
|
EXACTLY that many elements, so a guessed length is an error, not a short read;
|
||||||
|
names come from a table keyed by that length:
|
||||||
|
|
||||||
|
5 rows overflow fallbacks faults switch
|
||||||
|
10 rows overflow fallbacks faults rows_large ties_cut ties_dropped
|
||||||
|
faults_large switch switch_large
|
||||||
|
11 the ten above, plus decode_overflow
|
||||||
|
|
||||||
|
An unknown length degrades to `slot<i>` names rather than mislabelling.
|
||||||
|
|
||||||
|
WHY 11 AND NOT 10. The tie-determinism change numbered its one new counter around the large-path block
|
||||||
|
instead of after the medium-path one, so the integration renumbers nothing: slots 4-7 are
|
||||||
|
the large-path counters, 8-9 the two switch flags, 10 the decode overflow.
|
||||||
|
On the integrated build every one of the eleven is live; on a single-branch
|
||||||
|
build the other branch's slots are structurally zero and the line still
|
||||||
|
reports them by name -- they are the slots they will hold, not a
|
||||||
|
mislabelling. `decode_overflow` is the number that has to be read before any
|
||||||
|
claim that the DECODE path is deterministic: it counts rows whose buffered
|
||||||
|
candidates exceeded the 3,708-slot capacity before the tie ranking ran, which
|
||||||
|
ranking cannot repair.
|
||||||
|
|
||||||
|
HOW THE SMOKE READS IT. Set `MERLIN_TOPK_EXACT_COUNTERS_LOG=<seconds>` on the
|
||||||
|
engine; each GPU worker then emits, at most that often and once more at
|
||||||
|
shutdown, one line into the ordinary engine log:
|
||||||
|
|
||||||
|
topk-exact-counters rank=PP0_TP1 rows=... overflow=... fallbacks=...
|
||||||
|
faults=... switch=... [rows_large=... ... decode_overflow=...]
|
||||||
|
|
||||||
|
The first five fields are the medium-path line and keep their order on every build;
|
||||||
|
any further slot is appended after them, so an 11-slot build reads as the medium-path
|
||||||
|
line plus the four large-path fields, `switch_large` and `decode_overflow`.
|
||||||
|
|
||||||
|
Counts are cumulative per worker process, so the smoke takes the LATEST line
|
||||||
|
per rank -- `grep topk-exact-counters <engine log> | ...` keyed on the `rank=`
|
||||||
|
field -- and reads `faults` as the pass/fail number (it must stay 0) with
|
||||||
|
`rows` as the denominator. Earlier lines are only useful as a rate.
|
||||||
|
|
||||||
|
COST. `read()` passes `sync=1`, i.e. a full device synchronize, because a
|
||||||
|
stream-ordered snapshot can miss increments from work in flight on another
|
||||||
|
stream. That is acceptable ONLY because the cadence is coarse (>= 30 s in
|
||||||
|
practice): the time check happens before the op call, so a step that is not
|
||||||
|
due for a log touches nothing but `time.monotonic()`. The read is also refused
|
||||||
|
inside a cudagraph capture -- the op allocates and synchronizes, which would
|
||||||
|
abort the capture.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
_OP_NAME = "persistent_topk_exact_counters"
|
||||||
|
|
||||||
|
_MEDIUM_NAMES = ("rows", "overflow", "fallbacks", "faults", "switch")
|
||||||
|
_LARGEDET_NAMES = (
|
||||||
|
"rows",
|
||||||
|
"overflow",
|
||||||
|
"fallbacks",
|
||||||
|
"faults",
|
||||||
|
"rows_large",
|
||||||
|
"ties_cut",
|
||||||
|
"ties_dropped",
|
||||||
|
"faults_large",
|
||||||
|
"switch",
|
||||||
|
"switch_large",
|
||||||
|
)
|
||||||
|
# The decode-overflow counter comes after the large-path block; see the module
|
||||||
|
# docstring for why it is slot 10 and not slot 4.
|
||||||
|
_TIEDET_NAMES = _LARGEDET_NAMES + ("decode_overflow",)
|
||||||
|
_SLOT_NAMES: dict[int, tuple[str, ...]] = {
|
||||||
|
5: _MEDIUM_NAMES,
|
||||||
|
10: _LARGEDET_NAMES,
|
||||||
|
11: _TIEDET_NAMES,
|
||||||
|
}
|
||||||
|
|
||||||
|
# The medium-path line contract. Any further name is appended after these, in slot
|
||||||
|
# order, so a 10-slot build still starts with exactly the medium-path line.
|
||||||
|
_LINE_ORDER = _MEDIUM_NAMES
|
||||||
|
|
||||||
|
_warned_absent = False
|
||||||
|
_warned_failed = False
|
||||||
|
_last_log_monotonic = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _get_op(name: str):
|
||||||
|
try:
|
||||||
|
return getattr(torch.ops._C, name, None)
|
||||||
|
except (AttributeError, RuntimeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def slot_count() -> int:
|
||||||
|
"""How many int64 slots this build's counter array carries.
|
||||||
|
|
||||||
|
The build is asked first and the environment is only a fallback for builds
|
||||||
|
older than the slot-count op: a guessed length that does not match the extension is
|
||||||
|
rejected by the op's own length check, so the instrument would go dark
|
||||||
|
exactly when the layout changed -- which is when it matters most.
|
||||||
|
"""
|
||||||
|
query = _get_op(_OP_NAME + "_slots")
|
||||||
|
if query is not None:
|
||||||
|
return int(query())
|
||||||
|
return int(os.getenv("MERLIN_TOPK_COUNTER_SLOTS", "5"))
|
||||||
|
|
||||||
|
|
||||||
|
def slot_names(count: int) -> tuple[str, ...]:
|
||||||
|
names = _SLOT_NAMES.get(count)
|
||||||
|
return names if names is not None else tuple(f"slot{i}" for i in range(count))
|
||||||
|
|
||||||
|
|
||||||
|
def read(reset: bool = False, sync: bool = True) -> dict[str, int]:
|
||||||
|
"""Snapshot the counters as {slot name: value}; {} when unreadable."""
|
||||||
|
global _warned_absent
|
||||||
|
op = _get_op(_OP_NAME)
|
||||||
|
if op is None:
|
||||||
|
if not _warned_absent:
|
||||||
|
_warned_absent = True
|
||||||
|
logger.warning(
|
||||||
|
"torch.ops._C.%s is not in this build; the top-k exact-restart "
|
||||||
|
"counters cannot be read.",
|
||||||
|
_OP_NAME,
|
||||||
|
)
|
||||||
|
return {}
|
||||||
|
if reset and not sync:
|
||||||
|
raise ValueError("reset requires sync: zeroing races in-flight atomics")
|
||||||
|
if not (torch.cuda.is_available() and torch.cuda.is_initialized()):
|
||||||
|
return {}
|
||||||
|
if torch.cuda.is_current_stream_capturing():
|
||||||
|
# The op allocates and synchronizes; calling it here aborts the capture.
|
||||||
|
return {}
|
||||||
|
count = slot_count()
|
||||||
|
device = torch.device("cuda", torch.cuda.current_device())
|
||||||
|
# int64: the device counters are 64-bit and the op refuses any other
|
||||||
|
# width. See the module docstring -- this is protocol version 2.
|
||||||
|
out = torch.zeros(count, dtype=torch.int64, device=device)
|
||||||
|
op(out, 1 if reset else 0, 1 if sync else 0)
|
||||||
|
return dict(zip(slot_names(count), out.tolist(), strict=True))
|
||||||
|
|
||||||
|
|
||||||
|
def rank_tag() -> str:
|
||||||
|
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||||
|
|
||||||
|
try:
|
||||||
|
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
|
||||||
|
except Exception:
|
||||||
|
# Pre-init or single-process: say so rather than claim rank 0.
|
||||||
|
return "PP?_TP?"
|
||||||
|
|
||||||
|
|
||||||
|
def log_line(tag: str) -> str:
|
||||||
|
"""The one-line form the smoke greps; "" when the counters are unreadable."""
|
||||||
|
counters = read()
|
||||||
|
if not counters:
|
||||||
|
return ""
|
||||||
|
order = [name for name in _LINE_ORDER if name in counters]
|
||||||
|
order += [name for name in counters if name not in _LINE_ORDER]
|
||||||
|
fields = " ".join(f"{name}={counters[name]}" for name in order)
|
||||||
|
return f"topk-exact-counters rank={tag} {fields}"
|
||||||
|
|
||||||
|
|
||||||
|
def maybe_log(force: bool = False) -> str:
|
||||||
|
"""Emit the line if due (or forced). Time check first: when not due this
|
||||||
|
costs one clock read and never touches the device."""
|
||||||
|
global _last_log_monotonic, _warned_failed
|
||||||
|
period = envs.MERLIN_TOPK_EXACT_COUNTERS_LOG
|
||||||
|
if period <= 0:
|
||||||
|
return ""
|
||||||
|
now = time.monotonic()
|
||||||
|
if not force and now - _last_log_monotonic < period:
|
||||||
|
return ""
|
||||||
|
_last_log_monotonic = now
|
||||||
|
try:
|
||||||
|
line = log_line(rank_tag())
|
||||||
|
except Exception as exc: # never let an instrument break a step or shutdown
|
||||||
|
if not _warned_failed:
|
||||||
|
_warned_failed = True
|
||||||
|
logger.warning("top-k exact-restart counter read failed: %s", exc)
|
||||||
|
return ""
|
||||||
|
if line:
|
||||||
|
logger.info("%s", line)
|
||||||
|
return line
|
||||||
@@ -8,6 +8,7 @@ import torch
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import Qwen4ExpTextConfig
|
from transformers import Qwen4ExpTextConfig
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||||
from vllm.forward_context import get_forward_context
|
from vllm.forward_context import get_forward_context
|
||||||
@@ -18,6 +19,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
|
|||||||
MambaStateShapeCalculator,
|
MambaStateShapeCalculator,
|
||||||
is_conv_state_dim_first,
|
is_conv_state_dim_first,
|
||||||
)
|
)
|
||||||
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||||
from vllm.v1.attention.backends.short_conv_attn import (
|
from vllm.v1.attention.backends.short_conv_attn import (
|
||||||
PleShortConvAttentionBackend,
|
PleShortConvAttentionBackend,
|
||||||
@@ -91,16 +93,37 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
|||||||
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
|
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
|
||||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||||
self.activation = "silu"
|
self.activation = "silu"
|
||||||
self.ple_embedding = Qwen4ExpNGramEmbedding(
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
config,
|
# Fork: the E2 pull transport owns the table in a CPU-only
|
||||||
int(config.ple_embed_dim),
|
# offload process; upstream's device/UVA backends are never built.
|
||||||
self.ple_dense_layer_id,
|
# Pull transport with the compile-profile guard: no layer_name -- the pull module never routes
|
||||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
# through the n-gram custom ops. Its GPU placeholder dispatches
|
||||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
# through PleOffloadLayer.forward (the ple_offload_wait_copy op) and
|
||||||
prefix=f"{prefix}.ple_embedding",
|
# owns no table; the offload process calls forward_impl directly.
|
||||||
quant_config=quant_config,
|
from .ple_pull import Qwen4ExpNGramEmbeddingPull
|
||||||
params_dtype=model_config.dtype,
|
|
||||||
)
|
self.ple_embedding = Qwen4ExpNGramEmbeddingPull.build(
|
||||||
|
config,
|
||||||
|
int(config.ple_embed_dim),
|
||||||
|
self.ple_dense_layer_id,
|
||||||
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||||
|
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||||
|
prefix=f"{prefix}.ple_embedding",
|
||||||
|
quant_config=quant_config,
|
||||||
|
params_dtype=model_config.dtype,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.ple_embedding = Qwen4ExpNGramEmbedding(
|
||||||
|
config,
|
||||||
|
int(config.ple_embed_dim),
|
||||||
|
self.ple_dense_layer_id,
|
||||||
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||||
|
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||||
|
prefix=f"{prefix}.ple_embedding",
|
||||||
|
layer_name=prefix,
|
||||||
|
quant_config=quant_config,
|
||||||
|
params_dtype=model_config.dtype,
|
||||||
|
)
|
||||||
# The PLE cache is TP-replicated, so this merged projection is too.
|
# The PLE cache is TP-replicated, so this merged projection is too.
|
||||||
self.kv_proj = MergedColumnParallelLinear(
|
self.kv_proj = MergedColumnParallelLinear(
|
||||||
int(config.ple_embed_dim),
|
int(config.ple_embed_dim),
|
||||||
@@ -144,6 +167,11 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
|||||||
output_dtype: torch.dtype,
|
output_dtype: torch.dtype,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""Dequantize PLE lookup output."""
|
"""Dequantize PLE lookup output."""
|
||||||
|
|
||||||
|
dequantize_pulled = getattr(self.ple_embedding, "dequantize_pulled", None)
|
||||||
|
if dequantize_pulled is not None:
|
||||||
|
# Fork pull transport: no local table; FP8 scale kept here.
|
||||||
|
return dequantize_pulled(embeddings, output_dtype)
|
||||||
return self.ple_embedding.ngram_embedding.dequantize(
|
return self.ple_embedding.ngram_embedding.dequantize(
|
||||||
embeddings,
|
embeddings,
|
||||||
output_dtype,
|
output_dtype,
|
||||||
@@ -173,6 +201,14 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
def get_attn_backend(self) -> type[PleShortConvAttentionBackend]:
|
def get_attn_backend(self) -> type[PleShortConvAttentionBackend]:
|
||||||
|
from vllm.model_executor.layers.mamba.recoverssm_utils import uses_recoverssm
|
||||||
|
|
||||||
|
if uses_recoverssm(self.cache_config, self.num_spec_tokens):
|
||||||
|
from vllm.v1.attention.backends.ple_recoverssm import (
|
||||||
|
PleRecoverSSMAttentionBackend,
|
||||||
|
)
|
||||||
|
|
||||||
|
return PleRecoverSSMAttentionBackend
|
||||||
return PleShortConvAttentionBackend
|
return PleShortConvAttentionBackend
|
||||||
|
|
||||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||||
@@ -421,11 +457,44 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
|||||||
self.norm_key.eps,
|
self.norm_key.eps,
|
||||||
)
|
)
|
||||||
# State routing depends on runtime request metadata and remains outside
|
# State routing depends on runtime request metadata and remains outside
|
||||||
# the graph; short convolution accumulates into gated_output.
|
# the graph; short convolution accumulates into gated_output. Compile-profile guard:
|
||||||
self._short_conv(conv_input, gated_output, hidden_states)
|
# a splitting op again (as on v2.0.1), so VLLM_COMPILE never traces it.
|
||||||
|
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||||
|
conv_input, gated_output, hidden_states, self.prefix
|
||||||
|
)
|
||||||
return gated_output
|
return gated_output
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_ple_short_conv(
|
||||||
|
inputs: torch.Tensor,
|
||||||
|
residual_output: torch.Tensor,
|
||||||
|
outer_residual: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
"""PLE short convolution + state routing, outside piecewise graphs."""
|
||||||
|
layer = get_forward_context().no_compile_layers[layer_name]
|
||||||
|
if not isinstance(layer, Qwen4ExpPLELayer):
|
||||||
|
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE layer")
|
||||||
|
layer._short_conv(inputs, residual_output, outer_residual)
|
||||||
|
|
||||||
|
|
||||||
|
def _qwen4_exp_ple_short_conv_fake(
|
||||||
|
inputs: torch.Tensor,
|
||||||
|
residual_output: torch.Tensor,
|
||||||
|
outer_residual: torch.Tensor,
|
||||||
|
layer_name: str,
|
||||||
|
) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="qwen4_exp_ple_short_conv",
|
||||||
|
op_func=qwen4_exp_ple_short_conv,
|
||||||
|
mutates_args=["residual_output"],
|
||||||
|
fake_impl=_qwen4_exp_ple_short_conv_fake,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Qwen4ExpPLEGroupedNorm",
|
"Qwen4ExpPLEGroupedNorm",
|
||||||
"Qwen4ExpPLELayer",
|
"Qwen4ExpPLELayer",
|
||||||
|
|||||||
@@ -0,0 +1,343 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""The fork's PLE pull transport, re-seated on vLLM v0.30.0.
|
||||||
|
|
||||||
|
v0.30.0 split the E2 base's offload-aware ``Qwen4ExpNGramEmbedding`` into an
|
||||||
|
upstream-only module (``ngram_embedding.py``, device or pinned-host UVA table).
|
||||||
|
This module puts the E2 pull transport back WITHOUT editing that upstream
|
||||||
|
class: ``Qwen4ExpNGramEmbeddingPull`` inherits it and ``PleOffloadLayer``.
|
||||||
|
|
||||||
|
* In a GPU worker, ``PleOffloadLayer.__init_subclass__`` skips the constructor:
|
||||||
|
the module is an empty placeholder with no table. ``PleOffloadConnector``
|
||||||
|
binds it to a host-mapped flag/buffer; its forward is the captured
|
||||||
|
``ple_offload_wait_copy`` op. Only the FP8 global scale is kept on the GPU.
|
||||||
|
* In the CPU-only offload process the constructor runs in full, on CPU. The
|
||||||
|
table is upstream's ``Qwen4ExpPLEDeviceEmbedding`` allocated on CPU;
|
||||||
|
``forward_impl`` computes n-gram IDs and gathers raw (FP8) rows.
|
||||||
|
|
||||||
|
Selected when ``VLLM_E2_PLE_PULL_TRANSPORT`` is ``1`` or unset (the fork default
|
||||||
|
since v2.5); with ``=0`` this module is never imported by the model and the
|
||||||
|
tree is upstream.
|
||||||
|
|
||||||
|
Supported table formats: FP8 (global scale) and unquantized -- the two
|
||||||
|
``Qwen4ExpPLEEmbeddingMethod`` families v0.30.0 itself supports. The E2 NVFP4
|
||||||
|
path is not carried (v0.30.0 has no NVFP4 PLE method to select).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.config import get_current_vllm_config
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.model_executor.layers.ple_offload_layer import (
|
||||||
|
PleOffloadLayer,
|
||||||
|
is_offload_process,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||||
|
from transformers import Qwen4ExpTextConfig
|
||||||
|
|
||||||
|
from .ngram_embedding import (
|
||||||
|
Qwen4ExpNGramEmbedding,
|
||||||
|
Qwen4ExpPLEEmbeddingMethod,
|
||||||
|
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||||
|
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
#: The backend name this module reports in the PLE producer log line.
|
||||||
|
PULL_BACKEND_NAME = "E2PullTransport"
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen4ExpNGramEmbeddingPull(PleOffloadLayer, Qwen4ExpNGramEmbedding):
|
||||||
|
"""Upstream n-gram embedding whose table lives in the offload process."""
|
||||||
|
|
||||||
|
#: GPU placeholder: the method the offload process will use for the table,
|
||||||
|
#: resolved with upstream's own selector (set by ``build``).
|
||||||
|
_offload_quant_method: Qwen4ExpPLEEmbeddingMethod | None = None
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: Qwen4ExpTextConfig,
|
||||||
|
embedding_dim: int,
|
||||||
|
ple_dense_layer_id: int,
|
||||||
|
max_total_tokens: int,
|
||||||
|
*,
|
||||||
|
data_parallel_rank: int,
|
||||||
|
prefix: str,
|
||||||
|
quant_config: QuantizationConfig | None = None,
|
||||||
|
params_dtype: torch.dtype | None = None,
|
||||||
|
) -> None:
|
||||||
|
# Runs only in the offload process (guarded by PleOffloadLayer).
|
||||||
|
engram_config = get_current_vllm_config().engram_config
|
||||||
|
if engram_config is not None and engram_config.cpu_offload:
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE pull transport: the offload process resolved "
|
||||||
|
"cpu_offload=True; refusing to build upstream's UVA table here."
|
||||||
|
)
|
||||||
|
Qwen4ExpNGramEmbedding.__init__(
|
||||||
|
self,
|
||||||
|
config,
|
||||||
|
embedding_dim,
|
||||||
|
ple_dense_layer_id,
|
||||||
|
max_total_tokens,
|
||||||
|
data_parallel_rank=data_parallel_rank,
|
||||||
|
prefix=prefix,
|
||||||
|
quant_config=quant_config,
|
||||||
|
params_dtype=params_dtype,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def build(
|
||||||
|
cls,
|
||||||
|
config: Qwen4ExpTextConfig,
|
||||||
|
embedding_dim: int,
|
||||||
|
ple_dense_layer_id: int,
|
||||||
|
max_total_tokens: int,
|
||||||
|
*,
|
||||||
|
data_parallel_rank: int,
|
||||||
|
prefix: str,
|
||||||
|
quant_config: QuantizationConfig | None,
|
||||||
|
params_dtype: torch.dtype | None,
|
||||||
|
) -> "Qwen4ExpNGramEmbeddingPull":
|
||||||
|
"""Construct the placeholder (GPU) or the full CPU module (offload)."""
|
||||||
|
# The offload process builds the surrounding model on meta; this
|
||||||
|
# subtree must own real CPU storage. In a GPU worker the constructor is
|
||||||
|
# skipped, so the device context allocates nothing.
|
||||||
|
with torch.device(cls.get_target_device()):
|
||||||
|
module = cls(
|
||||||
|
config,
|
||||||
|
embedding_dim,
|
||||||
|
ple_dense_layer_id,
|
||||||
|
max_total_tokens,
|
||||||
|
data_parallel_rank=data_parallel_rank,
|
||||||
|
prefix=prefix,
|
||||||
|
quant_config=quant_config,
|
||||||
|
params_dtype=params_dtype,
|
||||||
|
)
|
||||||
|
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||||
|
method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||||
|
quant_config,
|
||||||
|
embedding_prefix,
|
||||||
|
getattr(config, "ple_embedding_dtype", None),
|
||||||
|
)
|
||||||
|
if not isinstance(
|
||||||
|
method,
|
||||||
|
(Qwen4ExpPLEFp8EmbeddingMethod, Qwen4ExpPLEUnquantizedEmbeddingMethod),
|
||||||
|
):
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"PLE pull transport does not support {type(method).__name__}"
|
||||||
|
)
|
||||||
|
module._offload_quant_method = method
|
||||||
|
module._pull_params_dtype = params_dtype or torch.get_default_dtype()
|
||||||
|
module._pull_embedding_prefix = embedding_prefix
|
||||||
|
if not is_offload_process():
|
||||||
|
# The GPU-rank producer line, in upstream's format plus backend=
|
||||||
|
# (the offload process logs its own table line from upstream's
|
||||||
|
# constructor). The table lives in the offload process's CPU memory
|
||||||
|
# and is not pinned; rows cross via host-mapped buffers.
|
||||||
|
logger.info(
|
||||||
|
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||||
|
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
|
||||||
|
embedding_prefix,
|
||||||
|
type(method).__name__,
|
||||||
|
module.get_offload_output_dtype(module._pull_params_dtype),
|
||||||
|
"cpu",
|
||||||
|
False,
|
||||||
|
PULL_BACKEND_NAME,
|
||||||
|
)
|
||||||
|
return module
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- formats
|
||||||
|
|
||||||
|
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
|
||||||
|
"""Rows cross the transport in their table storage dtype."""
|
||||||
|
embedding = getattr(self, "ngram_embedding", None)
|
||||||
|
weight = getattr(embedding, "weight", None)
|
||||||
|
if weight is not None:
|
||||||
|
return weight.dtype
|
||||||
|
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
|
||||||
|
return torch.float8_e4m3fn
|
||||||
|
return getattr(self, "_pull_params_dtype", default_dtype)
|
||||||
|
|
||||||
|
def get_offload_output_dim(self, default_dim: int) -> int:
|
||||||
|
return default_dim
|
||||||
|
|
||||||
|
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
|
||||||
|
"""Dummy loads skip checkpoint scales; give FP8 a unit scale."""
|
||||||
|
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
|
||||||
|
self.register_buffer(
|
||||||
|
"_offload_weight_scale",
|
||||||
|
torch.ones((), dtype=torch.float32, device=device),
|
||||||
|
persistent=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def dequantize_pulled(
|
||||||
|
self, embeddings: torch.Tensor, output_dtype: torch.dtype
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""GPU-side dequantization of rows DMA'd from the offload process.
|
||||||
|
|
||||||
|
Same arithmetic as upstream ``Qwen4ExpPLEFp8EmbeddingMethod.dequantize``
|
||||||
|
(and the E2 ``_dequantize_embeddings`` FP8 branch).
|
||||||
|
"""
|
||||||
|
method = self._offload_quant_method
|
||||||
|
if isinstance(method, Qwen4ExpPLEUnquantizedEmbeddingMethod):
|
||||||
|
return embeddings
|
||||||
|
weight_scale = getattr(self, "_offload_weight_scale", None)
|
||||||
|
if weight_scale is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"FP8 PLE pull transport is missing its global scale on the GPU"
|
||||||
|
)
|
||||||
|
if weight_scale.device != embeddings.device:
|
||||||
|
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
|
||||||
|
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- execution
|
||||||
|
|
||||||
|
def start_prefetch(
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
) -> None:
|
||||||
|
"""No prefetch: rows are published by the offload process."""
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _compute_ngram_ids_offload(
|
||||||
|
self,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""CPU n-gram IDs, carried unchanged from the E2 offload path.
|
||||||
|
|
||||||
|
Differs from upstream's CPU branch in two E2 fixes: the packed layout
|
||||||
|
is (num_reqs, max_seq_len) rather than (num_reqs, num_tokens), and
|
||||||
|
CUDA-graph padding rows beyond ``query_start_loc[-1]`` are not written,
|
||||||
|
so they cannot overwrite a request's final real token.
|
||||||
|
"""
|
||||||
|
input_ids = input_ids.reshape(-1).long()
|
||||||
|
query_start_loc = query_start_loc.long()
|
||||||
|
num_reqs = query_start_loc.numel() - 1
|
||||||
|
num_tokens = input_ids.shape[0]
|
||||||
|
if num_reqs <= 0:
|
||||||
|
raise ValueError("PLE requires at least one request")
|
||||||
|
max_seq_len = max(
|
||||||
|
1,
|
||||||
|
int((query_start_loc[1:] - query_start_loc[:-1]).max().item()),
|
||||||
|
)
|
||||||
|
num_valid_tokens = min(int(query_start_loc[-1].item()), num_tokens)
|
||||||
|
|
||||||
|
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
|
||||||
|
packed = torch.full(
|
||||||
|
(num_reqs, max_seq_len),
|
||||||
|
self.eos_token_id,
|
||||||
|
device=input_ids.device,
|
||||||
|
dtype=torch.int64,
|
||||||
|
)
|
||||||
|
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
|
||||||
|
request_indices.clamp_(max=num_reqs - 1)
|
||||||
|
columns = (positions - query_start_loc[request_indices]).clamp(
|
||||||
|
0, packed.shape[1] - 1
|
||||||
|
)
|
||||||
|
packed[request_indices[:num_valid_tokens], columns[:num_valid_tokens]] = (
|
||||||
|
input_ids[:num_valid_tokens]
|
||||||
|
)
|
||||||
|
ngram_context = ngram_context[:num_reqs].to(
|
||||||
|
device=input_ids.device, dtype=torch.long
|
||||||
|
)
|
||||||
|
|
||||||
|
context = torch.cat([ngram_context, packed], dim=-1)
|
||||||
|
positions_2d, position_in_segment = self._shift_precompute(
|
||||||
|
context, self.eos_token_id
|
||||||
|
)
|
||||||
|
shifted = [context]
|
||||||
|
for shift in range(1, self.ngram_size):
|
||||||
|
shifted.append(
|
||||||
|
self._shift_apply(
|
||||||
|
context,
|
||||||
|
positions_2d,
|
||||||
|
position_in_segment,
|
||||||
|
shift,
|
||||||
|
self.eos_token_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
adjusted_columns = columns + self.ngram_size - 1
|
||||||
|
id_blocks = []
|
||||||
|
for ngram in range(2, self.ngram_size + 1):
|
||||||
|
start = (ngram - 2) * self.heads_per_ngram
|
||||||
|
end = start + self.heads_per_ngram
|
||||||
|
mixed = shifted[0] * self.layer_multipliers[0]
|
||||||
|
for index in range(1, ngram):
|
||||||
|
mixed = torch.bitwise_xor(
|
||||||
|
mixed, shifted[index] * self.layer_multipliers[index]
|
||||||
|
)
|
||||||
|
sizes = self.ngram_heads_vocab_sizes[start:end]
|
||||||
|
offsets = self.ngram_heads_offsets[start:end]
|
||||||
|
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
|
||||||
|
id_blocks.append(ids[request_indices, adjusted_columns])
|
||||||
|
return torch.cat(id_blocks, dim=-1)
|
||||||
|
|
||||||
|
def forward_impl( # type: ignore[override]
|
||||||
|
self,
|
||||||
|
hidden_states: torch.Tensor,
|
||||||
|
input_ids: torch.Tensor,
|
||||||
|
query_start_loc: torch.Tensor,
|
||||||
|
ngram_context: torch.Tensor,
|
||||||
|
output_buffer: torch.Tensor | None = None,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Offload process only: gather raw table rows for these tokens."""
|
||||||
|
del hidden_states
|
||||||
|
if not is_offload_process():
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE pull transport: forward_impl ran outside the offload "
|
||||||
|
"process -- the GPU placeholder was never bound by the connector"
|
||||||
|
)
|
||||||
|
ngram_ids = self._compute_ngram_ids_offload(
|
||||||
|
input_ids, query_start_loc, ngram_context
|
||||||
|
)
|
||||||
|
num_tokens = input_ids.reshape(-1).shape[0]
|
||||||
|
weight = self.ngram_embedding.weight
|
||||||
|
if output_buffer is not None:
|
||||||
|
output = output_buffer[:num_tokens, : self.embedding_dim]
|
||||||
|
torch.index_select(
|
||||||
|
weight,
|
||||||
|
0,
|
||||||
|
ngram_ids.reshape(-1),
|
||||||
|
out=output.reshape(-1, self.head_dim),
|
||||||
|
)
|
||||||
|
return output
|
||||||
|
return torch.index_select(weight, 0, ngram_ids.reshape(-1)).reshape(
|
||||||
|
num_tokens, self.embedding_dim
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- loading
|
||||||
|
|
||||||
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
|
"""Offload process: upstream loading. GPU worker: keep the FP8 scale.
|
||||||
|
|
||||||
|
The GPU placeholder owns no table; every other checkpoint tensor for
|
||||||
|
this subtree is consumed and dropped here (the offload process loads
|
||||||
|
the table itself). Prefix grouping may call this more than once.
|
||||||
|
"""
|
||||||
|
if is_offload_process():
|
||||||
|
return Qwen4ExpNGramEmbedding.load_weights(self, weights)
|
||||||
|
retained: set[str] = set()
|
||||||
|
fp8 = isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod)
|
||||||
|
for name, loaded_weight in weights:
|
||||||
|
if fp8 and name == "ngram_embedding.weight_scale":
|
||||||
|
self.register_buffer(
|
||||||
|
"_offload_weight_scale",
|
||||||
|
loaded_weight.reshape(()).to(
|
||||||
|
device=torch.accelerator.current_accelerator(),
|
||||||
|
dtype=torch.float32,
|
||||||
|
),
|
||||||
|
persistent=False,
|
||||||
|
)
|
||||||
|
retained.add(name)
|
||||||
|
return retained
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["PULL_BACKEND_NAME", "Qwen4ExpNGramEmbeddingPull"]
|
||||||
@@ -4,7 +4,14 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Callable, Iterable
|
import atexit
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections.abc import Collection, Mapping
|
||||||
|
from pathlib import Path
|
||||||
from typing import ClassVar, cast
|
from typing import ClassVar, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -16,6 +23,7 @@ from vllm.config import VllmConfig
|
|||||||
from vllm.config.cache import CacheDType
|
from vllm.config.cache import CacheDType
|
||||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||||
from vllm.forward_context import get_forward_context
|
from vllm.forward_context import get_forward_context
|
||||||
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention.attention import (
|
from vllm.model_executor.layers.attention.attention import (
|
||||||
set_default_quant_scales,
|
set_default_quant_scales,
|
||||||
)
|
)
|
||||||
@@ -33,6 +41,10 @@ from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.platforms.interface import DeviceCapability
|
from vllm.platforms.interface import DeviceCapability
|
||||||
from vllm.utils.torch_utils import (
|
from vllm.utils.torch_utils import (
|
||||||
|
LayerNameType,
|
||||||
|
_encode_layer_name,
|
||||||
|
_resolve_layer_name,
|
||||||
|
direct_register_custom_op,
|
||||||
kv_cache_dtype_str_to_dtype,
|
kv_cache_dtype_str_to_dtype,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backend import (
|
from vllm.v1.attention.backend import (
|
||||||
@@ -58,6 +70,260 @@ from ..common.qsa_cache import QSAForwardMetadata
|
|||||||
from . import model
|
from . import model
|
||||||
from .indexer_qsa import QSAIndexer
|
from .indexer_qsa import QSAIndexer
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
# --- FP8 K/V scale calibration collector -----------------------
|
||||||
|
# Active ONLY when VLLM_QSA_KV_COLLECT names a writable directory. Records a
|
||||||
|
# GPU-side running absmax of the pre-quantization BF16 K/V per layer and dumps
|
||||||
|
# per-rank JSON (periodically + at exit) for an offline max-merge into the
|
||||||
|
# VLLM_QSA_KV_SCALES sidecar. Calibration must run with --enforce-eager: the
|
||||||
|
# collector mutates module-external state per step, which cudagraph capture
|
||||||
|
# would freeze and torch.compile may not retrace. With the env unset this is a
|
||||||
|
# constant-false branch and prod behaviour is unchanged.
|
||||||
|
_QSA_KV_COLLECT_ENV = "VLLM_QSA_KV_COLLECT"
|
||||||
|
_qsa_collect_dir = os.environ.get(_QSA_KV_COLLECT_ENV, "").strip()
|
||||||
|
_qsa_collect_state: dict[str, list[torch.Tensor]] = {}
|
||||||
|
_qsa_collect_calls = 0
|
||||||
|
|
||||||
|
|
||||||
|
_qsa_collect_ident: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_collect_identity() -> str:
|
||||||
|
"""Stable per-process dump name: (pp, tp) rank, resolved once and cached so
|
||||||
|
a teardown-time lookup failure cannot move the final flush to a PID-named
|
||||||
|
file. TP-only runs keep the historical name."""
|
||||||
|
global _qsa_collect_ident
|
||||||
|
if _qsa_collect_ident is not None:
|
||||||
|
return _qsa_collect_ident
|
||||||
|
try:
|
||||||
|
from vllm.distributed import get_pp_group, get_tensor_model_parallel_rank
|
||||||
|
|
||||||
|
rank = get_tensor_model_parallel_rank()
|
||||||
|
pp_group = get_pp_group()
|
||||||
|
pp_tag = f"pp{pp_group.rank_in_group}_" if pp_group.world_size > 1 else ""
|
||||||
|
_qsa_collect_ident = f"{pp_tag}rank{rank}"
|
||||||
|
except Exception:
|
||||||
|
return f"pid{os.getpid()}" # not cached: retry once the groups exist
|
||||||
|
return _qsa_collect_ident
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_collect_dump() -> None:
|
||||||
|
if not _qsa_collect_state:
|
||||||
|
return
|
||||||
|
out = {
|
||||||
|
name: {"k_absmax": float(st[0].item()), "v_absmax": float(st[1].item())}
|
||||||
|
for name, st in _qsa_collect_state.items()
|
||||||
|
}
|
||||||
|
path = Path(_qsa_collect_dir) / f"qsa_absmax_{_qsa_collect_identity()}.json"
|
||||||
|
tmp = path.with_suffix(".tmp")
|
||||||
|
tmp.write_text(json.dumps(out, indent=1), encoding="utf-8")
|
||||||
|
tmp.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_collect_absmax(
|
||||||
|
layer_name: str, key: torch.Tensor, value: torch.Tensor
|
||||||
|
) -> None:
|
||||||
|
global _qsa_collect_calls
|
||||||
|
k_abs = key.detach().abs().amax().float()
|
||||||
|
v_abs = value.detach().abs().amax().float()
|
||||||
|
st = _qsa_collect_state.get(layer_name)
|
||||||
|
if st is None:
|
||||||
|
if not _qsa_collect_state:
|
||||||
|
atexit.register(_qsa_collect_dump)
|
||||||
|
_qsa_collect_state[layer_name] = [k_abs, v_abs]
|
||||||
|
else:
|
||||||
|
torch.maximum(st[0], k_abs, out=st[0])
|
||||||
|
torch.maximum(st[1], v_abs, out=st[1])
|
||||||
|
_qsa_collect_calls += 1
|
||||||
|
if _qsa_collect_calls % 2000 == 0:
|
||||||
|
_qsa_collect_dump()
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# --- FP8 K/V runtime clipping counter --------------------------
|
||||||
|
# Under-coverage guard for the calibrated scales sidecar: counts K/V elements
|
||||||
|
# that would saturate E4M3 after scaling (|x| > 448 * scale), i.e. inputs that
|
||||||
|
# exceed the calibration-time absmax. Active ONLY when VLLM_QSA_KV_CLIP_COUNT
|
||||||
|
# is set to a positive read interval in seconds.
|
||||||
|
#
|
||||||
|
# Graph-safety: unlike the calibration collector above, this MUST work under
|
||||||
|
# FULL cudagraph replay, where the Python forward does not run per step. The
|
||||||
|
# counters are persistent per-layer GPU int64 tensors allocated on the first
|
||||||
|
# (eager, pre-capture) forward; the increment is pure captured GPU ops, so
|
||||||
|
# graph replay keeps counting. A daemon reader thread .item()s the counters on
|
||||||
|
# a cadence and logs: WARNING on any new clips, INFO heartbeat every 10th pass
|
||||||
|
# so a quiet log still proves the instrument is alive.
|
||||||
|
#
|
||||||
|
# Since v2.5 the reader thread is started AFTER warmup and cudagraph capture
|
||||||
|
# (guard_telemetry.register_post_capture_hook, run by start_reader() from
|
||||||
|
# gpu_worker), never from the first eager forward. Before, it started inside the
|
||||||
|
# first forward, i.e. before capture, and its device-to-host copies could run on
|
||||||
|
# a second thread while a capture was active -- the call class
|
||||||
|
# cudaStreamCaptureModeGlobal rejects. That is why COUNTERS shipped off.
|
||||||
|
_QSA_CLIP_ENV = "VLLM_QSA_KV_CLIP_COUNT"
|
||||||
|
_qsa_clip_raw = os.environ.get(_QSA_CLIP_ENV, "").strip()
|
||||||
|
try:
|
||||||
|
_qsa_clip_interval = float(_qsa_clip_raw) if _qsa_clip_raw else 0.0
|
||||||
|
except ValueError as exc: # fail closed: a set-but-broken gate must not be silent
|
||||||
|
raise ValueError(
|
||||||
|
f"{_QSA_CLIP_ENV} must be a read interval in seconds, got {_qsa_clip_raw!r}"
|
||||||
|
) from exc
|
||||||
|
if _qsa_clip_raw and not (math.isfinite(_qsa_clip_interval) and _qsa_clip_interval > 0.0):
|
||||||
|
raise ValueError(
|
||||||
|
f"{_QSA_CLIP_ENV} must be a finite positive number of seconds, got {_qsa_clip_raw!r}"
|
||||||
|
)
|
||||||
|
_qsa_clip_counters: dict[str, torch.Tensor] = {}
|
||||||
|
# v2 reader fix: the reader must never block on the compute
|
||||||
|
# stream; see guard_telemetry.NonBlockingReader.
|
||||||
|
from vllm.v1.worker.guard_telemetry import ( # noqa: E402
|
||||||
|
NonBlockingReader as _NonBlockingReader,
|
||||||
|
)
|
||||||
|
|
||||||
|
_qsa_clip_nb = _NonBlockingReader()
|
||||||
|
_qsa_clip_reader_started = False
|
||||||
|
_qsa_clip_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_clip_reader() -> None:
|
||||||
|
last: dict[str, tuple[int, int]] = {}
|
||||||
|
passes = 0
|
||||||
|
while True:
|
||||||
|
time.sleep(_qsa_clip_interval)
|
||||||
|
passes += 1
|
||||||
|
try:
|
||||||
|
totals_k = totals_v = 0
|
||||||
|
for name, ctr in list(_qsa_clip_counters.items()):
|
||||||
|
if not _qsa_clip_nb.registered(name):
|
||||||
|
_qsa_clip_nb.register(name, ctr) # reader thread: never capturing
|
||||||
|
ready = _qsa_clip_nb.poll()
|
||||||
|
for name, ctr in list(_qsa_clip_counters.items()):
|
||||||
|
row = ready.get(name)
|
||||||
|
if row is None:
|
||||||
|
# copy still in flight: report on the next pass
|
||||||
|
prev_k, prev_v = last.get(name, (0, 0))
|
||||||
|
totals_k += prev_k
|
||||||
|
totals_v += prev_v
|
||||||
|
continue
|
||||||
|
k_total = int(row[0])
|
||||||
|
v_total = int(row[1])
|
||||||
|
totals_k += k_total
|
||||||
|
totals_v += v_total
|
||||||
|
prev_k, prev_v = last.get(name, (0, 0))
|
||||||
|
if k_total > prev_k or v_total > prev_v:
|
||||||
|
logger.warning(
|
||||||
|
"QSA KV CLIPPING on %s: k +%d (total %d), v +%d (total %d)"
|
||||||
|
" -- inputs exceeded the calibration absmax; the scales"
|
||||||
|
" sidecar may need re-calibration",
|
||||||
|
name,
|
||||||
|
k_total - prev_k,
|
||||||
|
k_total,
|
||||||
|
v_total - prev_v,
|
||||||
|
v_total,
|
||||||
|
)
|
||||||
|
last[name] = (k_total, v_total)
|
||||||
|
if passes % 10 == 1:
|
||||||
|
logger.info(
|
||||||
|
"QSA clip counters alive: %d layer(s), totals k=%d v=%d",
|
||||||
|
len(_qsa_clip_counters),
|
||||||
|
totals_k,
|
||||||
|
totals_v,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("QSA clip reader pass failed; thread continues")
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_clip_start_reader() -> None:
|
||||||
|
"""Start the reader thread. Runs after warmup + capture (post-capture hook); idempotent."""
|
||||||
|
global _qsa_clip_reader_started
|
||||||
|
with _qsa_clip_lock:
|
||||||
|
if _qsa_clip_reader_started:
|
||||||
|
return
|
||||||
|
_qsa_clip_reader_started = True
|
||||||
|
threading.Thread(
|
||||||
|
target=_qsa_clip_reader,
|
||||||
|
name="qsa-clip-reader",
|
||||||
|
daemon=True,
|
||||||
|
).start()
|
||||||
|
# The leading "QSA clip counter armed (read interval Ns)" is the token the serving smoke greps
|
||||||
|
# (v2.2.0 logged it from the first forward); keep it verbatim when editing this line.
|
||||||
|
logger.info(
|
||||||
|
"QSA clip counter armed (read interval %.0fs); reader started after capture "
|
||||||
|
"(%d layer(s) registered)",
|
||||||
|
_qsa_clip_interval,
|
||||||
|
len(_qsa_clip_counters),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if _qsa_clip_interval > 0.0:
|
||||||
|
from vllm.v1.worker.guard_telemetry import register_post_capture_hook # noqa: E402
|
||||||
|
|
||||||
|
register_post_capture_hook(_qsa_clip_start_reader)
|
||||||
|
|
||||||
|
|
||||||
|
def _qsa_clip_count(
|
||||||
|
layer: Qwen4ExpQSAAttention,
|
||||||
|
key: torch.Tensor,
|
||||||
|
value: torch.Tensor,
|
||||||
|
slot_mapping: torch.Tensor | None = None,
|
||||||
|
) -> None:
|
||||||
|
ctr = _qsa_clip_counters.get(layer.layer_name)
|
||||||
|
if ctr is None:
|
||||||
|
# First forward is eager (profiling/warmup precede capture), so the
|
||||||
|
# allocation never happens inside cudagraph capture. The READER is not
|
||||||
|
# started here: _qsa_clip_start_reader runs after capture.
|
||||||
|
ctr = torch.zeros(2, dtype=torch.int64, device=key.device)
|
||||||
|
_qsa_clip_counters[layer.layer_name] = ctr # registered with the reader by the reader thread (never during capture)
|
||||||
|
# Thresholds are 1-D so the comparison promotes to float32; a 0-dim float32
|
||||||
|
# tensor promotes like a Python scalar and the threshold would be rounded to
|
||||||
|
# the BF16 grid of K/V, undercounting values just above the calibrated ceiling.
|
||||||
|
k_ceiling = (448.0 * layer._k_scale).to(torch.float32).view(1)
|
||||||
|
v_ceiling = (448.0 * layer._v_scale).to(torch.float32).view(1)
|
||||||
|
k_over = key.detach().abs() > k_ceiling
|
||||||
|
v_over = value.detach().abs() > v_ceiling
|
||||||
|
if slot_mapping is not None:
|
||||||
|
# The CUDA writer (reshape_and_cache_flash) skips rows with a negative
|
||||||
|
# slot: padding rows of a captured batch size and, under the V2 model
|
||||||
|
# runner, stale rows of the persistent input buffers. Count only what
|
||||||
|
# is actually written, with fixed-shape masking so the op is capturable.
|
||||||
|
n = min(slot_mapping.shape[0], key.shape[0])
|
||||||
|
written = (slot_mapping[:n] >= 0).view(-1, 1, 1)
|
||||||
|
k_over = k_over[:n] & written
|
||||||
|
v_over = v_over[:n] & written
|
||||||
|
ctr[0] += k_over.sum()
|
||||||
|
ctr[1] += v_over.sum()
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_QSA_FP8_CACHE_DTYPES = ("fp8", "fp8_e4m3")
|
||||||
|
_QSA_SUPPORTED_CACHE_DTYPES = ("auto", "bfloat16", *_QSA_FP8_CACHE_DTYPES)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_qsa_fp8_cache_dtype(cache_dtype: str) -> bool:
|
||||||
|
return cache_dtype in _QSA_FP8_CACHE_DTYPES
|
||||||
|
|
||||||
|
|
||||||
|
def _validated_qsa_fp8_dtype(cache_dtype: str) -> torch.dtype | None:
|
||||||
|
"""Return the platform E4M3 dtype after enforcing the validated HW gate."""
|
||||||
|
|
||||||
|
if not _is_qsa_fp8_cache_dtype(cache_dtype):
|
||||||
|
return None
|
||||||
|
if not current_platform.is_cuda():
|
||||||
|
raise ValueError(
|
||||||
|
"Qwen4Exp QSA E4M3 KV cache is supported only on CUDA; "
|
||||||
|
"ROCm and other platforms have not been validated"
|
||||||
|
)
|
||||||
|
|
||||||
|
capability = current_platform.get_device_capability()
|
||||||
|
if capability is None or capability.to_int() != 86:
|
||||||
|
cap_str = (
|
||||||
|
capability.as_version_str() if capability is not None else "unknown"
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
"Qwen4Exp QSA E4M3 KV cache is validated only on SM86, but "
|
||||||
|
f"{current_platform.get_device_name()} has compute capability "
|
||||||
|
f"{cap_str}. Re-run with --kv-cache-dtype bfloat16."
|
||||||
|
)
|
||||||
|
return current_platform.fp8_dtype()
|
||||||
|
|
||||||
|
|
||||||
class Qwen4ExpQSAQKVIndexerLinear(MergedColumnParallelLinear):
|
class Qwen4ExpQSAQKVIndexerLinear(MergedColumnParallelLinear):
|
||||||
"""Pack sharded Q/gate, K, V and replicated indexer Q/K in one GEMM."""
|
"""Pack sharded Q/gate, K, V and replicated indexer Q/K in one GEMM."""
|
||||||
@@ -135,10 +401,6 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
|||||||
"""FullAttentionSpec backend used by the merged QSA owner."""
|
"""FullAttentionSpec backend used by the merged QSA owner."""
|
||||||
|
|
||||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||||
# fp8/fp8_e4m3: e4m3 bytes in a uint8 cache, written by reshape_and_cache
|
|
||||||
# with the layer's per-tensor scales and dequantized on load inside the QSA
|
|
||||||
# Triton kernel. flash-attn never runs over this cache, so its fp8 probe
|
|
||||||
# does not apply (see supports_kv_cache_dtype and the impl constructor).
|
|
||||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||||
"auto",
|
"auto",
|
||||||
"bfloat16",
|
"bfloat16",
|
||||||
@@ -148,6 +410,12 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||||
|
# The parent additionally probes flash-attn for quantized-KV support
|
||||||
|
# and rejects fp8 wherever FA has no kernel for it (all of sm_86).
|
||||||
|
# QSA never runs flash-attn over this cache: reshape_and_cache_flash
|
||||||
|
# writes the e4m3 bytes and our Triton kernel dequantizes them on load,
|
||||||
|
# so the class list alone decides. See the impl constructor for the
|
||||||
|
# matching "auto"-then-restore of the same probe.
|
||||||
return kv_cache_dtype is None or kv_cache_dtype in cls.supported_kv_cache_dtypes
|
return kv_cache_dtype is None or kv_cache_dtype in cls.supported_kv_cache_dtypes
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -163,10 +431,36 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
|||||||
use_mm_prefix: bool,
|
use_mm_prefix: bool,
|
||||||
device_capability: DeviceCapability,
|
device_capability: DeviceCapability,
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
# QSA dequantizes the fp8 KV in its own Triton kernel and never runs
|
# Suppress ONLY the parent's fp8 clause ("FP8 KV cache requires FA3 on
|
||||||
# flash-attn over the quantized cache, so the parent's fp8-KV rejection
|
# SM90 or FA4 on SM100"), which is about flash-attn's own kernels and
|
||||||
# does not apply and every combination it is handed is accepted here.
|
# so does not apply to a cache QSA reads in Triton. The real device
|
||||||
return None
|
# gate for our fp8 path is `_validated_qsa_fp8_dtype`, which raises on
|
||||||
|
# any capability != 86.
|
||||||
|
#
|
||||||
|
# Everything else the parent rejects still holds and is returned
|
||||||
|
# unchanged: the sink/compute-capability clause and the
|
||||||
|
# mm-prefix/FA-version clause. Returning None for every combination
|
||||||
|
# would advertise fewer restrictions than `forward_qsa` enforces (it
|
||||||
|
# raises on ALiBi, sinks and sliding windows).
|
||||||
|
#
|
||||||
|
# Same shape as the impl constructor: hide the fp8 dtype from the
|
||||||
|
# probe, keep the rest of the parent's behaviour.
|
||||||
|
probe_kv_cache_dtype = kv_cache_dtype
|
||||||
|
if probe_kv_cache_dtype is not None and _is_qsa_fp8_cache_dtype(
|
||||||
|
probe_kv_cache_dtype
|
||||||
|
):
|
||||||
|
probe_kv_cache_dtype = None
|
||||||
|
return super().supports_combination(
|
||||||
|
head_size,
|
||||||
|
dtype,
|
||||||
|
probe_kv_cache_dtype,
|
||||||
|
block_size,
|
||||||
|
use_mla,
|
||||||
|
has_sink,
|
||||||
|
use_sparse,
|
||||||
|
use_mm_prefix,
|
||||||
|
device_capability,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_name() -> str:
|
def get_name() -> str:
|
||||||
@@ -214,14 +508,18 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
|||||||
kv_sharing_target_layer_name: str | None = None,
|
kv_sharing_target_layer_name: str | None = None,
|
||||||
sinks: torch.Tensor | None = None,
|
sinks: torch.Tensor | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
# The parent constructor probes flash-attn for quantized-KV support and
|
# QSA never dispatches to a FlashAttention kernel: attention runs through
|
||||||
# raises where it is unavailable (sm120), but QSA dequantizes fp8 inside
|
# forward_qsa -> qsa_sparse_paged_attention (our Triton sparse-GQA kernel)
|
||||||
# its own Triton kernel and never runs flash-attn over the cache. Hand
|
# and the cache write goes through the CUDA reshape_and_cache_flash op.
|
||||||
# the parent "auto" for that probe and restore the real dtype afterwards:
|
# FlashAttentionImpl.__init__ nevertheless rejects any quantized KV dtype
|
||||||
# the parent only uses it there, and do_kv_cache_update reads the
|
# that FA itself cannot read, and sm_86 has no FP8 FA kernel. Present
|
||||||
# attribute at call time.
|
# "auto" to the base class, then restore the real dtype. The only other
|
||||||
|
# kv_cache_dtype use in that constructor is an SM90/FA4 dequant path that
|
||||||
|
# cannot trigger here, and supports_quant_query_input is set below anyway.
|
||||||
|
# The parameters are spelled out rather than forwarded as *args/**kwargs
|
||||||
|
# so the substitution names the parameter instead of indexing position 6.
|
||||||
real_kv_cache_dtype = kv_cache_dtype
|
real_kv_cache_dtype = kv_cache_dtype
|
||||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
if _is_qsa_fp8_cache_dtype(kv_cache_dtype):
|
||||||
kv_cache_dtype = "auto"
|
kv_cache_dtype = "auto"
|
||||||
super().__init__(
|
super().__init__(
|
||||||
num_heads,
|
num_heads,
|
||||||
@@ -243,10 +541,12 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
|||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp QSA does not support decode context parallelism"
|
"Qwen4Exp QSA does not support decode context parallelism"
|
||||||
)
|
)
|
||||||
if self.kv_cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
|
if self.kv_cache_dtype not in _QSA_SUPPORTED_CACHE_DTYPES:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
"Qwen4Exp QSA supports BF16 or static per-tensor E4M3 main "
|
||||||
|
f"KV caches, not {self.kv_cache_dtype!r}"
|
||||||
)
|
)
|
||||||
|
self.qsa_fp8_dtype = _validated_qsa_fp8_dtype(self.kv_cache_dtype)
|
||||||
self.supports_quant_query_input = False
|
self.supports_quant_query_input = False
|
||||||
|
|
||||||
def forward_qsa(
|
def forward_qsa(
|
||||||
@@ -283,26 +583,42 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
|||||||
logical_indices = topk_buffer[:num_tokens]
|
logical_indices = topk_buffer[:num_tokens]
|
||||||
token_to_req = token_to_req[:num_tokens]
|
token_to_req = token_to_req[:num_tokens]
|
||||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||||
k_scale = v_scale = None
|
if query.dtype != torch.bfloat16:
|
||||||
if self.kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
raise NotImplementedError("Qwen4Exp QSA requires a BF16 query")
|
||||||
# The cache is allocated as uint8; reinterpret the e4m3 bytes
|
if key_cache.dtype != value_cache.dtype:
|
||||||
# (same itemsize, so shape and strides are preserved).
|
raise TypeError("Qwen4Exp QSA K and V cache dtypes must match")
|
||||||
key_cache = key_cache.view(torch.float8_e4m3fn)
|
if _is_qsa_fp8_cache_dtype(self.kv_cache_dtype):
|
||||||
value_cache = value_cache.view(torch.float8_e4m3fn)
|
if self.qsa_fp8_dtype is None:
|
||||||
# Host-side per-tensor dequant scales (Python floats), as used by
|
raise RuntimeError("QSA FP8 dtype was not initialized")
|
||||||
# other host-scale backends; folded into the kernel's scales.
|
if key_cache.dtype not in (torch.uint8, self.qsa_fp8_dtype):
|
||||||
k_scale = layer._k_scale_float
|
raise NotImplementedError(
|
||||||
v_scale = layer._v_scale_float
|
"Qwen4Exp QSA E4M3 cache storage must use raw uint8 or "
|
||||||
if query.dtype != torch.bfloat16 or key_cache.dtype not in (
|
f"the platform FP8 dtype, not {key_cache.dtype}"
|
||||||
torch.bfloat16,
|
)
|
||||||
torch.float8_e4m3fn,
|
elif key_cache.dtype != torch.bfloat16:
|
||||||
):
|
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp QSA requires BF16 Q and BF16 or FP8-e4m3 K/V"
|
"Qwen4Exp QSA BF16 mode requires a BF16 main KV cache"
|
||||||
)
|
)
|
||||||
|
|
||||||
from .ops.qsa import qsa_sparse_paged_attention
|
from .ops.qsa import qsa_sparse_paged_attention
|
||||||
|
|
||||||
|
# Announce the effective scales once per layer. load_qsa_static_kv_scales()
|
||||||
|
# is an explicit integration call, so a deployment that forgets it runs at
|
||||||
|
# the vLLM default of 1.0 -- which is a legitimate configuration but must
|
||||||
|
# never be a silent one. This makes the actual values greppable per rank.
|
||||||
|
if _is_qsa_fp8_cache_dtype(self.kv_cache_dtype) and not getattr(
|
||||||
|
self, "_qsa_scales_logged", False
|
||||||
|
):
|
||||||
|
self._qsa_scales_logged = True
|
||||||
|
logger.info(
|
||||||
|
"Qwen4Exp QSA E4M3 KV active on %s: k_scale=%.6g v_scale=%.6g%s",
|
||||||
|
getattr(layer, "layer_name", "<unnamed>"),
|
||||||
|
float(layer._k_scale),
|
||||||
|
float(layer._v_scale),
|
||||||
|
"" if float(layer._k_scale) != 1.0 or float(layer._v_scale) != 1.0
|
||||||
|
else " (DEFAULT 1.0 -- no static calibration loaded)",
|
||||||
|
)
|
||||||
|
|
||||||
qsa_sparse_paged_attention(
|
qsa_sparse_paged_attention(
|
||||||
query[:num_tokens],
|
query[:num_tokens],
|
||||||
key_cache,
|
key_cache,
|
||||||
@@ -311,9 +627,9 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
|||||||
attn_metadata.block_table,
|
attn_metadata.block_table,
|
||||||
token_to_req,
|
token_to_req,
|
||||||
use_prefill_config,
|
use_prefill_config,
|
||||||
output[:num_tokens],
|
out=output[:num_tokens],
|
||||||
k_scale=k_scale,
|
k_scale=layer._k_scale,
|
||||||
v_scale=v_scale,
|
v_scale=layer._v_scale,
|
||||||
output_gate=output_gate[:num_tokens],
|
output_gate=output_gate[:num_tokens],
|
||||||
)
|
)
|
||||||
return output
|
return output
|
||||||
@@ -341,17 +657,20 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|||||||
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
||||||
if model_config.dtype != torch.bfloat16:
|
if model_config.dtype != torch.bfloat16:
|
||||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||||
if cache_config.cache_dtype not in (
|
if cache_config.cache_dtype not in _QSA_SUPPORTED_CACHE_DTYPES:
|
||||||
"auto",
|
|
||||||
"bfloat16",
|
|
||||||
"fp8",
|
|
||||||
"fp8_e4m3",
|
|
||||||
):
|
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
"Qwen4Exp QSA supports BF16 or static per-tensor E4M3 main "
|
||||||
|
f"KV caches, not {cache_config.cache_dtype!r}"
|
||||||
|
)
|
||||||
|
qsa_fp8_dtype = _validated_qsa_fp8_dtype(cache_config.cache_dtype)
|
||||||
|
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
|
||||||
|
if kv_cache_scheme is not None:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"Qwen4Exp QSA does not implement kv_cache_scheme variants. "
|
||||||
|
"Dynamic, per-head, per-token, and other non-scalar KV scales "
|
||||||
|
"are unsupported; use static scalar per-layer K/V scales from "
|
||||||
|
"load_qsa_static_kv_scales()."
|
||||||
)
|
)
|
||||||
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
|
||||||
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
if (
|
if (
|
||||||
parallel_config.prefill_context_parallel_size > 1
|
parallel_config.prefill_context_parallel_size > 1
|
||||||
@@ -448,12 +767,23 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|||||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||||
self.kv_cache_dtype, model_config
|
self.kv_cache_dtype, model_config
|
||||||
)
|
)
|
||||||
if self.kv_cache_torch_dtype not in (torch.bfloat16, torch.uint8):
|
if _is_qsa_fp8_cache_dtype(self.kv_cache_dtype):
|
||||||
|
if qsa_fp8_dtype is None:
|
||||||
|
raise RuntimeError("QSA FP8 dtype was not initialized")
|
||||||
|
if self.kv_cache_torch_dtype not in (torch.uint8, qsa_fp8_dtype):
|
||||||
|
raise NotImplementedError(
|
||||||
|
"Qwen4Exp QSA E4M3 cache storage resolved to unsupported "
|
||||||
|
f"dtype {self.kv_cache_torch_dtype}"
|
||||||
|
)
|
||||||
|
elif self.kv_cache_torch_dtype != torch.bfloat16:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp QSA requires BF16 or FP8-e4m3 (uint8) cache storage"
|
"Qwen4Exp QSA BF16 mode requires BF16 cache storage"
|
||||||
)
|
)
|
||||||
self.kv_sharing_target_layer_name = None
|
self.kv_sharing_target_layer_name = None
|
||||||
self.kv_cache = torch.tensor([])
|
self.kv_cache = torch.tensor([])
|
||||||
|
# The checkpoint has no KV scale scheme. These registered scalar
|
||||||
|
# buffers therefore remain 1.0 unless explicitly replaced after
|
||||||
|
# weight loading by load_qsa_static_kv_scales().
|
||||||
set_default_quant_scales(self, register_buffer=True)
|
set_default_quant_scales(self, register_buffer=True)
|
||||||
|
|
||||||
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
||||||
@@ -578,6 +908,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|||||||
)
|
)
|
||||||
if selected.shape != (num_tokens, self.indexer.packed_output_width):
|
if selected.shape != (num_tokens, self.indexer.packed_output_width):
|
||||||
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
||||||
|
if _qsa_clip_interval > 0.0 and _is_qsa_fp8_cache_dtype(
|
||||||
|
self.kv_cache_dtype
|
||||||
|
):
|
||||||
|
_qsa_clip_count(self, key, value, main_metadata.slot_mapping)
|
||||||
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
||||||
if main_outputs is None:
|
if main_outputs is None:
|
||||||
assert key is not None and value is not None
|
assert key is not None and value is not None
|
||||||
@@ -617,34 +951,235 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|||||||
else:
|
else:
|
||||||
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
||||||
num_tokens = hidden_states.shape[0]
|
num_tokens = hidden_states.shape[0]
|
||||||
if not self.use_fused_qsa_prepare:
|
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||||
assert gate is not None
|
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||||
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
if _qsa_collect_dir:
|
||||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
_qsa_collect_absmax(self.layer_name, key, value)
|
||||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
attn_output = torch.empty_like(query)
|
||||||
|
# Keep the index projection outside the eager break / splitting op.
|
||||||
|
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
||||||
|
# Compile-profile guard: the QSA transaction reads per-step attention metadata from
|
||||||
|
# the forward context, so under VLLM_COMPILE it must be opaque to
|
||||||
|
# Dynamo and a piecewise split point (compilation.py _attention_ops).
|
||||||
|
# Restores the op #55272 removed, with #55309's fused output gate.
|
||||||
|
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||||
|
if current_platform.opaque_attention_op():
|
||||||
|
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
||||||
|
projected_qk,
|
||||||
|
positions,
|
||||||
|
query,
|
||||||
|
key,
|
||||||
|
value,
|
||||||
|
attn_output,
|
||||||
|
gate,
|
||||||
|
encoded_layer_name,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
# Norm/RoPE/gate and the K/V cache write happen inside _run_qsa.
|
qwen4_exp_qsa_with_output(
|
||||||
query = key = value = gate = None
|
projected_qk,
|
||||||
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
|
positions,
|
||||||
self._run_qsa(
|
query,
|
||||||
projected_qk,
|
key,
|
||||||
positions,
|
value,
|
||||||
query,
|
attn_output,
|
||||||
key,
|
gate,
|
||||||
value,
|
encoded_layer_name,
|
||||||
attn_output,
|
)
|
||||||
gate,
|
|
||||||
qkv,
|
|
||||||
)
|
|
||||||
flat_output = attn_output.view(num_tokens, -1)
|
flat_output = attn_output.view(num_tokens, -1)
|
||||||
output, _ = self.o_proj(flat_output)
|
output, _ = self.o_proj(flat_output)
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def load_qsa_static_kv_scales(
|
||||||
|
layers: nn.Module | Mapping[str, nn.Module],
|
||||||
|
sidecar_path: str | Path,
|
||||||
|
*,
|
||||||
|
strict: bool = True,
|
||||||
|
expected_layer_names: Collection[str] | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
"""Load static scalar QSA K/V scales into already-loaded model layers.
|
||||||
|
|
||||||
|
Call this on every model worker after checkpoint weight loading finishes
|
||||||
|
and before any profiling, warmup, or cudagraph capture. ``layers`` may be
|
||||||
|
the loaded root module or ``compilation_config.static_forward_context``.
|
||||||
|
|
||||||
|
The JSON schema is::
|
||||||
|
|
||||||
|
{
|
||||||
|
"model.layers.3.self_attn.attn": {
|
||||||
|
"k_scale": 0.25,
|
||||||
|
"v_scale": 2.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
``strict=False`` permits a deliberate subset; omitted layers keep the
|
||||||
|
vLLM default scale of 1.0. Unknown names and non-scalar values are always
|
||||||
|
errors.
|
||||||
|
|
||||||
|
Pipeline parallelism: a rank holds only its stage's layers, so the sidecar
|
||||||
|
legitimately names layers this rank does not own. Pass
|
||||||
|
``expected_layer_names`` = the QSA layer names of the WHOLE model; names are
|
||||||
|
validated against that set, ``strict`` requires the sidecar to cover all
|
||||||
|
of it (checkable on every rank), and the scales are applied to the local
|
||||||
|
intersection only. Without it (TP-only callers) the local layers are the
|
||||||
|
universe, as before.
|
||||||
|
"""
|
||||||
|
|
||||||
|
path = Path(sidecar_path)
|
||||||
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
raise ValueError(f"{path} must contain a JSON object keyed by layer name")
|
||||||
|
|
||||||
|
if isinstance(layers, nn.Module):
|
||||||
|
qsa_layers = {
|
||||||
|
layer.layer_name: layer
|
||||||
|
for layer in layers.modules()
|
||||||
|
if isinstance(layer, Qwen4ExpQSAAttention)
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
qsa_layers = {
|
||||||
|
name: layer
|
||||||
|
for name, layer in layers.items()
|
||||||
|
if isinstance(name, str)
|
||||||
|
and isinstance(layer, Qwen4ExpQSAAttention)
|
||||||
|
}
|
||||||
|
|
||||||
|
if not qsa_layers and expected_layer_names is None:
|
||||||
|
raise ValueError("No Qwen4Exp QSA attention layers were supplied")
|
||||||
|
non_fp8_layers = sorted(
|
||||||
|
name
|
||||||
|
for name, layer in qsa_layers.items()
|
||||||
|
if not _is_qsa_fp8_cache_dtype(layer.kv_cache_dtype)
|
||||||
|
)
|
||||||
|
if non_fp8_layers:
|
||||||
|
raise ValueError(
|
||||||
|
"Static QSA KV scales may only be injected into E4M3 layers; "
|
||||||
|
f"non-FP8 layers: {non_fp8_layers}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if expected_layer_names is None:
|
||||||
|
universe = set(qsa_layers)
|
||||||
|
else:
|
||||||
|
universe = set(expected_layer_names)
|
||||||
|
not_expected = sorted(name for name in qsa_layers if name not in universe)
|
||||||
|
if not_expected:
|
||||||
|
raise ValueError(
|
||||||
|
"Local QSA layers outside expected_layer_names: "
|
||||||
|
f"{not_expected}"
|
||||||
|
)
|
||||||
|
unknown = sorted(name for name in raw if name not in universe)
|
||||||
|
if unknown:
|
||||||
|
raise ValueError(f"Unknown QSA layer names in {path}: {unknown}")
|
||||||
|
missing = sorted(name for name in universe if name not in raw)
|
||||||
|
if strict and missing:
|
||||||
|
raise ValueError(f"Missing QSA layer scales in {path}: {missing}")
|
||||||
|
remote = sorted(name for name in raw if name not in qsa_layers)
|
||||||
|
if remote:
|
||||||
|
logger.info(
|
||||||
|
"Qwen4Exp QSA: %d sidecar layer(s) belong to other pipeline stages "
|
||||||
|
"and are skipped on this rank: %s%s",
|
||||||
|
len(remote),
|
||||||
|
", ".join(remote[:4]),
|
||||||
|
" ..." if len(remote) > 4 else "",
|
||||||
|
)
|
||||||
|
|
||||||
|
validated: dict[str, tuple[float, float]] = {}
|
||||||
|
for layer_name, entry in raw.items():
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise ValueError(f"{layer_name} scales must be a JSON object")
|
||||||
|
if set(entry) != {"k_scale", "v_scale"}:
|
||||||
|
raise ValueError(
|
||||||
|
f"{layer_name} must contain exactly k_scale and v_scale"
|
||||||
|
)
|
||||||
|
|
||||||
|
values: list[float] = []
|
||||||
|
for field in ("k_scale", "v_scale"):
|
||||||
|
value = entry[field]
|
||||||
|
if type(value) not in (int, float):
|
||||||
|
raise ValueError(
|
||||||
|
f"{layer_name}.{field} must be a scalar JSON number"
|
||||||
|
)
|
||||||
|
scale = float(value)
|
||||||
|
if not math.isfinite(scale) or scale <= 0.0:
|
||||||
|
raise ValueError(
|
||||||
|
f"{layer_name}.{field} must be finite and positive"
|
||||||
|
)
|
||||||
|
values.append(scale)
|
||||||
|
|
||||||
|
layer = qsa_layers.get(layer_name)
|
||||||
|
if layer is None:
|
||||||
|
continue # another pipeline stage's layer: schema-checked only
|
||||||
|
for field in ("_k_scale", "_v_scale"):
|
||||||
|
tensor = getattr(layer, field, None)
|
||||||
|
if not isinstance(tensor, torch.Tensor) or tensor.numel() != 1:
|
||||||
|
raise ValueError(
|
||||||
|
f"{layer_name}.{field} must be an existing scalar tensor"
|
||||||
|
)
|
||||||
|
validated[layer_name] = (values[0], values[1])
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
for layer_name, (k_scale, v_scale) in validated.items():
|
||||||
|
layer = qsa_layers[layer_name]
|
||||||
|
layer._k_scale.fill_(k_scale)
|
||||||
|
layer._v_scale.fill_(v_scale)
|
||||||
|
|
||||||
|
return sorted(validated)
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_qsa_with_output(
|
||||||
|
projected_qk: torch.Tensor,
|
||||||
|
positions: torch.Tensor,
|
||||||
|
query: torch.Tensor,
|
||||||
|
key: torch.Tensor,
|
||||||
|
value: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
output_gate: torch.Tensor,
|
||||||
|
layer_name: LayerNameType,
|
||||||
|
) -> None:
|
||||||
|
"""Run the complete QSA state/update/attend transaction."""
|
||||||
|
|
||||||
|
layer_name = _resolve_layer_name(layer_name)
|
||||||
|
layer = get_forward_context().no_compile_layers[layer_name]
|
||||||
|
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||||
|
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||||
|
layer._run_qsa(
|
||||||
|
projected_qk,
|
||||||
|
positions,
|
||||||
|
query,
|
||||||
|
key,
|
||||||
|
value,
|
||||||
|
output,
|
||||||
|
output_gate,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def qwen4_exp_qsa_with_output_fake(
|
||||||
|
projected_qk: torch.Tensor,
|
||||||
|
positions: torch.Tensor,
|
||||||
|
query: torch.Tensor,
|
||||||
|
key: torch.Tensor,
|
||||||
|
value: torch.Tensor,
|
||||||
|
output: torch.Tensor,
|
||||||
|
output_gate: torch.Tensor,
|
||||||
|
layer_name: LayerNameType,
|
||||||
|
) -> None:
|
||||||
|
del projected_qk, positions, query, key, value, output, output_gate, layer_name
|
||||||
|
|
||||||
|
|
||||||
|
direct_register_custom_op(
|
||||||
|
op_name="qwen4_exp_qsa_with_output",
|
||||||
|
op_func=qwen4_exp_qsa_with_output,
|
||||||
|
mutates_args=["output"],
|
||||||
|
fake_impl=qwen4_exp_qsa_with_output_fake,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"QSAIndexer",
|
"QSAIndexer",
|
||||||
"Qwen4ExpQSAAttention",
|
"Qwen4ExpQSAAttention",
|
||||||
"Qwen4ExpQSAFlashAttentionBackend",
|
"Qwen4ExpQSAFlashAttentionBackend",
|
||||||
"Qwen4ExpQSAFlashAttentionImpl",
|
"Qwen4ExpQSAFlashAttentionImpl",
|
||||||
|
"load_qsa_static_kv_scales",
|
||||||
|
"qwen4_exp_qsa_with_output",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -11,6 +11,12 @@
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.worker import guard_telemetry
|
||||||
|
|
||||||
|
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||||
|
# -- it MUST be a tl.constexpr: a plain module-level int is not
|
||||||
|
# readable from a @triton.jit body.
|
||||||
|
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||||
|
|
||||||
from .op import exp, log
|
from .op import exp, log
|
||||||
|
|
||||||
@@ -36,6 +42,7 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
|||||||
cu_seqlens,
|
cu_seqlens,
|
||||||
ssm_state_indices,
|
ssm_state_indices,
|
||||||
num_accepted_tokens,
|
num_accepted_tokens,
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
scale,
|
scale,
|
||||||
N: tl.int64, # num of sequences
|
N: tl.int64, # num of sequences
|
||||||
T: tl.int64, # num of tokens
|
T: tl.int64, # num of tokens
|
||||||
@@ -106,12 +113,36 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
|||||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||||
else:
|
else:
|
||||||
i_t = 0
|
i_t = 0
|
||||||
# Load state index and check for invalid entries
|
# PR #50021: i_t is num_accepted_tokens - 1 and is otherwise
|
||||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
# unbounded, while ssm_state_indices has only stride_indices_seq
|
||||||
tl.int64
|
# columns per request. A zero accepted count gives i_t == -1, a
|
||||||
)
|
# read before this request's row (and before the tensor for
|
||||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
# i_n == 0); a stale or too-large count reads past the row. The
|
||||||
|
# state_idx <= 0 guard below rejects only non-positive values, so
|
||||||
|
# an out-of-range read returning a garbage positive int flowed into
|
||||||
|
# h0 + state_idx * stride_init_state_token and faulted the SM.
|
||||||
|
#
|
||||||
|
# This is the SAME defect as the decode-state guard, in the twin kernel. An
|
||||||
|
# earlier revision left it unguarded and dropped the upstream test's arm for it;
|
||||||
|
# it is now ported rather than caveated.
|
||||||
|
idx_in_row = (i_t >= 0) & (i_t < stride_indices_seq)
|
||||||
|
if (i_t < 0) or (i_t >= stride_indices_seq):
|
||||||
|
tl.atomic_add(guard_ctr, 1)
|
||||||
|
tl.atomic_max(guard_ctr + 1, (i_n).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, (i_t).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
state_idx = tl.load(
|
||||||
|
ssm_state_indices + i_n * stride_indices_seq + i_t,
|
||||||
|
mask=idx_in_row,
|
||||||
|
other=0,
|
||||||
|
).to(tl.int64)
|
||||||
|
# Skip invalid state indices without exposing uninitialized output.
|
||||||
|
# o is q.new_empty in the entry below, so the previous early return
|
||||||
|
# left this request's rows holding whatever the allocator returned.
|
||||||
if state_idx <= 0:
|
if state_idx <= 0:
|
||||||
|
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||||
|
for _ in range(0, T):
|
||||||
|
tl.store(p_o, zero, mask=mask_v)
|
||||||
|
p_o += HV * V
|
||||||
return
|
return
|
||||||
p_h0 = h0 + state_idx * stride_init_state_token
|
p_h0 = h0 + state_idx * stride_init_state_token
|
||||||
else:
|
else:
|
||||||
@@ -227,6 +258,9 @@ def fused_recurrent_gated_delta_rule_fwd(
|
|||||||
cu_seqlens=cu_seqlens,
|
cu_seqlens=cu_seqlens,
|
||||||
ssm_state_indices=ssm_state_indices,
|
ssm_state_indices=ssm_state_indices,
|
||||||
num_accepted_tokens=num_accepted_tokens,
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
guard_ctr=guard_telemetry.counter(
|
||||||
|
guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device
|
||||||
|
),
|
||||||
scale=scale,
|
scale=scale,
|
||||||
N=N,
|
N=N,
|
||||||
T=T,
|
T=T,
|
||||||
|
|||||||
@@ -10,6 +10,14 @@
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.worker import guard_telemetry
|
||||||
|
|
||||||
|
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||||
|
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||||
|
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||||
|
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||||
|
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@triton.heuristics(
|
@triton.heuristics(
|
||||||
@@ -37,6 +45,7 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
|
|||||||
cu_seqlens,
|
cu_seqlens,
|
||||||
ssm_state_indices,
|
ssm_state_indices,
|
||||||
num_accepted_tokens,
|
num_accepted_tokens,
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
scale,
|
scale,
|
||||||
N: tl.int64, # num of sequences
|
N: tl.int64, # num of sequences
|
||||||
T: tl.int64, # num of tokens
|
T: tl.int64, # num of tokens
|
||||||
@@ -106,12 +115,32 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
|
|||||||
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1
|
||||||
else:
|
else:
|
||||||
i_t = 0
|
i_t = 0
|
||||||
# Load state index and check for invalid entries
|
# PR #50021: i_t is num_accepted_tokens - 1 and is otherwise
|
||||||
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to(
|
# unbounded against a stride_indices_seq-column tensor. A zero
|
||||||
tl.int64
|
# accepted count gives i_t == -1, a read before this request's row
|
||||||
)
|
# (and before the tensor for i_n == 0); a stale or too-large count
|
||||||
# Skip if state index is invalid (NULL_BLOCK_ID=0)
|
# reads past the row. The state_idx <= 0 guard below rejects only
|
||||||
|
# non-positive values, so an out-of-range read returning a garbage
|
||||||
|
# positive int flowed into h0 + state_idx * stride_init_state_token
|
||||||
|
# and faulted the SM.
|
||||||
|
idx_in_row = (i_t >= 0) & (i_t < stride_indices_seq)
|
||||||
|
if (i_t < 0) or (i_t >= stride_indices_seq):
|
||||||
|
tl.atomic_add(guard_ctr, 1)
|
||||||
|
tl.atomic_max(guard_ctr + 1, (i_n).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, (i_t).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
state_idx = tl.load(
|
||||||
|
ssm_state_indices + i_n * stride_indices_seq + i_t,
|
||||||
|
mask=idx_in_row,
|
||||||
|
other=0,
|
||||||
|
).to(tl.int64)
|
||||||
|
# Skip invalid state indices without exposing uninitialized output.
|
||||||
|
# o is q.new_empty in the entry below, so the previous early return
|
||||||
|
# left this request's rows holding whatever the allocator returned.
|
||||||
if state_idx <= 0:
|
if state_idx <= 0:
|
||||||
|
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
||||||
|
for _ in range(0, T):
|
||||||
|
tl.store(p_o, zero, mask=mask_v)
|
||||||
|
p_o += HV * V
|
||||||
return
|
return
|
||||||
p_h0 = h0 + state_idx * stride_init_state_token
|
p_h0 = h0 + state_idx * stride_init_state_token
|
||||||
else:
|
else:
|
||||||
@@ -255,6 +284,9 @@ def fused_sigmoid_gating_delta_rule_update(
|
|||||||
cu_seqlens=cu_seqlens,
|
cu_seqlens=cu_seqlens,
|
||||||
ssm_state_indices=ssm_state_indices,
|
ssm_state_indices=ssm_state_indices,
|
||||||
num_accepted_tokens=num_accepted_tokens,
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
guard_ctr=guard_telemetry.counter(
|
||||||
|
guard_telemetry.SITE_GDN_DECODE_STATE, q.device
|
||||||
|
),
|
||||||
scale=scale,
|
scale=scale,
|
||||||
N=N,
|
N=N,
|
||||||
T=T,
|
T=T,
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import torch.nn as nn
|
|||||||
|
|
||||||
from vllm.model_executor.custom_op import CustomOp
|
from vllm.model_executor.custom_op import CustomOp
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
|
from vllm.v1.worker import guard_telemetry
|
||||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||||
|
|
||||||
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
|
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
|
||||||
@@ -81,6 +82,12 @@ def fused_recurrent_kda_fwd(
|
|||||||
cu_seqlens=cu_seqlens,
|
cu_seqlens=cu_seqlens,
|
||||||
ssm_state_indices=ssm_state_indices,
|
ssm_state_indices=ssm_state_indices,
|
||||||
num_accepted_tokens=num_accepted_tokens,
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
# Bounds-guard instrumentation: the same kernel object as fused_recurrent.py:249, so it
|
||||||
|
# takes the same guard counter. Same site: the defect being guarded is
|
||||||
|
# the kernel's, not the caller's.
|
||||||
|
guard_ctr=guard_telemetry.counter(
|
||||||
|
guard_telemetry.SITE_GDN_RECURRENT_STATE, q.device
|
||||||
|
),
|
||||||
scale=scale,
|
scale=scale,
|
||||||
N=N,
|
N=N,
|
||||||
T=T,
|
T=T,
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""GDN attention metadata for RecoverSSM speculative decode (Qwen-style Gated DeltaNet).
|
||||||
|
|
||||||
|
Wraps the generic GDN builder: every active decode row runs through the spec path (a
|
||||||
|
draft-less step is a T=1 window), the spec state indices keep only the checkpoint
|
||||||
|
column, num_accepted_tokens is always 1 (the checkpoint and the compacted conv window
|
||||||
|
already hold the accepted state), and the metadata carries what the post-sampling commit
|
||||||
|
needs. Mirrors the Kimi-K3 KDA builder's RecoverSSM handling."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field, fields, replace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||||
|
from vllm.v1.attention.backends.gdn_attn import (
|
||||||
|
GDNAttentionBackend,
|
||||||
|
GDNAttentionMetadata,
|
||||||
|
GDNAttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||||
|
RecoverSSMMetadata,
|
||||||
|
RecoverSSMPostprocessMetadata,
|
||||||
|
clone_commit_tensors,
|
||||||
|
recoverssm_request_indices,
|
||||||
|
recoverssm_spec_rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GDNRecoverSSMCommitMetadata:
|
||||||
|
state_indices: torch.Tensor # [num_spec_decodes, 1]
|
||||||
|
query_start_loc: torch.Tensor # [num_spec_decodes + 1]
|
||||||
|
request_indices: (
|
||||||
|
torch.Tensor | None
|
||||||
|
) # batch rows of the spec requests (None: rows 0..n-1)
|
||||||
|
block_table: torch.Tensor | None # align mode
|
||||||
|
num_computed_tokens: torch.Tensor | None
|
||||||
|
block_size: int | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GDNRecoverSSMMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||||
|
recoverssm_commit: GDNRecoverSSMCommitMetadata | None = None
|
||||||
|
recoverssm_context: Any = field(default=None, repr=False, compare=False)
|
||||||
|
|
||||||
|
def commit_recoverssm_state(
|
||||||
|
self, num_accepted_tokens: torch.Tensor
|
||||||
|
) -> RecoverSSMPostprocessMetadata | None:
|
||||||
|
c = self.recoverssm_commit
|
||||||
|
if c is None:
|
||||||
|
return None
|
||||||
|
n = self.num_spec_decodes
|
||||||
|
self.recoverssm_context.commit(
|
||||||
|
num_accepted_tokens,
|
||||||
|
c.state_indices[:n, 0],
|
||||||
|
c.query_start_loc[: n + 1],
|
||||||
|
request_indices=c.request_indices,
|
||||||
|
block_table=c.block_table,
|
||||||
|
num_computed_tokens=c.num_computed_tokens,
|
||||||
|
mamba_block_size=c.block_size,
|
||||||
|
)
|
||||||
|
if c.block_table is None:
|
||||||
|
return None
|
||||||
|
assert c.block_size is not None and c.num_computed_tokens is not None
|
||||||
|
return RecoverSSMPostprocessMetadata(
|
||||||
|
num_spec_decodes=n,
|
||||||
|
request_indices=c.request_indices,
|
||||||
|
block_table=c.block_table,
|
||||||
|
num_computed_tokens=c.num_computed_tokens,
|
||||||
|
block_size=c.block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def snapshot_for_deferred_commit(self) -> "GDNRecoverSSMMetadata":
|
||||||
|
c = self.recoverssm_commit
|
||||||
|
if c is None:
|
||||||
|
return self
|
||||||
|
return replace(
|
||||||
|
self,
|
||||||
|
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_context_binding(context: Any, checkpoints: list[torch.Tensor]) -> None:
|
||||||
|
"""A cached commit context must point at the layers' live KV cache.
|
||||||
|
|
||||||
|
The context stores raw base addresses of every layer's conv, state and
|
||||||
|
replay tensors. One that outlived its cache (the profiling KV cache is
|
||||||
|
torn down and re-allocated before serving) would keep that cache alive
|
||||||
|
and, worse, commit into freed memory. Builders are re-created with the
|
||||||
|
real cache, so a mismatch is a lifecycle bug: fail loudly."""
|
||||||
|
bound = [t.data_ptr() for t in context.checkpoints]
|
||||||
|
live = [t.data_ptr() for t in checkpoints]
|
||||||
|
if bound != live:
|
||||||
|
raise RuntimeError(
|
||||||
|
"GDN RecoverSSM commit context is bound to a different KV cache than "
|
||||||
|
"its layers (stale context after a KV cache re-allocation)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||||
|
# Uniform spec-decode batches replay as FULL graphs: the base builder stages the
|
||||||
|
# state indices, query offsets and accepted counts into persistent buffers and pads
|
||||||
|
# the batch with null-state, zero-length rows, which the verify kernel skips. The
|
||||||
|
# commit runs after sampling, outside the graph, on the staged tensors.
|
||||||
|
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
|
def __init__(self, kv_cache_spec, layer_names, vllm_config, device) -> None:
|
||||||
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
|
max_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||||
|
self._ones = torch.ones(max_reqs, dtype=torch.int32, device=device)
|
||||||
|
self._context: Any = None
|
||||||
|
self._logged = False
|
||||||
|
self._capturing = False
|
||||||
|
|
||||||
|
def _get_context(self):
|
||||||
|
if self._context is not None:
|
||||||
|
fc = self.vllm_config.compilation_config.static_forward_context
|
||||||
|
_check_context_binding(
|
||||||
|
self._context, [fc[name].kv_cache[1] for name in self.layer_names]
|
||||||
|
)
|
||||||
|
if self._context is None:
|
||||||
|
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||||
|
GDNRecoverSSMCommitContext,
|
||||||
|
)
|
||||||
|
|
||||||
|
fc = self.vllm_config.compilation_config.static_forward_context
|
||||||
|
layers = [fc[name] for name in self.layer_names]
|
||||||
|
self._context = GDNRecoverSSMCommitContext.create(
|
||||||
|
layers,
|
||||||
|
spec_query_len=1 + self.num_spec,
|
||||||
|
max_num_reqs=self.vllm_config.scheduler_config.max_num_seqs,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"GDN RecoverSSM commit context: %d layers, spec_query_len %d",
|
||||||
|
len(layers),
|
||||||
|
1 + self.num_spec,
|
||||||
|
)
|
||||||
|
return self._context
|
||||||
|
|
||||||
|
def build( # type: ignore[override]
|
||||||
|
self,
|
||||||
|
common_prefix_len: int,
|
||||||
|
common_attn_metadata: CommonAttentionMetadata,
|
||||||
|
num_accepted_tokens: torch.Tensor | None = None,
|
||||||
|
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||||
|
fast_build: bool = False,
|
||||||
|
) -> "GDNRecoverSSMMetadata":
|
||||||
|
m = common_attn_metadata
|
||||||
|
rows = None
|
||||||
|
if self.use_spec_decode and num_decode_draft_tokens_cpu is not None:
|
||||||
|
if self._capturing and m.is_prefilling is None:
|
||||||
|
# CUDA graph capture: dummy decode batches, every row a spec window.
|
||||||
|
m = m.replace(is_prefilling=torch.zeros(m.num_reqs, dtype=torch.bool))
|
||||||
|
if m.is_prefilling is None or m.is_prefilling.device.type != "cpu":
|
||||||
|
raise ValueError("GDN RecoverSSM needs the CPU is_prefilling mask")
|
||||||
|
# Positive counts only defeat the base builder's "no drafts -> regular
|
||||||
|
# decode" shortcut: RecoverSSM keeps every post-prefill row on the spec path
|
||||||
|
# (extended conv window).
|
||||||
|
num_decode_draft_tokens_cpu, rows = recoverssm_spec_rows(
|
||||||
|
m.is_prefilling,
|
||||||
|
m.query_start_loc_cpu,
|
||||||
|
num_decode_draft_tokens_cpu,
|
||||||
|
m.num_reqs,
|
||||||
|
)
|
||||||
|
meta = super().build(
|
||||||
|
common_prefix_len,
|
||||||
|
m,
|
||||||
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||||
|
fast_build=fast_build,
|
||||||
|
)
|
||||||
|
base = {f.name: getattr(meta, f.name) for f in fields(meta)}
|
||||||
|
commit = None
|
||||||
|
if meta.num_spec_decodes > 0:
|
||||||
|
n = meta.num_spec_decodes
|
||||||
|
assert meta.spec_state_indices_tensor is not None
|
||||||
|
base["spec_state_indices_tensor"] = meta.spec_state_indices_tensor[:, :1]
|
||||||
|
# One accepted token per row: the checkpoint already holds the accepted
|
||||||
|
# state. Sized like the (possibly graph-padded) state indices.
|
||||||
|
base["num_accepted_tokens"] = self._ones[
|
||||||
|
: base["spec_state_indices_tensor"].shape[0]
|
||||||
|
]
|
||||||
|
if rows is None:
|
||||||
|
raise ValueError("GDN RecoverSSM: spec rows without draft counts")
|
||||||
|
request_indices = recoverssm_request_indices(
|
||||||
|
rows, n, m.query_start_loc.device
|
||||||
|
)
|
||||||
|
align = self.vllm_config.cache_config.mamba_cache_mode == "align"
|
||||||
|
commit = GDNRecoverSSMCommitMetadata(
|
||||||
|
state_indices=base["spec_state_indices_tensor"],
|
||||||
|
query_start_loc=meta.spec_query_start_loc,
|
||||||
|
request_indices=request_indices,
|
||||||
|
block_table=m.block_table_tensor if align else None,
|
||||||
|
num_computed_tokens=m.compute_num_computed_tokens() if align else None,
|
||||||
|
block_size=self.kv_cache_spec.block_size if align else None,
|
||||||
|
)
|
||||||
|
if not self._logged:
|
||||||
|
self._logged = True
|
||||||
|
logger.info(
|
||||||
|
"GDN RecoverSSM path taken: %d spec rows, align=%s", n, align
|
||||||
|
)
|
||||||
|
return GDNRecoverSSMMetadata(
|
||||||
|
**base,
|
||||||
|
recoverssm_commit=commit,
|
||||||
|
recoverssm_context=self._get_context() if commit is not None else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def build_for_cudagraph_capture(
|
||||||
|
self, common_attn_metadata: CommonAttentionMetadata
|
||||||
|
):
|
||||||
|
self._capturing = True
|
||||||
|
try:
|
||||||
|
return super().build_for_cudagraph_capture(common_attn_metadata)
|
||||||
|
finally:
|
||||||
|
self._capturing = False
|
||||||
|
|
||||||
|
|
||||||
|
class GDNRecoverSSMAttentionBackend(GDNAttentionBackend):
|
||||||
|
@staticmethod
|
||||||
|
def get_name() -> str:
|
||||||
|
return "GDN_RECOVERSSM"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_builder_cls() -> type[GDNRecoverSSMMetadataBuilder]:
|
||||||
|
return GDNRecoverSSMMetadataBuilder
|
||||||
@@ -104,7 +104,12 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
|||||||
self.compilation_config = vllm_config.compilation_config
|
self.compilation_config = vllm_config.compilation_config
|
||||||
self.num_spec_tokens: int = vllm_config.num_speculative_tokens
|
self.num_spec_tokens: int = vllm_config.num_speculative_tokens
|
||||||
self.use_spec_decode = self.num_spec_tokens > 0
|
self.use_spec_decode = self.num_spec_tokens > 0
|
||||||
self.use_replayssm = vllm_config.cache_config.use_replayssm
|
# RecoverSSM also rides on --use-replayssm but keeps its own metadata; the
|
||||||
|
# Mamba2 ReplaySSM ring-buffer workspaces below are not used then.
|
||||||
|
self.use_replayssm = (
|
||||||
|
vllm_config.cache_config.use_replayssm
|
||||||
|
and not vllm_config.cache_config.use_recoverssm
|
||||||
|
)
|
||||||
self.replayssm_buffer_len = vllm_config.cache_config.replayssm_buffer_len
|
self.replayssm_buffer_len = vllm_config.cache_config.replayssm_buffer_len
|
||||||
self.use_flashinfer_replayssm = (
|
self.use_flashinfer_replayssm = (
|
||||||
self.use_replayssm
|
self.use_replayssm
|
||||||
|
|||||||
@@ -0,0 +1,362 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""RecoverSSM protocol for the Qwen4Exp PLE short-conv layer.
|
||||||
|
|
||||||
|
When the GDN layers run RecoverSSM, the runner resets the shared num_accepted_tokens to
|
||||||
|
1 after each commit (align mode). The PLE dilated short conv keeps an extended window in
|
||||||
|
one state block and reads it at num_accepted - 1, so it must follow the same protocol:
|
||||||
|
every active decode row takes the spec path, the conv reads at offset 0, and after
|
||||||
|
sampling the accepted window is compacted to the front (the shared RecoverSSM compaction
|
||||||
|
kernel; in align mode it also writes the block-boundary window)."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field, fields, replace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.model_executor.layers.mamba.ops.recoverssm import (
|
||||||
|
compact_conv_state_kernel,
|
||||||
|
prepare_commit_plan_kernel,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.layers.mamba.recoverssm_utils import recoverssm_require
|
||||||
|
from vllm.triton_utils import triton
|
||||||
|
from vllm.v1.attention.backend import AttentionCGSupport
|
||||||
|
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||||
|
RecoverSSMMetadata,
|
||||||
|
RecoverSSMPostprocessMetadata,
|
||||||
|
clone_commit_tensors,
|
||||||
|
recoverssm_request_indices,
|
||||||
|
recoverssm_spec_rows,
|
||||||
|
)
|
||||||
|
from vllm.v1.attention.backends.short_conv_attn import (
|
||||||
|
PleShortConvAttentionBackend,
|
||||||
|
PleShortConvAttentionMetadata,
|
||||||
|
PleShortConvAttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _require(cond: bool, msg: str) -> None:
|
||||||
|
recoverssm_require(cond, msg, "PLE RecoverSSM")
|
||||||
|
|
||||||
|
|
||||||
|
class _PleConvCommit:
|
||||||
|
def __init__(self, conv_states, spec_query_len: int, max_num_reqs: int):
|
||||||
|
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
|
||||||
|
|
||||||
|
_require(len(conv_states) > 0, "PLE commit requires at least one layer")
|
||||||
|
if not is_conv_state_dim_first():
|
||||||
|
conv_states = [
|
||||||
|
s.transpose(-1, -2) for s in conv_states
|
||||||
|
] # -> [blocks, C, W]
|
||||||
|
self.conv_states = list(conv_states)
|
||||||
|
ref = self.conv_states[0]
|
||||||
|
_require(ref.ndim == 3, "PLE conv state must be [blocks, dim, window]")
|
||||||
|
dev = ref.device
|
||||||
|
for s in self.conv_states:
|
||||||
|
_require(
|
||||||
|
s.shape == ref.shape
|
||||||
|
and s.dtype == ref.dtype
|
||||||
|
and s.stride()[1:] == ref.stride()[1:]
|
||||||
|
and s.device == dev,
|
||||||
|
"PLE layers need matching conv states",
|
||||||
|
)
|
||||||
|
self.conv_dim, conv_len = ref.shape[1], ref.shape[2]
|
||||||
|
self.hist = conv_len - spec_query_len + 1
|
||||||
|
_require(self.hist > 0, "PLE conv state is shorter than its window")
|
||||||
|
self.spec_query_len = spec_query_len
|
||||||
|
t = lambda xs: torch.tensor(xs, dtype=torch.int64, device=dev)
|
||||||
|
self.base = t([s.data_ptr() for s in self.conv_states])
|
||||||
|
self.bstride = t([s.stride(0) for s in self.conv_states])
|
||||||
|
self.dstride = t([s.stride(1) for s in self.conv_states])
|
||||||
|
self.tstride = t([s.stride(2) for s in self.conv_states])
|
||||||
|
z = lambda: torch.empty(max_num_reqs, dtype=torch.int32, device=dev)
|
||||||
|
self.commit_lens, self.final_idx, self.bnd_idx, self.bnd_len = (
|
||||||
|
z(),
|
||||||
|
z(),
|
||||||
|
z(),
|
||||||
|
z(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def commit(
|
||||||
|
self,
|
||||||
|
num_accepted_tokens,
|
||||||
|
state_indices,
|
||||||
|
query_start_loc,
|
||||||
|
request_indices=None,
|
||||||
|
block_table=None,
|
||||||
|
num_computed_tokens=None,
|
||||||
|
mamba_block_size=None,
|
||||||
|
):
|
||||||
|
batch = state_indices.shape[0]
|
||||||
|
if batch == 0:
|
||||||
|
return
|
||||||
|
_require(state_indices.ndim == 1, "state indices must be one-dimensional")
|
||||||
|
_require(
|
||||||
|
batch <= self.commit_lens.shape[0],
|
||||||
|
"PLE commit batch exceeds its plan capacity",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
query_start_loc.ndim == 1 and query_start_loc.shape[0] == batch + 1,
|
||||||
|
"PLE commit metadata is incompatible",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
request_indices is None or request_indices.shape[0] >= batch,
|
||||||
|
"PLE request mapping is too short",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
num_accepted_tokens.ndim == 1
|
||||||
|
and (request_indices is not None or num_accepted_tokens.shape[0] >= batch),
|
||||||
|
"PLE accepted-token counts are too short",
|
||||||
|
)
|
||||||
|
_align = (block_table, num_computed_tokens, mamba_block_size)
|
||||||
|
_require(
|
||||||
|
all(x is None for x in _align) or all(x is not None for x in _align),
|
||||||
|
"PLE align metadata is incomplete",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
mamba_block_size is None or mamba_block_size >= self.spec_query_len,
|
||||||
|
"PLE align block size must cover one speculative window",
|
||||||
|
)
|
||||||
|
_require(
|
||||||
|
block_table is None or block_table.ndim == 2,
|
||||||
|
"PLE block table must be two-dimensional",
|
||||||
|
)
|
||||||
|
_dev = self.conv_states[0].device
|
||||||
|
_require(
|
||||||
|
all(
|
||||||
|
t_ is None or t_.device == _dev
|
||||||
|
for t_ in (
|
||||||
|
num_accepted_tokens,
|
||||||
|
state_indices,
|
||||||
|
query_start_loc,
|
||||||
|
request_indices,
|
||||||
|
block_table,
|
||||||
|
num_computed_tokens,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"PLE commit inputs must be on the same device",
|
||||||
|
)
|
||||||
|
bt = (0, 0) if block_table is None else block_table.stride()
|
||||||
|
prepare_commit_plan_kernel[(batch,)](
|
||||||
|
num_accepted_tokens,
|
||||||
|
request_indices,
|
||||||
|
state_indices,
|
||||||
|
query_start_loc,
|
||||||
|
block_table,
|
||||||
|
num_computed_tokens,
|
||||||
|
self.commit_lens,
|
||||||
|
self.final_idx,
|
||||||
|
self.bnd_idx,
|
||||||
|
self.bnd_len,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
mamba_block_size or 1,
|
||||||
|
block_table.shape[1] if block_table is not None else 1,
|
||||||
|
num_accepted_tokens.stride(0),
|
||||||
|
request_indices.stride(0) if request_indices is not None else 0,
|
||||||
|
state_indices.stride(0),
|
||||||
|
query_start_loc.stride(0),
|
||||||
|
bt[0],
|
||||||
|
bt[1],
|
||||||
|
0 if num_computed_tokens is None else num_computed_tokens.stride(0),
|
||||||
|
SPEC_QUERY_LEN=self.spec_query_len,
|
||||||
|
num_warps=1,
|
||||||
|
)
|
||||||
|
compact_conv_state_kernel[
|
||||||
|
(triton.cdiv(self.conv_dim, 256), batch, len(self.conv_states))
|
||||||
|
](
|
||||||
|
self.conv_states[0],
|
||||||
|
self.base,
|
||||||
|
self.bstride,
|
||||||
|
self.dstride,
|
||||||
|
self.tstride,
|
||||||
|
state_indices,
|
||||||
|
self.commit_lens,
|
||||||
|
self.final_idx,
|
||||||
|
self.bnd_idx,
|
||||||
|
self.bnd_len,
|
||||||
|
NULL_BLOCK_ID,
|
||||||
|
self.conv_dim,
|
||||||
|
self.hist,
|
||||||
|
state_indices.stride(0),
|
||||||
|
BLOCK_D=256,
|
||||||
|
BLOCK_HISTORY=triton.next_power_of_2(self.hist),
|
||||||
|
ALIGN_MODE=block_table is not None,
|
||||||
|
num_warps=4,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PleRecoverSSMCommitMetadata:
|
||||||
|
state_indices: torch.Tensor # [n]
|
||||||
|
query_start_loc: torch.Tensor # [n + 1]
|
||||||
|
request_indices: torch.Tensor | None
|
||||||
|
block_table: torch.Tensor | None
|
||||||
|
num_computed_tokens: torch.Tensor | None
|
||||||
|
block_size: int | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PleRecoverSSMMetadata(PleShortConvAttentionMetadata, RecoverSSMMetadata):
|
||||||
|
recoverssm_commit: PleRecoverSSMCommitMetadata | None = None
|
||||||
|
recoverssm_context: Any = field(default=None, repr=False, compare=False)
|
||||||
|
|
||||||
|
def commit_recoverssm_state(self, num_accepted_tokens):
|
||||||
|
c = self.recoverssm_commit
|
||||||
|
if c is None:
|
||||||
|
return None
|
||||||
|
n = self.num_spec_decodes
|
||||||
|
self.recoverssm_context.commit(
|
||||||
|
num_accepted_tokens,
|
||||||
|
c.state_indices[:n],
|
||||||
|
c.query_start_loc[: n + 1],
|
||||||
|
request_indices=c.request_indices,
|
||||||
|
block_table=c.block_table,
|
||||||
|
num_computed_tokens=c.num_computed_tokens,
|
||||||
|
mamba_block_size=c.block_size,
|
||||||
|
)
|
||||||
|
if c.block_table is None:
|
||||||
|
return None
|
||||||
|
assert c.block_size is not None and c.num_computed_tokens is not None
|
||||||
|
return RecoverSSMPostprocessMetadata(
|
||||||
|
num_spec_decodes=n,
|
||||||
|
request_indices=c.request_indices,
|
||||||
|
block_table=c.block_table,
|
||||||
|
num_computed_tokens=c.num_computed_tokens,
|
||||||
|
block_size=c.block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def snapshot_for_deferred_commit(self) -> "PleRecoverSSMMetadata":
|
||||||
|
c = self.recoverssm_commit
|
||||||
|
if c is None:
|
||||||
|
return self
|
||||||
|
return replace(
|
||||||
|
self,
|
||||||
|
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||||
|
# Same as the GDN RecoverSSM builder: uniform spec-decode batches replay as FULL
|
||||||
|
# graphs on the base builder's staged, null-padded buffers; the commit runs after
|
||||||
|
# sampling, outside the graph.
|
||||||
|
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||||
|
|
||||||
|
def __init__(self, kv_cache_spec, layer_names, vllm_config, device) -> None:
|
||||||
|
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||||
|
self._ones = torch.ones(
|
||||||
|
vllm_config.scheduler_config.max_num_seqs, dtype=torch.int32, device=device
|
||||||
|
)
|
||||||
|
self._ctx: _PleConvCommit | None = None
|
||||||
|
self._logged = False
|
||||||
|
self._capturing = False
|
||||||
|
|
||||||
|
def _get_ctx(self):
|
||||||
|
if self._ctx is not None:
|
||||||
|
fc = self.vllm_config.compilation_config.static_forward_context
|
||||||
|
live = [fc[name].kv_cache[0].data_ptr() for name in self.layer_names]
|
||||||
|
if [s.data_ptr() for s in self._ctx.conv_states] != live:
|
||||||
|
# Same lifecycle guard as the GDN builder: never commit into a
|
||||||
|
# KV cache that was torn down (profiling) and re-allocated.
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE RecoverSSM commit context is bound to a different KV "
|
||||||
|
"cache than its layers (stale context after a KV cache "
|
||||||
|
"re-allocation)"
|
||||||
|
)
|
||||||
|
if self._ctx is None:
|
||||||
|
fc = self.vllm_config.compilation_config.static_forward_context
|
||||||
|
convs = [fc[name].kv_cache[0] for name in self.layer_names]
|
||||||
|
ctx = _PleConvCommit(
|
||||||
|
convs, 1 + self.num_spec, self.vllm_config.scheduler_config.max_num_seqs
|
||||||
|
)
|
||||||
|
self._ctx = ctx
|
||||||
|
logger.info(
|
||||||
|
"PLE RecoverSSM commit context: %d layers, history %d",
|
||||||
|
len(convs),
|
||||||
|
ctx.hist,
|
||||||
|
)
|
||||||
|
return self._ctx
|
||||||
|
|
||||||
|
def build(
|
||||||
|
self,
|
||||||
|
common_prefix_len,
|
||||||
|
common_attn_metadata,
|
||||||
|
fast_build=False,
|
||||||
|
*,
|
||||||
|
num_accepted_tokens=None,
|
||||||
|
num_decode_draft_tokens_cpu=None,
|
||||||
|
**kwargs,
|
||||||
|
): # type: ignore[override]
|
||||||
|
m = common_attn_metadata
|
||||||
|
rows = None
|
||||||
|
if self.use_spec_decode and num_decode_draft_tokens_cpu is not None:
|
||||||
|
if self._capturing and m.is_prefilling is None:
|
||||||
|
# CUDA graph capture: dummy decode batches, every row a spec window.
|
||||||
|
m = m.replace(is_prefilling=torch.zeros(m.num_reqs, dtype=torch.bool))
|
||||||
|
if m.is_prefilling is None or m.is_prefilling.device.type != "cpu":
|
||||||
|
raise ValueError("PLE RecoverSSM needs the CPU is_prefilling mask")
|
||||||
|
# the same classification as the GDN builder, so both commits map the same
|
||||||
|
# requests
|
||||||
|
num_decode_draft_tokens_cpu, rows = recoverssm_spec_rows(
|
||||||
|
m.is_prefilling,
|
||||||
|
m.query_start_loc_cpu,
|
||||||
|
num_decode_draft_tokens_cpu,
|
||||||
|
m.num_reqs,
|
||||||
|
)
|
||||||
|
meta = super().build(
|
||||||
|
common_prefix_len,
|
||||||
|
m,
|
||||||
|
fast_build,
|
||||||
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not isinstance(meta, PleShortConvAttentionMetadata)
|
||||||
|
or meta.num_spec_decodes == 0
|
||||||
|
):
|
||||||
|
return meta
|
||||||
|
base = {f.name: getattr(meta, f.name) for f in fields(meta)}
|
||||||
|
n = meta.num_spec_decodes
|
||||||
|
# sized like the (possibly graph-padded) state indices
|
||||||
|
assert meta.spec_state_indices_tensor is not None
|
||||||
|
num_rows = meta.spec_state_indices_tensor.shape[0]
|
||||||
|
base["num_accepted_tokens"] = self._ones[:num_rows]
|
||||||
|
if rows is None:
|
||||||
|
raise ValueError("PLE RecoverSSM: spec rows without draft counts")
|
||||||
|
req_idx = recoverssm_request_indices(rows, n, m.query_start_loc.device)
|
||||||
|
align = self.vllm_config.cache_config.mamba_cache_mode == "align"
|
||||||
|
commit = PleRecoverSSMCommitMetadata(
|
||||||
|
state_indices=meta.spec_state_indices_tensor,
|
||||||
|
query_start_loc=meta.spec_query_start_loc,
|
||||||
|
request_indices=req_idx,
|
||||||
|
block_table=m.block_table_tensor if align else None,
|
||||||
|
num_computed_tokens=m.compute_num_computed_tokens() if align else None,
|
||||||
|
block_size=self.kv_cache_spec.block_size if align else None,
|
||||||
|
)
|
||||||
|
if not self._logged:
|
||||||
|
self._logged = True
|
||||||
|
logger.info("PLE RecoverSSM path taken: %d spec rows, align=%s", n, align)
|
||||||
|
return PleRecoverSSMMetadata(
|
||||||
|
**base, recoverssm_commit=commit, recoverssm_context=self._get_ctx()
|
||||||
|
)
|
||||||
|
|
||||||
|
def build_for_cudagraph_capture(self, common_attn_metadata):
|
||||||
|
self._capturing = True
|
||||||
|
try:
|
||||||
|
return super().build_for_cudagraph_capture(common_attn_metadata)
|
||||||
|
finally:
|
||||||
|
self._capturing = False
|
||||||
|
|
||||||
|
|
||||||
|
class PleRecoverSSMAttentionBackend(PleShortConvAttentionBackend):
|
||||||
|
@staticmethod
|
||||||
|
def get_name() -> str:
|
||||||
|
return "PLE_SHORT_CONV_RECOVERSSM"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_builder_cls() -> type[PleRecoverSSMMetadataBuilder]:
|
||||||
|
return PleRecoverSSMMetadataBuilder
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import abc
|
import abc
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, replace
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -24,3 +24,80 @@ class RecoverSSMMetadata(abc.ABC):
|
|||||||
self, num_accepted_tokens: torch.Tensor
|
self, num_accepted_tokens: torch.Tensor
|
||||||
) -> RecoverSSMPostprocessMetadata | None:
|
) -> RecoverSSMPostprocessMetadata | None:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def has_recoverssm_commit(self) -> bool:
|
||||||
|
"""Whether this step's metadata carries a commit (any spec row)."""
|
||||||
|
return getattr(self, "recoverssm_commit", None) is not None
|
||||||
|
|
||||||
|
def snapshot_for_deferred_commit(self) -> "RecoverSSMMetadata":
|
||||||
|
"""A copy whose commit inputs no longer alias per-step buffers.
|
||||||
|
|
||||||
|
Under pipeline parallelism a non-last rank learns the accepted counts
|
||||||
|
pp_size steps after its verify (the PP sampled-token broadcast), and
|
||||||
|
the steps in between rebuild the builders' persistent, graph-staged
|
||||||
|
buffers (spec state indices, query offsets, the batch-order block
|
||||||
|
table). The deferred commit must read this step's values, so they are
|
||||||
|
cloned here, on the current stream, before the next build overwrites
|
||||||
|
them. Implementations that do not support the PP deferral raise."""
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"{type(self).__name__} does not support the pipeline-parallel "
|
||||||
|
"RecoverSSM commit deferral"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def recoverssm_spec_rows(
|
||||||
|
is_prefilling_cpu: torch.Tensor,
|
||||||
|
query_start_loc_cpu: torch.Tensor,
|
||||||
|
num_decode_draft_tokens_cpu: torch.Tensor,
|
||||||
|
num_reqs: int,
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Which batch rows take the RecoverSSM spec path, shared by the RecoverSSM metadata
|
||||||
|
builders (GDN, PLE).
|
||||||
|
|
||||||
|
Every active decode row does (a draft-less step is a window of one token), plus
|
||||||
|
every row that has drafts. Only the first ``num_reqs`` rows are real; padding rows
|
||||||
|
never take the spec path. Returns the draft-count vector to hand to the base builder
|
||||||
|
(1 for spec rows, -1 otherwise, padding included) and the spec rows in batch order,
|
||||||
|
which is also the order of the base builder's spec state indices."""
|
||||||
|
query_lens = query_start_loc_cpu[: num_reqs + 1].diff()
|
||||||
|
active_decode = (~is_prefilling_cpu[:num_reqs]) & (query_lens > 0)
|
||||||
|
spec_mask = (num_decode_draft_tokens_cpu[:num_reqs] >= 0) | active_decode
|
||||||
|
drafts = torch.full_like(num_decode_draft_tokens_cpu, -1)
|
||||||
|
drafts[:num_reqs] = torch.where(spec_mask, 1, -1).to(drafts.dtype)
|
||||||
|
return drafts, spec_mask.nonzero().flatten()
|
||||||
|
|
||||||
|
|
||||||
|
def recoverssm_request_indices(
|
||||||
|
rows: torch.Tensor, num_spec_decodes: int, device: torch.device
|
||||||
|
) -> torch.Tensor | None:
|
||||||
|
"""None when the spec rows are exactly 0..n-1, else their batch rows (the commit
|
||||||
|
plan's request mapping). Raises if the base builder counted a different number of
|
||||||
|
spec rows than the classification above."""
|
||||||
|
if rows.numel() != num_spec_decodes:
|
||||||
|
raise ValueError(
|
||||||
|
f"RecoverSSM: {rows.numel()} spec rows classified, but the builder "
|
||||||
|
f"produced {num_spec_decodes}"
|
||||||
|
)
|
||||||
|
if num_spec_decodes == 0 or int(rows[-1]) == num_spec_decodes - 1:
|
||||||
|
return None
|
||||||
|
return rows.to(torch.int32).to(device, non_blocking=True)
|
||||||
|
|
||||||
|
|
||||||
|
def clone_commit_tensors(commit, num_spec_decodes: int):
|
||||||
|
"""Clone the commit inputs a deferred (PP) commit reads, trimmed to the real
|
||||||
|
spec rows. Works for every RecoverSSM commit dataclass with the fields
|
||||||
|
state_indices, query_start_loc, request_indices, block_table and
|
||||||
|
num_computed_tokens."""
|
||||||
|
n = num_spec_decodes
|
||||||
|
|
||||||
|
def c(t: torch.Tensor | None) -> torch.Tensor | None:
|
||||||
|
return None if t is None else t.clone()
|
||||||
|
|
||||||
|
return replace(
|
||||||
|
commit,
|
||||||
|
state_indices=commit.state_indices[:n].clone(),
|
||||||
|
query_start_loc=commit.query_start_loc[: n + 1].clone(),
|
||||||
|
request_indices=c(commit.request_indices),
|
||||||
|
block_table=c(commit.block_table),
|
||||||
|
num_computed_tokens=c(commit.num_computed_tokens),
|
||||||
|
)
|
||||||
|
|||||||
@@ -264,8 +264,9 @@ def resolve_kv_cache_layout(
|
|||||||
"""Resolve one KV cache layout for the whole model.
|
"""Resolve one KV cache layout for the whole model.
|
||||||
|
|
||||||
Runs once in the engine core. Every worker reports the layouts its backends
|
Runs once in the engine core. Every worker reports the layouts its backends
|
||||||
support, most preferred first (``get_supported_kv_cache_layouts``); all
|
support, most preferred first (``get_supported_kv_cache_layouts``); ranks
|
||||||
ranks run the same backends, so their lists must agree. Specs mixing HNC
|
may expose different sets under pipeline parallelism, but their lists must
|
||||||
|
share at least one layout. Specs mixing HNC
|
||||||
shapes narrow the candidates to block-compact layouts. An explicit
|
shapes narrow the candidates to block-compact layouts. An explicit
|
||||||
``VLLM_KV_CACHE_LAYOUT`` must be one of the candidates or resolution fails,
|
``VLLM_KV_CACHE_LAYOUT`` must be one of the candidates or resolution fails,
|
||||||
with the legacy ``NHD``/``HND`` names as aliases for ``LBNHC``/``LBHNC``; the
|
with the legacy ``NHD``/``HND`` names as aliases for ``LBNHC``/``LBHNC``; the
|
||||||
@@ -282,10 +283,20 @@ def resolve_kv_cache_layout(
|
|||||||
assert supported_layouts and all(supported_layouts), (
|
assert supported_layouts and all(supported_layouts), (
|
||||||
"No worker reported supported KV cache layouts."
|
"No worker reported supported KV cache layouts."
|
||||||
)
|
)
|
||||||
assert all(names == supported_layouts[0] for names in supported_layouts[1:]), (
|
# Fork: restore PP fix #54795 from the E2 base, dropped
|
||||||
f"Workers disagree on supported KV cache layouts: {supported_layouts}."
|
# in the v0.30.0 rebase. Under pipeline parallelism the stages run
|
||||||
)
|
# different backends (PP0 carries the PLE short-conv, PP1 the drafter), so
|
||||||
candidates = [_layout_from_name(name) for name in supported_layouts[0]]
|
# their lists need only share a layout, not be identical.
|
||||||
|
common_layout_names = [
|
||||||
|
name
|
||||||
|
for name in supported_layouts[0]
|
||||||
|
if all(name in names for names in supported_layouts[1:])
|
||||||
|
]
|
||||||
|
if not common_layout_names:
|
||||||
|
raise ValueError(
|
||||||
|
f"Workers share no supported KV cache layout: {supported_layouts}."
|
||||||
|
)
|
||||||
|
candidates = [_layout_from_name(name) for name in common_layout_names]
|
||||||
|
|
||||||
# A block-compact layout means the block is densely packed in memory, so any mix of
|
# A block-compact layout means the block is densely packed in memory, so any mix of
|
||||||
# specs can re-interpret HNC with different sizes as long as the total number of
|
# specs can re-interpret HNC with different sizes as long as the total number of
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from vllm.v1.core.kv_cache_utils import (
|
|||||||
KVCacheBlock,
|
KVCacheBlock,
|
||||||
)
|
)
|
||||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||||
|
CircularBufferManager,
|
||||||
CrossAttentionManager,
|
CrossAttentionManager,
|
||||||
MambaManager,
|
MambaManager,
|
||||||
SingleTypeKVCacheManager,
|
SingleTypeKVCacheManager,
|
||||||
@@ -147,6 +148,17 @@ class KVCacheCoordinator(ABC):
|
|||||||
)
|
)
|
||||||
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||||
)
|
)
|
||||||
|
# v2.5 side-cache layout: a slot-packed ring group shares each pool
|
||||||
|
# block between several requests (kv_cache_utils._choose_ring_slots).
|
||||||
|
for kv_cache_group, manager in zip(
|
||||||
|
self.kv_cache_config.kv_cache_groups, self.single_type_managers
|
||||||
|
):
|
||||||
|
if kv_cache_group.slots_per_block > 1:
|
||||||
|
assert type(manager) is CircularBufferManager, (
|
||||||
|
f"slot packing is only defined for the QSA ring, got "
|
||||||
|
f"{type(manager).__name__}"
|
||||||
|
)
|
||||||
|
manager.enable_slot_packing(kv_cache_group.slots_per_block)
|
||||||
# Match Mamba checkpoints to Eagle's attention replay boundary.
|
# Match Mamba checkpoints to Eagle's attention replay boundary.
|
||||||
if use_eagle:
|
if use_eagle:
|
||||||
for manager in self.single_type_managers:
|
for manager in self.single_type_managers:
|
||||||
@@ -233,6 +245,20 @@ class KVCacheCoordinator(ABC):
|
|||||||
apply_admission_cap=apply_admission_cap,
|
apply_admission_cap=apply_admission_cap,
|
||||||
prefill_end=prefill_end,
|
prefill_end=prefill_end,
|
||||||
)
|
)
|
||||||
|
if apply_admission_cap:
|
||||||
|
# Full-sequence admission: a retained Mamba checkpoint is
|
||||||
|
# held outside the block table for the request's
|
||||||
|
# lifetime, so reserve it here (v2.5). Per-step
|
||||||
|
# allocation needs no term: a held block is simply not in
|
||||||
|
# the free queue that check reads.
|
||||||
|
num_blocks_to_allocate += (
|
||||||
|
manager.get_num_retained_checkpoint_blocks_to_reserve(
|
||||||
|
request_id,
|
||||||
|
num_tokens_main_model,
|
||||||
|
new_computed_blocks[i],
|
||||||
|
num_local_computed_tokens,
|
||||||
|
)
|
||||||
|
)
|
||||||
return num_blocks_to_allocate
|
return num_blocks_to_allocate
|
||||||
|
|
||||||
def allocate_new_computed_blocks(
|
def allocate_new_computed_blocks(
|
||||||
@@ -395,26 +421,42 @@ class KVCacheCoordinator(ABC):
|
|||||||
request_id: The request ID.
|
request_id: The request ID.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
# Retained Mamba checkpoints go back last, after every group's table,
|
||||||
|
# so a finished request's checkpoint is never evicted ahead of the
|
||||||
|
# attention prefix it resumes (v2.5).
|
||||||
|
retained: list[KVCacheBlock] = []
|
||||||
for manager in self.single_type_managers:
|
for manager in self.single_type_managers:
|
||||||
|
retained.extend(manager.pop_retained_checkpoints(request_id))
|
||||||
manager.free(request_id)
|
manager.free(request_id)
|
||||||
|
if retained:
|
||||||
|
self.block_pool.free_blocks(retained)
|
||||||
|
|
||||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||||
"""Pop the request's bookkeeping from all single-type managers and
|
"""Pop the request's bookkeeping from all single-type managers and
|
||||||
return its blocks without returning them to the block pool. The
|
return its blocks without returning them to the block pool. The
|
||||||
caller must eventually pass the returned blocks to
|
caller must eventually pass the returned blocks to
|
||||||
`block_pool.free_blocks`, freeing them in reverse order (so that
|
`block_pool.free_blocks` as-is: they are already ordered so that
|
||||||
tail blocks are evicted first).
|
tail blocks are evicted first, within each KV cache group.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
request_id: The request ID.
|
request_id: The request ID.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The request's blocks in allocation order.
|
The request's blocks in eviction order (each group's blocks
|
||||||
|
reversed, then concatenated group by group).
|
||||||
|
|
||||||
"""
|
"""
|
||||||
blocks: list[KVCacheBlock] = []
|
blocks: list[KVCacheBlock] = []
|
||||||
|
retained: list[KVCacheBlock] = []
|
||||||
for manager in self.single_type_managers:
|
for manager in self.single_type_managers:
|
||||||
blocks.extend(manager.pop_blocks_for_free(request_id))
|
retained.extend(manager.pop_retained_checkpoints(request_id))
|
||||||
|
# Reverse within each group so the concatenation is already in
|
||||||
|
# eviction order, matching `free()`, which frees each group in its
|
||||||
|
# own reversed `free_blocks()` call. Reversing the flat list
|
||||||
|
# instead would emit whole groups in the wrong order.
|
||||||
|
blocks.extend(reversed(manager.pop_blocks_for_free(request_id)))
|
||||||
|
# Retained checkpoints last, as in `free()`.
|
||||||
|
blocks.extend(retained)
|
||||||
return blocks
|
return blocks
|
||||||
|
|
||||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
|
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
|
||||||
|
|||||||
@@ -223,6 +223,52 @@ class KVCacheManager:
|
|||||||
tuple(() for _ in range(self.num_kv_cache_groups))
|
tuple(() for _ in range(self.num_kv_cache_groups))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Prefix-cache hits pinned for requests waiting for admission
|
||||||
|
# ({request_id: blocks}). A waiting follow-up turn holds no ref on its
|
||||||
|
# cached prefix, and with a prefix unit finer than the block its
|
||||||
|
# whole hit depends on every one of those blocks (the resume key sits
|
||||||
|
# in the top block); under pool pressure other requests' allocations
|
||||||
|
# popped them from the free queue and the turn re-prefilled from 0
|
||||||
|
# (under a long-context multi-turn agent load: 4-5 total misses per
|
||||||
|
# 300 s at unit 64/512, none at 4096). Pinned blocks stay out of the
|
||||||
|
# free queue, so they count against admission and the watermark like
|
||||||
|
# any referenced block; they
|
||||||
|
# are released on admission (the table takes over), on free, and
|
||||||
|
# whenever a running request would otherwise fail to allocate
|
||||||
|
# (``yield_pinned_hits``), so a pin never causes a preemption.
|
||||||
|
self._pinned_hits: dict[str, list[KVCacheBlock]] = {}
|
||||||
|
# A finished request's hashed blocks (its attention prefix and
|
||||||
|
# the GDN states it holds) stay referenced as a "grace" pin until the
|
||||||
|
# conversation's next turn is admitted on them. With waiter pins alone,
|
||||||
|
# every remaining total miss had lost its top attention block BEFORE
|
||||||
|
# the next turn was first looked up -- in the gap between the
|
||||||
|
# previous turn's free and the next turn's arrival, or while its
|
||||||
|
# admission kept failing -- when nothing referenced the prefix and
|
||||||
|
# the running set's allocations popped it; once the hybrid hit is 0
|
||||||
|
# a waiter pin has nothing to pin. Grace pins yield first (oldest
|
||||||
|
# first) whenever anyone needs blocks, so they never cost a running
|
||||||
|
# request or an admission anything a free queue would have given.
|
||||||
|
# A grace is one extra ref per block, taken on whichever free path
|
||||||
|
# the request takes (immediate or deferred behind an in-flight step).
|
||||||
|
self._grace_pins: dict[str, list[KVCacheBlock]] = {}
|
||||||
|
self._grace_block_ids: dict[str, frozenset[int]] = {}
|
||||||
|
self._grace_prompt_tokens: dict[str, int] = {}
|
||||||
|
# Only for fine-grained hybrid hits (align Mamba hashed finer than its
|
||||||
|
# block): there the next turn's key sits in the top attention block
|
||||||
|
# and one lost block is a total miss; at a block-sized unit the top
|
||||||
|
# block is a spare and nothing changes. The scheduler also turns
|
||||||
|
# graces off when a KV connector is configured (as for waiter pins).
|
||||||
|
self.enable_grace_pins = (
|
||||||
|
self.enable_caching and self.coordinator.enable_partial_hash_hits
|
||||||
|
)
|
||||||
|
# Shortfall (blocks) of the last allocate_slots that returned None.
|
||||||
|
self.last_alloc_shortfall = 0
|
||||||
|
# A producer that moves its live GDN tail's hash to a CoW copy keeps
|
||||||
|
# writing the source; pins and graces follow the hash to the copy.
|
||||||
|
for manager in self.coordinator.single_type_managers:
|
||||||
|
if isinstance(manager, MambaManager):
|
||||||
|
manager.moved_hash_listeners.append(self._pins_follow_moved_hash)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def usage(self) -> float:
|
def usage(self) -> float:
|
||||||
"""Get the KV cache usage.
|
"""Get the KV cache usage.
|
||||||
@@ -536,6 +582,9 @@ class KVCacheManager:
|
|||||||
)
|
)
|
||||||
required_blocks = num_blocks_to_allocate + watermark_blocks
|
required_blocks = num_blocks_to_allocate + watermark_blocks
|
||||||
if required_blocks > self.block_pool.get_num_free_blocks():
|
if required_blocks > self.block_pool.get_num_free_blocks():
|
||||||
|
self.last_alloc_shortfall = (
|
||||||
|
required_blocks - self.block_pool.get_num_free_blocks()
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
num_tokens_main_model = total_computed_tokens + num_new_tokens
|
num_tokens_main_model = total_computed_tokens + num_new_tokens
|
||||||
@@ -576,7 +625,9 @@ class KVCacheManager:
|
|||||||
required_blocks = num_blocks_to_allocate + watermark_blocks
|
required_blocks = num_blocks_to_allocate + watermark_blocks
|
||||||
if required_blocks > available_blocks:
|
if required_blocks > available_blocks:
|
||||||
# Cannot allocate new blocks
|
# Cannot allocate new blocks
|
||||||
|
self.last_alloc_shortfall = required_blocks - available_blocks
|
||||||
return None
|
return None
|
||||||
|
self.last_alloc_shortfall = 0
|
||||||
|
|
||||||
if (
|
if (
|
||||||
new_computed_block_list is not self.empty_kv_cache_blocks.blocks
|
new_computed_block_list is not self.empty_kv_cache_blocks.blocks
|
||||||
@@ -602,6 +653,11 @@ class KVCacheManager:
|
|||||||
# P/D: delay caching blocks if we have to recv from
|
# P/D: delay caching blocks if we have to recv from
|
||||||
# remote. Update state for locally cached blocks.
|
# remote. Update state for locally cached blocks.
|
||||||
if delay_cache_blocks:
|
if delay_cache_blocks:
|
||||||
|
self.release_pinned_hit(request.request_id)
|
||||||
|
if self._grace_pins and num_new_computed_tokens > 0:
|
||||||
|
self._release_grace_for_hit(
|
||||||
|
new_computed_block_list, num_local_computed_tokens
|
||||||
|
)
|
||||||
return self.create_kv_cache_blocks(new_blocks)
|
return self.create_kv_cache_blocks(new_blocks)
|
||||||
|
|
||||||
# NOTE(woosuk): We want to commit (cache) up to num_local_computed_tokens
|
# NOTE(woosuk): We want to commit (cache) up to num_local_computed_tokens
|
||||||
@@ -615,6 +671,13 @@ class KVCacheManager:
|
|||||||
)
|
)
|
||||||
self.coordinator.cache_blocks(request, num_tokens_to_cache)
|
self.coordinator.cache_blocks(request, num_tokens_to_cache)
|
||||||
|
|
||||||
|
# Admitted: the table now references the hit, the pins are redundant.
|
||||||
|
self.release_pinned_hit(request.request_id)
|
||||||
|
if self._grace_pins and num_new_computed_tokens > 0:
|
||||||
|
# ... and so does the grace pin of the turn it resumes.
|
||||||
|
self._release_grace_for_hit(
|
||||||
|
new_computed_block_list, num_local_computed_tokens
|
||||||
|
)
|
||||||
return self.create_kv_cache_blocks(new_blocks)
|
return self.create_kv_cache_blocks(new_blocks)
|
||||||
|
|
||||||
def free(self, request: Request) -> None:
|
def free(self, request: Request) -> None:
|
||||||
@@ -626,7 +689,331 @@ class KVCacheManager:
|
|||||||
request: The request to free the blocks.
|
request: The request to free the blocks.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
self.coordinator.free(request.request_id)
|
self.release_pinned_hit(request.request_id)
|
||||||
|
if not self._grace_eligible(request):
|
||||||
|
self.coordinator.free(request.request_id)
|
||||||
|
return
|
||||||
|
# Same blocks and order as coordinator.free (the deferred path frees
|
||||||
|
# this list as-is too); the grace takes its ref before the table's
|
||||||
|
# goes, so the hashed blocks never pass through the free queue.
|
||||||
|
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
|
||||||
|
self._take_grace_pin(request, blocks)
|
||||||
|
self.block_pool.free_blocks(blocks)
|
||||||
|
|
||||||
|
# Finished requests whose hashed blocks stay pinned for the next turn
|
||||||
|
# (oldest released first beyond this). One per conversation is the
|
||||||
|
# need; twice the deployed max_num_seqs.
|
||||||
|
MAX_GRACE_PINS = 16
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_grace_pins(self) -> int:
|
||||||
|
return len(self._grace_pins)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_grace_pinned_blocks(self) -> int:
|
||||||
|
return sum(len(blocks) for blocks in self._grace_pins.values())
|
||||||
|
|
||||||
|
def grace_pinned_blocks(self, request_id: str) -> list[KVCacheBlock]:
|
||||||
|
return list(self._grace_pins.get(request_id, ()))
|
||||||
|
|
||||||
|
def _grace_eligible(self, request: Request) -> bool:
|
||||||
|
"""A turn that finished with output (stop, length, repetition, or
|
||||||
|
an abort after decoding) may have a next turn that resumes it. A
|
||||||
|
preemption (still RUNNING), an aborted waiter (no output), an error
|
||||||
|
or an ignored request leaves nothing worth holding."""
|
||||||
|
return (
|
||||||
|
self.enable_grace_pins
|
||||||
|
and self.enable_caching
|
||||||
|
and self.prefix_cache_lookup_enabled(request)
|
||||||
|
and request.is_finished()
|
||||||
|
and request.status
|
||||||
|
not in (RequestStatus.FINISHED_ERROR, RequestStatus.FINISHED_IGNORED)
|
||||||
|
and request.num_output_tokens > 0
|
||||||
|
)
|
||||||
|
|
||||||
|
def _take_grace_pin(self, request: Request, blocks: list[KVCacheBlock]) -> int:
|
||||||
|
"""Take one extra ref on every hashed block of a finished request's
|
||||||
|
just-popped blocks (``blocks`` is in eviction order: each group's
|
||||||
|
table tail first, the held GDN states last). The caller then drops
|
||||||
|
the table refs, now or after the in-flight fence; the hashed blocks
|
||||||
|
stay out of the free queue until the grace is released. Unhashed
|
||||||
|
blocks (running state, MTP scratch, an unkeyed last block) go back
|
||||||
|
with the table refs as before. Returns the number of blocks held."""
|
||||||
|
keep: list[KVCacheBlock] = []
|
||||||
|
seen: set[int] = set()
|
||||||
|
for block in blocks:
|
||||||
|
if block.is_null or block.block_hash is None or id(block) in seen:
|
||||||
|
continue
|
||||||
|
# Still referenced by the request's table or hold: the grace ref
|
||||||
|
# never moves a block in or out of a free queue (any pool).
|
||||||
|
assert block.ref_cnt > 0
|
||||||
|
seen.add(id(block))
|
||||||
|
keep.append(block)
|
||||||
|
if not keep:
|
||||||
|
return 0
|
||||||
|
request_id = request.request_id
|
||||||
|
self.release_grace_pin(request_id) # a repeated free: replace
|
||||||
|
for block in keep:
|
||||||
|
block.ref_cnt += 1
|
||||||
|
self._grace_pins[request_id] = keep
|
||||||
|
self._grace_block_ids[request_id] = frozenset(seen)
|
||||||
|
self._grace_prompt_tokens[request_id] = request.num_prompt_tokens
|
||||||
|
while len(self._grace_pins) > self.MAX_GRACE_PINS:
|
||||||
|
self.release_grace_pin(next(iter(self._grace_pins)))
|
||||||
|
return len(keep)
|
||||||
|
|
||||||
|
def release_grace_pin(self, request_id: str) -> int:
|
||||||
|
"""Return a finished request's grace-pinned blocks to the pool (in
|
||||||
|
the eviction order a free would have used). Returns the count."""
|
||||||
|
blocks = self._grace_pins.pop(request_id, None)
|
||||||
|
self._grace_block_ids.pop(request_id, None)
|
||||||
|
self._grace_prompt_tokens.pop(request_id, None)
|
||||||
|
if not blocks:
|
||||||
|
return 0
|
||||||
|
self.block_pool.free_blocks(blocks)
|
||||||
|
return len(blocks)
|
||||||
|
|
||||||
|
def _release_grace_for_hit(
|
||||||
|
self,
|
||||||
|
computed_blocks: tuple[Sequence[KVCacheBlock], ...],
|
||||||
|
num_hit_tokens: int,
|
||||||
|
) -> None:
|
||||||
|
"""An admitted hit now references its blocks through the table, so
|
||||||
|
the grace of the turn it resumes has done its job. That turn is a
|
||||||
|
grace that holds the deepest hit block of every group whose deepest
|
||||||
|
hit block any grace holds (the GDN state it resumes from, the
|
||||||
|
attention block holding the key), and whose prompt the hit reaches
|
||||||
|
to within two prefix units (the EAGLE-dropped tail key). A new turn
|
||||||
|
that only shares a system prompt with finished conversations
|
||||||
|
releases none of theirs."""
|
||||||
|
candidates: set[str] | None = None
|
||||||
|
for group in computed_blocks:
|
||||||
|
deepest = next((b for b in reversed(group) if not b.is_null), None)
|
||||||
|
if deepest is None:
|
||||||
|
continue
|
||||||
|
owners = {
|
||||||
|
request_id
|
||||||
|
for request_id, block_ids in self._grace_block_ids.items()
|
||||||
|
if id(deepest) in block_ids
|
||||||
|
}
|
||||||
|
if owners:
|
||||||
|
candidates = owners if candidates is None else candidates & owners
|
||||||
|
if not candidates:
|
||||||
|
return
|
||||||
|
reach = num_hit_tokens + 2 * self.block_pool.hash_block_size
|
||||||
|
for request_id in candidates:
|
||||||
|
if reach >= self._grace_prompt_tokens.get(request_id, 0):
|
||||||
|
self.release_grace_pin(request_id)
|
||||||
|
|
||||||
|
def _pins_follow_moved_hash(
|
||||||
|
self, source_block: KVCacheBlock, cow_block: KVCacheBlock
|
||||||
|
) -> None:
|
||||||
|
"""A Mamba producer moved ``source_block``'s hashes to ``cow_block``
|
||||||
|
and keeps writing ``source_block``. A grace or waiter pin on the
|
||||||
|
source follows to the copy (as a hold does), so it protects the
|
||||||
|
hashed state and never leaves an extra ref on a block its owner must
|
||||||
|
hold exclusively (speculative relocation asserts on that)."""
|
||||||
|
for request_id, blocks in self._grace_pins.items():
|
||||||
|
if id(source_block) in self._grace_block_ids[request_id]:
|
||||||
|
self._swap_pinned_block(blocks, source_block, cow_block)
|
||||||
|
self._grace_block_ids[request_id] = frozenset(
|
||||||
|
id(block) for block in blocks
|
||||||
|
)
|
||||||
|
for blocks in self._pinned_hits.values():
|
||||||
|
self._swap_pinned_block(blocks, source_block, cow_block)
|
||||||
|
|
||||||
|
def _swap_pinned_block(
|
||||||
|
self,
|
||||||
|
blocks: list[KVCacheBlock],
|
||||||
|
source_block: KVCacheBlock,
|
||||||
|
cow_block: KVCacheBlock,
|
||||||
|
) -> None:
|
||||||
|
for i, block in enumerate(blocks):
|
||||||
|
if block is source_block:
|
||||||
|
if any(other is cow_block for other in blocks):
|
||||||
|
del blocks[i]
|
||||||
|
else:
|
||||||
|
assert cow_block.ref_cnt > 0
|
||||||
|
cow_block.ref_cnt += 1
|
||||||
|
blocks[i] = cow_block
|
||||||
|
self.block_pool.free_blocks([source_block])
|
||||||
|
return
|
||||||
|
|
||||||
|
def num_yieldable_blocks(
|
||||||
|
self,
|
||||||
|
keep: str | None = None,
|
||||||
|
keep_hit: tuple[Sequence[KVCacheBlock], ...] | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Blocks that would return to the free queue if every grace and
|
||||||
|
every waiter pin except ``keep``'s were released now. With
|
||||||
|
``keep_hit`` (the hit ``keep`` is being admitted on), ``keep``'s
|
||||||
|
pinned blocks outside that hit (the proof block past it) count too:
|
||||||
|
``release_pinned_extras`` can give them back. Blocks of that hit
|
||||||
|
never count: freed, they are taken straight back by the admission
|
||||||
|
(a hit that grew past the pin snapshot would otherwise be counted
|
||||||
|
through the grace that still holds it)."""
|
||||||
|
pins: dict[int, list] = {}
|
||||||
|
holders = [*self._grace_pins.values()] + [
|
||||||
|
blocks
|
||||||
|
for request_id, blocks in self._pinned_hits.items()
|
||||||
|
if request_id != keep
|
||||||
|
]
|
||||||
|
hit_ids: set[int] = set()
|
||||||
|
if keep is not None and keep_hit is not None:
|
||||||
|
holders.append(self._pinned_extras(keep, keep_hit))
|
||||||
|
hit_ids = {id(block) for group in keep_hit for block in group}
|
||||||
|
for blocks in holders:
|
||||||
|
for block in blocks:
|
||||||
|
entry = pins.setdefault(id(block), [block, 0])
|
||||||
|
entry[1] += 1
|
||||||
|
return sum(
|
||||||
|
1
|
||||||
|
for block_id, (block, n) in pins.items()
|
||||||
|
if block.ref_cnt == n and block_id not in hit_ids
|
||||||
|
)
|
||||||
|
|
||||||
|
def _pinned_extras(
|
||||||
|
self, request_id: str, hit: tuple[Sequence[KVCacheBlock], ...]
|
||||||
|
) -> list[KVCacheBlock]:
|
||||||
|
hit_ids = {id(block) for group in hit for block in group}
|
||||||
|
return [
|
||||||
|
block
|
||||||
|
for block in self._pinned_hits.get(request_id, ())
|
||||||
|
if id(block) not in hit_ids
|
||||||
|
]
|
||||||
|
|
||||||
|
def release_pinned_extras(
|
||||||
|
self, request_id: str, hit: tuple[Sequence[KVCacheBlock], ...]
|
||||||
|
) -> int:
|
||||||
|
"""Drop from the request's pin the blocks outside ``hit``: the proof
|
||||||
|
block past it, and any hit block a later lookup added. The request
|
||||||
|
is being admitted on ``hit`` now, so they protect nothing it still
|
||||||
|
needs; they cost it blocks. Returns the number dropped."""
|
||||||
|
extras = self._pinned_extras(request_id, hit)
|
||||||
|
if not extras:
|
||||||
|
return 0
|
||||||
|
extra_ids = {id(block) for block in extras}
|
||||||
|
rest = [
|
||||||
|
block
|
||||||
|
for block in self._pinned_hits[request_id]
|
||||||
|
if id(block) not in extra_ids
|
||||||
|
]
|
||||||
|
if rest:
|
||||||
|
self._pinned_hits[request_id] = rest
|
||||||
|
else:
|
||||||
|
del self._pinned_hits[request_id]
|
||||||
|
self.block_pool.free_blocks(extras)
|
||||||
|
return len(extras)
|
||||||
|
|
||||||
|
# Waiting requests whose prefix-cache hit is pinned at once (arrival
|
||||||
|
# order). Pins never block anyone (they yield before a running request
|
||||||
|
# fails to allocate), so the bound only caps the lookup work per step;
|
||||||
|
# twice the deployed max_num_seqs covers every waiter seen under load.
|
||||||
|
MAX_PINNED_WAITERS = 16
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_pinned_waiters(self) -> int:
|
||||||
|
return len(self._pinned_hits)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_pinned_blocks(self) -> int:
|
||||||
|
return sum(len(blocks) for blocks in self._pinned_hits.values())
|
||||||
|
|
||||||
|
def pin_waiting_hit(self, request: Request) -> int:
|
||||||
|
"""Pin the request's current prefix-cache hit while it waits for
|
||||||
|
admission: every non-null hit block in every group (the attention
|
||||||
|
prefix up to the key and the GDN state it resumes from) takes a ref
|
||||||
|
and leaves the free queue. Idempotent. Returns the number of blocks
|
||||||
|
pinned now (0 if nothing hit, already pinned, caching is off, or the
|
||||||
|
waiter bound is reached).
|
||||||
|
|
||||||
|
Known gap: a pin is a snapshot of the hit at pin time. It is not
|
||||||
|
refreshed if the hit grows while the request waits (a sibling
|
||||||
|
computing more of the same prefix); the admission lookup still finds
|
||||||
|
whatever is cached then, so the only cost is a smaller hit, never a
|
||||||
|
wrong one. A producer moving the hash of a pinned live GDN block into
|
||||||
|
its CoW copy is followed (``_pins_follow_moved_hash``)."""
|
||||||
|
request_id = request.request_id
|
||||||
|
if (
|
||||||
|
request_id in self._pinned_hits
|
||||||
|
or not self.enable_caching
|
||||||
|
or not self.prefix_cache_lookup_enabled(request)
|
||||||
|
or len(self._pinned_hits) >= self.MAX_PINNED_WAITERS
|
||||||
|
):
|
||||||
|
return 0
|
||||||
|
computed, num_hit, _ = self.get_computed_blocks(request)
|
||||||
|
blocks = [
|
||||||
|
block
|
||||||
|
for group in computed.blocks
|
||||||
|
for block in group
|
||||||
|
if not block.is_null
|
||||||
|
]
|
||||||
|
if num_hit == 0 or not blocks:
|
||||||
|
return 0
|
||||||
|
# The EAGLE/MTP proof: fine-grained attention hits match one hash
|
||||||
|
# unit past the key and drop it, so the hit also depends on the block
|
||||||
|
# holding that unit -- a different block when the key ends a block
|
||||||
|
# (and never part of the returned hit). Pin it too.
|
||||||
|
unit = self.block_pool.hash_block_size
|
||||||
|
proof_idx = num_hit // unit
|
||||||
|
if proof_idx < len(request.block_hashes):
|
||||||
|
seen = {block.block_id for block in blocks}
|
||||||
|
for group_id in range(self.num_kv_cache_groups):
|
||||||
|
cached = self.block_pool.get_cached_block(
|
||||||
|
request.block_hashes[proof_idx], [group_id]
|
||||||
|
)
|
||||||
|
if cached and cached[0].block_id not in seen:
|
||||||
|
seen.add(cached[0].block_id)
|
||||||
|
blocks.append(cached[0])
|
||||||
|
# Blocks of a private-pool group (hisparse host tier) are touched in
|
||||||
|
# their own pool, as free_blocks routes them.
|
||||||
|
for block in blocks:
|
||||||
|
(block.pool or self.block_pool).touch([block])
|
||||||
|
self._pinned_hits[request_id] = blocks
|
||||||
|
return len(blocks)
|
||||||
|
|
||||||
|
def pinned_hit_blocks(self, request_id: str) -> list[KVCacheBlock]:
|
||||||
|
"""The blocks pinned for ``request_id`` (empty if none)."""
|
||||||
|
return list(self._pinned_hits.get(request_id, ()))
|
||||||
|
|
||||||
|
def release_pinned_hit(self, request_id: str) -> int:
|
||||||
|
"""Drop the request's pin (admission, abort, finish). Returns the
|
||||||
|
number of blocks released."""
|
||||||
|
blocks = self._pinned_hits.pop(request_id, None)
|
||||||
|
if not blocks:
|
||||||
|
return 0
|
||||||
|
# Reversed: the prefix stays an eviction candidate longer than the
|
||||||
|
# tail, as after a free.
|
||||||
|
self.block_pool.free_blocks(reversed(blocks))
|
||||||
|
return len(blocks)
|
||||||
|
|
||||||
|
def yield_pinned_hits(
|
||||||
|
self, keep: str | None = None, limit: int | None = None
|
||||||
|
) -> int:
|
||||||
|
"""Drop pins (except ``keep``'s) so a running request can allocate,
|
||||||
|
or a waiter ahead in line can be admitted: grace pins of finished
|
||||||
|
turns first (oldest first), then the most recently pinned waiters, at
|
||||||
|
most ``limit`` pins per call (all by default; the scheduler passes 1
|
||||||
|
and retries, so a decode step that needs a block costs one pin, not
|
||||||
|
every pin). Returns the number of blocks released; 0 means there was
|
||||||
|
nothing to yield."""
|
||||||
|
released = 0
|
||||||
|
yielded = 0
|
||||||
|
# Grace pins first (oldest finished conversation first), then the
|
||||||
|
# waiters' pins (most recently pinned first).
|
||||||
|
for request_id in list(self._grace_pins):
|
||||||
|
if limit is not None and yielded >= limit:
|
||||||
|
return released
|
||||||
|
released += self.release_grace_pin(request_id)
|
||||||
|
yielded += 1
|
||||||
|
for request_id in reversed(list(self._pinned_hits)):
|
||||||
|
if request_id == keep:
|
||||||
|
continue
|
||||||
|
if limit is not None and yielded >= limit:
|
||||||
|
break
|
||||||
|
released += self.release_pinned_hit(request_id)
|
||||||
|
yielded += 1
|
||||||
|
return released
|
||||||
|
|
||||||
def remove_skipped_blocks(
|
def remove_skipped_blocks(
|
||||||
self,
|
self,
|
||||||
@@ -651,16 +1038,26 @@ class KVCacheManager:
|
|||||||
def pop_blocks_for_free(self, request: Request) -> list[KVCacheBlock]:
|
def pop_blocks_for_free(self, request: Request) -> list[KVCacheBlock]:
|
||||||
"""Pop the request's bookkeeping and return its blocks without
|
"""Pop the request's bookkeeping and return its blocks without
|
||||||
returning them to the block pool. The caller must eventually free
|
returning them to the block pool. The caller must eventually free
|
||||||
them in reverse order (so that tail blocks are evicted first).
|
them as-is: they are already ordered so that tail blocks are
|
||||||
|
evicted first, within each KV cache group.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
request: The request to pop the blocks for.
|
request: The request to pop the blocks for.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The request's blocks in allocation order.
|
The request's blocks in eviction order (each group's blocks
|
||||||
|
reversed, then concatenated group by group).
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return self.coordinator.pop_blocks_for_free(request.request_id)
|
# A pin is never written by an in-flight step (the waiter has no
|
||||||
|
# table), so it is released now, not with the deferred blocks.
|
||||||
|
self.release_pinned_hit(request.request_id)
|
||||||
|
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
|
||||||
|
# A finished turn whose last step is still in flight: the grace takes
|
||||||
|
# its ref now; the table refs go when the fence drains.
|
||||||
|
if self._grace_eligible(request):
|
||||||
|
self._take_grace_pin(request, blocks)
|
||||||
|
return blocks
|
||||||
|
|
||||||
def evict_blocks(self, block_ids: set[int]) -> None:
|
def evict_blocks(self, block_ids: set[int]) -> None:
|
||||||
"""Evict blocks from the prefix cache by their block IDs.
|
"""Evict blocks from the prefix cache by their block IDs.
|
||||||
@@ -681,6 +1078,8 @@ class KVCacheManager:
|
|||||||
False otherwise.
|
False otherwise.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
# Pinned waiters' hits are about to be invalidated anyway.
|
||||||
|
self.yield_pinned_hits()
|
||||||
if not self.coordinator.reset_prefix_cache():
|
if not self.coordinator.reset_prefix_cache():
|
||||||
return False
|
return False
|
||||||
if self.log_stats:
|
if self.log_stats:
|
||||||
@@ -721,6 +1120,12 @@ class KVCacheManager:
|
|||||||
group.
|
group.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
if self._pinned_hits or self._grace_pins:
|
||||||
|
# A pinned or graced block carries a ref that is not a running
|
||||||
|
# request's, so the ref count can equal the running count for a
|
||||||
|
# block only some of them share. Report no common prefix (cascade
|
||||||
|
# attention is an optimisation; a wrong prefix would be read).
|
||||||
|
return [0] * len(self.kv_cache_config.kv_cache_groups)
|
||||||
return self.coordinator.get_num_common_prefix_blocks(running_request_id)
|
return self.coordinator.get_num_common_prefix_blocks(running_request_id)
|
||||||
|
|
||||||
def take_events(self) -> list[KVCacheEvent]:
|
def take_events(self) -> list[KVCacheEvent]:
|
||||||
@@ -867,6 +1272,21 @@ class KVCacheManager:
|
|||||||
ids.extend(mgr.take_new_block_ids())
|
ids.extend(mgr.take_new_block_ids())
|
||||||
return ids
|
return ids
|
||||||
|
|
||||||
|
def take_new_ring_block_ids(self) -> list[int]:
|
||||||
|
"""Drain ring blocks claimed since the last call (ring-row tag clear).
|
||||||
|
Independent of KV zeroing: every claim is returned."""
|
||||||
|
ids: list[int] = []
|
||||||
|
for mgr in self.coordinator.single_type_managers:
|
||||||
|
ids.extend(mgr.take_new_ring_block_ids())
|
||||||
|
return ids
|
||||||
|
|
||||||
|
def get_ring_block_ids(self, request_id: str) -> list[int]:
|
||||||
|
"""The request's ring (CircularBufferSpec) block ids, all groups."""
|
||||||
|
ids: list[int] = []
|
||||||
|
for mgr in self.coordinator.single_type_managers:
|
||||||
|
ids.extend(mgr.get_ring_block_ids(request_id))
|
||||||
|
return ids
|
||||||
|
|
||||||
def record_blocks_for_zeroing(self, request_id: str, start_token: int) -> None:
|
def record_blocks_for_zeroing(self, request_id: str, start_token: int) -> None:
|
||||||
"""Re-record the request's blocks from start_token onwards for
|
"""Re-record the request's blocks from start_token onwards for
|
||||||
zeroing, e.g. blocks a failed async KV load left unwritten.
|
zeroing, e.g. blocks a failed async KV load left unwritten.
|
||||||
|
|||||||
@@ -1121,7 +1121,19 @@ def get_max_concurrency_for_kv_cache_config(
|
|||||||
"""
|
"""
|
||||||
num_blocks_per_request = 0
|
num_blocks_per_request = 0
|
||||||
host_blocks_per_request = 0
|
host_blocks_per_request = 0
|
||||||
|
# v2.5 side-cache layout: a slot-packed ring group shares each pool block
|
||||||
|
# between `slots_per_block` requests, so it costs cdiv(max_num_seqs, slots)
|
||||||
|
# blocks for the whole pool instead of one block per request.
|
||||||
|
shared_blocks = 0
|
||||||
|
max_num_seqs = getattr(
|
||||||
|
getattr(vllm_config, "scheduler_config", None), "max_num_seqs", 1
|
||||||
|
)
|
||||||
|
if not isinstance(max_num_seqs, int) or max_num_seqs < 1:
|
||||||
|
max_num_seqs = 1
|
||||||
for group in kv_cache_config.kv_cache_groups:
|
for group in kv_cache_config.kv_cache_groups:
|
||||||
|
if group.slots_per_block > 1 and not group.host_resident:
|
||||||
|
shared_blocks += cdiv(max_num_seqs, group.slots_per_block)
|
||||||
|
continue
|
||||||
required = cdiv(
|
required = cdiv(
|
||||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||||
group.kv_cache_spec.page_size_bytes,
|
group.kv_cache_spec.page_size_bytes,
|
||||||
@@ -1130,7 +1142,9 @@ def get_max_concurrency_for_kv_cache_config(
|
|||||||
host_blocks_per_request += required
|
host_blocks_per_request += required
|
||||||
else:
|
else:
|
||||||
num_blocks_per_request += required
|
num_blocks_per_request += required
|
||||||
limits = [kv_cache_config.num_blocks / num_blocks_per_request]
|
limits = [
|
||||||
|
max(kv_cache_config.num_blocks - shared_blocks, 0) / num_blocks_per_request
|
||||||
|
]
|
||||||
if host_blocks_per_request:
|
if host_blocks_per_request:
|
||||||
assert kv_cache_config.hisparse_host_num_blocks is not None
|
assert kv_cache_config.hisparse_host_num_blocks is not None
|
||||||
limits.append(
|
limits.append(
|
||||||
@@ -1806,18 +1820,47 @@ def get_kv_cache_config_from_groups(
|
|||||||
layers_by_spec: defaultdict[KVCacheSpec, list[str]] = defaultdict(list)
|
layers_by_spec: defaultdict[KVCacheSpec, list[str]] = defaultdict(list)
|
||||||
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
if isinstance(group_spec, UniformTypeKVCacheSpecs):
|
||||||
for layer_name, spec in group_spec.kv_cache_specs.items():
|
for layer_name, spec in group_spec.kv_cache_specs.items():
|
||||||
layers_by_spec[spec].append(layer_name)
|
# Fork (E2 PP fix #54793, dropped in the v0.30.0 rebase, restored
|
||||||
|
# the vLLM 0.30 rebase): _project_kv_cache_groups_to_worker keeps the GLOBAL
|
||||||
|
# UniformTypeKVCacheSpecs for a group with no layer on this PP
|
||||||
|
# stage, so without this filter the worker gets a tensor for
|
||||||
|
# another stage's layer and allocate_kv_cache's group lookup
|
||||||
|
# raises StopIteration (Flash-Next PP1: the layer-2 PLE state).
|
||||||
|
if layer_name in group.layer_names:
|
||||||
|
layers_by_spec[spec].append(layer_name)
|
||||||
elif group.layer_names:
|
elif group.layer_names:
|
||||||
layers_by_spec[group_spec].extend(group.layer_names)
|
layers_by_spec[group_spec].extend(group.layer_names)
|
||||||
|
|
||||||
|
# v2.5 side-cache layout: a slot-packed group views every pool block as
|
||||||
|
# `slots` equal slots, so its block stride is one slot and its view has
|
||||||
|
# num_blocks * slots blocks. Pages of its layers must fit in one slot.
|
||||||
|
slots = group.slots_per_block
|
||||||
|
group_block_stride = interleaved_block_stride
|
||||||
|
group_num_blocks = num_blocks
|
||||||
|
if slots > 1:
|
||||||
|
assert layout.is_block_outermost, (
|
||||||
|
"Slot-packed KV cache groups need a block-outermost layout"
|
||||||
|
)
|
||||||
|
assert bytes_per_block % slots == 0, (bytes_per_block, slots)
|
||||||
|
group_block_stride = bytes_per_block // slots
|
||||||
|
group_num_blocks = num_blocks * slots
|
||||||
|
group_bytes = sum(
|
||||||
|
spec.page_size_bytes * len(names)
|
||||||
|
for spec, names in layers_by_spec.items()
|
||||||
|
)
|
||||||
|
assert group_bytes <= group_block_stride, (
|
||||||
|
f"slot-packed group needs {group_bytes} B per slot, slot is "
|
||||||
|
f"{group_block_stride} B"
|
||||||
|
)
|
||||||
|
|
||||||
byte_offset = 0
|
byte_offset = 0
|
||||||
for spec, layer_names in layers_by_spec.items():
|
for spec, layer_names in layers_by_spec.items():
|
||||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||||
spec,
|
spec,
|
||||||
num_blocks,
|
group_num_blocks,
|
||||||
len(layer_names),
|
len(layer_names),
|
||||||
layout,
|
layout,
|
||||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
fixed_strides=(None, group_block_stride, None, None, None),
|
||||||
)
|
)
|
||||||
offset = (
|
offset = (
|
||||||
byte_offset
|
byte_offset
|
||||||
@@ -1831,6 +1874,7 @@ def get_kv_cache_config_from_groups(
|
|||||||
layer_stride=layer_stride,
|
layer_stride=layer_stride,
|
||||||
block_stride=block_stride,
|
block_stride=block_stride,
|
||||||
offset=offset,
|
offset=offset,
|
||||||
|
slots_per_block=slots,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||||
@@ -2698,6 +2742,289 @@ def _project_kv_cache_groups_to_worker(
|
|||||||
return projected_groups
|
return projected_groups
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# v2.5 side-cache layout (VLLM_E2_SIDE_CACHE_LAYOUT, default ON).
|
||||||
|
#
|
||||||
|
# The packed block-outermost layout strides every pool block by the widest
|
||||||
|
# group, and every group draws whole blocks from the one pool. Flash-Next has
|
||||||
|
# two side caches whose real state is tiny next to that stride (12.9 MB on the
|
||||||
|
# binding PP stage): the TP-replicated PLE short-conv state (one layer, 2 + K
|
||||||
|
# blocks per request in align mode, a 1.6 MB padded page) and the QSA raw-key
|
||||||
|
# ring (one block per request, 16 KB for all of a stage's layers). Together
|
||||||
|
# they hold 6 of the 103 blocks a 262K request needs. Two layout-only changes
|
||||||
|
# remove that cost without changing any value a kernel reads:
|
||||||
|
#
|
||||||
|
# 1. PLE fold. The replicated Mamba group joins a sharded Mamba group with the
|
||||||
|
# same scheduling parameters whose block has a spare page on every worker
|
||||||
|
# (Flash-Next: a GDN group holds 6 of PP0's 7 pages), so the PLE state rides
|
||||||
|
# in that group's blocks under the block table it would have had on its
|
||||||
|
# own. No worker's stride changes, so the pool block count does not change.
|
||||||
|
# 2. Ring slot packing. The ring group splits every pool block into S equal
|
||||||
|
# slots and the ring manager hands out slots; a pool block goes back to the
|
||||||
|
# pool when its last slot is released. The ring then costs cdiv(requests, S)
|
||||||
|
# blocks in all instead of one per request.
|
||||||
|
#
|
||||||
|
# Off (=0) restores one group each, block for block.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Slot stride alignment for packed ring views (bytes) and the largest S tried.
|
||||||
|
_RING_SLOT_ALIGN = 256
|
||||||
|
_RING_MAX_SLOTS = 1024
|
||||||
|
|
||||||
|
|
||||||
|
def _side_cache_layout_enabled(vllm_config: VllmConfig) -> bool:
|
||||||
|
"""Whether the v2.5 side-cache layout may apply to this engine.
|
||||||
|
|
||||||
|
KV connectors (NIXL, offload, P/D) address groups and blocks themselves
|
||||||
|
(the NIXL worker requires exactly one single-layer replicated group), and
|
||||||
|
HiSparse has its own pool layout, so those engines keep the old layout.
|
||||||
|
"""
|
||||||
|
if not envs.VLLM_E2_SIDE_CACHE_LAYOUT:
|
||||||
|
return False
|
||||||
|
if getattr(vllm_config, "kv_transfer_config", None) is not None:
|
||||||
|
return False
|
||||||
|
attention_config = getattr(vllm_config, "attention_config", None)
|
||||||
|
if getattr(attention_config, "hisparse_config", None) is not None:
|
||||||
|
return False
|
||||||
|
parallel_config = getattr(vllm_config, "parallel_config", None)
|
||||||
|
if getattr(parallel_config, "decode_context_parallel_size", 1) not in (1, None):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||||
|
except ValueError:
|
||||||
|
# Unresolved layout: only reachable from callers that bypass the
|
||||||
|
# engine core; nothing to pack without a block-outermost layout.
|
||||||
|
return False
|
||||||
|
return bool(layout.is_block_outermost)
|
||||||
|
|
||||||
|
|
||||||
|
def _group_layer_specs(group: KVCacheGroupSpec) -> list[KVCacheSpec]:
|
||||||
|
spec = group.kv_cache_spec
|
||||||
|
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||||
|
return [spec.kv_cache_specs[name] for name in group.layer_names]
|
||||||
|
return [spec] * len(group.layer_names)
|
||||||
|
|
||||||
|
|
||||||
|
def _mamba_schedule_key(spec: MambaSpec) -> tuple:
|
||||||
|
"""Everything the scheduler's MambaManager and the worker's align-mode
|
||||||
|
bookkeeping read from a group's spec. Two Mamba groups with equal keys get
|
||||||
|
identical block tables for every request."""
|
||||||
|
return (
|
||||||
|
spec.block_size,
|
||||||
|
spec.page_size_bytes,
|
||||||
|
spec.num_speculative_blocks,
|
||||||
|
spec.num_prefill_checkpoint_blocks,
|
||||||
|
spec.prefill_checkpoint_alignment,
|
||||||
|
spec.mamba_cache_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_strides(
|
||||||
|
groups: list[KVCacheGroupSpec], worker_specs: list[dict[str, KVCacheSpec]]
|
||||||
|
) -> list[int]:
|
||||||
|
"""Pool block bytes of every worker for this global grouping."""
|
||||||
|
strides = []
|
||||||
|
for spec in worker_specs:
|
||||||
|
projected = _project_kv_cache_groups_to_worker(groups, spec)
|
||||||
|
if not any(g.layer_names for g in projected):
|
||||||
|
strides.append(0)
|
||||||
|
continue
|
||||||
|
strides.append(_get_kv_cache_bytes_per_block(projected))
|
||||||
|
return strides
|
||||||
|
|
||||||
|
|
||||||
|
def _fold_replicated_mamba_groups(
|
||||||
|
groups: list[KVCacheGroupSpec],
|
||||||
|
worker_specs: list[dict[str, KVCacheSpec]],
|
||||||
|
) -> tuple[list[KVCacheGroupSpec], list[str]]:
|
||||||
|
"""Fold each TP-replicated Mamba group into a compatible sharded one.
|
||||||
|
|
||||||
|
A target holds only sharded Mamba layers with the same scheduling key and
|
||||||
|
EAGLE flag, and the merged group must not widen the block of any worker
|
||||||
|
(checked on every worker's projection, so PP stages are covered). Among
|
||||||
|
valid targets the lowest index wins. Returns the new group list and one
|
||||||
|
log note per replicated group.
|
||||||
|
"""
|
||||||
|
groups = list(groups)
|
||||||
|
notes: list[str] = []
|
||||||
|
base_strides = _worker_strides(groups, worker_specs)
|
||||||
|
index = 0
|
||||||
|
while index < len(groups):
|
||||||
|
source = groups[index]
|
||||||
|
layer_specs = _group_layer_specs(source)
|
||||||
|
if not (
|
||||||
|
layer_specs
|
||||||
|
and all(
|
||||||
|
isinstance(spec, MambaSpec) and spec.tp_replicated
|
||||||
|
for spec in layer_specs
|
||||||
|
)
|
||||||
|
and len({_mamba_schedule_key(spec) for spec in layer_specs}) == 1
|
||||||
|
):
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
key = _mamba_schedule_key(cast(MambaSpec, layer_specs[0]))
|
||||||
|
chosen = None
|
||||||
|
for target_index, target in enumerate(groups):
|
||||||
|
if target_index == index or target.is_eagle_group != source.is_eagle_group:
|
||||||
|
continue
|
||||||
|
target_specs = _group_layer_specs(target)
|
||||||
|
if not target_specs or not all(
|
||||||
|
isinstance(spec, MambaSpec)
|
||||||
|
and not spec.tp_replicated
|
||||||
|
and _mamba_schedule_key(spec) == key
|
||||||
|
for spec in target_specs
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
merged_specs: dict[str, KVCacheSpec] = dict(
|
||||||
|
zip(target.layer_names, target_specs)
|
||||||
|
)
|
||||||
|
merged_specs.update(zip(source.layer_names, layer_specs))
|
||||||
|
# Built directly: is_uniform_type rejects mixed tp_replicated on
|
||||||
|
# purpose (NIXL), and connectors never reach this path.
|
||||||
|
merged = KVCacheGroupSpec(
|
||||||
|
target.layer_names + source.layer_names,
|
||||||
|
UniformTypeKVCacheSpecs(
|
||||||
|
block_size=target_specs[0].block_size,
|
||||||
|
kv_cache_specs=merged_specs,
|
||||||
|
),
|
||||||
|
is_eagle_group=target.is_eagle_group,
|
||||||
|
)
|
||||||
|
trial = [
|
||||||
|
merged if i == target_index else g
|
||||||
|
for i, g in enumerate(groups)
|
||||||
|
if i != index
|
||||||
|
]
|
||||||
|
strides = _worker_strides(trial, worker_specs)
|
||||||
|
if all(new <= old for new, old in zip(strides, base_strides)):
|
||||||
|
chosen = (target_index, merged, trial)
|
||||||
|
break
|
||||||
|
if chosen is None:
|
||||||
|
notes.append(
|
||||||
|
f"replicated Mamba group {index} ({len(source.layer_names)} "
|
||||||
|
"layer(s)) kept separate: no sharded group has a spare page on "
|
||||||
|
"every worker"
|
||||||
|
)
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
target_index, merged, trial = chosen
|
||||||
|
notes.append(
|
||||||
|
f"folded replicated Mamba group {index} "
|
||||||
|
f"({', '.join(source.layer_names)}) into Mamba group "
|
||||||
|
f"{target_index} (now {len(merged.layer_names)} layers); per-worker "
|
||||||
|
f"block bytes unchanged {sorted(set(base_strides))}"
|
||||||
|
)
|
||||||
|
groups = trial
|
||||||
|
# The group after the folded one now sits at `index`.
|
||||||
|
return groups, notes
|
||||||
|
|
||||||
|
|
||||||
|
def _is_ring_group(group: KVCacheGroupSpec) -> bool:
|
||||||
|
spec = group.kv_cache_spec
|
||||||
|
layer_specs = (
|
||||||
|
list(spec.kv_cache_specs.values())
|
||||||
|
if isinstance(spec, UniformTypeKVCacheSpecs)
|
||||||
|
else [spec]
|
||||||
|
)
|
||||||
|
return bool(layer_specs) and all(
|
||||||
|
type(layer_spec) is CircularBufferSpec for layer_spec in layer_specs
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _choose_ring_slots(
|
||||||
|
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||||
|
) -> dict[int, int]:
|
||||||
|
"""Slots per pool block for every ring group (group index -> S >= 2).
|
||||||
|
|
||||||
|
S must cut every worker's block stride into equal slots that are
|
||||||
|
`_RING_SLOT_ALIGN`-aligned and hold that worker's ring pages; the largest
|
||||||
|
such S up to `_RING_MAX_SLOTS` is taken. Block ids are global, so S is the
|
||||||
|
same on every worker. A ring group with no valid S keeps one pool block
|
||||||
|
per request.
|
||||||
|
"""
|
||||||
|
chosen: dict[int, int] = {}
|
||||||
|
if not projected_groups_per_worker or not projected_groups_per_worker[0]:
|
||||||
|
return chosen
|
||||||
|
num_groups = len(projected_groups_per_worker[0])
|
||||||
|
strides = [
|
||||||
|
_get_kv_cache_bytes_per_block(groups)
|
||||||
|
if any(g.layer_names for g in groups)
|
||||||
|
else 0
|
||||||
|
for groups in projected_groups_per_worker
|
||||||
|
]
|
||||||
|
for group_index in range(num_groups):
|
||||||
|
if not _is_ring_group(projected_groups_per_worker[0][group_index]):
|
||||||
|
continue
|
||||||
|
needs = []
|
||||||
|
for groups, stride in zip(projected_groups_per_worker, strides):
|
||||||
|
group = groups[group_index]
|
||||||
|
if not group.layer_names:
|
||||||
|
continue
|
||||||
|
ring_bytes = sum(spec.page_size_bytes for spec in _group_layer_specs(group))
|
||||||
|
needs.append((stride, ring_bytes))
|
||||||
|
if not needs:
|
||||||
|
continue
|
||||||
|
for slots in range(_RING_MAX_SLOTS, 1, -1):
|
||||||
|
if all(
|
||||||
|
stride % slots == 0
|
||||||
|
and (stride // slots) % _RING_SLOT_ALIGN == 0
|
||||||
|
and stride // slots >= ring_bytes
|
||||||
|
for stride, ring_bytes in needs
|
||||||
|
):
|
||||||
|
chosen[group_index] = slots
|
||||||
|
break
|
||||||
|
return chosen
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_side_cache_layout(
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
global_kv_cache_groups: list[KVCacheGroupSpec],
|
||||||
|
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||||
|
) -> tuple[list[KVCacheGroupSpec], list[list[KVCacheGroupSpec]]]:
|
||||||
|
"""Fold side Mamba groups, project per worker, then pick ring slots.
|
||||||
|
|
||||||
|
Returns the (possibly folded) global groups and every worker's projection,
|
||||||
|
with `slots_per_block` set on packed ring groups.
|
||||||
|
"""
|
||||||
|
relevant = any(
|
||||||
|
(isinstance(spec, MambaSpec) and spec.tp_replicated)
|
||||||
|
or type(spec) is CircularBufferSpec
|
||||||
|
for worker_spec in kv_cache_specs
|
||||||
|
for spec in worker_spec.values()
|
||||||
|
)
|
||||||
|
enabled = relevant and _side_cache_layout_enabled(vllm_config)
|
||||||
|
notes: list[str] = []
|
||||||
|
if enabled:
|
||||||
|
global_kv_cache_groups, notes = _fold_replicated_mamba_groups(
|
||||||
|
global_kv_cache_groups, kv_cache_specs
|
||||||
|
)
|
||||||
|
projected = [
|
||||||
|
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
||||||
|
for worker_spec in kv_cache_specs
|
||||||
|
]
|
||||||
|
if enabled and all(
|
||||||
|
_glm5_next_tensor_layout(groups) is None for groups in projected if groups
|
||||||
|
):
|
||||||
|
for group_index, slots in _choose_ring_slots(projected).items():
|
||||||
|
for groups in projected:
|
||||||
|
groups[group_index].slots_per_block = slots
|
||||||
|
notes.append(
|
||||||
|
f"QSA ring group {group_index} packs {slots} requests per pool block"
|
||||||
|
)
|
||||||
|
if relevant and envs.VLLM_E2_SIDE_CACHE_LAYOUT and not enabled:
|
||||||
|
notes.append(
|
||||||
|
"not applicable to this engine (KV connector, HiSparse, DCP or a "
|
||||||
|
"layer-outermost layout)"
|
||||||
|
)
|
||||||
|
if relevant:
|
||||||
|
# The A/B evidence line: boot logs must show which arm ran.
|
||||||
|
logger.info(
|
||||||
|
"Side-cache layout: %s (VLLM_E2_SIDE_CACHE_LAYOUT)%s",
|
||||||
|
"on" if enabled else "off",
|
||||||
|
"".join(f"; {note}" for note in notes),
|
||||||
|
)
|
||||||
|
return global_kv_cache_groups, projected
|
||||||
|
|
||||||
|
|
||||||
def get_kv_cache_configs(
|
def get_kv_cache_configs(
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||||
@@ -2767,10 +3094,11 @@ def get_kv_cache_configs(
|
|||||||
# If original_max_model_len was -1, automatically
|
# If original_max_model_len was -1, automatically
|
||||||
# determine the maximum model length that fits in available GPU memory.
|
# determine the maximum model length that fits in available GPU memory.
|
||||||
# We use per-worker projected groups to account for PP sharding.
|
# We use per-worker projected groups to account for PP sharding.
|
||||||
projected_groups_per_worker = [
|
# v2.5: the side-cache layout (PLE fold, ring slots) is applied here, where
|
||||||
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
# every worker's layers are known, so no PP stage's block can widen.
|
||||||
for worker_spec in kv_cache_specs
|
global_kv_cache_groups, projected_groups_per_worker = _apply_side_cache_layout(
|
||||||
]
|
vllm_config, global_kv_cache_groups, kv_cache_specs
|
||||||
|
)
|
||||||
|
|
||||||
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
||||||
# be allocated is decoupled from the profiled `available_memory`:
|
# be allocated is decoupled from the profiled `available_memory`:
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
"""Debug-only scheduler allocation log for KV-capacity probes (off unless
|
||||||
|
VLLM_E2_SCHED_LOG=<path>).
|
||||||
|
|
||||||
|
One JSON line per scheduler update while any request exists, and one "fin" line
|
||||||
|
per request as it is freed:
|
||||||
|
{"t": wall, "m": monotonic, "step": n, "total": pool blocks,
|
||||||
|
"free": free blocks, "running": r, "waiting": w,
|
||||||
|
"tot": tokens scheduled this step,
|
||||||
|
"reqs": [[id, prompt, computed, output, cached, preemptions,
|
||||||
|
spec_scheduled, status, [blocks per group],
|
||||||
|
tokens scheduled for this request this step]]}
|
||||||
|
{"t": wall, "m": monotonic,
|
||||||
|
"fin": [id, prompt, computed, output, cached, preemptions, status,
|
||||||
|
[blocks per group]]}
|
||||||
|
Blocks are counted per KV-cache group, null blocks excluded. A logging failure
|
||||||
|
writes an "error" line and never raises into the scheduler. Unset = the
|
||||||
|
scheduler only pays one env check per update.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
_PATH = os.environ.get("VLLM_E2_SCHED_LOG", "")
|
||||||
|
_state = {
|
||||||
|
"f": None,
|
||||||
|
"step": 0,
|
||||||
|
"flush": 0.0,
|
||||||
|
"wrapped": False,
|
||||||
|
"errors": 0,
|
||||||
|
"checked": False,
|
||||||
|
}
|
||||||
|
REQ_FIELDS = (
|
||||||
|
"request_id",
|
||||||
|
"num_prompt_tokens",
|
||||||
|
"num_computed_tokens",
|
||||||
|
"num_output_tokens",
|
||||||
|
"num_preemptions",
|
||||||
|
"status",
|
||||||
|
"prefill_stats",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def enabled() -> bool:
|
||||||
|
return bool(_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
def _blocks(sched, rid):
|
||||||
|
return [
|
||||||
|
sum(1 for b in m.req_to_blocks.get(rid, ()) if not b.is_null)
|
||||||
|
for m in sched.kv_cache_manager.coordinator.single_type_managers
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _status(r):
|
||||||
|
return getattr(r.status, "name", str(r.status))
|
||||||
|
|
||||||
|
|
||||||
|
def _cached(r):
|
||||||
|
"""Prefix-cache hit for this request. vLLM keeps it in
|
||||||
|
`request.prefill_stats` (PrefillStats.num_cached_tokens = local +
|
||||||
|
external); the output processor takes those stats after the first prefill
|
||||||
|
(`take_prefill_stats` sets the attribute to None), so -1 means "no longer
|
||||||
|
knowable from the scheduler", never a silent zero."""
|
||||||
|
ps = getattr(r, "prefill_stats", None)
|
||||||
|
return int(getattr(ps, "num_cached_tokens", -1)) if ps is not None else -1
|
||||||
|
|
||||||
|
|
||||||
|
def _contract(r):
|
||||||
|
"""Once per process: every field this log reads must exist on the real Request
|
||||||
|
object. A missing field is written once as a schema_error record — the evaluator
|
||||||
|
fails closed on it — instead of turning every row into an error."""
|
||||||
|
if _state["checked"]:
|
||||||
|
return
|
||||||
|
_state["checked"] = True
|
||||||
|
missing = [f for f in REQ_FIELDS if not hasattr(r, f)]
|
||||||
|
if missing:
|
||||||
|
_write(
|
||||||
|
{
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"schema_error": missing,
|
||||||
|
},
|
||||||
|
force=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write(rec, force=False):
|
||||||
|
f = _state["f"]
|
||||||
|
if f is None:
|
||||||
|
# noqa SIM115: one append handle for the process lifetime, kept in
|
||||||
|
# _state and flushed on a cadence; a context manager would reopen the
|
||||||
|
# log on every scheduler update.
|
||||||
|
f = _state["f"] = open(_PATH, "a", buffering=1 << 16) # noqa: SIM115
|
||||||
|
f.write(json.dumps(rec, separators=(",", ":")) + "\n")
|
||||||
|
if force or rec["t"] - _state["flush"] > 0.5:
|
||||||
|
f.flush()
|
||||||
|
_state["flush"] = rec["t"]
|
||||||
|
|
||||||
|
|
||||||
|
def _error(e):
|
||||||
|
_state["errors"] += 1
|
||||||
|
if _state["errors"] <= 20:
|
||||||
|
try: # noqa: SIM105
|
||||||
|
_write(
|
||||||
|
{
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"error": f"{type(e).__name__}: {e}"[:300],
|
||||||
|
},
|
||||||
|
force=True,
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
# SIM105: this module imports json, os and time only, so that an
|
||||||
|
# enabled log costs the scheduler nothing it can see; the
|
||||||
|
# last-resort writer does not earn a contextlib import.
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _wrap_free(sched):
|
||||||
|
orig = sched._free_request
|
||||||
|
|
||||||
|
def _free_request(request, *args, **kwargs):
|
||||||
|
try:
|
||||||
|
_write(
|
||||||
|
{
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"fin": [
|
||||||
|
request.request_id,
|
||||||
|
request.num_prompt_tokens,
|
||||||
|
request.num_computed_tokens,
|
||||||
|
request.num_output_tokens,
|
||||||
|
_cached(request),
|
||||||
|
request.num_preemptions,
|
||||||
|
_status(request),
|
||||||
|
_blocks(sched, request.request_id),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
force=True,
|
||||||
|
)
|
||||||
|
except Exception as e: # noqa: BLE001 - the debug log must never break the scheduler
|
||||||
|
_error(e)
|
||||||
|
return orig(request, *args, **kwargs)
|
||||||
|
|
||||||
|
sched._free_request = _free_request
|
||||||
|
_state["wrapped"] = True
|
||||||
|
|
||||||
|
|
||||||
|
def log_step(sched, scheduler_output) -> None:
|
||||||
|
try:
|
||||||
|
if not _state["wrapped"]:
|
||||||
|
_wrap_free(sched)
|
||||||
|
if not sched.requests:
|
||||||
|
return
|
||||||
|
_state["step"] += 1
|
||||||
|
spec = getattr(scheduler_output, "scheduled_spec_decode_tokens", None) or {}
|
||||||
|
pool = sched.kv_cache_manager.block_pool
|
||||||
|
nsched = getattr(scheduler_output, "num_scheduled_tokens", None) or {}
|
||||||
|
_contract(next(iter(sched.requests.values())))
|
||||||
|
rows = [
|
||||||
|
[
|
||||||
|
rid,
|
||||||
|
r.num_prompt_tokens,
|
||||||
|
r.num_computed_tokens,
|
||||||
|
r.num_output_tokens,
|
||||||
|
_cached(r),
|
||||||
|
r.num_preemptions,
|
||||||
|
len(spec.get(rid, ())),
|
||||||
|
_status(r),
|
||||||
|
_blocks(sched, rid),
|
||||||
|
int(nsched.get(rid, 0)),
|
||||||
|
]
|
||||||
|
for rid, r in sched.requests.items()
|
||||||
|
]
|
||||||
|
_write(
|
||||||
|
{
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"step": _state["step"],
|
||||||
|
"total": pool.num_gpu_blocks,
|
||||||
|
"free": pool.get_num_free_blocks(),
|
||||||
|
"running": len(sched.running),
|
||||||
|
"waiting": len(sched.waiting),
|
||||||
|
"tot": int(
|
||||||
|
getattr(scheduler_output, "total_num_scheduled_tokens", 0) or 0
|
||||||
|
),
|
||||||
|
"reqs": rows,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
_error(e)
|
||||||
@@ -301,6 +301,13 @@ class SchedulerOutput:
|
|||||||
# preventing stale NaN/data from corrupting attention or SSM computation.
|
# preventing stale NaN/data from corrupting attention or SSM computation.
|
||||||
new_block_ids_to_zero: list[int] | None = None
|
new_block_ids_to_zero: list[int] | None = None
|
||||||
|
|
||||||
|
# v2.2 ring-row tag fix: ring (CircularBufferSpec) blocks claimed during this
|
||||||
|
# scheduling step. Independent of new_block_ids_to_zero and of
|
||||||
|
# needs_kv_cache_zeroing: the worker clears only the rows' tag words, after
|
||||||
|
# the zeroer and before any copy or forward of this step, i.e. before the
|
||||||
|
# new owner's first read or write of the ring.
|
||||||
|
new_ring_block_ids_to_clear: list[int] | None = None
|
||||||
|
|
||||||
# CoW copies to apply after zeroing new blocks and before forward.
|
# CoW copies to apply after zeroing new blocks and before forward.
|
||||||
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
|
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
|
||||||
|
|
||||||
@@ -314,6 +321,10 @@ class SchedulerOutput:
|
|||||||
# Number of spec tokens to schedule for the next step.
|
# Number of spec tokens to schedule for the next step.
|
||||||
num_spec_tokens_to_schedule: int = 0
|
num_spec_tokens_to_schedule: int = 0
|
||||||
|
|
||||||
|
# Identity of this schedule() call (`Scheduler.current_step`). Used to
|
||||||
|
# retrieve the matching consumption-time draft-token snapshot.
|
||||||
|
scheduler_step: int = 0
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def make_empty(cls) -> "SchedulerOutput":
|
def make_empty(cls) -> "SchedulerOutput":
|
||||||
return cls(
|
return cls(
|
||||||
@@ -338,5 +349,10 @@ class GrammarOutput:
|
|||||||
# Per request, ordered as structured_output_request_ids: how many leading
|
# Per request, ordered as structured_output_request_ids: how many leading
|
||||||
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
||||||
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
||||||
|
# from this index on must not be accepted. None means nothing to invalidate.
|
||||||
|
num_acceptable_drafts: list[int] | None = None
|
||||||
|
# Per request, ordered as structured_output_request_ids: how many leading
|
||||||
|
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
||||||
|
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
||||||
# from this index on must not be accepted. None invalidates every draft.
|
# from this index on must not be accepted. None invalidates every draft.
|
||||||
num_acceptable_drafts: list[int] | None = None
|
num_acceptable_drafts: list[int] | None = None
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import itertools
|
import itertools
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
from collections import defaultdict, deque
|
from collections import defaultdict, deque
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
@@ -38,6 +39,7 @@ from vllm.v1.core.encoder_cache_manager import (
|
|||||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||||
|
from vllm.v1.core.sched import e2_alloc_log as _e2_alloc_log
|
||||||
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
|
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
|
||||||
from vllm.v1.core.sched.output import (
|
from vllm.v1.core.sched.output import (
|
||||||
CachedRequestData,
|
CachedRequestData,
|
||||||
@@ -55,10 +57,12 @@ from vllm.v1.core.sched.request_queue import (
|
|||||||
from vllm.v1.core.sched.utils import check_stop, remove_all
|
from vllm.v1.core.sched.utils import check_stop, remove_all
|
||||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
|
CircularBufferSpec,
|
||||||
KVCacheConfig,
|
KVCacheConfig,
|
||||||
MambaSpec,
|
MambaSpec,
|
||||||
get_mamba_prefill_checkpoint_position,
|
get_mamba_prefill_checkpoint_position,
|
||||||
is_mamba_prefill_checkpoint_valid,
|
is_mamba_prefill_checkpoint_valid,
|
||||||
|
iter_layer_specs,
|
||||||
)
|
)
|
||||||
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
|
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
|
||||||
from vllm.v1.metrics.stats import (
|
from vllm.v1.metrics.stats import (
|
||||||
@@ -76,10 +80,15 @@ from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
|||||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||||
from vllm.v1.structured_output import StructuredOutputManager
|
from vllm.v1.structured_output import StructuredOutputManager
|
||||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||||
|
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||||
from vllm.v1.utils import record_function_or_nullcontext
|
from vllm.v1.utils import record_function_or_nullcontext
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
# Bounds-guard instrumentation: verify KVBlockZeroer's exclusive-ownership precondition before a
|
||||||
|
# block is zeroed. Off by default, so it costs nothing.
|
||||||
|
_ZERO_OWNERSHIP_CHECK = os.getenv("VLLM_E2_ZERO_OWNERSHIP_CHECK") == "1"
|
||||||
|
|
||||||
|
|
||||||
class Scheduler(SchedulerInterface):
|
class Scheduler(SchedulerInterface):
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -187,6 +196,26 @@ class Scheduler(SchedulerInterface):
|
|||||||
|
|
||||||
self.requires_kv_delivery = self.connector.requires_kv_delivery
|
self.requires_kv_delivery = self.connector.requires_kv_delivery
|
||||||
|
|
||||||
|
# Ring-skip telemetry: the QSA ring block (CircularBufferSpec) is rewritten
|
||||||
|
# by every step that touches the request, so with overlapping batches
|
||||||
|
# (async scheduling or PP) an in-flight step may still be committing a
|
||||||
|
# freed/preempted request's suffix rows when the scheduler hands that
|
||||||
|
# block to another request. A KV connector is not the only writer that
|
||||||
|
# can race the stale in-flight write; the next request's own first
|
||||||
|
# step is. Defer the return of freed blocks to the pool until the
|
||||||
|
# newest scheduled step's output has been processed whenever a ring
|
||||||
|
# group exists. Same mechanism and fence as the connector gate above.
|
||||||
|
if (
|
||||||
|
not self.defer_block_free
|
||||||
|
and self.vllm_config.max_concurrent_batches > 1
|
||||||
|
and any(
|
||||||
|
isinstance(spec, CircularBufferSpec)
|
||||||
|
for group in kv_cache_config.kv_cache_groups
|
||||||
|
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
self.defer_block_free = True
|
||||||
|
|
||||||
self.kv_event_publisher = EventPublisherFactory.create(
|
self.kv_event_publisher = EventPublisherFactory.create(
|
||||||
self.kv_events_config,
|
self.kv_events_config,
|
||||||
self.parallel_config.data_parallel_index,
|
self.parallel_config.data_parallel_index,
|
||||||
@@ -337,6 +366,16 @@ class Scheduler(SchedulerInterface):
|
|||||||
# Bind after construction so connectors can access the cache manager.
|
# Bind after construction so connectors can access the cache manager.
|
||||||
if self.connector is not None:
|
if self.connector is not None:
|
||||||
self.connector.bind_kv_cache_manager(self.kv_cache_manager)
|
self.connector.bind_kv_cache_manager(self.kv_cache_manager)
|
||||||
|
# Finished turns' grace pins are off with a connector, as
|
||||||
|
# waiter pins are (_pin_waiting_hits).
|
||||||
|
self.kv_cache_manager.enable_grace_pins = False
|
||||||
|
# Ring blocks an async KV load is about to fill. Their tag clear
|
||||||
|
# is skipped for the same reason as zeroing (it could land after the
|
||||||
|
# out-of-band write and wipe the producer's committed tags). Tracked
|
||||||
|
# whatever needs_kv_cache_zeroing says, like the clear itself.
|
||||||
|
self._skip_ring_clear_block_ids: set[int] = set()
|
||||||
|
|
||||||
|
self._log_prefix_cache_features()
|
||||||
|
|
||||||
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
|
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
|
||||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||||
@@ -535,6 +574,54 @@ class Scheduler(SchedulerInterface):
|
|||||||
end = min((s for s in stops if start < s < end), default=end)
|
end = min((s for s in stops if start < s < end), default=end)
|
||||||
return max(end - start, 0)
|
return max(end - start, 0)
|
||||||
|
|
||||||
|
def _log_prefix_cache_features(self) -> None:
|
||||||
|
"""One startup line per prefix-cache retention feature that is active
|
||||||
|
(nothing when it is off)."""
|
||||||
|
manager = self.kv_cache_manager
|
||||||
|
if manager.enable_caching and self.connector is None:
|
||||||
|
logger.info(
|
||||||
|
"Prefix cache: waiting requests' cached prefixes are pinned "
|
||||||
|
"until admission (up to %d waiters; hash unit %d, block %d)",
|
||||||
|
manager.MAX_PINNED_WAITERS,
|
||||||
|
self.hash_block_size,
|
||||||
|
self.block_size,
|
||||||
|
)
|
||||||
|
if manager.enable_grace_pins:
|
||||||
|
logger.info(
|
||||||
|
"Prefix cache: a finished turn's cached prefix stays referenced "
|
||||||
|
"until its next turn is admitted (up to %d turns; hash unit %d, "
|
||||||
|
"block %d)",
|
||||||
|
manager.MAX_GRACE_PINS,
|
||||||
|
self.hash_block_size,
|
||||||
|
self.block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _pin_waiting_hits(self) -> None:
|
||||||
|
"""Pin the prefix-cache hits of the requests at the front of the
|
||||||
|
waiting queue (arrival order, bounded by the manager) so the
|
||||||
|
admissions and decode growth of other requests cannot evict them
|
||||||
|
while they wait -- also while the running set is full, when they are
|
||||||
|
not even looked up. Released on admission, free, and whenever a
|
||||||
|
running request would otherwise fail to allocate.
|
||||||
|
"""
|
||||||
|
manager = self.kv_cache_manager
|
||||||
|
if (
|
||||||
|
not manager.enable_caching
|
||||||
|
or self.connector is not None
|
||||||
|
or manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS
|
||||||
|
):
|
||||||
|
return
|
||||||
|
for request in self.waiting:
|
||||||
|
if manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS:
|
||||||
|
break
|
||||||
|
if (
|
||||||
|
request.num_computed_tokens != 0
|
||||||
|
or request.status
|
||||||
|
not in (RequestStatus.WAITING, RequestStatus.PREEMPTED)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
manager.pin_waiting_hit(request)
|
||||||
|
|
||||||
def _get_local_prefix_cache_hit(
|
def _get_local_prefix_cache_hit(
|
||||||
self, request: Request
|
self, request: Request
|
||||||
) -> tuple[KVCacheBlocks, int, int, bool]:
|
) -> tuple[KVCacheBlocks, int, int, bool]:
|
||||||
@@ -769,6 +856,12 @@ class Scheduler(SchedulerInterface):
|
|||||||
):
|
):
|
||||||
break
|
break
|
||||||
|
|
||||||
|
# Waiting requests' pinned prefix hits yield
|
||||||
|
# before anyone is preempted (a pin never costs a running
|
||||||
|
# request its blocks); retry with them in the free queue.
|
||||||
|
if self.kv_cache_manager.yield_pinned_hits(limit=1):
|
||||||
|
continue
|
||||||
|
|
||||||
# The request cannot be scheduled.
|
# The request cannot be scheduled.
|
||||||
# Preempt the lowest-priority request.
|
# Preempt the lowest-priority request.
|
||||||
if self.policy == SchedulingPolicy.PRIORITY:
|
if self.policy == SchedulingPolicy.PRIORITY:
|
||||||
@@ -780,6 +873,13 @@ class Scheduler(SchedulerInterface):
|
|||||||
preempted_req = self.running[-1]
|
preempted_req = self.running[-1]
|
||||||
|
|
||||||
# A deferred free will not help with immediate allocation.
|
# A deferred free will not help with immediate allocation.
|
||||||
|
# (Upstream #49675. It subsumes our earlier post-hoc guard,
|
||||||
|
# which stopped after ONE deferred victim: a victim that
|
||||||
|
# passes this check is freed immediately by
|
||||||
|
# _preempt_request, so deferred_frees cannot grow below.
|
||||||
|
# The requester stays running, unscheduled, and retries
|
||||||
|
# after the fence drains; with every running request in
|
||||||
|
# flight (PP / async) that is zero victims, not the batch.)
|
||||||
if not self._request_blocks_can_be_freed(preempted_req):
|
if not self._request_blocks_can_be_freed(preempted_req):
|
||||||
break
|
break
|
||||||
|
|
||||||
@@ -1238,6 +1338,103 @@ class Scheduler(SchedulerInterface):
|
|||||||
has_scheduled_reqs=bool(self.running),
|
has_scheduled_reqs=bool(self.running),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Finished turns' grace pins (oldest first), then the
|
||||||
|
# pins of waiters behind this one, yield to it one at a time
|
||||||
|
# (arrival order). Its own hit is pinned first (a turn that
|
||||||
|
# arrived this step is not pinned yet and its hit may sit in
|
||||||
|
# a grace that yields), so only blocks it does not resume
|
||||||
|
# from go back. And only when releasing them all would cover
|
||||||
|
# the shortfall: otherwise yielding costs other conversations
|
||||||
|
# their prefix and still does not admit this one.
|
||||||
|
kv_cache_manager = self.kv_cache_manager
|
||||||
|
yield_for_head = new_blocks is None and (
|
||||||
|
kv_cache_manager.last_alloc_shortfall > 0
|
||||||
|
and (
|
||||||
|
kv_cache_manager.num_grace_pins > 0
|
||||||
|
or kv_cache_manager.num_pinned_waiters > 0
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if yield_for_head:
|
||||||
|
if (
|
||||||
|
self.connector is None
|
||||||
|
and request.num_computed_tokens == 0
|
||||||
|
and kv_cache_manager.pin_waiting_hit(request) > 0
|
||||||
|
):
|
||||||
|
# The pin just took blocks out of the free queue that
|
||||||
|
# the failed attempt counted as free: the proof block
|
||||||
|
# past the hit, and any hit block the lookup above did
|
||||||
|
# not return. Recompute the shortfall on the pinned
|
||||||
|
# state before measuring it against the others' pins.
|
||||||
|
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||||
|
request,
|
||||||
|
num_new_tokens,
|
||||||
|
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||||
|
new_computed_blocks=new_computed_blocks,
|
||||||
|
num_lookahead_tokens=effective_lookahead_tokens,
|
||||||
|
num_external_computed_tokens=num_external_computed_tokens,
|
||||||
|
delay_cache_blocks=load_kv_async,
|
||||||
|
num_encoder_tokens=num_encoder_tokens,
|
||||||
|
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||||
|
reserved_blocks=reserved_blocks,
|
||||||
|
has_scheduled_reqs=bool(self.running),
|
||||||
|
)
|
||||||
|
yield_for_head = new_blocks is None and (
|
||||||
|
kv_cache_manager.last_alloc_shortfall
|
||||||
|
<= kv_cache_manager.num_yieldable_blocks(
|
||||||
|
keep=request_id, keep_hit=new_computed_blocks.blocks
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
yield_for_head
|
||||||
|
and kv_cache_manager.release_pinned_extras(
|
||||||
|
request_id, new_computed_blocks.blocks
|
||||||
|
)
|
||||||
|
> 0
|
||||||
|
):
|
||||||
|
# Its own pinned blocks outside the hit it is admitted
|
||||||
|
# on (the proof block past it) go first: they cost
|
||||||
|
# nobody else anything, and at an exact fit they are
|
||||||
|
# the blocks it needs. Retry on them before the others
|
||||||
|
# yield.
|
||||||
|
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||||
|
request,
|
||||||
|
num_new_tokens,
|
||||||
|
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||||
|
new_computed_blocks=new_computed_blocks,
|
||||||
|
num_lookahead_tokens=effective_lookahead_tokens,
|
||||||
|
num_external_computed_tokens=num_external_computed_tokens,
|
||||||
|
delay_cache_blocks=load_kv_async,
|
||||||
|
num_encoder_tokens=num_encoder_tokens,
|
||||||
|
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||||
|
reserved_blocks=reserved_blocks,
|
||||||
|
has_scheduled_reqs=bool(self.running),
|
||||||
|
)
|
||||||
|
# Coverage is re-read before every yield: a retry can fail
|
||||||
|
# by more than the attempt before it (the full-sequence gate
|
||||||
|
# passes, the lookahead gate does not).
|
||||||
|
while (
|
||||||
|
yield_for_head
|
||||||
|
and new_blocks is None
|
||||||
|
and kv_cache_manager.last_alloc_shortfall
|
||||||
|
<= kv_cache_manager.num_yieldable_blocks(
|
||||||
|
keep=request_id, keep_hit=new_computed_blocks.blocks
|
||||||
|
)
|
||||||
|
and kv_cache_manager.yield_pinned_hits(keep=request_id, limit=1)
|
||||||
|
):
|
||||||
|
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||||
|
request,
|
||||||
|
num_new_tokens,
|
||||||
|
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||||
|
new_computed_blocks=new_computed_blocks,
|
||||||
|
num_lookahead_tokens=effective_lookahead_tokens,
|
||||||
|
num_external_computed_tokens=num_external_computed_tokens,
|
||||||
|
delay_cache_blocks=load_kv_async,
|
||||||
|
num_encoder_tokens=num_encoder_tokens,
|
||||||
|
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||||
|
reserved_blocks=reserved_blocks,
|
||||||
|
has_scheduled_reqs=bool(self.running),
|
||||||
|
)
|
||||||
|
|
||||||
if new_blocks is None:
|
if new_blocks is None:
|
||||||
# The request cannot be scheduled.
|
# The request cannot be scheduled.
|
||||||
|
|
||||||
@@ -1293,6 +1490,12 @@ class Scheduler(SchedulerInterface):
|
|||||||
request.num_computed_tokens = num_computed_tokens
|
request.num_computed_tokens = num_computed_tokens
|
||||||
self._inflight_prefills.add(request)
|
self._inflight_prefills.add(request)
|
||||||
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
|
self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS)
|
||||||
|
# Same race for the ring's tag clear; the load
|
||||||
|
# brings the producer's rows and tags (or none, and the
|
||||||
|
# reader then fails closed on the missing history).
|
||||||
|
self._skip_ring_clear_block_ids.update(
|
||||||
|
self.kv_cache_manager.get_ring_block_ids(request.request_id)
|
||||||
|
)
|
||||||
skip_request(request_queue)
|
skip_request(request_queue)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -1354,6 +1557,9 @@ class Scheduler(SchedulerInterface):
|
|||||||
if step_skipped_waiting:
|
if step_skipped_waiting:
|
||||||
self.waiting.prepend_requests(step_skipped_waiting)
|
self.waiting.prepend_requests(step_skipped_waiting)
|
||||||
|
|
||||||
|
# Whoever is still waiting keeps its prefix-cache hit.
|
||||||
|
self._pin_waiting_hits()
|
||||||
|
|
||||||
# DP prefill balancing: on a step that admitted prefills (release),
|
# DP prefill balancing: on a step that admitted prefills (release),
|
||||||
# record whether it was capacity-bound.
|
# record whether it was capacity-bound.
|
||||||
if not defer_prefills:
|
if not defer_prefills:
|
||||||
@@ -1485,11 +1691,14 @@ class Scheduler(SchedulerInterface):
|
|||||||
finished_req_ids=self.finished_req_ids,
|
finished_req_ids=self.finished_req_ids,
|
||||||
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
||||||
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
|
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
|
||||||
|
# Drained every step, never gated on KV zeroing.
|
||||||
|
new_ring_block_ids_to_clear=self._get_new_ring_block_ids_to_clear(),
|
||||||
has_sync_kv_loads=has_sync_kv_loads,
|
has_sync_kv_loads=has_sync_kv_loads,
|
||||||
kv_cache_block_copies=pending_kv_cache_block_copies,
|
kv_cache_block_copies=pending_kv_cache_block_copies,
|
||||||
kv_connector_block_state=kv_connector_block_state,
|
kv_connector_block_state=kv_connector_block_state,
|
||||||
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
||||||
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
|
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
|
||||||
|
scheduler_step=self.current_step,
|
||||||
)
|
)
|
||||||
|
|
||||||
# NOTE(Kuntai): this function is designed for multiple purposes:
|
# NOTE(Kuntai): this function is designed for multiple purposes:
|
||||||
@@ -1531,14 +1740,62 @@ class Scheduler(SchedulerInterface):
|
|||||||
) -> KVConnectorMetadata:
|
) -> KVConnectorMetadata:
|
||||||
return connector.build_connector_meta(scheduler_output)
|
return connector.build_connector_meta(scheduler_output)
|
||||||
|
|
||||||
|
def _get_new_ring_block_ids_to_clear(self) -> list[int] | None:
|
||||||
|
"""Ring claims of this step whose tag words the worker clears.
|
||||||
|
Unconditional (not under needs_kv_cache_zeroing); only blocks an async
|
||||||
|
KV load is about to write are left out."""
|
||||||
|
ids = self.kv_cache_manager.take_new_ring_block_ids()
|
||||||
|
if self._skip_ring_clear_block_ids:
|
||||||
|
skip = self._skip_ring_clear_block_ids
|
||||||
|
ids = [b for b in ids if b not in skip]
|
||||||
|
skip.clear()
|
||||||
|
return ids or None
|
||||||
|
|
||||||
def _get_new_block_ids_to_zero(self) -> list[int] | None:
|
def _get_new_block_ids_to_zero(self) -> list[int] | None:
|
||||||
# Drain new attention block ids every step so the manager-side list
|
# Drain new attention block ids every step so the manager-side list
|
||||||
# does not grow unbounded; only kv-cache zeroing consumes them.
|
# does not grow unbounded; only kv-cache zeroing consumes them.
|
||||||
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
|
new_block_ids_to_zero = self.kv_cache_manager.take_new_block_ids()
|
||||||
if not self.needs_kv_cache_zeroing:
|
if not self.needs_kv_cache_zeroing:
|
||||||
return None
|
return None
|
||||||
|
if new_block_ids_to_zero and _ZERO_OWNERSHIP_CHECK:
|
||||||
|
self._check_zero_block_ownership(new_block_ids_to_zero)
|
||||||
|
|
||||||
return new_block_ids_to_zero or None
|
return new_block_ids_to_zero or None
|
||||||
|
|
||||||
|
def _check_zero_block_ownership(self, block_ids: list[int]) -> None:
|
||||||
|
"""Test KVBlockZeroer's stated precondition, host side, off by default.
|
||||||
|
|
||||||
|
worker/utils.py justifies zeroing a whole tile with "block IDs are
|
||||||
|
global pool indices and a newly allocated block owns its whole tile".
|
||||||
|
Ownership is visible only here, in the scheduler process, so the check
|
||||||
|
lives here rather than in the zeroer. A block about to be zeroed should
|
||||||
|
be referenced by exactly one live request. More than one means a stale
|
||||||
|
reference survived into the zeroing, and the kernel is about to destroy
|
||||||
|
state another request still points at -- the stale-reference shape.
|
||||||
|
|
||||||
|
VLLM_E2_ZERO_OWNERSHIP_CHECK=1. O(live blocks) per step.
|
||||||
|
"""
|
||||||
|
pending = set(block_ids)
|
||||||
|
owners: dict[int, set[str]] = {}
|
||||||
|
for mgr in self.kv_cache_manager.coordinator.single_type_managers:
|
||||||
|
if getattr(mgr, "slot_pool", None) is not None:
|
||||||
|
# v2.5 slot-packed ring: its block ids are slot ids, not pool
|
||||||
|
# block ids, and its pages are never zeroed.
|
||||||
|
continue
|
||||||
|
for req_id, blocks in mgr.req_to_blocks.items():
|
||||||
|
for blk in blocks:
|
||||||
|
if blk.block_id in pending:
|
||||||
|
owners.setdefault(blk.block_id, set()).add(req_id)
|
||||||
|
for block_id, req_ids in sorted(owners.items()):
|
||||||
|
if len(req_ids) > 1:
|
||||||
|
logger.warning(
|
||||||
|
"KV zeroing ownership violation: block %d is about to be "
|
||||||
|
"zeroed but is referenced by %d live requests (%s).",
|
||||||
|
block_id,
|
||||||
|
len(req_ids),
|
||||||
|
", ".join(sorted(req_ids)[:4]),
|
||||||
|
)
|
||||||
|
|
||||||
def _preempt_request(
|
def _preempt_request(
|
||||||
self, request: Request, timestamp: float, drop_stale_output: bool = False
|
self, request: Request, timestamp: float, drop_stale_output: bool = False
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -2373,6 +2630,8 @@ class Scheduler(SchedulerInterface):
|
|||||||
engine_core_outputs[0] = eco = EngineCoreOutputs()
|
engine_core_outputs[0] = eco = EngineCoreOutputs()
|
||||||
eco.scheduler_stats = stats
|
eco.scheduler_stats = stats
|
||||||
|
|
||||||
|
if _e2_alloc_log.enabled():
|
||||||
|
_e2_alloc_log.log_step(self, scheduler_output)
|
||||||
return engine_core_outputs
|
return engine_core_outputs
|
||||||
|
|
||||||
def _ec_transfer_pending(self, request: Request, num_computed_tokens: int) -> bool:
|
def _ec_transfer_pending(self, request: Request, num_computed_tokens: int) -> bool:
|
||||||
@@ -2735,7 +2994,9 @@ class Scheduler(SchedulerInterface):
|
|||||||
if not self.defer_block_free or fence_seq <= self.processed_step_seq:
|
if not self.defer_block_free or fence_seq <= self.processed_step_seq:
|
||||||
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||||
return
|
return
|
||||||
self.deferred_frees.append((fence_seq, blocks[::-1]))
|
# The immediate path above frees these as-is, so enqueue them as-is:
|
||||||
|
# the drain no longer reverses.
|
||||||
|
self.deferred_frees.append((fence_seq, blocks))
|
||||||
|
|
||||||
def _drain_deferred_frees(self):
|
def _drain_deferred_frees(self):
|
||||||
"""Return deferred blocks whose fence step has completed.
|
"""Return deferred blocks whose fence step has completed.
|
||||||
@@ -2749,8 +3010,9 @@ class Scheduler(SchedulerInterface):
|
|||||||
if fence > self.processed_step_seq:
|
if fence > self.processed_step_seq:
|
||||||
break
|
break
|
||||||
_, blocks = self.deferred_frees.popleft()
|
_, blocks = self.deferred_frees.popleft()
|
||||||
# Free in reverse order so that the tail blocks are evicted first.
|
# Already in eviction order (reversed per KV cache group by
|
||||||
self.kv_cache_manager.block_pool.free_blocks(reversed(blocks))
|
# `pop_blocks_for_free`), so free as-is.
|
||||||
|
self.kv_cache_manager.block_pool.free_blocks(blocks)
|
||||||
|
|
||||||
def get_num_unfinished_requests(self) -> int:
|
def get_num_unfinished_requests(self) -> int:
|
||||||
if self._pause_state == PauseState.PAUSED_ALL:
|
if self._pause_state == PauseState.PAUSED_ALL:
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -3,9 +3,10 @@
|
|||||||
import itertools
|
import itertools
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Sequence
|
from collections.abc import Callable, Sequence
|
||||||
from typing import TYPE_CHECKING, ClassVar
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
|
from vllm import envs
|
||||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.utils.math_utils import cdiv
|
from vllm.utils.math_utils import cdiv
|
||||||
@@ -121,6 +122,12 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
and not isinstance(kv_cache_spec, CircularBufferSpec)
|
and not isinstance(kv_cache_spec, CircularBufferSpec)
|
||||||
)
|
)
|
||||||
self.new_block_ids: list[int] = []
|
self.new_block_ids: list[int] = []
|
||||||
|
# v2.2 ring-row tag fix: ring blocks claimed since the last drain. The ring stays
|
||||||
|
# OUT of whole-block zeroing above; instead every claim is recorded
|
||||||
|
# UNCONDITIONALLY (never under needs_kv_cache_zeroing) so the worker
|
||||||
|
# clears the rows' tag words before the new owner's first step. Only
|
||||||
|
# CircularBufferManager appends here.
|
||||||
|
self.new_ring_block_ids: list[int] = []
|
||||||
|
|
||||||
# Mapping from request ID to blocks to track the blocks allocated
|
# Mapping from request ID to blocks to track the blocks allocated
|
||||||
# for each request, so that we can free the blocks when the request
|
# for each request, so that we can free the blocks when the request
|
||||||
@@ -422,6 +429,17 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
self.new_block_ids = []
|
self.new_block_ids = []
|
||||||
return ids
|
return ids
|
||||||
|
|
||||||
|
def take_new_ring_block_ids(self) -> list[int]:
|
||||||
|
"""Drain ring blocks claimed since the last call; empty for
|
||||||
|
every manager but CircularBufferManager."""
|
||||||
|
ids = self.new_ring_block_ids
|
||||||
|
self.new_ring_block_ids = []
|
||||||
|
return ids
|
||||||
|
|
||||||
|
def get_ring_block_ids(self, request_id: str) -> list[int]:
|
||||||
|
"""The request's ring block ids; empty for non-ring managers."""
|
||||||
|
return []
|
||||||
|
|
||||||
def take_pending_cow_copies(
|
def take_pending_cow_copies(
|
||||||
self,
|
self,
|
||||||
) -> list[tuple[KVCacheBlock, KVCacheBlock]]:
|
) -> list[tuple[KVCacheBlock, KVCacheBlock]]:
|
||||||
@@ -591,6 +609,26 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
# Free blocks in reverse order so that the tail blocks are freed first.
|
# Free blocks in reverse order so that the tail blocks are freed first.
|
||||||
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
|
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
|
||||||
|
|
||||||
|
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
|
||||||
|
"""Pop the prefix-cache checkpoints held for ``request_id`` outside its
|
||||||
|
block table, without freeing them. Only align-mode Mamba holds any
|
||||||
|
(see ``MambaManager._retire_state_blocks``). The coordinator frees them
|
||||||
|
after every group's table, so they are the request's last blocks to
|
||||||
|
become eviction candidates."""
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||||
|
self,
|
||||||
|
request_id: str,
|
||||||
|
num_tokens: int,
|
||||||
|
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||||
|
num_local_computed_tokens: int = 0,
|
||||||
|
) -> int:
|
||||||
|
"""Blocks the admission gate must add for checkpoints this request
|
||||||
|
will hold outside its block table (0 for every manager but align-mode
|
||||||
|
Mamba with retention on)."""
|
||||||
|
return 0
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||||
"""Get the number of common prefix blocks for all requests with allocated
|
"""Get the number of common prefix blocks for all requests with allocated
|
||||||
@@ -1199,23 +1237,160 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
class _RingSlotPage:
|
||||||
|
"""One pool block shared by up to ``slots`` requests' rings."""
|
||||||
|
|
||||||
|
__slots__ = ("block", "free_slots", "live")
|
||||||
|
|
||||||
|
def __init__(self, block: KVCacheBlock, slots: int) -> None:
|
||||||
|
self.block = block
|
||||||
|
# Highest first, so pop() hands out the lowest free slot.
|
||||||
|
self.free_slots = list(range(slots - 1, -1, -1))
|
||||||
|
self.live = 0
|
||||||
|
|
||||||
|
|
||||||
|
class RingSlotPool:
|
||||||
|
"""v2.5 side-cache layout: ring blocks as slots of shared pool blocks.
|
||||||
|
|
||||||
|
The QSA ring needs one block per request for the request's lifetime, but
|
||||||
|
its pages for every layer of a stage are ~16 KB against a 12.9 MB pool
|
||||||
|
block. This pool takes whole blocks from the main ``BlockPool`` and cuts
|
||||||
|
each into ``slots_per_block`` slots; a slot's block id is
|
||||||
|
``pool_block_id * slots_per_block + slot``, which is exactly the index of
|
||||||
|
the slot in the worker's ring view (``KVCacheTensor.slots_per_block``).
|
||||||
|
|
||||||
|
The slot blocks it hands out carry ``pool=self``, so every existing free
|
||||||
|
path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred
|
||||||
|
``pop_blocks_for_free`` and the scheduler's fenced drain, which all call
|
||||||
|
``BlockPool.free_blocks`` -- routes them back here, at the time the old
|
||||||
|
per-request block would have gone back (the deferred free is kept). A pool
|
||||||
|
block returns to the main pool when its last slot is freed. Rings are never
|
||||||
|
prefix-cached, so a slot block carries no hash.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None:
|
||||||
|
assert slots_per_block > 1
|
||||||
|
self.block_pool = block_pool
|
||||||
|
self.slots_per_block = slots_per_block
|
||||||
|
# Pool block id -> page, in claim order (the oldest page fills first).
|
||||||
|
self._pages: dict[int, _RingSlotPage] = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_pages(self) -> int:
|
||||||
|
"""Pool blocks currently held for rings."""
|
||||||
|
return len(self._pages)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_live_slots(self) -> int:
|
||||||
|
return sum(page.live for page in self._pages.values())
|
||||||
|
|
||||||
|
def has_free_slot(self) -> bool:
|
||||||
|
return any(page.free_slots for page in self._pages.values())
|
||||||
|
|
||||||
|
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||||
|
out: list[KVCacheBlock] = []
|
||||||
|
for _ in range(num_blocks):
|
||||||
|
page = next((p for p in self._pages.values() if p.free_slots), None)
|
||||||
|
if page is None:
|
||||||
|
(pool_block,) = self.block_pool.get_new_blocks(1)
|
||||||
|
page = _RingSlotPage(pool_block, self.slots_per_block)
|
||||||
|
self._pages[pool_block.block_id] = page
|
||||||
|
slot = page.free_slots.pop()
|
||||||
|
page.live += 1
|
||||||
|
out.append(
|
||||||
|
KVCacheBlock(
|
||||||
|
page.block.block_id * self.slots_per_block + slot,
|
||||||
|
ref_cnt=1,
|
||||||
|
pool=self, # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def free_blocks(self, ordered_blocks) -> None:
|
||||||
|
released: list[KVCacheBlock] = []
|
||||||
|
for block in ordered_blocks:
|
||||||
|
assert block.pool is self, "not a ring slot of this pool"
|
||||||
|
block.ref_cnt -= 1
|
||||||
|
assert block.ref_cnt == 0, "ring slots have a single owner"
|
||||||
|
page_id, slot = divmod(block.block_id, self.slots_per_block)
|
||||||
|
page = self._pages[page_id]
|
||||||
|
assert slot not in page.free_slots, f"slot {block.block_id} freed twice"
|
||||||
|
page.free_slots.append(slot)
|
||||||
|
page.free_slots.sort(reverse=True)
|
||||||
|
page.live -= 1
|
||||||
|
if page.live == 0:
|
||||||
|
del self._pages[page_id]
|
||||||
|
released.append(page.block)
|
||||||
|
if released:
|
||||||
|
self.block_pool.free_blocks(released)
|
||||||
|
|
||||||
|
|
||||||
class CircularBufferManager(FullAttentionManager):
|
class CircularBufferManager(FullAttentionManager):
|
||||||
"""Claims the ring's single block per request; prefix caching disabled."""
|
"""Claims the ring's single block per request; prefix caching disabled."""
|
||||||
|
|
||||||
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||||
|
|
||||||
|
# v2.5 side-cache layout: set by the coordinator for a slot-packed group.
|
||||||
|
_slot_pool: RingSlotPool | None = None
|
||||||
|
|
||||||
|
def enable_slot_packing(self, slots_per_block: int) -> None:
|
||||||
|
"""Claim ring slots of shared pool blocks instead of whole blocks."""
|
||||||
|
assert not self.req_to_blocks, "enable slot packing before any claim"
|
||||||
|
self._slot_pool = RingSlotPool(self.block_pool, slots_per_block)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def slot_pool(self) -> RingSlotPool | None:
|
||||||
|
return self._slot_pool
|
||||||
|
|
||||||
|
if self._slot_pool is not None:
|
||||||
|
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||||
|
else:
|
||||||
|
new_blocks = self.block_pool.get_new_blocks(1)
|
||||||
|
|
||||||
def _claim_ring_block(
|
def _claim_ring_block(
|
||||||
self, request_id: str, record_for_zeroing: bool = True
|
self, request_id: str, record_for_zeroing: bool = True
|
||||||
) -> list[KVCacheBlock]:
|
) -> list[KVCacheBlock]:
|
||||||
req_blocks = self.req_to_blocks[request_id]
|
req_blocks = self.req_to_blocks[request_id]
|
||||||
if req_blocks:
|
if req_blocks:
|
||||||
return []
|
return []
|
||||||
new_blocks = self.block_pool.get_new_blocks(1)
|
if self._slot_pool is not None:
|
||||||
|
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||||
|
else:
|
||||||
|
new_blocks = self.block_pool.get_new_blocks(1)
|
||||||
req_blocks.extend(new_blocks)
|
req_blocks.extend(new_blocks)
|
||||||
if self._record_new_block_ids and record_for_zeroing:
|
if self._record_new_block_ids and record_for_zeroing:
|
||||||
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
||||||
|
# Unconditional -- not gated on needs_kv_cache_zeroing.
|
||||||
|
self.new_ring_block_ids.extend(block.block_id for block in new_blocks)
|
||||||
return new_blocks
|
return new_blocks
|
||||||
|
|
||||||
|
def take_new_ring_block_ids(self) -> list[int]:
|
||||||
|
"""Drain the ring blocks claimed since the last call, keeping only
|
||||||
|
blocks a request of this manager still holds, in claim order, once.
|
||||||
|
|
||||||
|
A block claimed and then freed within the same scheduling pass (the
|
||||||
|
request was preempted after its allocation) may already belong to
|
||||||
|
another group by the time the worker acts; clearing its tag words
|
||||||
|
there would write into that group's page, so it is dropped here.
|
||||||
|
"""
|
||||||
|
claimed = self.new_ring_block_ids
|
||||||
|
self.new_ring_block_ids = []
|
||||||
|
if not claimed:
|
||||||
|
return []
|
||||||
|
held = {
|
||||||
|
block.block_id for blocks in self.req_to_blocks.values() for block in blocks
|
||||||
|
}
|
||||||
|
out: list[int] = []
|
||||||
|
seen: set[int] = set()
|
||||||
|
for block_id in claimed:
|
||||||
|
if block_id in held and block_id not in seen:
|
||||||
|
seen.add(block_id)
|
||||||
|
out.append(block_id)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def get_ring_block_ids(self, request_id: str) -> list[int]:
|
||||||
|
return [block.block_id for block in self.req_to_blocks.get(request_id, ())]
|
||||||
|
|
||||||
def get_num_blocks_to_allocate(
|
def get_num_blocks_to_allocate(
|
||||||
self,
|
self,
|
||||||
request_id: str,
|
request_id: str,
|
||||||
@@ -1227,7 +1402,12 @@ class CircularBufferManager(FullAttentionManager):
|
|||||||
apply_admission_cap: bool = False,
|
apply_admission_cap: bool = False,
|
||||||
prefill_end: int = 0,
|
prefill_end: int = 0,
|
||||||
) -> int:
|
) -> int:
|
||||||
return 0 if self.req_to_blocks.get(request_id) else 1
|
if self.req_to_blocks.get(request_id):
|
||||||
|
return 0
|
||||||
|
# A free slot in a page the ring already holds costs no pool block.
|
||||||
|
if self._slot_pool is not None and self._slot_pool.has_free_slot():
|
||||||
|
return 0
|
||||||
|
return 1
|
||||||
|
|
||||||
def allocate_new_blocks(
|
def allocate_new_blocks(
|
||||||
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
||||||
@@ -1489,6 +1669,191 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
# connector; a request that finishes first hands off this table
|
# connector; a request that finishes first hands off this table
|
||||||
# source directly.
|
# source directly.
|
||||||
self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {}
|
self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {}
|
||||||
|
# v2.5. In align mode a state block leaves the block table (it is
|
||||||
|
# nulled and released) as soon as the request computes one block past
|
||||||
|
# it -- `_remove_blocks_in_range` / `remove_skipped_blocks`. When that
|
||||||
|
# block carries a prefix-cache hash it is the request's GDN checkpoint
|
||||||
|
# (the replay boundary under retention interval 0), and releasing it
|
||||||
|
# mid-request puts it in the free queue AHEAD of the request's own
|
||||||
|
# attention blocks, which stay referenced until finish. Concurrent
|
||||||
|
# prefills then pop it first; since the hybrid hit is the shortest
|
||||||
|
# group's, that single block zeroes the next turn's reuse. With
|
||||||
|
# retention on, the newest hashed retired state stays referenced here
|
||||||
|
# (at most one per request per group) and is released only when the
|
||||||
|
# request is freed, after its attention blocks (coordinator.free).
|
||||||
|
self._retain_checkpoint: bool = (
|
||||||
|
self.mamba_cache_mode == "align"
|
||||||
|
and self.enable_caching
|
||||||
|
and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT
|
||||||
|
)
|
||||||
|
# {request_id: (hashed token position, block)}: the newest hashed
|
||||||
|
# state the request resumed from or produced, held with one ref of
|
||||||
|
# its own (ordered by ``block_hash_num_tokens`` rather than
|
||||||
|
# block index so a sub-block partial-tail key sorts correctly against
|
||||||
|
# full-block boundary states).
|
||||||
|
#
|
||||||
|
# With ``--prefix-match-unit`` below the block size a request
|
||||||
|
# that resumed at a partial tail (mid-block) never hashes a boundary
|
||||||
|
# state: the replay boundary lies below its resume point. Its only
|
||||||
|
# reusable GDN state is its own partial-tail checkpoint, which the
|
||||||
|
# next chunk moves into a CoW block in no block table; the scheduler
|
||||||
|
# drops the copy's retention refs once the copy has run, so that
|
||||||
|
# block sat at ref 0 in the free queue for the rest of the request,
|
||||||
|
# ahead of its attention blocks, and concurrent prefills evicted it
|
||||||
|
# (measured: 7/61 follow-ups with cached_tokens 0 at unit 64, every
|
||||||
|
# one after a partial-tail resume; none at unit 4096). The CoW paths
|
||||||
|
# therefore hold their hashed endpoint here too: the moved tail for
|
||||||
|
# a running request, and the hit source a new request copies from.
|
||||||
|
self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {}
|
||||||
|
# Callbacks (source_block, cow_block) run after a running
|
||||||
|
# request moved its live tail's hash to the CoW copy, for refs held
|
||||||
|
# outside this manager (the KVCacheManager's grace and waiter pins).
|
||||||
|
self.moved_hash_listeners: list[
|
||||||
|
Callable[[KVCacheBlock, KVCacheBlock], None]
|
||||||
|
] = []
|
||||||
|
if self.mamba_cache_mode == "align" and self.enable_caching:
|
||||||
|
# The A/B evidence line: boot logs must show which arm ran.
|
||||||
|
logger.info_once(
|
||||||
|
"Mamba align prefix-checkpoint retention: %s "
|
||||||
|
"(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)",
|
||||||
|
"on" if self._retain_checkpoint else "off",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
self._retain_checkpoint
|
||||||
|
and self.block_size > self.block_pool.hash_block_size
|
||||||
|
):
|
||||||
|
logger.info_once(
|
||||||
|
"Mamba align: each request holds its partial-tail CoW checkpoint "
|
||||||
|
"until it is freed (hash unit %d, block %d)",
|
||||||
|
self.block_pool.hash_block_size,
|
||||||
|
self.block_size,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
self.mamba_cache_mode == "align"
|
||||||
|
and self.enable_caching
|
||||||
|
and self.block_size > self.block_pool.hash_block_size
|
||||||
|
):
|
||||||
|
logger.info_once(
|
||||||
|
"Mamba align: a prompt tail key that lands on a block end is "
|
||||||
|
"hashed too (hash unit %d, block %d)",
|
||||||
|
self.block_pool.hash_block_size,
|
||||||
|
self.block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _retire_state_blocks(
|
||||||
|
self, request_id: str, retired: list[tuple[int, KVCacheBlock]]
|
||||||
|
) -> None:
|
||||||
|
"""Release align-mode state blocks the request no longer reads.
|
||||||
|
|
||||||
|
``retired`` holds (logical index, block) pairs already nulled out of
|
||||||
|
the block table, in the eviction order the caller wants. With
|
||||||
|
retention on, the newest hashed one is kept referenced as the request's
|
||||||
|
checkpoint instead, and the checkpoint it supersedes (if any) is
|
||||||
|
released now -- so the request never holds more than one.
|
||||||
|
"""
|
||||||
|
if self._retain_checkpoint:
|
||||||
|
newest: tuple[int, KVCacheBlock] | None = None
|
||||||
|
for _, block in retired:
|
||||||
|
if block.block_hash is None:
|
||||||
|
continue
|
||||||
|
position = block.block_hash_num_tokens
|
||||||
|
assert position is not None
|
||||||
|
if newest is None or position > newest[0]:
|
||||||
|
newest = (position, block)
|
||||||
|
held = self._retained_checkpoints.get(request_id)
|
||||||
|
if newest is not None and (held is None or newest[0] > held[0]):
|
||||||
|
# The retired block's table ref becomes the hold's ref.
|
||||||
|
self._retained_checkpoints[request_id] = newest
|
||||||
|
retired = [entry for entry in retired if entry[1] is not newest[1]]
|
||||||
|
if held is not None:
|
||||||
|
# Superseded: it ages from here like any cached free block.
|
||||||
|
retired.append(held)
|
||||||
|
if retired:
|
||||||
|
self.block_pool.free_blocks([block for _, block in retired])
|
||||||
|
|
||||||
|
def _hold_checkpoint(self, request_id: str, block: KVCacheBlock) -> bool:
|
||||||
|
"""Hold ``block`` (a hashed state outside the request's table) as the
|
||||||
|
request's prefix checkpoint, taking one extra ref for the hold.
|
||||||
|
|
||||||
|
Only the newest position is held: an older one is left alone (it ages
|
||||||
|
from wherever it is), a superseded one is released here. The hold's
|
||||||
|
ref is returned by ``pop_retained_checkpoints`` -> ``free_blocks``.
|
||||||
|
Returns whether the block is now held.
|
||||||
|
"""
|
||||||
|
if not self._retain_checkpoint or block.is_null or block.block_hash is None:
|
||||||
|
return False
|
||||||
|
position = block.block_hash_num_tokens
|
||||||
|
assert position is not None
|
||||||
|
held = self._retained_checkpoints.get(request_id)
|
||||||
|
if held is not None and (held[1] is block or held[0] >= position):
|
||||||
|
return False
|
||||||
|
self.block_pool.touch([block])
|
||||||
|
self._retained_checkpoints[request_id] = (position, block)
|
||||||
|
if held is not None:
|
||||||
|
self.block_pool.free_blocks([held[1]])
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _follow_moved_hash(
|
||||||
|
self, owner_id: str, source_block: KVCacheBlock, cow_block: KVCacheBlock
|
||||||
|
) -> None:
|
||||||
|
"""``owner_id`` just moved ``source_block``'s hashes to ``cow_block``
|
||||||
|
and keeps writing ``source_block``. A request that resumed at that
|
||||||
|
tail while it was still the owner's live block holds the source for
|
||||||
|
those hashes; move its hold to the copy, so it pins the hashed state
|
||||||
|
and not another request's running block.
|
||||||
|
"""
|
||||||
|
for request_id, (position, block) in list(self._retained_checkpoints.items()):
|
||||||
|
if request_id != owner_id and block is source_block:
|
||||||
|
self.block_pool.touch([cow_block])
|
||||||
|
self._retained_checkpoints[request_id] = (position, cow_block)
|
||||||
|
self.block_pool.free_blocks([source_block])
|
||||||
|
for listener in self.moved_hash_listeners:
|
||||||
|
listener(source_block, cow_block)
|
||||||
|
|
||||||
|
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
|
||||||
|
if self.mamba_cache_mode != "align":
|
||||||
|
return []
|
||||||
|
held = self._retained_checkpoints.pop(request_id, None)
|
||||||
|
return [] if held is None else [held[1]]
|
||||||
|
|
||||||
|
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||||
|
self,
|
||||||
|
request_id: str,
|
||||||
|
num_tokens: int,
|
||||||
|
new_computed_blocks: Sequence[KVCacheBlock] = (),
|
||||||
|
num_local_computed_tokens: int = 0,
|
||||||
|
) -> int:
|
||||||
|
# A request that already holds a checkpoint only ever swaps it. One
|
||||||
|
# that does not will hold one once a hashed state retires, which needs
|
||||||
|
# at least one state block behind the running one.
|
||||||
|
if not self._retain_checkpoint or request_id in self._retained_checkpoints:
|
||||||
|
return 0
|
||||||
|
# A sub-block (partial-tail) hit holds the hit source only
|
||||||
|
# until the request's own tail is moved into a CoW block that is held
|
||||||
|
# in its place; at that step both are referenced, so the request's
|
||||||
|
# lifetime peak is one block above what the hit brought (whatever the
|
||||||
|
# prompt length: the tail CoW happens inside the first block too).
|
||||||
|
if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens):
|
||||||
|
return 1
|
||||||
|
if num_tokens <= self.block_size:
|
||||||
|
# No state block retires behind the running one, but a prompt
|
||||||
|
# whose tail is keyed inside the block still moves that tail into
|
||||||
|
# a held CoW block when the hash unit is finer than the block.
|
||||||
|
return int(self.block_pool.hash_block_size < self.block_size)
|
||||||
|
# A prefix hit already brings the block that becomes the held
|
||||||
|
# checkpoint -- the hit state, counted by get_num_blocks_to_allocate as
|
||||||
|
# an evictable computed block (or already referenced elsewhere) -- and
|
||||||
|
# the same holds for a hashed state still in the request's table.
|
||||||
|
# Reserving another one double-counts every hit admission by one block
|
||||||
|
# per group, which only delays the follow-up turns the fix is for.
|
||||||
|
if any(not blk.is_null for blk in new_computed_blocks):
|
||||||
|
return 0
|
||||||
|
if any(
|
||||||
|
not blk.is_null and blk.block_hash is not None
|
||||||
|
for blk in self.req_to_blocks.get(request_id, ())
|
||||||
|
):
|
||||||
|
return 0
|
||||||
|
return 1
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def find_longest_cache_hit(
|
def find_longest_cache_hit(
|
||||||
@@ -1544,6 +1909,15 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
return computed_blocks, hit_length
|
return computed_blocks, hit_length
|
||||||
|
|
||||||
max_num_blocks = max_length // block_size
|
max_num_blocks = max_length // block_size
|
||||||
|
if drop_eagle_block and max_num_blocks > 0:
|
||||||
|
# EAGLE/MTP: drop the final matched page. Its recurrent-state snapshot
|
||||||
|
# may have been taken over draft tokens that verification later rejects,
|
||||||
|
# so the state must be restored from the previous page boundary and
|
||||||
|
# recomputed forward. FullAttentionManager (see find_longest_cache_hit
|
||||||
|
# above) pops the last block; Mamba keeps only the rightmost real block
|
||||||
|
# with the prefix null-padded, so a literal pop would delete the state
|
||||||
|
# block itself -- lower the search ceiling by one page instead.
|
||||||
|
max_num_blocks -= 1
|
||||||
# Search from right to left and early stop when a match is found.
|
# Search from right to left and early stop when a match is found.
|
||||||
for i in range(max_num_blocks - 1, -1, -1):
|
for i in range(max_num_blocks - 1, -1, -1):
|
||||||
if cached_block := block_pool.get_cached_block(
|
if cached_block := block_pool.get_cached_block(
|
||||||
@@ -1639,15 +2013,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
last_block = min(last_block, len(blocks))
|
last_block = min(last_block, len(blocks))
|
||||||
if first_block >= last_block:
|
if first_block >= last_block:
|
||||||
return
|
return
|
||||||
freed: list[KVCacheBlock] = []
|
freed: list[tuple[int, KVCacheBlock]] = []
|
||||||
# Mamba prefill leaves null gaps between states awaiting retirement.
|
# Mamba prefill leaves null gaps between states awaiting retirement.
|
||||||
for i in range(last_block - 1, first_block - 1, -1):
|
for i in range(last_block - 1, first_block - 1, -1):
|
||||||
if blocks[i].is_null:
|
if blocks[i].is_null:
|
||||||
continue
|
continue
|
||||||
freed.append(blocks[i])
|
freed.append((i, blocks[i]))
|
||||||
blocks[i] = self._null_block
|
blocks[i] = self._null_block
|
||||||
if freed:
|
self._retire_state_blocks(request_id, freed)
|
||||||
self.block_pool.free_blocks(freed)
|
|
||||||
self._num_retired_blocks[request_id] = last_block
|
self._num_retired_blocks[request_id] = last_block
|
||||||
|
|
||||||
def remove_skipped_blocks(
|
def remove_skipped_blocks(
|
||||||
@@ -1677,8 +2050,11 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
):
|
):
|
||||||
blocks = self.req_to_blocks[request_id]
|
blocks = self.req_to_blocks[request_id]
|
||||||
if blocks[last_state_block_idx] != self._null_block:
|
if blocks[last_state_block_idx] != self._null_block:
|
||||||
self.block_pool.free_blocks([blocks[last_state_block_idx]])
|
block = blocks[last_state_block_idx]
|
||||||
blocks[last_state_block_idx] = self._null_block
|
blocks[last_state_block_idx] = self._null_block
|
||||||
|
self._retire_state_blocks(
|
||||||
|
request_id, [(last_state_block_idx, block)]
|
||||||
|
)
|
||||||
|
|
||||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||||
"""Cascade attention is not supported by mamba"""
|
"""Cascade attention is not supported by mamba"""
|
||||||
@@ -1915,6 +2291,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
# overwrites source_block.
|
# overwrites source_block.
|
||||||
assert req_blocks[block_idx] is source_block
|
assert req_blocks[block_idx] is source_block
|
||||||
self.block_pool.move_block_hashes(source_block, cow_block)
|
self.block_pool.move_block_hashes(source_block, cow_block)
|
||||||
|
self._follow_moved_hash(request_id, source_block, cow_block)
|
||||||
self._pending_cow_copies.append((source_block, cow_block))
|
self._pending_cow_copies.append((source_block, cow_block))
|
||||||
source_block.ref_cnt += 1
|
source_block.ref_cnt += 1
|
||||||
producer_tail = self._producer_partial_tail_reqs.pop(
|
producer_tail = self._producer_partial_tail_reqs.pop(
|
||||||
@@ -1938,9 +2315,20 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
# The moved entry is only filled by this step's
|
# The moved entry is only filled by this step's
|
||||||
# copy, so defer same-step hits on it.
|
# copy, so defer same-step hits on it.
|
||||||
self.cached_blocks_this_step.add(cow_block.block_hash)
|
self.cached_blocks_this_step.add(cow_block.block_hash)
|
||||||
|
# The moved tail is the request's newest
|
||||||
|
# reusable state and lives in no block table; hold it
|
||||||
|
# past the copy's retention so the request's next
|
||||||
|
# turn can resume at it (see __init__).
|
||||||
|
self._hold_checkpoint(request_id, cow_block)
|
||||||
else:
|
else:
|
||||||
self._apply_cow(request_id, block_idx, source_block, cow_block)
|
self._apply_cow(request_id, block_idx, source_block, cow_block)
|
||||||
returned_blocks = [cow_block] + returned_blocks
|
returned_blocks = [cow_block] + returned_blocks
|
||||||
|
# The hit source keeps the hash while the
|
||||||
|
# request advances its private copy; hold it until
|
||||||
|
# the request's own tail supersedes it, so a request
|
||||||
|
# that stops early (preempted, aborted, block-aligned
|
||||||
|
# prompt) still leaves the state it resumed from.
|
||||||
|
self._hold_checkpoint(request_id, source_block)
|
||||||
req_blocks.extend(new_blocks)
|
req_blocks.extend(new_blocks)
|
||||||
self._allocated_block_reqs.add(request_id)
|
self._allocated_block_reqs.add(request_id)
|
||||||
self._partial_hit_reqs.pop(request_id, None)
|
self._partial_hit_reqs.pop(request_id, None)
|
||||||
@@ -1991,6 +2379,12 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
for entry in self._pending_boundary_state_offloads
|
for entry in self._pending_boundary_state_offloads
|
||||||
if entry[0] != request_id
|
if entry[0] != request_id
|
||||||
]
|
]
|
||||||
|
# The coordinator pops the retained checkpoint first and frees it
|
||||||
|
# after every group. A direct caller still gets it back, first in
|
||||||
|
# allocation order, so the caller's reversed free releases it last.
|
||||||
|
retained = self.pop_retained_checkpoints(request_id)
|
||||||
|
if retained:
|
||||||
|
return retained + super().pop_blocks_for_free(request_id)
|
||||||
return super().pop_blocks_for_free(request_id)
|
return super().pop_blocks_for_free(request_id)
|
||||||
|
|
||||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||||
@@ -2009,6 +2403,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
replay_boundaries: Sequence[int],
|
replay_boundaries: Sequence[int],
|
||||||
) -> None:
|
) -> None:
|
||||||
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
|
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
|
||||||
|
tail = self._block_aligned_prompt_tail(request)
|
||||||
|
if tail:
|
||||||
|
# With a hash unit finer than the block, the prompt's tail
|
||||||
|
# key (where the next turn resumes; `_cache_partial_tail_block`)
|
||||||
|
# can land exactly on a block end. It is then a full-block key,
|
||||||
|
# not a partial one, and the replay-boundary mask alone (one block
|
||||||
|
# below it under the eagle drop) would never hash that state.
|
||||||
|
replay_boundaries = (*replay_boundaries, tail)
|
||||||
super().cache_blocks(
|
super().cache_blocks(
|
||||||
request,
|
request,
|
||||||
num_tokens,
|
num_tokens,
|
||||||
@@ -2050,6 +2452,22 @@ class MambaManager(SingleTypeKVCacheManager):
|
|||||||
def new_step_starts(self) -> None:
|
def new_step_starts(self) -> None:
|
||||||
self.cached_blocks_this_step.clear()
|
self.cached_blocks_this_step.clear()
|
||||||
|
|
||||||
|
def _block_aligned_prompt_tail(self, request: Request) -> int:
|
||||||
|
"""The prompt's tail key when it falls on a block end, else 0.
|
||||||
|
|
||||||
|
Only with a hash unit finer than the block (at the block unit the
|
||||||
|
tail IS the replay boundary, which the mask already keeps). The key
|
||||||
|
is placed exactly as `_cache_partial_tail_block` places a sub-block
|
||||||
|
tail, eagle shift included.
|
||||||
|
"""
|
||||||
|
hash_block_size = self.block_pool.hash_block_size
|
||||||
|
if self.mamba_cache_mode != "align" or self.block_size <= hash_block_size:
|
||||||
|
return 0
|
||||||
|
tail = request.num_prompt_tokens // hash_block_size * hash_block_size
|
||||||
|
if self.drop_eagle_checkpoint_block:
|
||||||
|
tail = max(tail - hash_block_size, 0)
|
||||||
|
return tail if tail % self.block_size == 0 else 0
|
||||||
|
|
||||||
def _cache_partial_tail_block(
|
def _cache_partial_tail_block(
|
||||||
self,
|
self,
|
||||||
request: Request,
|
request: Request,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,547 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Fork startup guards that fail closed on a graph profile we never qualified.
|
||||||
|
|
||||||
|
These run at model construction, on every rank, for the target model and for
|
||||||
|
the MTP drafter, before profiling or capture.
|
||||||
|
|
||||||
|
Compile-profile guard: this tree restores torch.compile on the
|
||||||
|
NVIDIA Qwen4Exp path (reverting vLLM #55272) and removes Qwen4Exp from
|
||||||
|
``DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES``. The ONE qualified profile is the
|
||||||
|
v2.0.1 production identity:
|
||||||
|
|
||||||
|
* ``VLLM_USE_BREAKABLE_CUDAGRAPH`` explicitly pinned to exactly ``0``, and
|
||||||
|
resolved OFF;
|
||||||
|
* ``CompilationMode.VLLM_COMPILE``;
|
||||||
|
* ``CUDAGraphMode.FULL_AND_PIECEWISE``;
|
||||||
|
* every Qwen4Exp request-dependent custom op in ``splitting_ops``
|
||||||
|
(:data:`REQUIRED_SPLITTING_OPS`);
|
||||||
|
* a live ``@support_torch_compile`` submodule
|
||||||
|
(:func:`assert_compiled_submodule`), which is the property
|
||||||
|
``vllm/v1/worker/gpu/cudagraph_utils.py:605`` actually tests at capture.
|
||||||
|
|
||||||
|
Anything else -- eager, NONE + FULL, FULL_DECODE_ONLY, breakable PIECEWISE --
|
||||||
|
fails loud. ``VLLM_E2_ALLOW_BREAKABLE=1`` is the single, logged opt-out for
|
||||||
|
deliberate non-production boots (debug, probes); a production launch must not set it.
|
||||||
|
|
||||||
|
Timing limit: this runs AFTER ``VllmConfig.__post_init__`` resolved the
|
||||||
|
profile. It stops the boot before profiling and capture; it cannot recover
|
||||||
|
the originally requested profile if the config layer rewrote it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
BREAKABLE_ENV = "VLLM_USE_BREAKABLE_CUDAGRAPH"
|
||||||
|
#: Opt-out for every guard in this module. Recorded in the launch identity and
|
||||||
|
#: PROHIBITED in production profiles; a production launch must not set it.
|
||||||
|
ALLOW_BREAKABLE_ENV = "VLLM_E2_ALLOW_BREAKABLE"
|
||||||
|
#: Optional expected resolved cudagraph mode, by name. When set, a mismatch is
|
||||||
|
#: an error (in addition to the fixed requirement below).
|
||||||
|
EXPECTED_MODE_ENV = "VLLM_E2_EXPECTED_CUDAGRAPH_MODE"
|
||||||
|
|
||||||
|
#: The qualified profile (the compile-profile guard).
|
||||||
|
REQUIRED_COMPILATION_MODE_NAME = "VLLM_COMPILE"
|
||||||
|
REQUIRED_CUDAGRAPH_MODE_NAME = "FULL_AND_PIECEWISE"
|
||||||
|
#: The only raw VLLM_USE_BREAKABLE_CUDAGRAPH value accepted as the production
|
||||||
|
#: pin. Resolution semantics stay upstream's (_upstream_truth).
|
||||||
|
PRODUCTION_BREAKABLE_PIN = "0"
|
||||||
|
|
||||||
|
#: Custom ops that must stay graph breaks on the NVIDIA Qwen4Exp path. Each
|
||||||
|
#: reads per-step request metadata (or forks/joins a side stream) and would be
|
||||||
|
#: baked into a piecewise graph otherwise.
|
||||||
|
REQUIRED_SPLITTING_OPS = (
|
||||||
|
"vllm::qwen4_exp_qsa_with_output",
|
||||||
|
"vllm::qwen4_exp_ple_short_conv",
|
||||||
|
"vllm::qwen4_exp_compute_ple_ngram_ids",
|
||||||
|
"vllm::qwen4_exp_ple_start_prefetch",
|
||||||
|
"vllm::qwen4_exp_ple_finalize_prefetch",
|
||||||
|
"vllm::qwen_gdn_attention_core",
|
||||||
|
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||||
|
"vllm::ple_offload_wait_copy",
|
||||||
|
)
|
||||||
|
|
||||||
|
#: Upstream v0.30.0 auto-enabled breakable cudagraphs for these; this tree
|
||||||
|
#: does not (vllm/config/vllm.py). Kept to name the mechanism in messages.
|
||||||
|
OUR_AUTO_BREAKABLE_ARCHITECTURES = frozenset(
|
||||||
|
{
|
||||||
|
"Qwen4ExpForCausalLM",
|
||||||
|
"Qwen4ExpForConditionalGeneration",
|
||||||
|
"Qwen4ExpMTP",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _upstream_truth(raw: str | None) -> bool:
|
||||||
|
"""Upstream's exact semantics for the breakable switch.
|
||||||
|
|
||||||
|
``envs.py:937`` is ``os.environ.get(..., "0") == "1"``. Nothing else is on.
|
||||||
|
"""
|
||||||
|
return raw == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def breakable_resolved() -> bool:
|
||||||
|
"""Whether breakable cudagraphs are on, as the engine will see it.
|
||||||
|
|
||||||
|
The raw environment and the (cached) envs accessor are two readings of one
|
||||||
|
source; either saying "on" means on. No exception is swallowed.
|
||||||
|
"""
|
||||||
|
if _upstream_truth(os.environ.get(BREAKABLE_ENV)):
|
||||||
|
return True
|
||||||
|
import vllm.envs as envs
|
||||||
|
|
||||||
|
return bool(envs.VLLM_USE_BREAKABLE_CUDAGRAPH)
|
||||||
|
|
||||||
|
|
||||||
|
def _name(value) -> str:
|
||||||
|
return getattr(value, "name", str(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _profile(
|
||||||
|
vllm_config, role: str = "target"
|
||||||
|
) -> tuple[list[str], object, object, list[str] | None]:
|
||||||
|
"""Resolved (architectures, mode, cudagraph mode, splitting ops).
|
||||||
|
|
||||||
|
For ``role="draft"`` the architectures are the DRAFT model's
|
||||||
|
(``speculative_config.draft_model_config``): the drafter is constructed
|
||||||
|
with the target's VllmConfig, whose model_config names the target. The reference host
|
||||||
|
logged the target architecture on the draft line before this fix.
|
||||||
|
The graph profile is the engine's single CompilationConfig for both roles.
|
||||||
|
"""
|
||||||
|
model_config = getattr(vllm_config, "model_config", None)
|
||||||
|
if role == "draft":
|
||||||
|
spec = getattr(vllm_config, "speculative_config", None)
|
||||||
|
draft_config = getattr(spec, "draft_model_config", None)
|
||||||
|
if draft_config is not None:
|
||||||
|
model_config = draft_config
|
||||||
|
if model_config is None:
|
||||||
|
raise RuntimeError("e2-guard: vllm_config has no model_config to check")
|
||||||
|
compilation_config = getattr(vllm_config, "compilation_config", None)
|
||||||
|
if compilation_config is None:
|
||||||
|
raise RuntimeError("e2-guard: vllm_config has no compilation_config to check")
|
||||||
|
architectures = list(getattr(model_config, "architectures", None) or [])
|
||||||
|
return (
|
||||||
|
architectures,
|
||||||
|
compilation_config.mode,
|
||||||
|
compilation_config.cudagraph_mode,
|
||||||
|
getattr(compilation_config, "splitting_ops", None),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _opted_out(role: str, detail: str) -> bool:
|
||||||
|
if os.environ.get(ALLOW_BREAKABLE_ENV, "") != "1":
|
||||||
|
return False
|
||||||
|
logger.warning(
|
||||||
|
"e2-guard[%s]: %s=1 -- guards disabled by request (%s). This boot is "
|
||||||
|
"NOT a promotion-grade boot; a production launch must not set this flag.",
|
||||||
|
role,
|
||||||
|
ALLOW_BREAKABLE_ENV,
|
||||||
|
detail,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def assert_graph_profile_is_executable(vllm_config, *, role: str = "target") -> None:
|
||||||
|
"""Require the qualified profile: VLLM_COMPILE + FULL_AND_PIECEWISE.
|
||||||
|
|
||||||
|
``role`` is ``"target"`` or ``"draft"``; both are checked, because
|
||||||
|
``Qwen4ExpMTP`` builds ``Qwen4ExpMultiTokenPredictor`` rather than
|
||||||
|
``Qwen4ExpForCausalLM``, from a config rebuilt in ``mtp.py``.
|
||||||
|
"""
|
||||||
|
architectures, mode, cudagraph_mode, splitting_ops = _profile(vllm_config, role)
|
||||||
|
detail = (
|
||||||
|
f"architectures={architectures} breakable={os.environ.get(BREAKABLE_ENV)} "
|
||||||
|
f"compilation_mode={_name(mode)} cudagraph_mode={_name(cudagraph_mode)}"
|
||||||
|
)
|
||||||
|
if _opted_out(role, detail):
|
||||||
|
return
|
||||||
|
|
||||||
|
# 1. The pin must be explicit, so a future upstream merge that re-adds us
|
||||||
|
# to DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES cannot flip it silently.
|
||||||
|
raw_breakable = os.environ.get(BREAKABLE_ENV)
|
||||||
|
if raw_breakable is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: {BREAKABLE_ENV} is not set. Pin it in the "
|
||||||
|
f"launcher: export {BREAKABLE_ENV}=0, or set {ALLOW_BREAKABLE_ENV}=1 "
|
||||||
|
"to take an unpinned boot on purpose."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Breakable capture is not the qualified identity.
|
||||||
|
if raw_breakable == "1" or breakable_resolved():
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: {BREAKABLE_ENV} resolved ON ({detail}). The "
|
||||||
|
"qualified profile is breakable-off VLLM_COMPILE + "
|
||||||
|
f"FULL_AND_PIECEWISE. Export {BREAKABLE_ENV}=0, or set "
|
||||||
|
f"{ALLOW_BREAKABLE_ENV}=1 to take it deliberately."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2b. The production pin is the exact string "0".
|
||||||
|
# Upstream's accessor reads anything but "1" as off, so "2", "true",
|
||||||
|
# "" or "00" would resolve off -- but a mistyped launcher value must
|
||||||
|
# not be recorded as a qualified breakable-off profile. The logged
|
||||||
|
# debug opt-out (VLLM_E2_ALLOW_BREAKABLE=1, handled above) is separate.
|
||||||
|
if raw_breakable != PRODUCTION_BREAKABLE_PIN:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: {BREAKABLE_ENV}={raw_breakable!r} is not the "
|
||||||
|
f"production pin; it must be exactly {PRODUCTION_BREAKABLE_PIN!r}. "
|
||||||
|
"Upstream would read it as off, but the launch profile does not "
|
||||||
|
f"say what was meant. Export {BREAKABLE_ENV}=0, or set "
|
||||||
|
f"{ALLOW_BREAKABLE_ENV}=1 for a deliberate non-production boot."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. The profile itself: mode AND cudagraph mode, by name.
|
||||||
|
if (
|
||||||
|
_name(mode) != REQUIRED_COMPILATION_MODE_NAME
|
||||||
|
or _name(cudagraph_mode) != REQUIRED_CUDAGRAPH_MODE_NAME
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: resolved compilation_mode={_name(mode)} "
|
||||||
|
f"cudagraph_mode={_name(cudagraph_mode)}; the qualified profile is "
|
||||||
|
f"compilation_mode={REQUIRED_COMPILATION_MODE_NAME} "
|
||||||
|
f"cudagraph_mode={REQUIRED_CUDAGRAPH_MODE_NAME} (compile-profile guard). Check "
|
||||||
|
"-O / --compilation-config / --enforce-eager / TORCH_COMPILE_DISABLE, "
|
||||||
|
f"or set {ALLOW_BREAKABLE_ENV}=1 for a deliberate non-production boot."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4. The graph breaks that make PIECEWISE correct on this model.
|
||||||
|
if splitting_ops is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: compilation_config.splitting_ops is unresolved; "
|
||||||
|
"cannot confirm the Qwen4Exp graph breaks."
|
||||||
|
)
|
||||||
|
missing = [op for op in REQUIRED_SPLITTING_OPS if op not in splitting_ops]
|
||||||
|
if missing:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: splitting_ops is missing {missing}. Each of "
|
||||||
|
"these reads per-step request metadata (or forks/joins a side "
|
||||||
|
"stream) and would be frozen into a piecewise CUDA graph."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 5. Optional exact pin of the resolved mode, for a launch profile.
|
||||||
|
expected = os.environ.get(EXPECTED_MODE_ENV, "").strip()
|
||||||
|
if expected and _name(cudagraph_mode) != expected:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: resolved cudagraph_mode={_name(cudagraph_mode)} "
|
||||||
|
f"but {EXPECTED_MODE_ENV}={expected}. The launch profile and the "
|
||||||
|
"engine disagree about the graph identity."
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"e2-guard[%s]: architectures=%s breakable=OFF compilation_mode=%s "
|
||||||
|
"cudagraph_mode=%s splitting_ops=+%d required. Recorded, not "
|
||||||
|
"certified -- this checks startup state, not qualification.",
|
||||||
|
role,
|
||||||
|
architectures,
|
||||||
|
_name(mode),
|
||||||
|
_name(cudagraph_mode),
|
||||||
|
len(REQUIRED_SPLITTING_OPS),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_compiled_submodule(module, vllm_config, *, role: str = "target") -> None:
|
||||||
|
"""Require a live ``@support_torch_compile`` wrapper on ``module``.
|
||||||
|
|
||||||
|
This is the property ``has_compiled_submodule()`` checks before PIECEWISE
|
||||||
|
capture (``gpu/cudagraph_utils.py:132``, ``:605``). Checking it at
|
||||||
|
construction turns a mid-capture failure into a startup one.
|
||||||
|
"""
|
||||||
|
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||||
|
|
||||||
|
compiled = isinstance(module, TorchCompileWithNoGuardsWrapper) and not getattr(
|
||||||
|
module, "do_not_compile", True
|
||||||
|
)
|
||||||
|
if compiled:
|
||||||
|
return
|
||||||
|
detail = (
|
||||||
|
f"{type(module).__name__} is_wrapper="
|
||||||
|
f"{isinstance(module, TorchCompileWithNoGuardsWrapper)} "
|
||||||
|
f"do_not_compile={getattr(module, 'do_not_compile', '<absent>')}"
|
||||||
|
)
|
||||||
|
if _opted_out(role, detail):
|
||||||
|
return
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[{role}]: {detail}: no live torch.compile wrapper, so "
|
||||||
|
"PIECEWISE capture would raise at gpu/cudagraph_utils.py:605. The "
|
||||||
|
"@support_torch_compile decorator (compile-profile guard) is missing or disabled."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Resolved runtime graph modes.
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#: Target architectures whose V2 capture is checked. The construction guard
|
||||||
|
#: above lives in the NVIDIA Qwen4Exp model; this tree targets sm_86 only.
|
||||||
|
QWEN4EXP_TARGET_ARCHITECTURES = frozenset(
|
||||||
|
{"Qwen4ExpForCausalLM", "Qwen4ExpForConditionalGeneration"}
|
||||||
|
)
|
||||||
|
|
||||||
|
#: What each V2 cudagraph manager may hold immediately before capture.
|
||||||
|
#: The construction guard checks ``compilation_config.cudagraph_mode``; the
|
||||||
|
#: V2 runner later calls ``resolve_cudagraph_mode_and_sizes`` (and, with
|
||||||
|
#: adaptive verification, ``resolve_adaptive_cudagraph_mode``), which may
|
||||||
|
#: downgrade to FULL_DECODE_ONLY / PIECEWISE / NONE or drop the piecewise half
|
||||||
|
#: (attention CG support, spec-decode uniform batches, no compiled submodule).
|
||||||
|
#: No such downgrade is allowed for the target. The drafter's managers are
|
||||||
|
#: derived from the target's resolved mode by the speculator itself:
|
||||||
|
#: * autoregressive / MTP (``AutoRegressiveSpeculator.init_cudagraph_manager``):
|
||||||
|
#: prefill = the target's mode unchanged; decode = FULL_DECODE_ONLY
|
||||||
|
#: ("PIECEWISE cudagraphs are not supported for draft decodes");
|
||||||
|
#: * multi-module MTP (``MultiModuleMTPSpeculator``): one manager,
|
||||||
|
#: FULL_AND_PIECEWISE -> FULL_DECODE_ONLY by design.
|
||||||
|
#: Those decode-specific modes are the only drafter deviations accepted.
|
||||||
|
EXPECTED_RESOLVED_MODES = {
|
||||||
|
"target": REQUIRED_CUDAGRAPH_MODE_NAME,
|
||||||
|
"draft.prefill_cudagraph_manager": REQUIRED_CUDAGRAPH_MODE_NAME,
|
||||||
|
"draft.decode_cudagraph_manager": "FULL_DECODE_ONLY",
|
||||||
|
"draft.cudagraph_manager": "FULL_DECODE_ONLY",
|
||||||
|
}
|
||||||
|
_DRAFT_MANAGER_ATTRS = (
|
||||||
|
"prefill_cudagraph_manager",
|
||||||
|
"decode_cudagraph_manager",
|
||||||
|
"cudagraph_manager",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_resolved_cudagraph_modes(
|
||||||
|
vllm_config, target_manager, speculator=None
|
||||||
|
) -> None:
|
||||||
|
"""Fail loud if the V2 runner is about to capture a downgraded mode.
|
||||||
|
|
||||||
|
Called from ``GPUModelRunner.capture_model`` (``vllm/v1/worker/gpu``)
|
||||||
|
before any capture, including the ``profile_only`` capture and the
|
||||||
|
"nothing to capture" early return, so a resolver downgrade to NONE cannot
|
||||||
|
skip it. Applies only to a Qwen4Exp target; any other model is untouched.
|
||||||
|
``VLLM_E2_ALLOW_BREAKABLE=1`` skips it, logged, like the other graph
|
||||||
|
guards.
|
||||||
|
"""
|
||||||
|
model_config = getattr(vllm_config, "model_config", None)
|
||||||
|
architectures = list(getattr(model_config, "architectures", None) or [])
|
||||||
|
if not QWEN4EXP_TARGET_ARCHITECTURES & set(architectures):
|
||||||
|
return
|
||||||
|
|
||||||
|
observed: dict[str, str] = {}
|
||||||
|
if target_manager is None:
|
||||||
|
observed["target"] = "<no manager>"
|
||||||
|
else:
|
||||||
|
observed["target"] = _name(getattr(target_manager, "cudagraph_mode", None))
|
||||||
|
if speculator is not None:
|
||||||
|
found = False
|
||||||
|
for attr in _DRAFT_MANAGER_ATTRS:
|
||||||
|
if not hasattr(speculator, attr):
|
||||||
|
continue
|
||||||
|
found = True
|
||||||
|
manager = getattr(speculator, attr)
|
||||||
|
observed[f"draft.{attr}"] = (
|
||||||
|
"<no manager>"
|
||||||
|
if manager is None
|
||||||
|
else _name(getattr(manager, "cudagraph_mode", None))
|
||||||
|
)
|
||||||
|
if not found:
|
||||||
|
observed["draft"] = f"<unrecognised speculator {type(speculator).__name__}>"
|
||||||
|
|
||||||
|
detail = " ".join(f"{key}={value}" for key, value in observed.items())
|
||||||
|
if _opted_out("runtime", detail):
|
||||||
|
return
|
||||||
|
wrong = {
|
||||||
|
key: value
|
||||||
|
for key, value in observed.items()
|
||||||
|
if EXPECTED_RESOLVED_MODES.get(key) != value
|
||||||
|
}
|
||||||
|
if wrong:
|
||||||
|
expected = {key: EXPECTED_RESOLVED_MODES.get(key, "<none>") for key in wrong}
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard[runtime]: resolved cudagraph modes {wrong} differ from "
|
||||||
|
f"the qualified profile {expected} ({detail}). The V2 runner's "
|
||||||
|
"resolver downgraded the graph identity after the construction "
|
||||||
|
"guard passed (attention CG support, spec-decode uniform batch, "
|
||||||
|
"or no compiled submodule); see the resolver's warning above. Set "
|
||||||
|
f"{ALLOW_BREAKABLE_ENV}=1 only for a deliberate non-production boot."
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"e2-guard[runtime]: resolved cudagraph modes before capture: %s.", detail
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# PLE transport: carry the pull transport, UVA disabled.
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#: The fork's pull-transport switch (strict "0"/"1"; registered and hashed in
|
||||||
|
#: vllm/envs.py). Renamed from the E2 base's VLLM_PLE_CPU_OFFLOAD.
|
||||||
|
PULL_TRANSPORT_ENV = "VLLM_E2_PLE_PULL_TRANSPORT"
|
||||||
|
#: Upstream v0.30.0's switch for the pinned-host UVA PLE backend. Same name as
|
||||||
|
#: the E2 base's pull switch, opposite default (upstream: "1").
|
||||||
|
UPSTREAM_UVA_ENV = "VLLM_PLE_CPU_OFFLOAD"
|
||||||
|
|
||||||
|
|
||||||
|
def pull_transport_enabled() -> bool:
|
||||||
|
"""Whether the fork's pull transport is on. Strict; raises on bad values.
|
||||||
|
|
||||||
|
One source: the registered accessor ``envs.VLLM_E2_PLE_PULL_TRANSPORT``,
|
||||||
|
whose parser (``envs._e2_strict_switch``) accepts exactly unset/"0"/"1".
|
||||||
|
"""
|
||||||
|
import vllm.envs as envs
|
||||||
|
|
||||||
|
return bool(envs.VLLM_E2_PLE_PULL_TRANSPORT)
|
||||||
|
|
||||||
|
|
||||||
|
def check_upstream_uva_env_against_pull() -> None:
|
||||||
|
"""Fail loud if the old name would turn upstream's UVA path on.
|
||||||
|
|
||||||
|
With the pull transport on, ``VLLM_PLE_CPU_OFFLOAD`` may be unset (the
|
||||||
|
fork then supplies ``cpu_offload=False`` as the Engram default) or exactly
|
||||||
|
``"0"``. ``"1"`` -- the E2-era spelling of "enable our offload" -- would
|
||||||
|
enable upstream's pinned-host UVA backend beside ours: a double offload.
|
||||||
|
Any other value is rejected rather than interpreted. With the pull
|
||||||
|
transport off this function does nothing: the name is upstream's.
|
||||||
|
"""
|
||||||
|
if not pull_transport_enabled():
|
||||||
|
return
|
||||||
|
raw = os.environ.get(UPSTREAM_UVA_ENV)
|
||||||
|
if raw is None or raw == "0":
|
||||||
|
return
|
||||||
|
if raw == "1":
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}=1. On "
|
||||||
|
f"vLLM v0.30.0 {UPSTREAM_UVA_ENV} is UPSTREAM's switch for the "
|
||||||
|
"pinned-host UVA PLE backend (default on), not the E2 pull "
|
||||||
|
"transport, so this launch asks for both transports at once. Unset "
|
||||||
|
f"{UPSTREAM_UVA_ENV} or export {UPSTREAM_UVA_ENV}=0 "
|
||||||
|
"(the pull transport replaces that switch)."
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}={raw!r}. "
|
||||||
|
f'With the pull transport on, {UPSTREAM_UVA_ENV} must be unset or '
|
||||||
|
'exactly "0"; any other value is refused rather than guessed.'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def pull_transport_cpu_offload_default(upstream_default: bool) -> bool:
|
||||||
|
"""The Engram ``cpu_offload`` default, as this fork resolves it.
|
||||||
|
|
||||||
|
Pull off: ``upstream_default`` unchanged (the tree behaves as upstream).
|
||||||
|
Pull on: the old-name check above, then ``False`` -- upstream's UVA
|
||||||
|
backend is never the default beside the pull transport.
|
||||||
|
"""
|
||||||
|
if not pull_transport_enabled():
|
||||||
|
return upstream_default
|
||||||
|
check_upstream_uva_env_against_pull()
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def verify_engram_against_pull(engram_config) -> None:
|
||||||
|
"""Refuse an explicit ``cpu_offload=True`` while the pull transport is on.
|
||||||
|
|
||||||
|
``--engram-config '{"cpu_offload": true}'`` takes precedence over every
|
||||||
|
default, so it is checked on the resolved object, not the environment.
|
||||||
|
"""
|
||||||
|
if not pull_transport_enabled():
|
||||||
|
return
|
||||||
|
check_upstream_uva_env_against_pull()
|
||||||
|
if engram_config is not None and bool(engram_config.cpu_offload):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard: {PULL_TRANSPORT_ENV}=1 but the resolved EngramConfig "
|
||||||
|
"has cpu_offload=True (upstream's pinned-host UVA PLE backend). "
|
||||||
|
"Both transports would own the PLE table. Drop cpu_offload from "
|
||||||
|
"--engram-config (or set it false), or turn the pull transport off."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
#: Effective PLE modes, as the guard names them.
|
||||||
|
PLE_MODE_PULL = "pull" # E2 pull transport (VLLM_E2_PLE_PULL_TRANSPORT=1)
|
||||||
|
PLE_MODE_UVA = "uva" # upstream Qwen4ExpPLEPinnedHostEmbedding
|
||||||
|
PLE_MODE_DEVICE = "device" # upstream Qwen4ExpPLEDeviceEmbedding
|
||||||
|
PLE_MODE_NONE = "none" # no Engram config resolved (no PLE layers / not CUDA)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_ple_mode(vllm_config) -> str:
|
||||||
|
"""The PLE transport this process will build, or an error if mixed.
|
||||||
|
|
||||||
|
Reads the two inputs that decide it -- the pull switch (strict) and the
|
||||||
|
RESOLVED ``engram_config.cpu_offload`` -- and nothing else. Mixed states
|
||||||
|
raise; there is no fallback from one transport to another.
|
||||||
|
"""
|
||||||
|
pull = pull_transport_enabled()
|
||||||
|
check_upstream_uva_env_against_pull()
|
||||||
|
engram_config = getattr(vllm_config, "engram_config", None)
|
||||||
|
cpu_offload = (
|
||||||
|
None if engram_config is None else bool(engram_config.cpu_offload)
|
||||||
|
)
|
||||||
|
if pull:
|
||||||
|
if cpu_offload:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"e2-guard: {PULL_TRANSPORT_ENV}=1 with a resolved Engram "
|
||||||
|
"cpu_offload=True: the pull transport and upstream's UVA "
|
||||||
|
"backend would both own the PLE table."
|
||||||
|
)
|
||||||
|
return PLE_MODE_PULL
|
||||||
|
if cpu_offload is None:
|
||||||
|
return PLE_MODE_NONE
|
||||||
|
return PLE_MODE_UVA if cpu_offload else PLE_MODE_DEVICE
|
||||||
|
|
||||||
|
|
||||||
|
def _role_ple_layer_ids(vllm_config, role: str) -> list[int]:
|
||||||
|
"""The 1-based PLE layer ids the role's own decoder layers carry.
|
||||||
|
|
||||||
|
``role="target"``: ``model_config.hf_text_config.ple_layer_ids``.
|
||||||
|
``role="draft"``: the drafter is constructed with the target's VllmConfig
|
||||||
|
(the trap an earlier commit fixed for the architectures), so the target's
|
||||||
|
model_config names the TARGET's PLE layers. The drafter's decoder layers
|
||||||
|
are ``num_hidden_layers + i`` for ``i < mtp_num_hidden_layers``
|
||||||
|
(``Qwen4ExpMultiTokenPredictor``), and a layer builds a PLE block only if
|
||||||
|
``layer_idx + 1`` is in ``ple_layer_ids`` (``Qwen4ExpDecoderLayer``). The
|
||||||
|
draft line reports exactly that intersection, read from
|
||||||
|
``speculative_config.draft_model_config`` -- empty for this checkpoint,
|
||||||
|
whose ids must name backbone layers (``Qwen4ExpTextConfig``).
|
||||||
|
"""
|
||||||
|
model_config = getattr(vllm_config, "model_config", None)
|
||||||
|
if role == "draft":
|
||||||
|
spec = getattr(vllm_config, "speculative_config", None)
|
||||||
|
draft_config = getattr(spec, "draft_model_config", None)
|
||||||
|
if draft_config is not None:
|
||||||
|
model_config = draft_config
|
||||||
|
text_config = getattr(model_config, "hf_text_config", None)
|
||||||
|
ids = list(getattr(text_config, "ple_layer_ids", None) or [])
|
||||||
|
if role != "draft":
|
||||||
|
return ids
|
||||||
|
num_hidden_layers = getattr(text_config, "num_hidden_layers", None)
|
||||||
|
if num_hidden_layers is None:
|
||||||
|
return ids
|
||||||
|
num_mtp = int(getattr(text_config, "mtp_num_hidden_layers", 1))
|
||||||
|
own = range(int(num_hidden_layers) + 1, int(num_hidden_layers) + num_mtp + 1)
|
||||||
|
return [layer_id for layer_id in ids if layer_id in own]
|
||||||
|
|
||||||
|
|
||||||
|
def assert_ple_mode(vllm_config, *, role: str = "target") -> str:
|
||||||
|
"""Startup guard: resolve, assert and log the effective PLE mode.
|
||||||
|
|
||||||
|
Runs beside ``assert_graph_profile_is_executable`` at model construction
|
||||||
|
for ``role="target"`` and ``role="draft"`` (the MTP drafter carries the
|
||||||
|
target's EngramConfig and the same process environment, so a mixed
|
||||||
|
configuration fails there too). NOT disabled by VLLM_E2_ALLOW_BREAKABLE:
|
||||||
|
that opt-out is about graph identity, not about owning the PLE table twice.
|
||||||
|
The logged ``ple_layer_ids`` are the role's own (:func:`_role_ple_layer_ids`).
|
||||||
|
"""
|
||||||
|
mode = resolve_ple_mode(vllm_config)
|
||||||
|
ple_layer_ids = _role_ple_layer_ids(vllm_config, role)
|
||||||
|
engram_config = getattr(vllm_config, "engram_config", None)
|
||||||
|
logger.info(
|
||||||
|
"e2-guard[%s]: ple_mode=%s pull_transport=%d engram_cpu_offload=%s "
|
||||||
|
"%s=%s ple_layer_ids=%s.",
|
||||||
|
role,
|
||||||
|
mode,
|
||||||
|
1 if mode == PLE_MODE_PULL else 0,
|
||||||
|
None if engram_config is None else bool(engram_config.cpu_offload),
|
||||||
|
UPSTREAM_UVA_ENV,
|
||||||
|
os.environ.get(UPSTREAM_UVA_ENV, "<unset>"),
|
||||||
|
ple_layer_ids,
|
||||||
|
)
|
||||||
|
return mode
|
||||||
|
|
||||||
|
|
||||||
|
#: Back-compatible name for the earlier call sites.
|
||||||
|
assert_breakable_cudagraph_pinned = assert_graph_profile_is_executable
|
||||||
+123
-6
@@ -4,12 +4,13 @@ import gc
|
|||||||
import os
|
import os
|
||||||
import queue
|
import queue
|
||||||
import signal
|
import signal
|
||||||
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from collections import defaultdict, deque
|
from collections import defaultdict, deque
|
||||||
from collections.abc import Callable, Generator, Sequence
|
from collections.abc import Callable, Generator, Sequence
|
||||||
from concurrent.futures import Future
|
from concurrent.futures import Future
|
||||||
from contextlib import ExitStack, contextmanager
|
from contextlib import ExitStack, contextmanager, suppress
|
||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from inspect import isclass, signature
|
from inspect import isclass, signature
|
||||||
@@ -108,6 +109,103 @@ HANDSHAKE_TIMEOUT_MINS = 5
|
|||||||
_R = TypeVar("_R") # Return type for collective_rpc
|
_R = TypeVar("_R") # Return type for collective_rpc
|
||||||
|
|
||||||
|
|
||||||
|
# How long EngineCore teardown may run after it deferred a late SIGTERM/SIGINT
|
||||||
|
# before the process exits anyway. Covers the whole of engine_core.shutdown(),
|
||||||
|
# not only the executor's worker ladder (grace + 4 s): a hang in later teardown
|
||||||
|
# (scheduler, distributed cleanup) must not make the process unkillable.
|
||||||
|
_TEARDOWN_SIGNAL_BOUND_S = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
# How long the guard's forced-exit paths wait for their diagnostic line. The
|
||||||
|
# line is written from a helper thread; a blocked handler, a held logging lock
|
||||||
|
# or a full stderr pipe costs at most this, never the exit.
|
||||||
|
_TEARDOWN_EXIT_LOG_BOUND_S = 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def _log_bounded(emit: Callable[[], None], bound_s: float) -> None:
|
||||||
|
"""Run ``emit`` (logging, flushes) on a daemon thread; wait <= bound_s."""
|
||||||
|
try:
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=emit, daemon=True, name="EngineCoreTeardownLog"
|
||||||
|
)
|
||||||
|
thread.start()
|
||||||
|
thread.join(bound_s)
|
||||||
|
except BaseException: # noqa: BLE001 -- the caller's exit must still run
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _TeardownSignalGuard:
|
||||||
|
"""EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded.
|
||||||
|
|
||||||
|
Dying on the first late signal would SIGKILL every worker
|
||||||
|
(PR_SET_PDEATHSIG) before its shutdown drain; see the finally block of
|
||||||
|
EngineCoreProc.run_engine_core. The deferral is bounded two ways:
|
||||||
|
- a SECOND late signal restores the default disposition and re-delivers
|
||||||
|
itself, so the process dies of it (the maintainer's escalation);
|
||||||
|
- after the first deferred signal a watchdog ends the process with
|
||||||
|
exit code 1 if teardown is still running _TEARDOWN_SIGNAL_BOUND_S later.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, bound_s: float | None = None) -> None:
|
||||||
|
self.bound_s = _TEARDOWN_SIGNAL_BOUND_S if bound_s is None else bound_s
|
||||||
|
self.deferred = 0
|
||||||
|
self._done = threading.Event()
|
||||||
|
self._watchdog: threading.Thread | None = None
|
||||||
|
|
||||||
|
def __call__(self, signum, frame) -> None:
|
||||||
|
name = signal.Signals(signum).name
|
||||||
|
if self._done.is_set():
|
||||||
|
return
|
||||||
|
self.deferred += 1
|
||||||
|
if self.deferred == 1:
|
||||||
|
logger.warning(
|
||||||
|
"[shutdown] EngineCore: %s during resource teardown deferred; "
|
||||||
|
"workers finish their shutdown (FINAL DRAIN) first (a second "
|
||||||
|
"signal, or %.0f s of teardown, ends the process)",
|
||||||
|
name,
|
||||||
|
self.bound_s,
|
||||||
|
)
|
||||||
|
self._watchdog = threading.Thread(
|
||||||
|
target=self._expire,
|
||||||
|
daemon=True,
|
||||||
|
name="EngineCoreTeardownWatchdog",
|
||||||
|
)
|
||||||
|
self._watchdog.start()
|
||||||
|
return
|
||||||
|
# Default disposition first, so a third signal also ends the process
|
||||||
|
# while the (bounded) diagnostic line is written.
|
||||||
|
signal.signal(signum, signal.SIG_DFL)
|
||||||
|
_log_bounded(
|
||||||
|
lambda: logger.warning(
|
||||||
|
"[shutdown] EngineCore: second late signal (%s) during "
|
||||||
|
"resource teardown; not deferring again",
|
||||||
|
name,
|
||||||
|
),
|
||||||
|
_TEARDOWN_EXIT_LOG_BOUND_S,
|
||||||
|
)
|
||||||
|
os.kill(os.getpid(), signum)
|
||||||
|
|
||||||
|
def _expire(self) -> None:
|
||||||
|
if self._done.wait(self.bound_s):
|
||||||
|
return
|
||||||
|
|
||||||
|
def emit() -> None:
|
||||||
|
logger.error(
|
||||||
|
"[shutdown] EngineCore: resource teardown still running %.0f s "
|
||||||
|
"after a deferred signal; exiting (code 1)",
|
||||||
|
self.bound_s,
|
||||||
|
)
|
||||||
|
for stream in (sys.stdout, sys.stderr):
|
||||||
|
with suppress(Exception):
|
||||||
|
stream.flush()
|
||||||
|
|
||||||
|
_log_bounded(emit, _TEARDOWN_EXIT_LOG_BOUND_S)
|
||||||
|
os._exit(1)
|
||||||
|
|
||||||
|
def disarm(self) -> None:
|
||||||
|
self._done.set()
|
||||||
|
|
||||||
|
|
||||||
class EngineCore:
|
class EngineCore:
|
||||||
"""Inner loop of vLLM's Engine."""
|
"""Inner loop of vLLM's Engine."""
|
||||||
|
|
||||||
@@ -764,7 +862,9 @@ class EngineCore:
|
|||||||
# When draft tokens are used with structured output, validate them
|
# When draft tokens are used with structured output, validate them
|
||||||
# before computing the grammar bitmask for the deferred request.
|
# before computing the grammar bitmask for the deferred request.
|
||||||
if self.check_for_draft_tokens:
|
if self.check_for_draft_tokens:
|
||||||
draft_token_ids = self.model_executor.take_draft_token_ids()
|
draft_token_ids = self.model_executor.take_draft_token_ids(
|
||||||
|
step_id=deferred_scheduler_output.scheduler_step
|
||||||
|
)
|
||||||
if draft_token_ids is not None:
|
if draft_token_ids is not None:
|
||||||
# Update the draft token ids in the scheduler output to
|
# Update the draft token ids in the scheduler output to
|
||||||
# filter out the invalid spec tokens, which will be padded
|
# filter out the invalid spec tokens, which will be padded
|
||||||
@@ -1439,12 +1539,29 @@ class EngineCoreProc(EngineCore):
|
|||||||
engine_core._send_engine_dead()
|
engine_core._send_engine_dead()
|
||||||
raise e
|
raise e
|
||||||
finally:
|
finally:
|
||||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
# Workers carry PR_SET_PDEATHSIG=SIGKILL (fail-closed PLE teardown),
|
||||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
# so if this process dies before model_executor.shutdown() has reaped
|
||||||
|
# them, every worker is SIGKILLed mid-shutdown and its bounds-guard
|
||||||
|
# FINAL DRAIN never runs. The API server's process manager sends a
|
||||||
|
# second SIGTERM right after the signal that started this shutdown;
|
||||||
|
# with SIG_DFL here it killed teardown before the executor waited for
|
||||||
|
# the workers. Defer ONE late signal until teardown ends; a second
|
||||||
|
# signal or _TEARDOWN_SIGNAL_BOUND_S of teardown after the deferral
|
||||||
|
# still ends the process (_TeardownSignalGuard), as do the
|
||||||
|
# executor's terminate/kill ladder and the process manager's timeout
|
||||||
|
# (kill_process_tree -> SIGKILL).
|
||||||
|
teardown_guard = _TeardownSignalGuard()
|
||||||
|
signal.signal(signal.SIGTERM, teardown_guard)
|
||||||
|
signal.signal(signal.SIGINT, teardown_guard)
|
||||||
if signal_callback is not None:
|
if signal_callback is not None:
|
||||||
signal_callback.stop()
|
signal_callback.stop()
|
||||||
if engine_core is not None:
|
try:
|
||||||
engine_core.shutdown()
|
if engine_core is not None:
|
||||||
|
engine_core.shutdown()
|
||||||
|
finally:
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||||
|
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||||
|
teardown_guard.disarm()
|
||||||
if clean_shutdown:
|
if clean_shutdown:
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
|
|||||||
@@ -265,8 +265,13 @@ class Executor(ABC):
|
|||||||
def execute_dummy_batch(self) -> None:
|
def execute_dummy_batch(self) -> None:
|
||||||
self.collective_rpc("execute_dummy_batch")
|
self.collective_rpc("execute_dummy_batch")
|
||||||
|
|
||||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||||
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
|
# Omit step_id when unset so plugin workers with the documented
|
||||||
|
# no-argument take_draft_token_ids() keep working.
|
||||||
|
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||||
|
output: list[DraftTokenIds] = self.collective_rpc(
|
||||||
|
"take_draft_token_ids", kwargs=kwargs
|
||||||
|
)
|
||||||
return output[0]
|
return output[0]
|
||||||
|
|
||||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import os
|
|||||||
import pickle
|
import pickle
|
||||||
import queue
|
import queue
|
||||||
import signal
|
import signal
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
@@ -70,11 +72,192 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
|||||||
from vllm.v1.executor.abstract import Executor, FailureCallback
|
from vllm.v1.executor.abstract import Executor, FailureCallback
|
||||||
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
|
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
|
||||||
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
|
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
|
||||||
|
from vllm.v1.ple_offload.fatal import set_fatal_notify_fd
|
||||||
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
_WORKER_PARENT_DEATH_SIGNAL = signal.SIGKILL
|
||||||
|
_PR_SET_PDEATHSIG = 1
|
||||||
|
|
||||||
|
|
||||||
|
def _install_worker_parent_death_signal(expected_parent_pid: int) -> None:
|
||||||
|
"""Have Linux kill this worker if its EngineCore parent exits."""
|
||||||
|
if not sys.platform.startswith("linux"):
|
||||||
|
return
|
||||||
|
|
||||||
|
import ctypes
|
||||||
|
|
||||||
|
libc = ctypes.CDLL(None, use_errno=True)
|
||||||
|
prctl = libc.prctl
|
||||||
|
prctl.argtypes = [
|
||||||
|
ctypes.c_int,
|
||||||
|
ctypes.c_ulong,
|
||||||
|
ctypes.c_ulong,
|
||||||
|
ctypes.c_ulong,
|
||||||
|
ctypes.c_ulong,
|
||||||
|
]
|
||||||
|
prctl.restype = ctypes.c_int
|
||||||
|
if (
|
||||||
|
prctl(
|
||||||
|
_PR_SET_PDEATHSIG,
|
||||||
|
int(_WORKER_PARENT_DEATH_SIGNAL),
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
!= 0
|
||||||
|
):
|
||||||
|
error_number = ctypes.get_errno()
|
||||||
|
raise OSError(error_number, os.strerror(error_number))
|
||||||
|
|
||||||
|
# The parent can exit between spawning this process and the prctl call.
|
||||||
|
# Once prctl is installed, a later exit is covered by the kernel.
|
||||||
|
if os.getppid() != expected_parent_pid:
|
||||||
|
os.kill(os.getpid(), _WORKER_PARENT_DEATH_SIGNAL)
|
||||||
|
os._exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
# Grace and post-SIGTERM wait, after a worker's death, for survivors that never
|
||||||
|
# started their shutdown (see MultiprocExecutor._ensure_worker_termination).
|
||||||
|
_FAILED_WORKER_WAIT_S = 1.0
|
||||||
|
# After a worker's death, the bounded grace for survivors whose shutdown has
|
||||||
|
# STARTED (FINAL DRAIN in progress). Every FINAL DRAIN measured on the reference host took
|
||||||
|
# < 0.1 s (drain-path tests: the drain line lands within ~0.05 s of the
|
||||||
|
# shutdown request); 2 s is 20x that, so a draining rank is not cut
|
||||||
|
# (a shorter cap did), while one blocked inside its shutdown on the dead rank is ended.
|
||||||
|
_FAILED_DRAIN_GRACE_S = 2.0
|
||||||
|
# Per-worker shared byte (WorkerProcHandle.shutdown_state): written by the
|
||||||
|
# worker in worker_main's finally, read by the executor's termination ladder.
|
||||||
|
_SHUTDOWN_NOT_STARTED = 0
|
||||||
|
_SHUTDOWN_STARTED = 1
|
||||||
|
_SHUTDOWN_DONE = 2
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerShutdownRequested(RuntimeError):
|
||||||
|
"""The busy loop's dequeue was cancelled by a shutdown request.
|
||||||
|
|
||||||
|
Raised only by WorkerProc.worker_busy_loop when the RPC broadcast queue
|
||||||
|
reports its own cancellation (shm_broadcast: RuntimeError("cancelled")
|
||||||
|
after MessageQueue.shutdown(), which the death-pipe monitor calls once the
|
||||||
|
parent closes the pipe). worker_main takes the orderly exit path on this
|
||||||
|
type only; any other exception is a failure, even one that races a
|
||||||
|
shutdown request. Subclasses RuntimeError so existing catchers still match.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _exit_worker_process(exit_code: int) -> None:
|
||||||
|
"""End a worker process whose orderly shutdown has already run.
|
||||||
|
|
||||||
|
Called after WorkerProc.shutdown(destroy_distributed=False): the FINAL
|
||||||
|
DRAIN, the instrumentation flushes and the PLE offload / shared-memory
|
||||||
|
cleanup are done. What would remain is process-exit-only teardown --
|
||||||
|
destroying the NCCL communicators (PyNcclCommunicator.destroy joins its
|
||||||
|
ncclCommAbort thread for up to 5 s per communicator while CUDA graphs that
|
||||||
|
captured NCCL ops are still alive) and interpreter finalization (CUDA
|
||||||
|
context, allocator pools). The driver and kernel reclaim all of it at
|
||||||
|
process exit, and without this the executor's 5 s grace expired and the
|
||||||
|
worker was SIGKILLed at ~9 s anyway. Flush the logs,
|
||||||
|
then leave.
|
||||||
|
"""
|
||||||
|
logger.info(
|
||||||
|
"[shutdown] WorkerProc: shutdown complete; exiting process (code %d) "
|
||||||
|
"without distributed teardown",
|
||||||
|
exit_code,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logging.shutdown()
|
||||||
|
except Exception: # noqa: BLE001 -- the exit must happen regardless
|
||||||
|
pass
|
||||||
|
for stream in (sys.stdout, sys.stderr):
|
||||||
|
with suppress(Exception):
|
||||||
|
stream.flush()
|
||||||
|
os._exit(exit_code)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_exit_waitables(
|
||||||
|
procs: Sequence[BaseProcess],
|
||||||
|
) -> list[tuple[Any, BaseProcess, int | None]]:
|
||||||
|
"""What the worker monitor waits on to learn that a worker process exited.
|
||||||
|
|
||||||
|
A pidfd (Linux >= 5.3) where possible: it becomes readable when the process
|
||||||
|
itself exits. proc.sentinel is only the read end of a pipe whose write end
|
||||||
|
the worker holds, so it reports the exit of the LAST process holding that
|
||||||
|
write end -- and anything the worker fork+execs inherits it. Every PLE-
|
||||||
|
owning worker does: registering its host buffers under the "file_system"
|
||||||
|
sharing strategy starts a torch_shm_manager child, which keeps the write
|
||||||
|
end, and which lives while any client -- rank 0's PLE child, for a TP1
|
||||||
|
worker's buffers -- still maps them. In a rank-1 fault test PP0_TP1 died
|
||||||
|
in release_outputs, sat as a zombie, its sentinel never fired and the
|
||||||
|
engine hung; rank 0's death only looked detected because its PLE child
|
||||||
|
(and so both managers) died with it, ~8 s later.
|
||||||
|
|
||||||
|
Returns (waitable, proc, pidfd-to-close-or-None) per process, falling back
|
||||||
|
to the sentinel where no pidfd can be opened.
|
||||||
|
"""
|
||||||
|
pidfd_open = getattr(os, "pidfd_open", None)
|
||||||
|
waitables: list[tuple[Any, BaseProcess, int | None]] = []
|
||||||
|
try:
|
||||||
|
for proc in procs:
|
||||||
|
pidfd: int | None = None
|
||||||
|
pid = getattr(proc, "pid", None)
|
||||||
|
if pidfd_open is not None and pid is not None:
|
||||||
|
try:
|
||||||
|
pidfd = pidfd_open(pid)
|
||||||
|
except OSError:
|
||||||
|
# ESRCH (already reaped) or no pidfd support: the sentinel.
|
||||||
|
pidfd = None
|
||||||
|
try:
|
||||||
|
waitable = pidfd if pidfd is not None else proc.sentinel
|
||||||
|
waitables.append((waitable, proc, pidfd))
|
||||||
|
except BaseException:
|
||||||
|
# Not yet in `waitables`: the outer handler would miss it.
|
||||||
|
if pidfd is not None:
|
||||||
|
os.close(pidfd)
|
||||||
|
raise
|
||||||
|
except BaseException:
|
||||||
|
# Nothing is handed out on failure: close what was opened.
|
||||||
|
for _, _, opened in waitables:
|
||||||
|
if opened is not None:
|
||||||
|
with suppress(OSError):
|
||||||
|
os.close(opened)
|
||||||
|
raise
|
||||||
|
return waitables
|
||||||
|
|
||||||
|
|
||||||
|
def _make_inherited_pipes_close_on_exec() -> None:
|
||||||
|
"""Keep this worker's inherited pipe fds out of the programs it execs.
|
||||||
|
|
||||||
|
Under the spawn start method multiprocessing hands the worker its end of
|
||||||
|
the sentinel pipe (and the ready / death pipes) as INHERITABLE fds, so any
|
||||||
|
program the worker fork+execs keeps them open: the torch_shm_manager that
|
||||||
|
the PLE host-buffer registration starts did, and outlived the worker, so
|
||||||
|
neither the sentinel nor the ready pipe reported its death (rank-1 fault injection).
|
||||||
|
Nothing this process execs needs them -- multiprocessing children get
|
||||||
|
their fds passed explicitly, subprocess closes them by default -- so mark
|
||||||
|
every inherited pipe above stderr close-on-exec. The executor's pidfd
|
||||||
|
wait is the primary detection; this only stops the leak at its source.
|
||||||
|
"""
|
||||||
|
if not sys.platform.startswith("linux"):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
fds = [int(name) for name in os.listdir("/proc/self/fd")]
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return
|
||||||
|
for fd in fds:
|
||||||
|
if fd <= 2:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if stat.S_ISFIFO(os.fstat(fd).st_mode) and os.get_inheritable(fd):
|
||||||
|
os.set_inheritable(fd, False)
|
||||||
|
except OSError:
|
||||||
|
# The listing's own directory fd, or one closed since.
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
class FutureWrapper(Future):
|
class FutureWrapper(Future):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -206,6 +389,7 @@ class MultiprocExecutor(Executor):
|
|||||||
if inherited_fds is not None:
|
if inherited_fds is not None:
|
||||||
inherited_fds.append(unready_worker_handle.death_writer.fileno())
|
inherited_fds.append(unready_worker_handle.death_writer.fileno())
|
||||||
inherited_fds.append(unready_worker_handle.ready_pipe.fileno())
|
inherited_fds.append(unready_worker_handle.ready_pipe.fileno())
|
||||||
|
inherited_fds.append(unready_worker_handle.fault_reader.fileno())
|
||||||
|
|
||||||
# Workers must be created before wait_for_ready to avoid
|
# Workers must be created before wait_for_ready to avoid
|
||||||
# deadlock, since worker.init_device() does a device sync.
|
# deadlock, since worker.init_device() does a device sync.
|
||||||
@@ -302,21 +486,78 @@ class MultiprocExecutor(Executor):
|
|||||||
# Monitors worker process liveness. If any die unexpectedly,
|
# Monitors worker process liveness. If any die unexpectedly,
|
||||||
# logs an error, shuts down the executor and invokes the failure
|
# logs an error, shuts down the executor and invokes the failure
|
||||||
# callback to inform the engine.
|
# callback to inform the engine.
|
||||||
|
#
|
||||||
|
# Two wake sources per worker:
|
||||||
|
# - its exit: a pidfd, readable when the process itself has exited
|
||||||
|
# (fallback: proc.sentinel, which a fork+exec'd child holding the
|
||||||
|
# write end -- torch_shm_manager from the PLE registration -- keeps
|
||||||
|
# silent; see _worker_exit_waitables);
|
||||||
|
# - its fault pipe: one byte from the fail-closed path right before
|
||||||
|
# os._exit, so the failure ladder starts -- and ends the peers --
|
||||||
|
# without waiting for the process teardown, which can itself wait
|
||||||
|
# on a live peer. A worker closing its end on the way out (EOF, no
|
||||||
|
# byte) is not a fault; its exit follows.
|
||||||
|
# In a rank-1 fault test a fault on PP0_TP1 was never detected and the
|
||||||
|
# engine hung on the response queue for 300 s.
|
||||||
|
# Opened here, before the thread starts, so a worker that dies early
|
||||||
|
# is still a zombie (pidfds work on zombies) rather than reaped.
|
||||||
|
exit_waitables = _worker_exit_waitables([h.proc for h in workers])
|
||||||
|
|
||||||
def monitor_workers():
|
def monitor_workers():
|
||||||
sentinels = [h.proc.sentinel for h in workers]
|
exit_to_handle = {
|
||||||
died = multiprocessing.connection.wait(sentinels)
|
w: h for (w, _, _), h in zip(exit_waitables, workers)
|
||||||
|
}
|
||||||
|
fault_to_handle = {
|
||||||
|
h.fault_reader: h
|
||||||
|
for h in workers
|
||||||
|
if getattr(h, "fault_reader", None) is not None
|
||||||
|
}
|
||||||
|
waiting: list[Any] = list(exit_to_handle) + list(fault_to_handle)
|
||||||
|
handle = None
|
||||||
|
reported_fatal = False
|
||||||
|
try:
|
||||||
|
while handle is None:
|
||||||
|
ready = multiprocessing.connection.wait(waiting)
|
||||||
|
for obj in ready:
|
||||||
|
if obj in exit_to_handle:
|
||||||
|
handle = exit_to_handle[obj]
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
data = os.read(obj.fileno(), 1)
|
||||||
|
except OSError:
|
||||||
|
data = b""
|
||||||
|
if data:
|
||||||
|
handle = fault_to_handle[obj]
|
||||||
|
reported_fatal = True
|
||||||
|
break
|
||||||
|
waiting.remove(obj)
|
||||||
|
finally:
|
||||||
|
for _, _, pidfd in exit_waitables:
|
||||||
|
if pidfd is not None:
|
||||||
|
with suppress(OSError):
|
||||||
|
os.close(pidfd)
|
||||||
_self = self_ref()
|
_self = self_ref()
|
||||||
if not _self or getattr(_self, "shutting_down", False):
|
if not _self or getattr(_self, "shutting_down", False):
|
||||||
logger.debug("MultiprocWorkerMonitor: shutdown already initiated")
|
logger.debug("MultiprocWorkerMonitor: shutdown already initiated")
|
||||||
return
|
return
|
||||||
_self.is_failed = True
|
_self.is_failed = True
|
||||||
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
|
proc = handle.proc
|
||||||
logger.error(
|
if reported_fatal:
|
||||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
logger.error(
|
||||||
"shutting down executor.",
|
"Worker proc %s reported a fatal fault (fail-closed) and is "
|
||||||
proc.name,
|
"exiting; shutting down executor without waiting for its "
|
||||||
proc.exitcode,
|
"process teardown. executor_shutdown_monotonic=%.9f",
|
||||||
)
|
proc.name,
|
||||||
|
time.monotonic(),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(
|
||||||
|
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||||
|
"shutting down executor. executor_shutdown_monotonic=%.9f",
|
||||||
|
proc.name,
|
||||||
|
proc.exitcode,
|
||||||
|
time.monotonic(),
|
||||||
|
)
|
||||||
_self.shutdown()
|
_self.shutdown()
|
||||||
callback = _self.failure_callback
|
callback = _self.failure_callback
|
||||||
if callback is not None:
|
if callback is not None:
|
||||||
@@ -324,9 +565,16 @@ class MultiprocExecutor(Executor):
|
|||||||
callback()
|
callback()
|
||||||
|
|
||||||
if not inline:
|
if not inline:
|
||||||
Thread(
|
try:
|
||||||
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
|
Thread(
|
||||||
).start()
|
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
|
||||||
|
).start()
|
||||||
|
except BaseException:
|
||||||
|
for _, _, pidfd in exit_waitables:
|
||||||
|
if pidfd is not None:
|
||||||
|
with suppress(OSError):
|
||||||
|
os.close(pidfd)
|
||||||
|
raise
|
||||||
return
|
return
|
||||||
|
|
||||||
monitor_workers()
|
monitor_workers()
|
||||||
@@ -366,12 +614,16 @@ class MultiprocExecutor(Executor):
|
|||||||
def execute_dummy_batch(self) -> None:
|
def execute_dummy_batch(self) -> None:
|
||||||
self.collective_rpc("execute_dummy_batch", unique_reply_rank=self.output_rank)
|
self.collective_rpc("execute_dummy_batch", unique_reply_rank=self.output_rank)
|
||||||
|
|
||||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
def take_draft_token_ids(
|
||||||
|
self, step_id: int | None = None
|
||||||
|
) -> DraftTokenIds | None:
|
||||||
# OPTIMIZATION: Get output only from a single worker (output_rank)
|
# OPTIMIZATION: Get output only from a single worker (output_rank)
|
||||||
|
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||||
return self.collective_rpc(
|
return self.collective_rpc(
|
||||||
"take_draft_token_ids",
|
"take_draft_token_ids",
|
||||||
unique_reply_rank=self.output_rank,
|
unique_reply_rank=self.output_rank,
|
||||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||||
|
kwargs=kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
def collective_rpc( # type: ignore[override]
|
def collective_rpc( # type: ignore[override]
|
||||||
@@ -450,18 +702,45 @@ class MultiprocExecutor(Executor):
|
|||||||
return future if non_block else future.result()
|
return future if non_block else future.result()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _ensure_worker_termination(worker_procs: list[BaseProcess]):
|
def _ensure_worker_termination(
|
||||||
|
worker_procs: list[BaseProcess],
|
||||||
|
*,
|
||||||
|
after_failure: bool = False,
|
||||||
|
shutdown_states: list[Any] | None = None,
|
||||||
|
):
|
||||||
"""Ensure that all worker processes are terminated. Assumes workers have
|
"""Ensure that all worker processes are terminated. Assumes workers have
|
||||||
received termination requests. Waits for processing, then sends
|
received termination requests. Waits for processing, then sends
|
||||||
termination and kill signals if needed."""
|
termination and kill signals if needed.
|
||||||
|
|
||||||
|
after_failure: a worker already died. Survivors split in two. Those
|
||||||
|
that never started WorkerProc.shutdown() are parked in a collective
|
||||||
|
with the dead rank (NCCL / gloo, in C, where neither the grace nor a
|
||||||
|
SIGTERM reaches them -- in a fault-injection test two ranks ran the full
|
||||||
|
5 s + 4 s ladder to SIGKILL): they get _FAILED_WORKER_WAIT_S to start
|
||||||
|
their shutdown, then SIGTERM, then SIGKILL after as long again. Those
|
||||||
|
whose shutdown_states entry shows the shutdown started (FINAL DRAIN,
|
||||||
|
flushes) get _FAILED_DRAIN_GRACE_S from the start of the ladder, then
|
||||||
|
SIGTERM, then SIGKILL after _FAILED_WORKER_WAIT_S: a started shutdown
|
||||||
|
can itself block on the dead rank (in a drain fault-injection test one
|
||||||
|
survivor had started its shutdown and never logged its FINAL DRAIN --
|
||||||
|
the drain's device synchronize waits on work paired with the dead
|
||||||
|
rank -- and ran the full orderly ladder, workers gone 17.6 s after the
|
||||||
|
fault). Before the drain-order fix the engine died on the late SIGTERM and
|
||||||
|
PDEATHSIG killed all survivors at once. A worker with no state (None,
|
||||||
|
or no shutdown_states) counts as parked. The orderly ladder (no
|
||||||
|
failure) is unchanged.
|
||||||
|
"""
|
||||||
|
grace_s = float(envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS)
|
||||||
|
term_wait_s = 4.0
|
||||||
|
|
||||||
def wait_for_termination(procs, timeout):
|
def wait_for_termination(procs, timeout):
|
||||||
if not time:
|
if not time:
|
||||||
# If we are in late stage shutdown, the interpreter may replace
|
# If we are in late stage shutdown, the interpreter may replace
|
||||||
# `time` with `None`.
|
# `time` with `None`.
|
||||||
return all(not proc.is_alive() for proc in procs)
|
return all(not proc.is_alive() for proc in procs)
|
||||||
start_time = time.time()
|
# Monotonic: a wall-clock step must not cut or stretch the grace.
|
||||||
while time.time() - start_time < timeout:
|
start_time = time.monotonic()
|
||||||
|
while time.monotonic() - start_time < timeout:
|
||||||
if all(not proc.is_alive() for proc in procs):
|
if all(not proc.is_alive() for proc in procs):
|
||||||
return True
|
return True
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
@@ -475,9 +754,42 @@ class MultiprocExecutor(Executor):
|
|||||||
"[shutdown] Executor: waiting for worker exit count=%d",
|
"[shutdown] Executor: waiting for worker exit count=%d",
|
||||||
initial_count,
|
initial_count,
|
||||||
)
|
)
|
||||||
if wait_for_termination(
|
if after_failure and time:
|
||||||
active_procs(), timeout=envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS
|
start = time.monotonic()
|
||||||
):
|
states = list(shutdown_states or [None] * len(worker_procs))
|
||||||
|
|
||||||
|
def parked() -> list[BaseProcess]:
|
||||||
|
return [
|
||||||
|
proc
|
||||||
|
for proc, state in zip(worker_procs, states)
|
||||||
|
if proc.is_alive()
|
||||||
|
and (state is None or state.value == _SHUTDOWN_NOT_STARTED)
|
||||||
|
]
|
||||||
|
|
||||||
|
if not wait_for_termination(active_procs(), _FAILED_WORKER_WAIT_S):
|
||||||
|
stuck = parked()
|
||||||
|
if stuck:
|
||||||
|
logger.warning(
|
||||||
|
"[shutdown] Executor: workers parked after a worker "
|
||||||
|
"failure (shutdown not started); sending SIGTERM count=%d",
|
||||||
|
len(stuck),
|
||||||
|
)
|
||||||
|
for p in stuck:
|
||||||
|
p.terminate()
|
||||||
|
if not wait_for_termination(stuck, _FAILED_WORKER_WAIT_S):
|
||||||
|
stuck = [p for p in stuck if p.is_alive()]
|
||||||
|
logger.warning(
|
||||||
|
"[shutdown] Executor: parked workers still running "
|
||||||
|
"after SIGTERM; sending SIGKILL count=%d",
|
||||||
|
len(stuck),
|
||||||
|
)
|
||||||
|
for p in stuck:
|
||||||
|
p.kill()
|
||||||
|
# Workers that started their shutdown get a BOUNDED grace, counted
|
||||||
|
# from the start of this ladder, then SIGTERM, then SIGKILL.
|
||||||
|
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.monotonic() - start))
|
||||||
|
term_wait_s = _FAILED_WORKER_WAIT_S
|
||||||
|
if wait_for_termination(active_procs(), timeout=grace_s):
|
||||||
logger.info_once("[shutdown] Executor: all workers exited gracefully")
|
logger.info_once("[shutdown] Executor: all workers exited gracefully")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -490,7 +802,7 @@ class MultiprocExecutor(Executor):
|
|||||||
)
|
)
|
||||||
for p in remaining:
|
for p in remaining:
|
||||||
p.terminate()
|
p.terminate()
|
||||||
if not wait_for_termination(active_procs(), 4):
|
if not wait_for_termination(active_procs(), term_wait_s):
|
||||||
# Send SIGKILL if still running
|
# Send SIGKILL if still running
|
||||||
remaining = active_procs()
|
remaining = active_procs()
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -518,7 +830,13 @@ class MultiprocExecutor(Executor):
|
|||||||
if w.death_writer is not None:
|
if w.death_writer is not None:
|
||||||
w.death_writer.close()
|
w.death_writer.close()
|
||||||
w.death_writer = None
|
w.death_writer = None
|
||||||
self._ensure_worker_termination([w.proc for w in workers])
|
self._ensure_worker_termination(
|
||||||
|
[w.proc for w in workers],
|
||||||
|
after_failure=getattr(self, "is_failed", False),
|
||||||
|
shutdown_states=[
|
||||||
|
getattr(w, "shutdown_state", None) for w in workers
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
for w in workers:
|
for w in workers:
|
||||||
# Shutdown response queues
|
# Shutdown response queues
|
||||||
@@ -569,6 +887,8 @@ class UnreadyWorkerProcHandle:
|
|||||||
rank: int
|
rank: int
|
||||||
ready_pipe: Connection
|
ready_pipe: Connection
|
||||||
death_writer: Connection | None = None
|
death_writer: Connection | None = None
|
||||||
|
shutdown_state: Any = None
|
||||||
|
fault_reader: Connection | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -582,6 +902,10 @@ class WorkerProcHandle:
|
|||||||
# `peer_worker_response_mqs[i]`
|
# `peer_worker_response_mqs[i]`
|
||||||
peer_worker_response_mqs: list[MessageQueue | None]
|
peer_worker_response_mqs: list[MessageQueue | None]
|
||||||
death_writer: Connection | None = None
|
death_writer: Connection | None = None
|
||||||
|
# Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes).
|
||||||
|
shutdown_state: Any = None
|
||||||
|
# Read end of the worker's fault pipe (see start_worker_monitor).
|
||||||
|
fault_reader: Connection | None = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_unready_handle(
|
def from_unready_handle(
|
||||||
@@ -596,6 +920,8 @@ class WorkerProcHandle:
|
|||||||
worker_response_mq=worker_response_mq,
|
worker_response_mq=worker_response_mq,
|
||||||
peer_worker_response_mqs=peer_worker_response_mqs,
|
peer_worker_response_mqs=peer_worker_response_mqs,
|
||||||
death_writer=unready_handle.death_writer,
|
death_writer=unready_handle.death_writer,
|
||||||
|
shutdown_state=unready_handle.shutdown_state,
|
||||||
|
fault_reader=unready_handle.fault_reader,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -672,6 +998,10 @@ class WorkerProc:
|
|||||||
|
|
||||||
# Load model
|
# Load model
|
||||||
self.worker.init_device()
|
self.worker.init_device()
|
||||||
|
# PLE pull transport: DP0/TP0 starts the CPU-only PLE process before
|
||||||
|
# load_model, so its weight load overlaps ours; READY is awaited after.
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
self.worker.spawn_ple_offload()
|
||||||
# Update process title now that parallel groups are initialized
|
# Update process title now that parallel groups are initialized
|
||||||
self.setup_proc_title_and_log_prefix(
|
self.setup_proc_title_and_log_prefix(
|
||||||
enable_ep=vllm_config.parallel_config.enable_expert_parallel
|
enable_ep=vllm_config.parallel_config.enable_expert_parallel
|
||||||
@@ -680,6 +1010,8 @@ class WorkerProc:
|
|||||||
self.worker.elastic_ep_execute("load_model")
|
self.worker.elastic_ep_execute("load_model")
|
||||||
else:
|
else:
|
||||||
self.worker.load_model()
|
self.worker.load_model()
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
self.worker.wait_ple_offload_ready()
|
||||||
|
|
||||||
scheduler_config = vllm_config.scheduler_config
|
scheduler_config = vllm_config.scheduler_config
|
||||||
self.use_async_scheduling = scheduler_config.async_scheduling
|
self.use_async_scheduling = scheduler_config.async_scheduling
|
||||||
@@ -715,14 +1047,31 @@ class WorkerProc:
|
|||||||
inherited_fds: list[int] | None = None,
|
inherited_fds: list[int] | None = None,
|
||||||
) -> UnreadyWorkerProcHandle:
|
) -> UnreadyWorkerProcHandle:
|
||||||
context = get_mp_context()
|
context = get_mp_context()
|
||||||
|
# Workers arm PR_SET_PDEATHSIG against this process and verify it via
|
||||||
|
# getppid(). Under forkserver the OS parent is the forkserver, so a
|
||||||
|
# healthy worker would kill itself; refuse before creating anything.
|
||||||
|
start_method = context.get_start_method()
|
||||||
|
if start_method not in ("fork", "spawn"):
|
||||||
|
raise RuntimeError(
|
||||||
|
"vLLM worker processes require the 'fork' or 'spawn' "
|
||||||
|
f"multiprocessing start method (got {start_method!r})"
|
||||||
|
)
|
||||||
# Ready pipe to communicate readiness from child to parent
|
# Ready pipe to communicate readiness from child to parent
|
||||||
ready_reader, ready_writer = context.Pipe(duplex=False)
|
ready_reader, ready_writer = context.Pipe(duplex=False)
|
||||||
# Death pipe to let child detect parent process exit
|
# Death pipe to let child detect parent process exit
|
||||||
death_reader, death_writer = context.Pipe(duplex=False)
|
death_reader, death_writer = context.Pipe(duplex=False)
|
||||||
|
# Fault pipe: the worker's fail-closed path writes one byte before
|
||||||
|
# os._exit; the executor's monitor waits on it next to the sentinel.
|
||||||
|
fault_reader, fault_writer = context.Pipe(duplex=False)
|
||||||
|
# One shared byte: has this worker started / finished its shutdown?
|
||||||
|
shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED)
|
||||||
if inherited_fds is not None:
|
if inherited_fds is not None:
|
||||||
inherited_fds = inherited_fds.copy()
|
inherited_fds = inherited_fds.copy()
|
||||||
inherited_fds.extend((ready_reader.fileno(), death_writer.fileno()))
|
inherited_fds.extend(
|
||||||
|
(ready_reader.fileno(), death_writer.fileno(), fault_reader.fileno())
|
||||||
|
)
|
||||||
process_kwargs = {
|
process_kwargs = {
|
||||||
|
"expected_parent_pid": os.getpid(),
|
||||||
"vllm_config": vllm_config,
|
"vllm_config": vllm_config,
|
||||||
"local_rank": local_rank,
|
"local_rank": local_rank,
|
||||||
"rank": rank,
|
"rank": rank,
|
||||||
@@ -730,6 +1079,8 @@ class WorkerProc:
|
|||||||
"input_shm_handle": input_shm_handle,
|
"input_shm_handle": input_shm_handle,
|
||||||
"ready_pipe": ready_writer,
|
"ready_pipe": ready_writer,
|
||||||
"death_pipe": death_reader,
|
"death_pipe": death_reader,
|
||||||
|
"fault_pipe": fault_writer,
|
||||||
|
"shutdown_state": shutdown_state,
|
||||||
"shared_worker_lock": shared_worker_lock,
|
"shared_worker_lock": shared_worker_lock,
|
||||||
"is_driver_worker": is_driver_worker,
|
"is_driver_worker": is_driver_worker,
|
||||||
# Have the worker close parent end of this worker's pipes too
|
# Have the worker close parent end of this worker's pipes too
|
||||||
@@ -752,9 +1103,12 @@ class WorkerProc:
|
|||||||
# Close child ends of pipes here in the parent
|
# Close child ends of pipes here in the parent
|
||||||
ready_writer.close()
|
ready_writer.close()
|
||||||
death_reader.close()
|
death_reader.close()
|
||||||
|
fault_writer.close()
|
||||||
# Keep death_writer open in parent - when parent exits,
|
# Keep death_writer open in parent - when parent exits,
|
||||||
# death_reader in child will get EOFError
|
# death_reader in child will get EOFError
|
||||||
return UnreadyWorkerProcHandle(proc, rank, ready_reader, death_writer)
|
return UnreadyWorkerProcHandle(
|
||||||
|
proc, rank, ready_reader, death_writer, shutdown_state, fault_reader
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def wait_for_response_handle_ready(
|
def wait_for_response_handle_ready(
|
||||||
@@ -790,10 +1144,59 @@ class WorkerProc:
|
|||||||
ready_proc_handles: list[WorkerProcHandle | None] = [None] * len(
|
ready_proc_handles: list[WorkerProcHandle | None] = [None] * len(
|
||||||
unready_proc_handles
|
unready_proc_handles
|
||||||
)
|
)
|
||||||
while pipes:
|
# A worker that dies before READY need not close its ready pipe: a
|
||||||
ready = multiprocessing.connection.wait(pipes.keys())
|
# child it fork+exec'd (torch_shm_manager, started by the PLE
|
||||||
for pipe in ready:
|
# registration) can hold the write end, so no EOF ever comes. Wait on
|
||||||
assert isinstance(pipe, Connection)
|
# each pending worker's exit (pidfd, else its sentinel -- see
|
||||||
|
# _worker_exit_waitables) and on its fault pipe (one byte from the
|
||||||
|
# fail-closed path; EOF is not a fault), as the post-READY monitor does.
|
||||||
|
exit_waitables = _worker_exit_waitables(
|
||||||
|
[h.proc for h in unready_proc_handles]
|
||||||
|
)
|
||||||
|
exits = {
|
||||||
|
w: handle
|
||||||
|
for (w, _, _), handle in zip(exit_waitables, unready_proc_handles)
|
||||||
|
}
|
||||||
|
faults = {
|
||||||
|
handle.fault_reader: handle
|
||||||
|
for handle in unready_proc_handles
|
||||||
|
if getattr(handle, "fault_reader", None) is not None
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
while pipes:
|
||||||
|
waiting: list[Any] = [*pipes.keys()]
|
||||||
|
# EVERY worker's exit and fault pipe, READY or not: the live
|
||||||
|
# monitor starts only after all are READY, so a READY worker
|
||||||
|
# that faults or dies while a sibling initialises must fail
|
||||||
|
# initialization here.
|
||||||
|
waiting += list(exits)
|
||||||
|
waiting += list(faults)
|
||||||
|
ready = multiprocessing.connection.wait(waiting)
|
||||||
|
WorkerProc._receive_ready(
|
||||||
|
ready, pipes, exits, faults, ready_proc_handles, e
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
for _, _, pidfd in exit_waitables:
|
||||||
|
if pidfd is not None:
|
||||||
|
with suppress(OSError):
|
||||||
|
os.close(pidfd)
|
||||||
|
|
||||||
|
return cast(list[WorkerProcHandle], ready_proc_handles)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _receive_ready(
|
||||||
|
ready: list[Any],
|
||||||
|
pipes: dict[Connection, UnreadyWorkerProcHandle],
|
||||||
|
exits: dict[Any, UnreadyWorkerProcHandle],
|
||||||
|
faults: dict[Any, UnreadyWorkerProcHandle],
|
||||||
|
ready_proc_handles: list[WorkerProcHandle | None],
|
||||||
|
e: Exception,
|
||||||
|
) -> None:
|
||||||
|
# Ready pipes first: a READY sent before an exit or fault still counts.
|
||||||
|
ordered = sorted(ready, key=lambda obj: 0 if obj in pipes else 1)
|
||||||
|
for obj in ordered:
|
||||||
|
if obj in pipes:
|
||||||
|
pipe = obj
|
||||||
try:
|
try:
|
||||||
# Wait until the WorkerProc is ready.
|
# Wait until the WorkerProc is ready.
|
||||||
unready_proc_handle = pipes.pop(pipe)
|
unready_proc_handle = pipes.pop(pipe)
|
||||||
@@ -802,20 +1205,59 @@ class WorkerProc:
|
|||||||
raise e
|
raise e
|
||||||
|
|
||||||
idx = unready_proc_handle.rank % len(ready_proc_handles)
|
idx = unready_proc_handle.rank % len(ready_proc_handles)
|
||||||
ready_proc_handles[idx] = WorkerProc.wait_for_response_handle_ready(
|
ready_proc_handles[idx] = (
|
||||||
response, unready_proc_handle
|
WorkerProc.wait_for_response_handle_ready(
|
||||||
|
response, unready_proc_handle
|
||||||
|
)
|
||||||
)
|
)
|
||||||
except EOFError:
|
except EOFError:
|
||||||
e.__suppress_context__ = True
|
e.__suppress_context__ = True
|
||||||
raise e from None
|
raise e from None
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
# Close connection.
|
# Close connection.
|
||||||
pipe.close()
|
pipe.close()
|
||||||
|
continue
|
||||||
|
handle = exits.get(obj) if obj in exits else faults.get(obj)
|
||||||
|
if handle is None:
|
||||||
|
continue
|
||||||
|
pending = handle.ready_pipe in pipes
|
||||||
|
if pending and handle.ready_pipe.poll(0):
|
||||||
|
continue # its READY is queued: read it next round
|
||||||
|
phase = (
|
||||||
|
"before READY"
|
||||||
|
if pending
|
||||||
|
else "after READY, while other workers were initializing"
|
||||||
|
)
|
||||||
|
if obj in exits:
|
||||||
|
logger.error(
|
||||||
|
"Worker proc %s exited (code %s) %s",
|
||||||
|
handle.proc.name,
|
||||||
|
handle.proc.exitcode,
|
||||||
|
phase,
|
||||||
|
)
|
||||||
|
raise e
|
||||||
|
try:
|
||||||
|
data = os.read(obj.fileno(), 1)
|
||||||
|
except OSError:
|
||||||
|
data = b""
|
||||||
|
if data:
|
||||||
|
logger.error(
|
||||||
|
"Worker proc %s reported a fatal fault (fail-closed) %s",
|
||||||
|
handle.proc.name,
|
||||||
|
phase,
|
||||||
|
)
|
||||||
|
raise e
|
||||||
|
# EOF on the fault pipe is not a fault; its exit follows.
|
||||||
|
del faults[obj]
|
||||||
|
|
||||||
return cast(list[WorkerProcHandle], ready_proc_handles)
|
def shutdown(self, *, destroy_distributed: bool = True):
|
||||||
|
if not destroy_distributed:
|
||||||
def shutdown(self):
|
# worker_main os._exits right after this: the PLE offload handle
|
||||||
|
# (rank 0 owns the child) must not wait for the child's reap,
|
||||||
|
# which trails its exit by seconds for a large host table.
|
||||||
|
ple_worker = sys.modules.get("vllm.v1.ple_offload.worker")
|
||||||
|
if ple_worker is not None:
|
||||||
|
ple_worker.mark_process_exit_imminent()
|
||||||
if self.rpc_broadcast_mq is not None:
|
if self.rpc_broadcast_mq is not None:
|
||||||
self.rpc_broadcast_mq.shutdown()
|
self.rpc_broadcast_mq.shutdown()
|
||||||
if self.worker_response_mq is not None:
|
if self.worker_response_mq is not None:
|
||||||
@@ -823,8 +1265,12 @@ class WorkerProc:
|
|||||||
self.worker.shutdown()
|
self.worker.shutdown()
|
||||||
self.rpc_broadcast_mq = None
|
self.rpc_broadcast_mq = None
|
||||||
self.worker_response_mq = None
|
self.worker_response_mq = None
|
||||||
destroy_model_parallel()
|
# worker_main passes False when the process exits right after this
|
||||||
destroy_distributed_environment()
|
# (see _exit_worker_process): the NCCL destroy is exit-only work that
|
||||||
|
# can block for 5 s per communicator.
|
||||||
|
if destroy_distributed:
|
||||||
|
destroy_model_parallel()
|
||||||
|
destroy_distributed_environment()
|
||||||
|
|
||||||
def monitor_death_pipe(self, death_pipe, shutdown_requested: threading.Event):
|
def monitor_death_pipe(self, death_pipe, shutdown_requested: threading.Event):
|
||||||
if death_pipe is None:
|
if death_pipe is None:
|
||||||
@@ -858,6 +1304,9 @@ class WorkerProc:
|
|||||||
if logging_config := getattr(kwargs["vllm_config"], "logging_config", None):
|
if logging_config := getattr(kwargs["vllm_config"], "logging_config", None):
|
||||||
configure_logging(logging_config)
|
configure_logging(logging_config)
|
||||||
|
|
||||||
|
expected_parent_pid = kwargs.pop("expected_parent_pid")
|
||||||
|
_install_worker_parent_death_signal(expected_parent_pid)
|
||||||
|
|
||||||
# Signal handler used for graceful termination.
|
# Signal handler used for graceful termination.
|
||||||
# SystemExit exception is only raised once to allow this and worker
|
# SystemExit exception is only raised once to allow this and worker
|
||||||
# processes to terminate without error
|
# processes to terminate without error
|
||||||
@@ -890,8 +1339,30 @@ class WorkerProc:
|
|||||||
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
|
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
|
||||||
|
|
||||||
worker = None
|
worker = None
|
||||||
|
# Set once a READY worker leaves its busy loop because shutdown was
|
||||||
|
# requested (death pipe EOF or SIGTERM/SIGINT): the process then exits
|
||||||
|
# straight after WorkerProc.shutdown (see _exit_worker_process).
|
||||||
|
orderly_shutdown = False
|
||||||
|
# Set when a READY worker's busy loop FAILED (anything but its own
|
||||||
|
# shutdown-request cancellation): full teardown, then exit code 1.
|
||||||
|
# Never keyed to shutdown_requested -- the death-pipe monitor can set
|
||||||
|
# it just after the check, which made a failure exit 0.
|
||||||
|
busy_loop_failed = False
|
||||||
ready_writer = kwargs.pop("ready_pipe")
|
ready_writer = kwargs.pop("ready_pipe")
|
||||||
death_pipe = kwargs.pop("death_pipe", None)
|
death_pipe = kwargs.pop("death_pipe", None)
|
||||||
|
# Fault pipe: registered for the PLE fail-closed path
|
||||||
|
# (ple_offload.fatal.notify_parent_fatal); kept open for the life of
|
||||||
|
# the process so the byte written right before os._exit reaches the
|
||||||
|
# executor's monitor. Closed only on the orderly way out (EOF, which
|
||||||
|
# the monitor does not count as a fault).
|
||||||
|
fault_pipe = kwargs.pop("fault_pipe", None)
|
||||||
|
if fault_pipe is not None:
|
||||||
|
set_fatal_notify_fd(fault_pipe.fileno())
|
||||||
|
shutdown_state = kwargs.pop("shutdown_state", None)
|
||||||
|
|
||||||
|
def mark_shutdown(value: int) -> None:
|
||||||
|
if shutdown_state is not None:
|
||||||
|
shutdown_state.value = value
|
||||||
|
|
||||||
# Close inherited pipes from parent (incl. other worker pipes)
|
# Close inherited pipes from parent (incl. other worker pipes)
|
||||||
# Explicitly passing in existing pipes and closing them makes the pipe
|
# Explicitly passing in existing pipes and closing them makes the pipe
|
||||||
@@ -902,6 +1373,8 @@ class WorkerProc:
|
|||||||
os.close(fd)
|
os.close(fd)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Error closing inherited connection: %s: %s", type(e), e)
|
logger.warning("Error closing inherited connection: %s: %s", type(e), e)
|
||||||
|
# Before anything below can fork+exec a helper (torch_shm_manager).
|
||||||
|
_make_inherited_pipes_close_on_exec()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Initialize tracer
|
# Initialize tracer
|
||||||
@@ -938,7 +1411,7 @@ class WorkerProc:
|
|||||||
|
|
||||||
worker.worker_busy_loop()
|
worker.worker_busy_loop()
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# NOTE: if an Exception arises in busy_loop, we send
|
# NOTE: if an Exception arises in busy_loop, we send
|
||||||
# a FAILURE message over the MQ RPC to notify the Executor,
|
# a FAILURE message over the MQ RPC to notify the Executor,
|
||||||
# which triggers system shutdown.
|
# which triggers system shutdown.
|
||||||
@@ -946,12 +1419,23 @@ class WorkerProc:
|
|||||||
|
|
||||||
if ready_writer is not None:
|
if ready_writer is not None:
|
||||||
logger.exception("WorkerProc failed to start.")
|
logger.exception("WorkerProc failed to start.")
|
||||||
elif shutdown_requested.is_set():
|
elif isinstance(e, WorkerShutdownRequested) and (
|
||||||
|
shutdown_requested.is_set()
|
||||||
|
):
|
||||||
|
# The busy loop left BECAUSE of the shutdown request (its
|
||||||
|
# dequeue was cancelled), not because something failed.
|
||||||
logger.debug_once(
|
logger.debug_once(
|
||||||
"[shutdown] WorkerProc: exiting after shutdown request"
|
"[shutdown] WorkerProc: exiting after shutdown request"
|
||||||
)
|
)
|
||||||
|
orderly_shutdown = True
|
||||||
else:
|
else:
|
||||||
logger.exception("WorkerProc failed.")
|
# A real failure, whether or not a shutdown request is pending
|
||||||
|
# (yet): full teardown, then a non-zero exit.
|
||||||
|
if shutdown_requested.is_set():
|
||||||
|
logger.exception("WorkerProc failed during shutdown.")
|
||||||
|
else:
|
||||||
|
logger.exception("WorkerProc failed.")
|
||||||
|
busy_loop_failed = True
|
||||||
|
|
||||||
# The parent sends a SIGTERM to all worker processes if
|
# The parent sends a SIGTERM to all worker processes if
|
||||||
# any worker dies. Set this value so we don't re-throw
|
# any worker dies. Set this value so we don't re-throw
|
||||||
@@ -965,6 +1449,7 @@ class WorkerProc:
|
|||||||
logger.debug_once(
|
logger.debug_once(
|
||||||
"[shutdown] WorkerProc: terminated by shutdown signal"
|
"[shutdown] WorkerProc: terminated by shutdown signal"
|
||||||
)
|
)
|
||||||
|
orderly_shutdown = ready_writer is None
|
||||||
else:
|
else:
|
||||||
logger.warning("WorkerProc was terminated")
|
logger.warning("WorkerProc was terminated")
|
||||||
# SystemExit must never be ignored
|
# SystemExit must never be ignored
|
||||||
@@ -975,14 +1460,60 @@ class WorkerProc:
|
|||||||
ready_writer.close()
|
ready_writer.close()
|
||||||
if death_pipe is not None:
|
if death_pipe is not None:
|
||||||
death_pipe.close()
|
death_pipe.close()
|
||||||
# Clean up once worker exits busy loop
|
# The fault fd stays registered through the whole teardown: a PLE
|
||||||
if worker is not None:
|
# fail-closed inside worker.shutdown() must still reach the
|
||||||
worker.shutdown()
|
# executor. Unregistered after it; on the orderly
|
||||||
|
# os._exit path the process exit closes it.
|
||||||
|
try:
|
||||||
|
# Clean up once worker exits busy loop
|
||||||
|
if worker is not None:
|
||||||
|
# Tells the executor's ladder this rank is draining, not parked
|
||||||
|
# in a collective (see _ensure_worker_termination).
|
||||||
|
mark_shutdown(_SHUTDOWN_STARTED)
|
||||||
|
if orderly_shutdown:
|
||||||
|
# Everything that must happen before exit (FINAL DRAIN,
|
||||||
|
# instrumentation flushes, PLE cleanup) is in
|
||||||
|
# worker.shutdown(); then exit instead of paying the NCCL
|
||||||
|
# destroy + finalization past the executor's grace.
|
||||||
|
exit_code = 0
|
||||||
|
try:
|
||||||
|
worker.shutdown(destroy_distributed=False)
|
||||||
|
except BaseException:
|
||||||
|
logger.exception("WorkerProc shutdown failed")
|
||||||
|
exit_code = 1
|
||||||
|
mark_shutdown(_SHUTDOWN_DONE)
|
||||||
|
_exit_worker_process(exit_code)
|
||||||
|
worker.shutdown()
|
||||||
|
mark_shutdown(_SHUTDOWN_DONE)
|
||||||
|
if busy_loop_failed:
|
||||||
|
sys.exit(1)
|
||||||
|
finally:
|
||||||
|
if fault_pipe is not None:
|
||||||
|
set_fatal_notify_fd(None)
|
||||||
|
fault_pipe.close()
|
||||||
|
|
||||||
class ResponseStatus(Enum):
|
class ResponseStatus(Enum):
|
||||||
SUCCESS = auto()
|
SUCCESS = auto()
|
||||||
FAILURE = auto()
|
FAILURE = auto()
|
||||||
|
|
||||||
|
def _fail_ple_rpc_if_pending(self, where: str, error: BaseException) -> None:
|
||||||
|
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
|
||||||
|
owners = (self.worker, getattr(self.worker, "worker", None))
|
||||||
|
for owner in owners:
|
||||||
|
if owner is None:
|
||||||
|
continue
|
||||||
|
runner = getattr(owner, "model_runner", None)
|
||||||
|
connector = getattr(runner, "_ple_offload_connector", None)
|
||||||
|
if connector is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pending = connector.has_pending_completions
|
||||||
|
except BaseException as state_error:
|
||||||
|
connector._fail_closed(f"{where}:pending-state", state_error)
|
||||||
|
if pending:
|
||||||
|
connector._fail_closed(where, error)
|
||||||
|
return
|
||||||
|
|
||||||
def enqueue_output(self, output: Any):
|
def enqueue_output(self, output: Any):
|
||||||
"""Prepares output from the worker and enqueues it to the
|
"""Prepares output from the worker and enqueues it to the
|
||||||
worker_response_mq. If the output is an Exception, it is
|
worker_response_mq. If the output is an Exception, it is
|
||||||
@@ -993,6 +1524,7 @@ class WorkerProc:
|
|||||||
output = output.get_output()
|
output = output.get_output()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Error getting async model runner output")
|
logger.exception("Error getting async model runner output")
|
||||||
|
self._fail_ple_rpc_if_pending("async-output-copy", e)
|
||||||
output = e
|
output = e
|
||||||
|
|
||||||
if isinstance(output, Exception):
|
if isinstance(output, Exception):
|
||||||
@@ -1032,8 +1564,20 @@ class WorkerProc:
|
|||||||
def worker_busy_loop(self):
|
def worker_busy_loop(self):
|
||||||
"""Main busy loop for Multiprocessing Workers."""
|
"""Main busy loop for Multiprocessing Workers."""
|
||||||
assert self.rpc_broadcast_mq is not None
|
assert self.rpc_broadcast_mq is not None
|
||||||
|
mq = self.rpc_broadcast_mq
|
||||||
while True:
|
while True:
|
||||||
self._execute_worker_rpc(self.rpc_broadcast_mq.dequeue(indefinite=True))
|
# No local holds the request across the next dequeue: its payload
|
||||||
|
# must be released first (test_multiproc_executor).
|
||||||
|
self._execute_worker_rpc(self._dequeue_worker_rpc(mq))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _dequeue_worker_rpc(mq: MessageQueue):
|
||||||
|
try:
|
||||||
|
return mq.dequeue(indefinite=True)
|
||||||
|
except RuntimeError as e:
|
||||||
|
if getattr(mq, "shutting_down", False) and str(e) == "cancelled":
|
||||||
|
raise WorkerShutdownRequested("RPC broadcast queue shut down") from e
|
||||||
|
raise
|
||||||
|
|
||||||
def _execute_worker_rpc(
|
def _execute_worker_rpc(
|
||||||
self,
|
self,
|
||||||
@@ -1052,6 +1596,7 @@ class WorkerProc:
|
|||||||
if output_rank is None or self.rank == output_rank:
|
if output_rank is None or self.rank == output_rank:
|
||||||
self.handle_output(output)
|
self.handle_output(output)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
self._fail_ple_rpc_if_pending(f"worker-rpc:{method}", e)
|
||||||
# Notes have been introduced in python 3.11
|
# Notes have been introduced in python 3.11
|
||||||
if hasattr(e, "add_note"):
|
if hasattr(e, "add_note"):
|
||||||
e.add_note(traceback.format_exc())
|
e.add_note(traceback.format_exc())
|
||||||
|
|||||||
@@ -67,11 +67,15 @@ class UniProcExecutor(Executor):
|
|||||||
|
|
||||||
self.driver_worker.init_worker(all_kwargs=[kwargs])
|
self.driver_worker.init_worker(all_kwargs=[kwargs])
|
||||||
self.driver_worker.init_device()
|
self.driver_worker.init_device()
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
self.driver_worker.spawn_ple_offload()
|
||||||
|
|
||||||
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||||
self.driver_worker.elastic_ep_execute("load_model")
|
self.driver_worker.elastic_ep_execute("load_model")
|
||||||
else:
|
else:
|
||||||
self.driver_worker.load_model()
|
self.driver_worker.load_model()
|
||||||
|
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||||
|
self.driver_worker.wait_ple_offload_ready()
|
||||||
current_platform.update_block_size_for_backend(self.vllm_config)
|
current_platform.update_block_size_for_backend(self.vllm_config)
|
||||||
|
|
||||||
def _distributed_args(self) -> tuple[str, int, int]:
|
def _distributed_args(self) -> tuple[str, int, int]:
|
||||||
@@ -141,8 +145,13 @@ class UniProcExecutor(Executor):
|
|||||||
single_value=True,
|
single_value=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||||
return self.collective_rpc("take_draft_token_ids", single_value=True)
|
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||||
|
return self.collective_rpc(
|
||||||
|
"take_draft_token_ids",
|
||||||
|
kwargs=kwargs,
|
||||||
|
single_value=True,
|
||||||
|
)
|
||||||
|
|
||||||
def check_health(self) -> None:
|
def check_health(self) -> None:
|
||||||
# UniProcExecutor will always be healthy as long as
|
# UniProcExecutor will always be healthy as long as
|
||||||
|
|||||||
@@ -1431,6 +1431,11 @@ class KVCacheTensor:
|
|||||||
block_stride: int
|
block_stride: int
|
||||||
offset: int = 0 # byte offset of layers[0]'s block 0
|
offset: int = 0 # byte offset of layers[0]'s block 0
|
||||||
host_resident: bool = False
|
host_resident: bool = False
|
||||||
|
# v2.5 side-cache layout: a slot-packed ring tensor views every pool block
|
||||||
|
# as ``slots_per_block`` equal slots (``block_stride`` is then one slot), so
|
||||||
|
# its block ids are ``pool_block * slots_per_block + slot``. 1 = one view
|
||||||
|
# block per pool block, the ordinary case.
|
||||||
|
slots_per_block: int = 1
|
||||||
|
|
||||||
|
|
||||||
class KVCacheGroupRole(str, Enum):
|
class KVCacheGroupRole(str, Enum):
|
||||||
@@ -1458,6 +1463,10 @@ class KVCacheGroupSpec:
|
|||||||
# source group after a connector load.
|
# source group after a connector load.
|
||||||
enable_kv_transfer: bool = True
|
enable_kv_transfer: bool = True
|
||||||
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
||||||
|
# v2.5 side-cache layout: > 1 when this one-block-per-request ring group
|
||||||
|
# shares each pool block between that many requests (see
|
||||||
|
# ``CircularBufferManager.enable_slot_packing``); 1 = one pool block each.
|
||||||
|
slots_per_block: int = 1
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -1545,6 +1554,18 @@ class KVCacheConfig:
|
|||||||
)
|
)
|
||||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def slot_packed_group_ids(self) -> tuple[int, ...]:
|
||||||
|
"""Groups whose blocks are slots of shared pool blocks (v2.5 side-cache
|
||||||
|
layout). Their views are not one block per pool block, so whole-block
|
||||||
|
zeroing must skip them; their bytes lie inside pool blocks the ring
|
||||||
|
manager owns and that no other group writes."""
|
||||||
|
return tuple(
|
||||||
|
group_id
|
||||||
|
for group_id, group in enumerate(self.kv_cache_groups)
|
||||||
|
if group.slots_per_block > 1
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def host_group_ids(self) -> tuple[int, ...]:
|
def host_group_ids(self) -> tuple[int, ...]:
|
||||||
return tuple(
|
return tuple(
|
||||||
@@ -1556,7 +1577,7 @@ class KVCacheConfig:
|
|||||||
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
||||||
"""Number of blocks addressable by the pool backing ``tensor``."""
|
"""Number of blocks addressable by the pool backing ``tensor``."""
|
||||||
if not tensor.host_resident:
|
if not tensor.host_resident:
|
||||||
return self.num_blocks
|
return self.num_blocks * tensor.slots_per_block
|
||||||
assert self.hisparse_host_num_blocks is not None
|
assert self.hisparse_host_num_blocks is not None
|
||||||
return self.hisparse_host_num_blocks
|
return self.hisparse_host_num_blocks
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
@@ -0,0 +1,997 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Exchange PLE data between a GPU worker and the CPU-offload process."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from multiprocessing.reduction import ForkingPickler
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import msgspec
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
import zmq
|
||||||
|
from cuda.bindings import driver as cuda_driver
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
|
from vllm.config import VllmConfig
|
||||||
|
from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.model_executor.layers.ple_offload_layer import (
|
||||||
|
CpuGpuSemaphore,
|
||||||
|
PleOffloadLayer,
|
||||||
|
flag_check_is_authoritative,
|
||||||
|
stale_offload_flags,
|
||||||
|
)
|
||||||
|
from vllm.v1.ple_offload.fatal import notify_parent_fatal, ple_fail_closed
|
||||||
|
from vllm.v1.ple_offload.progress import PleProgressMonitor
|
||||||
|
from vllm.v1.ple_offload.protocol import (
|
||||||
|
PleOffloadRegistration,
|
||||||
|
PleOffloadRequest,
|
||||||
|
)
|
||||||
|
from vllm.v1.utils import record_function_or_nullcontext
|
||||||
|
from vllm_ple_test_fault import _FAULT
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
# PLE diagnostics / vllm#53899 class: read the pull-transport semaphore back before a real
|
||||||
|
# launch and refuse to publish into a flag that is already set. Costs a
|
||||||
|
# model-stream synchronize, so it is a diagnostic, not a hot-path guard, and is
|
||||||
|
# OFF by default -- the same split VLLM_E2_ZERO_OWNERSHIP_CHECK established in
|
||||||
|
# the scheduler (VLLM_E2_ZERO_OWNERSHIP_CHECK).
|
||||||
|
#
|
||||||
|
# off | warn | raise (1 is accepted as warn). Default off. The default action on
|
||||||
|
# a violation is WARN, not raise: this diagnostic could
|
||||||
|
# terminate a healthy run, and a diagnostic that kills the qualification boot
|
||||||
|
# intended to establish safety is worse than no diagnostic. raise is for a
|
||||||
|
# targeted repro.
|
||||||
|
#
|
||||||
|
# Read through the envs registry, not os.getenv: the registry owns the
|
||||||
|
# normalization, so the value reported by
|
||||||
|
# `vllm.envs` and the value this check applies cannot drift apart. It is
|
||||||
|
# registered as a non-compile factor there. A set-but-unrecognised value
|
||||||
|
# raises on first read -- fail closed, because a broken gate that silently
|
||||||
|
# disarms its own instrument is the failure mode these checks exist to avoid.
|
||||||
|
_PLE_FLAG_CHECK_MODE: str = envs.VLLM_E2_PLE_FLAG_CHECK
|
||||||
|
# Liveness cadence for the diagnostic checks: the first check and every Nth
|
||||||
|
# are logged with a running count, so "zero findings" can be told apart from
|
||||||
|
# "never ran" in a reference-host log.
|
||||||
|
_E2_CHECK_HEARTBEAT = 4096
|
||||||
|
|
||||||
|
# CUDA registration alone does not own Python storage. Captured graphs retain
|
||||||
|
# raw addresses, so every successfully registered host allocation remains
|
||||||
|
# strongly referenced until process exit.
|
||||||
|
_PROCESS_LIFETIME_REGISTERED_STORAGE: list[torch.Tensor] = []
|
||||||
|
|
||||||
|
|
||||||
|
def _unlink_filename_shared_storage(
|
||||||
|
tensor: torch.Tensor,
|
||||||
|
reserved_filenames: set[bytes],
|
||||||
|
) -> None:
|
||||||
|
"""Reserve and unlink one already filename-shared CPU storage."""
|
||||||
|
try:
|
||||||
|
storage = tensor.untyped_storage()
|
||||||
|
if storage.device.type != "cpu" or not storage.is_shared():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
storage._get_shared_fd()
|
||||||
|
except RuntimeError:
|
||||||
|
# Filename-shared storages do not expose a shared fd. Only now is
|
||||||
|
# it safe to query the filename: doing so on ordinary or fd-shared
|
||||||
|
# storage would convert it to a new filename-backed allocation.
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
_, filename, _ = storage._share_filename_cpu_()
|
||||||
|
if not isinstance(filename, bytes):
|
||||||
|
filename = os.fsencode(filename)
|
||||||
|
path = os.path.join("/dev/shm", os.fsdecode(filename).lstrip("/"))
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to inspect a PLE shared-memory storage during teardown",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# RefcountedMapAllocator unlinks the name when its shared counter reaches
|
||||||
|
# zero. Reserve one count before manual unlink so later storage destruction
|
||||||
|
# cannot try to unlink the already-removed name. The filename identifies
|
||||||
|
# the shared counter and also deduplicates tensor aliases and repeated close.
|
||||||
|
if filename in reserved_filenames:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
storage._shared_incref()
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to reserve the PLE shared-memory counter for %s",
|
||||||
|
path,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
reserved_filenames.add(filename)
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.unlink(path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
# Another teardown participant already removed the name. Retain the
|
||||||
|
# reservation because this allocator must still avoid a later unlink.
|
||||||
|
pass
|
||||||
|
except OSError as error:
|
||||||
|
# The name remains available, so restore the allocator's original
|
||||||
|
# counter and permit a later release attempt to retry.
|
||||||
|
try:
|
||||||
|
storage._shared_decref()
|
||||||
|
except Exception:
|
||||||
|
# The reservation state is uncertain after a failed private call.
|
||||||
|
# Keep it marked to avoid adding another reservation on retry.
|
||||||
|
logger.warning(
|
||||||
|
"Failed to roll back the PLE shared-memory counter for %s",
|
||||||
|
path,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
reserved_filenames.discard(filename)
|
||||||
|
logger.warning("Failed to unlink PLE shared memory %s: %s", path, error)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _PendingPleOffloadRequest:
|
||||||
|
"""Bind request metadata to its MRV2 D2H completion event."""
|
||||||
|
|
||||||
|
request: PleOffloadRequest
|
||||||
|
d2h_done_event: torch.cuda.Event
|
||||||
|
eligible: bool = False
|
||||||
|
transaction_id: str | None = None
|
||||||
|
correlation_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _cuda_check(result: Any, operation: str) -> Any:
|
||||||
|
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
|
||||||
|
error = result[0] if isinstance(result, tuple) else result
|
||||||
|
if error.value != 0:
|
||||||
|
raise RuntimeError(f"{operation} failed: {error}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class PleOffloadConnector:
|
||||||
|
"""Connect a model runner V2 GPU worker to the PLE CPU worker."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
model: nn.Module,
|
||||||
|
device: torch.device,
|
||||||
|
ipc_addr: str,
|
||||||
|
*,
|
||||||
|
input_ids_source: torch.Tensor,
|
||||||
|
query_start_loc_source: torch.Tensor,
|
||||||
|
ngram_context_source: torch.Tensor | None,
|
||||||
|
) -> None:
|
||||||
|
self.device = device
|
||||||
|
self.rank = int(vllm_config.parallel_config.rank)
|
||||||
|
self.dp_rank = get_dp_group().rank_in_group
|
||||||
|
self.tp_rank = get_tp_group().rank_in_group
|
||||||
|
self.pp_rank = get_pp_group().rank_in_group
|
||||||
|
# Host-mapped pull design: per layer a shared host output buffer and a
|
||||||
|
# shared host int32 flag (element 0), filled/stored by the offload
|
||||||
|
# process with plain CPU writes. Page-locked + device-mapped here only
|
||||||
|
# after registration, because serialising for registration may move
|
||||||
|
# the shared storage to a new address.
|
||||||
|
self._host_output_bufs: dict[str, torch.Tensor] = {}
|
||||||
|
self._host_flags: dict[str, torch.Tensor] = {}
|
||||||
|
self._reserved_host_storage_filenames: set[bytes] = set()
|
||||||
|
self._gpu_output_bufs: dict[str, torch.Tensor] = {}
|
||||||
|
self._layers = self._setup_layers(vllm_config, model)
|
||||||
|
self._progress = PleProgressMonitor.from_environment(
|
||||||
|
rank=self.rank,
|
||||||
|
layer_names=tuple(self._layers),
|
||||||
|
)
|
||||||
|
|
||||||
|
# TP0 registers shared buffers with CUDA for asynchronous D2H copies.
|
||||||
|
scheduler_config = vllm_config.scheduler_config
|
||||||
|
self._input_ids_buf = torch.empty(
|
||||||
|
scheduler_config.max_num_batched_tokens,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device="cpu",
|
||||||
|
).share_memory_()
|
||||||
|
self._query_start_loc_buf = torch.empty(
|
||||||
|
scheduler_config.max_num_seqs + 1,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device="cpu",
|
||||||
|
).share_memory_()
|
||||||
|
self._ngram_context_buf = None
|
||||||
|
config = vllm_config.model_config.hf_text_config
|
||||||
|
ngram_context_len = int(config.ngram_size) - 1
|
||||||
|
if ngram_context_len > 0:
|
||||||
|
self._ngram_context_buf = torch.empty(
|
||||||
|
scheduler_config.max_num_seqs,
|
||||||
|
ngram_context_len,
|
||||||
|
dtype=torch.int32,
|
||||||
|
device="cpu",
|
||||||
|
).share_memory_()
|
||||||
|
|
||||||
|
# Runner input allocations are address-stable, so bind them once and
|
||||||
|
# pass only batch sizes through the per-forward request queue.
|
||||||
|
self._input_ids_source = input_ids_source
|
||||||
|
self._query_start_loc_source = query_start_loc_source
|
||||||
|
self._ngram_context_source = ngram_context_source
|
||||||
|
self._validate_input_sources()
|
||||||
|
|
||||||
|
self._pinned_input_buffers: list[torch.Tensor] = []
|
||||||
|
# Device-mapped host outputs/flags referenced by raw address from
|
||||||
|
# captured graphs: never unregistered while the process lives
|
||||||
|
# (unregistering under a pending graph wait unmaps it).
|
||||||
|
self._mapped_host_buffers: list[torch.Tensor] = []
|
||||||
|
# True while dummy/capture work (seeds, dummy forwards, capture
|
||||||
|
# cleanup resets) has run since the last real forward. Releases in
|
||||||
|
# that state complete synchronously, and the next real forward first
|
||||||
|
# passes a TP barrier, so no rank can publish a real request while
|
||||||
|
# another rank still has a dummy/cleanup reset pending.
|
||||||
|
# In this tree real-path kernel warm-ups run BEFORE
|
||||||
|
# capture and capture adds seeds/resets after them, so this must hold
|
||||||
|
# at every dummy->real transition, not only at the first real one.
|
||||||
|
self._after_dummy = True
|
||||||
|
self._request_queue: queue.Queue[_PendingPleOffloadRequest | None] = (
|
||||||
|
queue.Queue(maxsize=vllm_config.max_concurrent_batches)
|
||||||
|
)
|
||||||
|
self._dropped_events: list[torch.cuda.Event] = []
|
||||||
|
self._request_thread: threading.Thread | None = None
|
||||||
|
self._request_thread_ready = threading.Event()
|
||||||
|
self._zmq_ctx: zmq.Context | None = None
|
||||||
|
self._registration_socket: zmq.Socket | None = None
|
||||||
|
self._d2h_event_pool: queue.Queue[torch.cuda.Event] | None = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
self._zmq_ctx = zmq.Context()
|
||||||
|
self._registration_socket = self._zmq_ctx.socket(zmq.PUSH)
|
||||||
|
self._registration_socket.connect(ipc_addr)
|
||||||
|
payload = self._serialize_registration(vllm_config)
|
||||||
|
|
||||||
|
# Serialising moved the shared storages to their final addresses.
|
||||||
|
# Page-lock/map exactly those, THEN publish, so a received
|
||||||
|
# registration always means "storage usable".
|
||||||
|
with torch.accelerator.device_index(self.device.index):
|
||||||
|
self._map_host_outputs(vllm_config)
|
||||||
|
if self.tp_rank == 0:
|
||||||
|
self._pin_input_buffers()
|
||||||
|
self._send_registration(payload, ipc_addr)
|
||||||
|
|
||||||
|
if self.tp_rank == 0:
|
||||||
|
with torch.accelerator.device_index(self.device.index):
|
||||||
|
self._d2h_event_pool = queue.Queue(
|
||||||
|
maxsize=vllm_config.max_concurrent_batches
|
||||||
|
)
|
||||||
|
for _ in range(vllm_config.max_concurrent_batches):
|
||||||
|
self._d2h_event_pool.put_nowait(torch.cuda.Event())
|
||||||
|
self._start_request_thread(ipc_addr)
|
||||||
|
except Exception:
|
||||||
|
self.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _setup_layers(
|
||||||
|
self,
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
model: nn.Module,
|
||||||
|
) -> dict[str, PleOffloadLayer]:
|
||||||
|
"""Attach output buffers and semaphores to GPU PLE placeholders."""
|
||||||
|
layers = {
|
||||||
|
name: module
|
||||||
|
for name, module in model.named_modules()
|
||||||
|
if isinstance(module, PleOffloadLayer)
|
||||||
|
}
|
||||||
|
if not layers:
|
||||||
|
raise RuntimeError(
|
||||||
|
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no "
|
||||||
|
"PleOffloadLayer"
|
||||||
|
)
|
||||||
|
|
||||||
|
config = vllm_config.model_config.hf_text_config
|
||||||
|
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||||
|
for name, layer in layers.items():
|
||||||
|
if vllm_config.load_config.load_format == "dummy":
|
||||||
|
layer.initialize_dummy_offload_metadata(self.device)
|
||||||
|
out_dim = layer.get_offload_output_dim(int(config.ple_embed_dim))
|
||||||
|
out_dtype = layer.get_offload_output_dtype(vllm_config.model_config.dtype)
|
||||||
|
# The captured forward DMAs the host result into this device
|
||||||
|
# buffer after the host flag is set; consumers read it as before.
|
||||||
|
self._gpu_output_bufs[name] = torch.empty(
|
||||||
|
max_num_tokens, out_dim, dtype=out_dtype, device=self.device
|
||||||
|
)
|
||||||
|
# Zero-initialised so dummy/profile forwards copy finite values.
|
||||||
|
self._host_output_bufs[name] = torch.zeros(
|
||||||
|
max_num_tokens, out_dim, dtype=out_dtype, device="cpu"
|
||||||
|
).share_memory_()
|
||||||
|
# 16 x int32 = one 64-byte line; element 0 is the flag.
|
||||||
|
self._host_flags[name] = torch.zeros(
|
||||||
|
16, dtype=torch.int32, device="cpu"
|
||||||
|
).share_memory_()
|
||||||
|
return layers
|
||||||
|
|
||||||
|
def _map_host_outputs(self, vllm_config: VllmConfig) -> None:
|
||||||
|
"""Page-lock + device-map host outputs/flags and arm the placeholders."""
|
||||||
|
for name, layer in self._layers.items():
|
||||||
|
layer._ple_offload_name = name
|
||||||
|
host_out = self._host_output_bufs[name]
|
||||||
|
host_flag = self._host_flags[name]
|
||||||
|
for buffer in (host_out, host_flag):
|
||||||
|
self._register_host_buffer(buffer, device_map=True)
|
||||||
|
flag_dev_ptr = _cuda_check(
|
||||||
|
cuda_driver.cuMemHostGetDevicePointer(host_flag.data_ptr(), 0),
|
||||||
|
"cuMemHostGetDevicePointer(PLE host flag)",
|
||||||
|
)[1]
|
||||||
|
out_dev_ptr = _cuda_check(
|
||||||
|
cuda_driver.cuMemHostGetDevicePointer(host_out.data_ptr(), 0),
|
||||||
|
"cuMemHostGetDevicePointer(PLE host output)",
|
||||||
|
)[1]
|
||||||
|
row_bytes = host_out.shape[1] * host_out.element_size()
|
||||||
|
gpu_buf = self._gpu_output_bufs[name]
|
||||||
|
if (
|
||||||
|
gpu_buf.shape[0] != host_out.shape[0]
|
||||||
|
or gpu_buf.shape[1] * gpu_buf.element_size() != row_bytes
|
||||||
|
):
|
||||||
|
raise RuntimeError(f"PLE host/GPU buffer geometry mismatch for {name}")
|
||||||
|
# Stream memops use the flag's DEVICE pointer; the H2D copy uses
|
||||||
|
# this worker's HOST pointer (no raw addresses cross
|
||||||
|
# processes; the offload writes through its own mapping).
|
||||||
|
layer.setup_cross_process_offload(
|
||||||
|
gpu_buf,
|
||||||
|
CpuGpuSemaphore.from_host_mapped(host_flag, int(flag_dev_ptr)),
|
||||||
|
host_output_ptr=host_out.data_ptr(),
|
||||||
|
host_row_bytes=row_bytes,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"PleOffload: host-mapped pull for %s (rows %d x %d B, flag dev "
|
||||||
|
"0x%x, out dev 0x%x, host==dev %s)",
|
||||||
|
name,
|
||||||
|
host_out.shape[0],
|
||||||
|
row_bytes,
|
||||||
|
int(flag_dev_ptr),
|
||||||
|
int(out_dev_ptr),
|
||||||
|
int(out_dev_ptr) == host_out.data_ptr(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _pin_input_buffers(self) -> None:
|
||||||
|
"""Page-lock shared input allocations without replacing their storage."""
|
||||||
|
buffers = [self._input_ids_buf, self._query_start_loc_buf]
|
||||||
|
if self._ngram_context_buf is not None:
|
||||||
|
buffers.append(self._ngram_context_buf)
|
||||||
|
for buffer in buffers:
|
||||||
|
self._register_host_buffer(buffer, device_map=False)
|
||||||
|
|
||||||
|
def _register_host_buffer(self, buffer: torch.Tensor, device_map: bool) -> None:
|
||||||
|
"""Page-lock (and optionally device-map) a shared CPU allocation."""
|
||||||
|
if buffer.device.type != "cpu" or not buffer.is_shared():
|
||||||
|
raise RuntimeError("PLE host buffers must be shared CPU tensors")
|
||||||
|
if not buffer.is_contiguous():
|
||||||
|
raise RuntimeError("PLE host buffers must be contiguous")
|
||||||
|
flags = cuda_driver.CU_MEMHOSTREGISTER_PORTABLE
|
||||||
|
if device_map:
|
||||||
|
flags |= cuda_driver.CU_MEMHOSTREGISTER_DEVICEMAP
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuMemHostRegister(
|
||||||
|
buffer.data_ptr(),
|
||||||
|
buffer.numel() * buffer.element_size(),
|
||||||
|
flags,
|
||||||
|
),
|
||||||
|
"cuMemHostRegister(PLE host buffer)",
|
||||||
|
)
|
||||||
|
_PROCESS_LIFETIME_REGISTERED_STORAGE.append(buffer)
|
||||||
|
if device_map:
|
||||||
|
self._mapped_host_buffers.append(buffer)
|
||||||
|
else:
|
||||||
|
self._pinned_input_buffers.append(buffer)
|
||||||
|
if not buffer.is_pinned():
|
||||||
|
raise RuntimeError("CUDA did not page-lock a PLE host buffer")
|
||||||
|
|
||||||
|
def _unpin_input_buffers(self) -> None:
|
||||||
|
"""Release CUDA registrations after the request thread has stopped."""
|
||||||
|
for buffer in reversed(self._pinned_input_buffers):
|
||||||
|
try:
|
||||||
|
_cuda_check(
|
||||||
|
cuda_driver.cuMemHostUnregister(buffer.data_ptr()),
|
||||||
|
"cuMemHostUnregister(PLE input buffer)",
|
||||||
|
)
|
||||||
|
except RuntimeError:
|
||||||
|
logger.exception("Failed to unregister a PLE input buffer")
|
||||||
|
self._pinned_input_buffers.clear()
|
||||||
|
|
||||||
|
def _serialize_registration(self, vllm_config: VllmConfig) -> Any:
|
||||||
|
"""Build and serialise the registration (may move shared storages)."""
|
||||||
|
# Each GPU worker owns distinct host output buffers and flags, while
|
||||||
|
# TP0's shared inputs become the request source for its DP rank.
|
||||||
|
registration = PleOffloadRegistration(
|
||||||
|
worker_id=(
|
||||||
|
self.dp_rank * vllm_config.parallel_config.world_size
|
||||||
|
+ vllm_config.parallel_config.rank
|
||||||
|
),
|
||||||
|
tp_rank=self.tp_rank,
|
||||||
|
dp_rank=self.dp_rank,
|
||||||
|
host_output_buffers=dict(self._host_output_bufs),
|
||||||
|
host_flag_tensors=dict(self._host_flags),
|
||||||
|
input_ids_buf=self._input_ids_buf,
|
||||||
|
query_start_loc_buf=self._query_start_loc_buf,
|
||||||
|
ngram_context_buf=self._ngram_context_buf,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ForkingPickler transmits tensors through shared-memory and CUDA IPC.
|
||||||
|
import torch.multiprocessing as torch_mp
|
||||||
|
|
||||||
|
original_strategy = torch_mp.get_sharing_strategy()
|
||||||
|
torch_mp.set_sharing_strategy("file_system")
|
||||||
|
try:
|
||||||
|
payload = ForkingPickler.dumps(registration)
|
||||||
|
finally:
|
||||||
|
torch_mp.set_sharing_strategy(original_strategy)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
def _send_registration(self, payload: Any, ipc_addr: str) -> None:
|
||||||
|
"""Publish the serialised registration (storages already mapped)."""
|
||||||
|
assert self._registration_socket is not None
|
||||||
|
self._registration_socket.send(payload)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"PleOffload: registered %d PleOffloadLayer(s) "
|
||||||
|
"(dp_rank=%d, tp_rank=%d, ipc_addr=%s): %s",
|
||||||
|
len(self._layers),
|
||||||
|
self.dp_rank,
|
||||||
|
self.tp_rank,
|
||||||
|
ipc_addr,
|
||||||
|
sorted(self._layers),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _start_request_thread(self, ipc_addr: str) -> None:
|
||||||
|
"""Start the thread that publishes batches after inputs are ready."""
|
||||||
|
self._request_thread = threading.Thread(
|
||||||
|
target=self._request_loop,
|
||||||
|
args=(ipc_addr,),
|
||||||
|
name=f"ple-offload-dp{self.dp_rank}",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
self._request_thread.start()
|
||||||
|
if not self._request_thread_ready.wait(timeout=10):
|
||||||
|
raise RuntimeError("Timed out starting the PLE request thread")
|
||||||
|
|
||||||
|
def _request_loop(self, ipc_addr: str) -> None:
|
||||||
|
"""Wait for staged inputs, then notify the CPU worker."""
|
||||||
|
socket: zmq.Socket | None = None
|
||||||
|
try:
|
||||||
|
if self._zmq_ctx is None:
|
||||||
|
raise RuntimeError("PLE ZMQ context closed before thread startup")
|
||||||
|
socket = self._zmq_ctx.socket(zmq.PUSH)
|
||||||
|
socket.connect(ipc_addr)
|
||||||
|
self._request_thread_ready.set()
|
||||||
|
while True:
|
||||||
|
request = self._request_queue.get()
|
||||||
|
if request is None:
|
||||||
|
return
|
||||||
|
self._process_request(request, socket)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("PLE request thread failed")
|
||||||
|
notify_parent_fatal()
|
||||||
|
os._exit(1)
|
||||||
|
finally:
|
||||||
|
self._request_thread_ready.set()
|
||||||
|
if socket is not None:
|
||||||
|
socket.close(linger=0)
|
||||||
|
|
||||||
|
def _process_request(
|
||||||
|
self, pending: _PendingPleOffloadRequest, socket: zmq.Socket
|
||||||
|
) -> None:
|
||||||
|
"""Wait for one staged batch and publish its request."""
|
||||||
|
request = pending.request
|
||||||
|
event_pool = self._d2h_event_pool
|
||||||
|
event = pending.d2h_done_event
|
||||||
|
assert event_pool is not None, "PLE D2H event pool is not initialized"
|
||||||
|
if _FAULT is not None and _FAULT.fires(
|
||||||
|
"block_input_completion",
|
||||||
|
role="gpu",
|
||||||
|
unit="connector-request",
|
||||||
|
eligible=pending.eligible,
|
||||||
|
tp_rank=self.tp_rank,
|
||||||
|
pp_rank=self.pp_rank,
|
||||||
|
dp_rank=self.dp_rank,
|
||||||
|
transaction_id=pending.transaction_id,
|
||||||
|
correlation_id=pending.correlation_id,
|
||||||
|
):
|
||||||
|
threading.Event().wait()
|
||||||
|
|
||||||
|
with torch.accelerator.device_index(self.device.index):
|
||||||
|
event.synchronize()
|
||||||
|
|
||||||
|
payload = msgspec.msgpack.encode(request)
|
||||||
|
if _FAULT is None:
|
||||||
|
socket.send(payload)
|
||||||
|
else:
|
||||||
|
# Armed runs only: correlation metadata rides as a second frame.
|
||||||
|
metadata = msgspec.msgpack.encode(
|
||||||
|
{
|
||||||
|
"serving": pending.eligible,
|
||||||
|
"transaction": pending.transaction_id,
|
||||||
|
"correlation": pending.correlation_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
socket.send_multipart([payload, metadata])
|
||||||
|
event_pool.put_nowait(event)
|
||||||
|
|
||||||
|
def _validate_input_sources(self) -> None:
|
||||||
|
"""Validate fixed runner sources against shared input buffers."""
|
||||||
|
sources = [
|
||||||
|
("input_ids", self._input_ids_source, self._input_ids_buf),
|
||||||
|
(
|
||||||
|
"query_start_loc",
|
||||||
|
self._query_start_loc_source,
|
||||||
|
self._query_start_loc_buf,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
if (self._ngram_context_source is None) != (self._ngram_context_buf is None):
|
||||||
|
raise ValueError("PLE ngram_context source and buffer must match")
|
||||||
|
if self._ngram_context_source is not None:
|
||||||
|
assert self._ngram_context_buf is not None
|
||||||
|
sources.append(
|
||||||
|
(
|
||||||
|
"ngram_context",
|
||||||
|
self._ngram_context_source,
|
||||||
|
self._ngram_context_buf,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for name, source, buffer in sources:
|
||||||
|
if (
|
||||||
|
source.device != self.device
|
||||||
|
or source.dtype != buffer.dtype
|
||||||
|
or source.ndim != buffer.ndim
|
||||||
|
or source.shape[0] < buffer.shape[0]
|
||||||
|
or source.shape[1:] != buffer.shape[1:]
|
||||||
|
):
|
||||||
|
raise ValueError(f"PLE {name} source is incompatible")
|
||||||
|
|
||||||
|
def _enqueue_cuda_inputs(
|
||||||
|
self,
|
||||||
|
request: PleOffloadRequest,
|
||||||
|
d2h_done_event: torch.cuda.Event,
|
||||||
|
) -> None:
|
||||||
|
"""Stage inputs on the model stream and record D2H completion."""
|
||||||
|
with (
|
||||||
|
record_function_or_nullcontext("ple_offload: enqueue_cuda_inputs"),
|
||||||
|
torch.accelerator.device_index(self.device.index),
|
||||||
|
):
|
||||||
|
stream = torch.cuda.current_stream(self.device)
|
||||||
|
self._input_ids_buf[: request.num_tokens].copy_(
|
||||||
|
self._input_ids_source[: request.num_tokens],
|
||||||
|
non_blocking=True,
|
||||||
|
)
|
||||||
|
self._query_start_loc_buf[: request.num_reqs + 1].copy_(
|
||||||
|
self._query_start_loc_source[: request.num_reqs + 1],
|
||||||
|
non_blocking=True,
|
||||||
|
)
|
||||||
|
if self._ngram_context_buf is not None:
|
||||||
|
assert self._ngram_context_source is not None
|
||||||
|
self._ngram_context_buf[: request.num_reqs].copy_(
|
||||||
|
self._ngram_context_source[: request.num_reqs],
|
||||||
|
non_blocking=True,
|
||||||
|
)
|
||||||
|
d2h_done_event.record(stream)
|
||||||
|
|
||||||
|
def _launch(
|
||||||
|
self,
|
||||||
|
num_reqs: int,
|
||||||
|
num_tokens: int,
|
||||||
|
*,
|
||||||
|
serving: bool = False,
|
||||||
|
transaction_id: str | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Stage or queue one batch for request publication."""
|
||||||
|
# Inputs are replicated across TP ranks. One request per DP rank drives
|
||||||
|
# the CPU result fan-out to every registered TP output buffer.
|
||||||
|
if self.tp_rank != 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
request = PleOffloadRequest(
|
||||||
|
dp_rank=self.dp_rank,
|
||||||
|
num_tokens=num_tokens,
|
||||||
|
num_reqs=num_reqs,
|
||||||
|
)
|
||||||
|
assert self._d2h_event_pool is not None, "PLE D2H event pool is not initialized"
|
||||||
|
try:
|
||||||
|
d2h_done_event = self._d2h_event_pool.get_nowait()
|
||||||
|
except queue.Empty as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE has more requests than configured concurrent batches"
|
||||||
|
) from exc
|
||||||
|
self._enqueue_cuda_inputs(request, d2h_done_event)
|
||||||
|
if _FAULT is not None and _FAULT.fires(
|
||||||
|
"drop_request",
|
||||||
|
role="gpu",
|
||||||
|
unit="connector-request",
|
||||||
|
eligible=serving,
|
||||||
|
tp_rank=self.tp_rank,
|
||||||
|
pp_rank=self.pp_rank,
|
||||||
|
dp_rank=self.dp_rank,
|
||||||
|
transaction_id=transaction_id,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
):
|
||||||
|
# Deliberately retain the event: the fault is loss after D2H, not
|
||||||
|
# premature event-pool reuse while that D2H may still be in flight.
|
||||||
|
self._dropped_events.append(d2h_done_event)
|
||||||
|
return
|
||||||
|
self._request_queue.put_nowait(
|
||||||
|
_PendingPleOffloadRequest(
|
||||||
|
request,
|
||||||
|
d2h_done_event,
|
||||||
|
eligible=serving,
|
||||||
|
transaction_id=transaction_id,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _fail_closed(self, where: str, error: BaseException) -> None:
|
||||||
|
ple_fail_closed(
|
||||||
|
where=where,
|
||||||
|
rank=self.rank,
|
||||||
|
layer=",".join(self._layers),
|
||||||
|
sequence=self._progress.sequence,
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_pending_completions(self) -> bool:
|
||||||
|
return self._progress.has_pending_completions
|
||||||
|
|
||||||
|
def begin_forward(self, *, dummy_run: bool, num_tokens: int) -> int:
|
||||||
|
"""Begin the bounded transaction before any runner fault boundary."""
|
||||||
|
try:
|
||||||
|
return self._progress.begin(
|
||||||
|
startup=dummy_run,
|
||||||
|
num_tokens=num_tokens,
|
||||||
|
)
|
||||||
|
except Exception as error:
|
||||||
|
self._fail_closed("prepare_forward", error)
|
||||||
|
|
||||||
|
def fault_transaction(self, sequence: int) -> str:
|
||||||
|
"""Read diagnostic state only when the caller has established arming."""
|
||||||
|
return self._progress.describe_submission(sequence)
|
||||||
|
|
||||||
|
def completion_is_pending(self, sequence: int) -> bool:
|
||||||
|
"""Return exact-sequence incomplete completion state for cut 6."""
|
||||||
|
return self._progress.completion_is_pending(sequence)
|
||||||
|
|
||||||
|
def prepare_forward(
|
||||||
|
self,
|
||||||
|
num_reqs: int,
|
||||||
|
num_tokens: int,
|
||||||
|
dummy_run: bool,
|
||||||
|
*,
|
||||||
|
consumer_num_tokens: int | None = None,
|
||||||
|
serving: bool = False,
|
||||||
|
transaction_id: str | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Prepare one PLE transaction; any ordinary exception is fatal."""
|
||||||
|
try:
|
||||||
|
self._prepare_forward_impl(
|
||||||
|
num_reqs,
|
||||||
|
num_tokens,
|
||||||
|
dummy_run,
|
||||||
|
consumer_num_tokens=consumer_num_tokens,
|
||||||
|
serving=serving,
|
||||||
|
transaction_id=transaction_id,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
)
|
||||||
|
except Exception as error:
|
||||||
|
self._fail_closed("prepare_forward", error)
|
||||||
|
|
||||||
|
def _prepare_forward_impl(
|
||||||
|
self,
|
||||||
|
num_reqs: int,
|
||||||
|
num_tokens: int,
|
||||||
|
dummy_run: bool,
|
||||||
|
*,
|
||||||
|
consumer_num_tokens: int | None = None,
|
||||||
|
serving: bool = False,
|
||||||
|
transaction_id: str | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Submit real inputs or satisfy the PLE wait for a dummy forward."""
|
||||||
|
if consumer_num_tokens is None:
|
||||||
|
consumer_num_tokens = num_tokens
|
||||||
|
if int(num_tokens) != int(consumer_num_tokens):
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE request/consumer token-count mismatch: "
|
||||||
|
f"request={num_tokens}, consumer={consumer_num_tokens}"
|
||||||
|
)
|
||||||
|
if not 0 <= int(num_tokens) <= self._input_ids_buf.shape[0]:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE token count {num_tokens} exceeds connector capacity "
|
||||||
|
f"{self._input_ids_buf.shape[0]}"
|
||||||
|
)
|
||||||
|
if not 0 <= int(num_reqs) < self._query_start_loc_buf.shape[0]:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE request count {num_reqs} exceeds connector capacity "
|
||||||
|
f"{self._query_start_loc_buf.shape[0] - 1}"
|
||||||
|
)
|
||||||
|
if dummy_run:
|
||||||
|
self._signal_dummy_outputs_impl(num_tokens)
|
||||||
|
return
|
||||||
|
if self._after_dummy:
|
||||||
|
# Every dummy/cleanup reset on THIS rank already completed
|
||||||
|
# (synchronous releases); the barrier extends that to every
|
||||||
|
# stage-0 TP rank before TP0 can publish (CPU group, no GPU work;
|
||||||
|
# all TP ranks run the same forwards in the same order).
|
||||||
|
get_tp_group().barrier()
|
||||||
|
self._after_dummy = False
|
||||||
|
logger.info(
|
||||||
|
"PleOffload: dummy->real transition (dp_rank=%d, tp_rank=%d): "
|
||||||
|
"releases were synchronous, TP barrier passed",
|
||||||
|
self.dp_rank,
|
||||||
|
self.tp_rank,
|
||||||
|
)
|
||||||
|
if _PLE_FLAG_CHECK_MODE != "off":
|
||||||
|
self._check_flags_clear_before_launch(num_tokens)
|
||||||
|
self._launch(
|
||||||
|
num_reqs,
|
||||||
|
num_tokens,
|
||||||
|
serving=serving,
|
||||||
|
transaction_id=transaction_id,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _check_flags_clear_before_launch(self, num_tokens: int) -> None:
|
||||||
|
"""Test the transport's staleness precondition, off by default.
|
||||||
|
|
||||||
|
A real launch publishes rows that the forward will pick up with
|
||||||
|
``cuStreamWaitValue32(flag == DONE_VALUE)``
|
||||||
|
(``ple_offload_layer.py:279``). That wait is only meaningful if the
|
||||||
|
flag is CLEAR when we publish: a leftover ``DONE_VALUE`` satisfies it
|
||||||
|
from the PREVIOUS step's copy and the forward silently consumes the
|
||||||
|
previous step's PLE outputs -- vllm#53899, where the semaphore stayed
|
||||||
|
one step ahead for the life of the server and ``cudagraph_mode=NONE``
|
||||||
|
was the only correct configuration.
|
||||||
|
|
||||||
|
Nothing here can be inferred from ``_after_dummy``: that is a bool
|
||||||
|
tracking a transition, not the flag's value. The structural argument
|
||||||
|
(dummy signal consumed by the dummy forward, ``_release_outputs_impl``
|
||||||
|
resetting every layer and synchronizing while ``_after_dummy`` is set,
|
||||||
|
a TP barrier at the transition) holds only while every
|
||||||
|
``prepare_forward`` is matched by a ``release_outputs``. An exception
|
||||||
|
path, an aborted step, or a future capture that signals without a
|
||||||
|
consuming forward leaves the flag at 1, and the #53899 thread's own
|
||||||
|
trap applies: identical sequential requests produce two bit-identical
|
||||||
|
classes, so a "same prompt twice" check passes in the broken world
|
||||||
|
(an instrument must discriminate both worlds).
|
||||||
|
|
||||||
|
PUBLISHING RANK ONLY. ``flag == 0`` before a launch is
|
||||||
|
NOT an invariant on a non-publishing rank, and the first version of
|
||||||
|
this check could terminate a HEALTHY TP2 run because of it. ``_launch``
|
||||||
|
publishes only on TP0 (``if self.tp_rank != 0: return``), and the
|
||||||
|
offload worker then fans the result out to EVERY registered target,
|
||||||
|
each gated only on that target's own flag having been reset
|
||||||
|
(``worker.py``, ``for target in targets: _wait_flag_released(target)
|
||||||
|
... target.host_flag[0] = 1``). So this legal interleaving exists:
|
||||||
|
|
||||||
|
1. TP0 checks its flag (clear), publishes step N.
|
||||||
|
2. The worker fills TP1's buffer and sets TP1's flag for step N.
|
||||||
|
3. TP1 reaches its own check for step N and reads ``DONE``.
|
||||||
|
4. TP1 reports a stale signal -- for a signal that belongs to the
|
||||||
|
CURRENT step.
|
||||||
|
|
||||||
|
The dummy->real TP barrier is before the checks and does not order this.
|
||||||
|
On TP0 the property does hold, and by construction rather than by
|
||||||
|
timing: the worker learns of a request only from ``_launch``, which is
|
||||||
|
after this check, and ``PleProgressMonitor.begin`` refuses overlapping
|
||||||
|
serving submissions ("Overlapping PLE host submissions violate the
|
||||||
|
serialized single-slot protocol"), so at most one transaction is
|
||||||
|
outstanding and the stream synchronize below has already retired its
|
||||||
|
release. A non-publishing rank cannot distinguish "this step's fan-out
|
||||||
|
arrived early" from "last step's signal was never consumed" with a
|
||||||
|
BINARY flag -- that is the same representational limit that rules out
|
||||||
|
an epoch word (see ``ple_offload_layer.py``), so those ranks are
|
||||||
|
skipped rather than checked weakly.
|
||||||
|
|
||||||
|
VLLM_E2_PLE_FLAG_CHECK=off|warn|raise, default off, default ACTION warn.
|
||||||
|
|
||||||
|
SCOPE. The skip below tests whether a capture
|
||||||
|
is ACTIVE, which is not the same as "this step's forward is eager". A
|
||||||
|
captured replay is not an active capture, and ``prepare_forward`` is
|
||||||
|
host Python that runs ahead of the forward either way, so this check
|
||||||
|
DOES run for steps whose forward is a graph replay. What it does not do
|
||||||
|
is run *inside* a capture, where a host synchronize is illegal. The
|
||||||
|
earlier "eager only" wording confused those two lifecycle states.
|
||||||
|
|
||||||
|
LIVENESS (PLE diagnostics on 0.30). A zero-findings result is only evidence if the
|
||||||
|
check demonstrably RAN, so every rank says once what it does with the
|
||||||
|
check, and the publishing rank logs a running count of checks and
|
||||||
|
findings on its first check and every ``_E2_CHECK_HEARTBEAT`` checks
|
||||||
|
after. A reference-host test arm greps these lines; their absence voids the arm.
|
||||||
|
"""
|
||||||
|
if not flag_check_is_authoritative(self.tp_rank):
|
||||||
|
if not getattr(self, "_flag_check_skip_logged", False):
|
||||||
|
self._flag_check_skip_logged = True
|
||||||
|
logger.info(
|
||||||
|
"PLE flag check (mode=%s): non-publishing rank, not judged "
|
||||||
|
"here by design (dp_rank=%d, tp_rank=%d, pp_rank=%d)",
|
||||||
|
_PLE_FLAG_CHECK_MODE,
|
||||||
|
self.dp_rank,
|
||||||
|
self.tp_rank,
|
||||||
|
self.pp_rank,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if torch.cuda.is_current_stream_capturing():
|
||||||
|
return
|
||||||
|
stream = torch.cuda.current_stream(self.device)
|
||||||
|
# Order the read against any reset still queued on the model stream.
|
||||||
|
# Without this the check reports the pre-reset value and fires on a
|
||||||
|
# correct run -- a guard that fires on correct metadata is worse than
|
||||||
|
# no guard.
|
||||||
|
stream.synchronize()
|
||||||
|
stale = stale_offload_flags(self._layers)
|
||||||
|
runs = self._flag_checks_run = getattr(self, "_flag_checks_run", 0) + 1
|
||||||
|
findings = getattr(self, "_flag_check_findings", 0) + (1 if stale else 0)
|
||||||
|
self._flag_check_findings = findings
|
||||||
|
if runs == 1 or runs % _E2_CHECK_HEARTBEAT == 0:
|
||||||
|
logger.info(
|
||||||
|
"PLE flag check (mode=%s): %d checks run, %d with findings "
|
||||||
|
"(dp_rank=%d, tp_rank=%d, pp_rank=%d)",
|
||||||
|
_PLE_FLAG_CHECK_MODE,
|
||||||
|
runs,
|
||||||
|
findings,
|
||||||
|
self.dp_rank,
|
||||||
|
self.tp_rank,
|
||||||
|
self.pp_rank,
|
||||||
|
)
|
||||||
|
if not stale:
|
||||||
|
return
|
||||||
|
message = (
|
||||||
|
"PLE pull transport: the output semaphore was already signalled "
|
||||||
|
"before this publishing rank published a real request, so the "
|
||||||
|
"forward's cuStreamWaitValue32(flag == 1) would be satisfied by "
|
||||||
|
"the PREVIOUS step's copy and this forward would consume the "
|
||||||
|
"previous step's PLE rows (vllm#53899 one-step-lag class). Stale "
|
||||||
|
f"layers (name, flag): {stale}; expected "
|
||||||
|
f"{CpuGpuSemaphore.RESET_VALUE} for all {len(self._layers)} "
|
||||||
|
f"layer(s). num_tokens={num_tokens}, dp_rank={self.dp_rank}, "
|
||||||
|
f"tp_rank={self.tp_rank}, pp_rank={self.pp_rank}."
|
||||||
|
)
|
||||||
|
if _PLE_FLAG_CHECK_MODE == "raise":
|
||||||
|
# Reaches _fail_closed through prepare_forward, which is the
|
||||||
|
# transport's stated contract for an uncertain correspondence.
|
||||||
|
raise RuntimeError(message)
|
||||||
|
logger.warning("%s", message)
|
||||||
|
|
||||||
|
def signal_dummy_outputs(self, num_tokens: int) -> None:
|
||||||
|
"""Locally satisfy PLE waits for dummy and capture forwards."""
|
||||||
|
self._progress.begin(startup=True, num_tokens=num_tokens)
|
||||||
|
self._signal_dummy_outputs_impl(num_tokens)
|
||||||
|
|
||||||
|
def _signal_dummy_outputs_impl(self, num_tokens: int) -> None:
|
||||||
|
"""Signal dummy outputs after the progress transaction is armed."""
|
||||||
|
# Dummy and capture forwards do not send CPU requests, but every PLE
|
||||||
|
# placeholder still waits for a completed output semaphore.
|
||||||
|
self._after_dummy = True
|
||||||
|
stream = torch.cuda.current_stream(self.device)
|
||||||
|
for layer in self._layers.values():
|
||||||
|
layer._gpu_output_buffer[:num_tokens].zero_()
|
||||||
|
layer._sem.signal(stream)
|
||||||
|
|
||||||
|
def release_outputs(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
serving: bool = False,
|
||||||
|
transaction_id: str | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Release one PLE transaction; any ordinary exception is fatal."""
|
||||||
|
try:
|
||||||
|
self._release_outputs_impl(
|
||||||
|
serving=serving,
|
||||||
|
transaction_id=transaction_id,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
)
|
||||||
|
self._progress.complete()
|
||||||
|
except Exception as error:
|
||||||
|
self._fail_closed("release_outputs", error)
|
||||||
|
|
||||||
|
def _release_outputs_impl(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
serving: bool = False,
|
||||||
|
transaction_id: str | None = None,
|
||||||
|
correlation_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Mark GPU output buffers reusable after the model consumes them."""
|
||||||
|
if _FAULT is not None and _FAULT.fires(
|
||||||
|
"exc_in_release",
|
||||||
|
role="gpu",
|
||||||
|
unit="connector-release",
|
||||||
|
eligible=serving,
|
||||||
|
tp_rank=self.tp_rank,
|
||||||
|
pp_rank=self.pp_rank,
|
||||||
|
dp_rank=self.dp_rank,
|
||||||
|
transaction_id=transaction_id,
|
||||||
|
correlation_id=correlation_id,
|
||||||
|
):
|
||||||
|
raise RuntimeError("PLE test fault: exc_in_release")
|
||||||
|
# Reset only after the consumer forward so the CPU worker cannot
|
||||||
|
# overwrite an output that a GPU PLE placeholder may still read.
|
||||||
|
stream = torch.cuda.current_stream(self.device)
|
||||||
|
for layer in self._layers.values():
|
||||||
|
layer.release_offloaded_output(stream)
|
||||||
|
# Retain the serving deadline until the model stream has actually
|
||||||
|
# executed every reset above. The monitor thread is the sole query site.
|
||||||
|
completion_event = torch.cuda.Event()
|
||||||
|
completion_event.record(stream)
|
||||||
|
self._progress.track_completion(completion_event)
|
||||||
|
# After dummy/capture work (profile, warm-up sizes, capture seed,
|
||||||
|
# adaptive-verification dummy replays, capture cleanup): complete the
|
||||||
|
# reset before returning, so it cannot still be pending when a real
|
||||||
|
# request lets the offload publish. Real-forward releases stay async.
|
||||||
|
# Never inside a stream capture.
|
||||||
|
if self._after_dummy and not torch.cuda.is_current_stream_capturing():
|
||||||
|
stream.synchronize()
|
||||||
|
|
||||||
|
def release_host_storage(self) -> None:
|
||||||
|
"""Unlink host-storage names while retaining mappings and tensors."""
|
||||||
|
buffers = [
|
||||||
|
*self._host_output_bufs.values(),
|
||||||
|
*self._host_flags.values(),
|
||||||
|
self._input_ids_buf,
|
||||||
|
self._query_start_loc_buf,
|
||||||
|
]
|
||||||
|
if self._ngram_context_buf is not None:
|
||||||
|
buffers.append(self._ngram_context_buf)
|
||||||
|
for buffer in buffers:
|
||||||
|
_unlink_filename_shared_storage(
|
||||||
|
buffer,
|
||||||
|
self._reserved_host_storage_filenames,
|
||||||
|
)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Stop transport and unlink names, retaining host mappings."""
|
||||||
|
request_thread = self._request_thread
|
||||||
|
if request_thread is not None and request_thread.is_alive():
|
||||||
|
try:
|
||||||
|
self._request_queue.put(None, timeout=5)
|
||||||
|
except queue.Full:
|
||||||
|
logger.error("Timed out stopping the PLE request thread")
|
||||||
|
request_thread.join(timeout=5)
|
||||||
|
if request_thread is not None and request_thread.is_alive():
|
||||||
|
# The thread may still access the registered buffers or ZMQ context.
|
||||||
|
# It may also still race another process opening a shared name, so
|
||||||
|
# this deferred-cleanup branch must not unlink host storage.
|
||||||
|
logger.error("PLE request thread did not stop; deferring resource cleanup")
|
||||||
|
self._progress.close()
|
||||||
|
return
|
||||||
|
self._request_thread = None
|
||||||
|
|
||||||
|
# Inputs, outputs, and flags remain registered and strongly referenced.
|
||||||
|
# A captured graph or delayed request can still contain their addresses;
|
||||||
|
# the CUDA driver releases registrations when the process exits.
|
||||||
|
registered = len(self._pinned_input_buffers) + len(self._mapped_host_buffers)
|
||||||
|
if registered:
|
||||||
|
logger.debug(
|
||||||
|
"PleOffload: %d registered host buffers retained until exit",
|
||||||
|
registered,
|
||||||
|
)
|
||||||
|
self._d2h_event_pool = None
|
||||||
|
if self._registration_socket is not None:
|
||||||
|
self._registration_socket.close(linger=0)
|
||||||
|
self._registration_socket = None
|
||||||
|
if self._zmq_ctx is not None:
|
||||||
|
self._zmq_ctx.term()
|
||||||
|
self._zmq_ctx = None
|
||||||
|
self._progress.close()
|
||||||
|
|
||||||
|
# __init__'s failure handler can reach here before the child has rebuilt
|
||||||
|
# a registration. Startup is already failing in that case; if a payload
|
||||||
|
# was sent, a late rebuild may also observe the removed name. Successful
|
||||||
|
# startup accepts every registration before an external close begins.
|
||||||
|
#
|
||||||
|
# NOTE: Graceful shutdown unlinks the PLE shared-memory names and the
|
||||||
|
# PLE ipc socket. The remaining 17 /tmp UUID sockets per boot are
|
||||||
|
# upstream vLLM VLLM_RPC_BASE_PATH behaviour; a tmpfiles/launch-profile
|
||||||
|
# sweep is recommended.
|
||||||
|
# Keep this last: unlinking preserves every existing CPU/CUDA mapping,
|
||||||
|
# but no transport user may still need to open a name at this point.
|
||||||
|
self.release_host_storage()
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Fail-closed helpers for the PLE single-slot transport."""
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
|
from types import TracebackType
|
||||||
|
from typing import Any, NoReturn, TypeVar
|
||||||
|
|
||||||
|
_F = TypeVar("_F", bound=Callable[..., Any])
|
||||||
|
|
||||||
|
_TRUSTED_ERROR_LABELS = (
|
||||||
|
(RuntimeError, "RuntimeError"),
|
||||||
|
(TimeoutError, "TimeoutError"),
|
||||||
|
(ValueError, "ValueError"),
|
||||||
|
(OSError, "OSError"),
|
||||||
|
(EOFError, "EOFError"),
|
||||||
|
(BrokenPipeError, "BrokenPipeError"),
|
||||||
|
(AssertionError, "AssertionError"),
|
||||||
|
(KeyError, "KeyError"),
|
||||||
|
(IndexError, "IndexError"),
|
||||||
|
(TypeError, "TypeError"),
|
||||||
|
(MemoryError, "MemoryError"),
|
||||||
|
(KeyboardInterrupt, "KeyboardInterrupt"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Slot readers for the dying path. Each is the C getter of the base type,
|
||||||
|
# called directly, so no __getattribute__, property, metaclass attribute or
|
||||||
|
# descriptor defined by the exception (or its class) can run. The traceback
|
||||||
|
# module is NOT safe here: traceback.format_exception calls str(error) (and
|
||||||
|
# str() on every chained exception) and reads error.__traceback__ through the
|
||||||
|
# instance's own __getattribute__, so an exception whose __str__ or
|
||||||
|
# __getattribute__ blocks kept the process alive forever -- the fail-closed
|
||||||
|
# path hung instead of dying (found by the blocking-exception transport tests).
|
||||||
|
_EXC_ARGS = BaseException.args.__get__
|
||||||
|
_EXC_TRACEBACK = BaseException.__traceback__.__get__
|
||||||
|
_EXC_CAUSE = BaseException.__cause__.__get__
|
||||||
|
_EXC_CONTEXT = BaseException.__context__.__get__
|
||||||
|
_EXC_SUPPRESS_CONTEXT = BaseException.__suppress_context__.__get__
|
||||||
|
_TYPE_QUALNAME = type.__dict__["__qualname__"].__get__
|
||||||
|
_TYPE_MODULE = type.__dict__["__module__"].__get__
|
||||||
|
_MAX_CHAIN = 16
|
||||||
|
_MAX_ARG_CHARS = 1024
|
||||||
|
|
||||||
|
# Write end of the per-worker fault pipe (WorkerProc.make_worker_process); the
|
||||||
|
# executor's worker monitor waits on the read end next to the process
|
||||||
|
# sentinels. None outside a worker process (tests, the offload child).
|
||||||
|
_FATAL_NOTIFY_FD: int | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def set_fatal_notify_fd(fd: int | None) -> None:
|
||||||
|
"""Register the fault pipe worker_main received from the executor."""
|
||||||
|
global _FATAL_NOTIFY_FD
|
||||||
|
_FATAL_NOTIFY_FD = fd
|
||||||
|
|
||||||
|
|
||||||
|
def notify_parent_fatal() -> None:
|
||||||
|
"""Tell the executor this worker is dying, before the process is gone.
|
||||||
|
|
||||||
|
The executor's only other signal is the process sentinel, which the kernel
|
||||||
|
closes when the LAST task of the worker leaves -- after the address-space
|
||||||
|
teardown, not at os._exit. On a rank whose GPU work is paired with a live
|
||||||
|
peer that teardown can wait for the peer's exit (in a rank-1 fault test, a
|
||||||
|
fail-closed fault on PP0_TP1 left its leader Z and unreapable for 300 s
|
||||||
|
while rank 0 waited on it in turn; the engine hung on the response
|
||||||
|
queue). One byte here wakes the monitor at once so the failure ladder ends
|
||||||
|
the peers, which is what frees the teardown. Nonblocking, never raises:
|
||||||
|
the caller os._exits right after, whatever happens here.
|
||||||
|
"""
|
||||||
|
fd = _FATAL_NOTIFY_FD
|
||||||
|
if fd is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
os.set_blocking(fd, False)
|
||||||
|
os.write(fd, b"F")
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_type_label(cls: type) -> str:
|
||||||
|
"""``module.qualname`` of ``cls`` without running class-defined code."""
|
||||||
|
try:
|
||||||
|
qualname = _TYPE_QUALNAME(cls)
|
||||||
|
module = _TYPE_MODULE(cls)
|
||||||
|
except BaseException:
|
||||||
|
return "<unknown type>"
|
||||||
|
if type(qualname) is not str:
|
||||||
|
qualname = "<unknown type>"
|
||||||
|
if type(module) is str and module != "builtins":
|
||||||
|
return f"{module}.{qualname}"
|
||||||
|
return qualname
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_exception_line(error: BaseException) -> str:
|
||||||
|
"""Type label plus the exact-``str``/``int`` args; nothing else is shown."""
|
||||||
|
label = _safe_type_label(type(error))
|
||||||
|
args = _EXC_ARGS(error)
|
||||||
|
if type(args) is not tuple or len(args) == 0:
|
||||||
|
return label
|
||||||
|
parts = []
|
||||||
|
for arg in args[:4]:
|
||||||
|
if type(arg) is str:
|
||||||
|
parts.append(arg[:_MAX_ARG_CHARS])
|
||||||
|
elif type(arg) is int and -(10**18) < arg < 10**18:
|
||||||
|
parts.append(str(arg))
|
||||||
|
else:
|
||||||
|
parts.append(f"<{_safe_type_label(type(arg))}>")
|
||||||
|
if len(args) > 4:
|
||||||
|
parts.append("...")
|
||||||
|
if len(args) == 1:
|
||||||
|
return f"{label}: {parts[0]}"
|
||||||
|
return f"{label}: ({', '.join(parts)})"
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_frames(error: BaseException) -> list[str]:
|
||||||
|
lines = []
|
||||||
|
tb = _EXC_TRACEBACK(error)
|
||||||
|
while type(tb) is TracebackType:
|
||||||
|
code = tb.tb_frame.f_code
|
||||||
|
filename = code.co_filename
|
||||||
|
name = code.co_qualname
|
||||||
|
lineno = tb.tb_lineno
|
||||||
|
lines.append(
|
||||||
|
f' File "{filename if type(filename) is str else "?"}", '
|
||||||
|
f"line {lineno if type(lineno) is int else '?'}, "
|
||||||
|
f"in {name if type(name) is str else '?'}\n"
|
||||||
|
)
|
||||||
|
tb = tb.tb_next
|
||||||
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_format_exception(error: BaseException) -> str:
|
||||||
|
"""A traceback.format_exception look-alike that runs no exception code.
|
||||||
|
|
||||||
|
Frames come from the traceback objects (file, line, function; no source
|
||||||
|
lines, which would mean file I/O on the dying path); the message is the
|
||||||
|
exception's exact-``str`` args, never ``str(error)``. Chains follow
|
||||||
|
``__cause__`` / ``__context__`` like the interpreter, bounded and
|
||||||
|
cycle-safe.
|
||||||
|
"""
|
||||||
|
chain: list[tuple[BaseException, str]] = []
|
||||||
|
seen: set[int] = set()
|
||||||
|
current: BaseException | None = error
|
||||||
|
link = ""
|
||||||
|
while current is not None and len(chain) < _MAX_CHAIN:
|
||||||
|
if id(current) in seen:
|
||||||
|
break
|
||||||
|
seen.add(id(current))
|
||||||
|
chain.append((current, link))
|
||||||
|
cause = _EXC_CAUSE(current)
|
||||||
|
if cause is not None:
|
||||||
|
current = cause
|
||||||
|
link = (
|
||||||
|
"\nThe above exception was the direct cause of the "
|
||||||
|
"following exception:\n\n"
|
||||||
|
)
|
||||||
|
elif _EXC_SUPPRESS_CONTEXT(current) is not True:
|
||||||
|
current = _EXC_CONTEXT(current)
|
||||||
|
link = (
|
||||||
|
"\nDuring handling of the above exception, another "
|
||||||
|
"exception occurred:\n\n"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
current = None
|
||||||
|
out: list[str] = []
|
||||||
|
# Innermost first, as the interpreter prints; each outer exception's link
|
||||||
|
# text sits between it and the exception it chains to.
|
||||||
|
for index in range(len(chain) - 1, -1, -1):
|
||||||
|
exc, _ = chain[index]
|
||||||
|
out.append("Traceback (most recent call last):\n")
|
||||||
|
out.extend(_safe_frames(exc))
|
||||||
|
out.append(_safe_exception_line(exc) + "\n")
|
||||||
|
if index > 0:
|
||||||
|
out.append(chain[index][1])
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
|
def ple_fail_closed(
|
||||||
|
*,
|
||||||
|
where: str,
|
||||||
|
rank: int,
|
||||||
|
layer: str,
|
||||||
|
sequence: int,
|
||||||
|
error: BaseException,
|
||||||
|
) -> NoReturn:
|
||||||
|
"""Emit one best-effort nonblocking diagnostic and terminate immediately."""
|
||||||
|
try:
|
||||||
|
os.set_blocking(2, False)
|
||||||
|
error_type = type(error)
|
||||||
|
error_name = "Exception"
|
||||||
|
trusted_error = False
|
||||||
|
for trusted_type, trusted_name in _TRUSTED_ERROR_LABELS:
|
||||||
|
if error_type is trusted_type:
|
||||||
|
error_name = trusted_name
|
||||||
|
trusted_error = True
|
||||||
|
break
|
||||||
|
|
||||||
|
safe_where = where[:128] if type(where) is str else "unknown"
|
||||||
|
safe_layer = layer[:256] if type(layer) is str else "unknown"
|
||||||
|
safe_rank = rank if type(rank) is int else -1
|
||||||
|
safe_sequence = sequence if type(sequence) is int else -1
|
||||||
|
detail_suffix = ""
|
||||||
|
if trusted_error:
|
||||||
|
args = BaseException.args.__get__(error, error_type)
|
||||||
|
if type(args) is tuple and len(args) == 1 and type(args[0]) is str:
|
||||||
|
detail = args[0][:512].replace("\n", "\\n")
|
||||||
|
detail_suffix = f":{detail}"
|
||||||
|
message = (
|
||||||
|
f"PLE fatal: where={safe_where} rank={safe_rank} layer={safe_layer} "
|
||||||
|
f"sequence={safe_sequence} error={error_name}{detail_suffix} "
|
||||||
|
f"pid={os.getpid()} "
|
||||||
|
f"monotonic={time.monotonic():.9f} wall={time.time():.9f}\n"
|
||||||
|
)
|
||||||
|
os.write(2, message.encode("utf-8", "backslashreplace")[:2048])
|
||||||
|
except BaseException:
|
||||||
|
pass
|
||||||
|
# THE TRACEBACK, unconditionally, in its own try so it cannot cost us the
|
||||||
|
# line above.
|
||||||
|
#
|
||||||
|
# This path os._exit(1)s, which runs no handlers and flushes nothing, so
|
||||||
|
# whatever is not written here is gone. The line above names the TYPE only
|
||||||
|
# when the type is one of _TRUSTED_ERROR_LABELS and flattens everything
|
||||||
|
# else to the literal string "Exception" -- and the flattened set includes
|
||||||
|
# the two that matter most here: torch.AcceleratorError (a RuntimeError
|
||||||
|
# SUBCLASS, so the `is` comparison misses it) and Triton CompilationError.
|
||||||
|
# An earlier boot died exactly that way: both PP0 ranks
|
||||||
|
# reported "error=Exception" and there was no traceback anywhere in the
|
||||||
|
# log, in any rank, so the cause could not be named at all and the next
|
||||||
|
# step was a re-run rather than a fix.
|
||||||
|
#
|
||||||
|
# A fail-closed path is where evidence matters most, not least: it fires
|
||||||
|
# once, it kills the process, and it is the only witness. Printing the type
|
||||||
|
# of a wrapped exception without the frames that raised it is not a
|
||||||
|
# diagnostic.
|
||||||
|
try:
|
||||||
|
# Three-arg form: it does not depend on being inside an active except
|
||||||
|
# block, and this function is called from one frame further out than
|
||||||
|
# the raise.
|
||||||
|
# _safe_format_exception, not traceback.format_exception: see the note
|
||||||
|
# above _EXC_ARGS -- the stdlib formatter calls str(error), which is
|
||||||
|
# exception-defined code and can block the dying process forever.
|
||||||
|
tb = _safe_format_exception(error)
|
||||||
|
raw = tb.encode("utf-8", "backslashreplace")
|
||||||
|
# Truncate from the END, not the start. format_exception puts the
|
||||||
|
# frames first and the exception TYPE AND MESSAGE last, so a head-side
|
||||||
|
# cut throws away the one line that names what went wrong and keeps the
|
||||||
|
# import-time frames nobody needs. A long chained traceback is exactly
|
||||||
|
# the case where that matters.
|
||||||
|
clipped = len(raw) > 8192
|
||||||
|
raw = raw[-8192:]
|
||||||
|
os.write(
|
||||||
|
2,
|
||||||
|
b"PLE fatal traceback (tail) follows:\n"
|
||||||
|
if clipped
|
||||||
|
else b"PLE fatal traceback follows:\n",
|
||||||
|
)
|
||||||
|
os.write(2, raw)
|
||||||
|
if not raw.endswith(b"\n"):
|
||||||
|
os.write(2, b"\n")
|
||||||
|
except BaseException:
|
||||||
|
# Never silent HERE of all places: without this the reader cannot tell
|
||||||
|
# "the wrapper predates the traceback fix" from "the traceback itself
|
||||||
|
# could not be produced", and those call for different next steps.
|
||||||
|
# Still guarded, because os._exit below must run whatever happens: an
|
||||||
|
# escape from this handler would leave the worker alive in the state
|
||||||
|
# the fail-closed path exists to end.
|
||||||
|
try: # noqa: SIM105
|
||||||
|
os.write(2, b"PLE fatal: traceback unavailable\n")
|
||||||
|
except BaseException:
|
||||||
|
# SIM105 declined: an earlier revision made this helper call no
|
||||||
|
# exception-defined code on the dying path. contextlib.suppress
|
||||||
|
# runs a Python __exit__ between here and os._exit below; a bare
|
||||||
|
# except keeps that window empty.
|
||||||
|
pass
|
||||||
|
notify_parent_fatal()
|
||||||
|
os._exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def fail_ple_execute_model(func: _F) -> _F:
|
||||||
|
"""Fail a non-shutdown worker-step escape only on PLE-owning ranks.
|
||||||
|
|
||||||
|
This includes ``sample_tokens``, the second half of the split execute-model
|
||||||
|
step. A rank without a PLE connector rethrows the original exception.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapped(self: Any, *args: Any, **kwargs: Any) -> Any:
|
||||||
|
try:
|
||||||
|
return func(self, *args, **kwargs)
|
||||||
|
except SystemExit:
|
||||||
|
# WorkerProc's signal handler uses SystemExit for clean shutdown.
|
||||||
|
raise
|
||||||
|
except BaseException as error:
|
||||||
|
connector = getattr(self, "_ple_offload_connector", None)
|
||||||
|
if connector is None:
|
||||||
|
model_runner = getattr(self, "model_runner", None)
|
||||||
|
connector = getattr(model_runner, "_ple_offload_connector", None)
|
||||||
|
if connector is None:
|
||||||
|
raise
|
||||||
|
connector._fail_closed(func.__name__, error)
|
||||||
|
|
||||||
|
return wrapped # type: ignore[return-value]
|
||||||
@@ -0,0 +1,305 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Independent progress deadline for PLE transactions."""
|
||||||
|
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import deque
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||||
|
|
||||||
|
_DEFAULT_SERVING_DEADLINE_S = 300.0
|
||||||
|
_DEFAULT_STARTUP_DEADLINE_S = 1800.0
|
||||||
|
_MAX_PROGRESS_DEADLINE_S = 86400.0
|
||||||
|
_POLL_INTERVAL_S = 0.1
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_deadline(value: float, name: str) -> float:
|
||||||
|
value = float(value)
|
||||||
|
if not math.isfinite(value) or value <= 0 or value > _MAX_PROGRESS_DEADLINE_S:
|
||||||
|
raise ValueError(
|
||||||
|
f"{name} must be finite, greater than zero, and no greater than "
|
||||||
|
f"{_MAX_PROGRESS_DEADLINE_S:g} seconds"
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _environment_deadline(name: str, default: float) -> float:
|
||||||
|
value = float(os.environ.get(name, str(default)))
|
||||||
|
return _validate_deadline(value, name)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _Submission:
|
||||||
|
sequence: int
|
||||||
|
started: float
|
||||||
|
startup: bool
|
||||||
|
num_tokens: int
|
||||||
|
deadline: float
|
||||||
|
depth: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _PendingCompletion:
|
||||||
|
sequence: int
|
||||||
|
started: float
|
||||||
|
startup: bool
|
||||||
|
num_tokens: int
|
||||||
|
deadline: float
|
||||||
|
event: Any
|
||||||
|
|
||||||
|
|
||||||
|
class PleProgressMonitor:
|
||||||
|
"""Bound host submission and the resulting ordered GPU completions."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
rank: int,
|
||||||
|
layer_names: tuple[str, ...],
|
||||||
|
serving_deadline_s: float,
|
||||||
|
startup_deadline_s: float,
|
||||||
|
) -> None:
|
||||||
|
serving_deadline_s = _validate_deadline(
|
||||||
|
serving_deadline_s, "serving_deadline_s"
|
||||||
|
)
|
||||||
|
startup_deadline_s = _validate_deadline(
|
||||||
|
startup_deadline_s, "startup_deadline_s"
|
||||||
|
)
|
||||||
|
self._rank = rank
|
||||||
|
self._layer = ",".join(layer_names)
|
||||||
|
self._serving_deadline_s = serving_deadline_s
|
||||||
|
self._startup_deadline_s = startup_deadline_s
|
||||||
|
self._condition = threading.Condition()
|
||||||
|
self._submission: _Submission | None = None
|
||||||
|
self._pending: deque[_PendingCompletion] = deque()
|
||||||
|
self._sequence = 0
|
||||||
|
self._stopping = False
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=self._run,
|
||||||
|
name=f"ple-progress-rank{rank}",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_environment(
|
||||||
|
cls, *, rank: int, layer_names: tuple[str, ...]
|
||||||
|
) -> "PleProgressMonitor":
|
||||||
|
return cls(
|
||||||
|
rank=rank,
|
||||||
|
layer_names=layer_names,
|
||||||
|
serving_deadline_s=_environment_deadline(
|
||||||
|
"VLLM_PLE_PROGRESS_DEADLINE_S",
|
||||||
|
_DEFAULT_SERVING_DEADLINE_S,
|
||||||
|
),
|
||||||
|
startup_deadline_s=_environment_deadline(
|
||||||
|
"VLLM_PLE_STARTUP_DEADLINE_S",
|
||||||
|
_DEFAULT_STARTUP_DEADLINE_S,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sequence(self) -> int:
|
||||||
|
with self._condition:
|
||||||
|
return self._sequence
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_pending_completions(self) -> bool:
|
||||||
|
with self._condition:
|
||||||
|
return bool(self._pending)
|
||||||
|
|
||||||
|
def describe_submission(self, sequence: int) -> str:
|
||||||
|
"""Return the armed transaction descriptor for the active sequence."""
|
||||||
|
with self._condition:
|
||||||
|
active = self._submission
|
||||||
|
if active is None or active.sequence != sequence:
|
||||||
|
raise RuntimeError("PLE fault correlation has no active submission")
|
||||||
|
return f"sequence:{active.sequence}@begin:{active.started:.9f}"
|
||||||
|
|
||||||
|
def completion_is_pending(self, sequence: int) -> bool:
|
||||||
|
"""Whether this serving sequence has a retained, incomplete event.
|
||||||
|
|
||||||
|
This method is called only behind the armed fault guard. An event that
|
||||||
|
has completed on the GPU but has not yet been dequeued is not counted.
|
||||||
|
"""
|
||||||
|
with self._condition:
|
||||||
|
pending = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in self._pending
|
||||||
|
if item.sequence == sequence and not item.startup
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if pending is None:
|
||||||
|
return False
|
||||||
|
return not bool(pending.event.query())
|
||||||
|
|
||||||
|
def begin(self, *, startup: bool, num_tokens: int) -> int:
|
||||||
|
now = time.monotonic()
|
||||||
|
budget = self._startup_deadline_s if startup else self._serving_deadline_s
|
||||||
|
with self._condition:
|
||||||
|
active = self._submission
|
||||||
|
if active is not None:
|
||||||
|
if not startup or not active.startup:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Overlapping PLE host submissions violate the serialized "
|
||||||
|
"single-slot protocol"
|
||||||
|
)
|
||||||
|
sequence = active.sequence
|
||||||
|
self._submission = _Submission(
|
||||||
|
sequence=sequence,
|
||||||
|
started=active.started,
|
||||||
|
startup=True,
|
||||||
|
num_tokens=num_tokens,
|
||||||
|
deadline=now + budget,
|
||||||
|
depth=active.depth + 1,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self._sequence += 1
|
||||||
|
sequence = self._sequence
|
||||||
|
self._submission = _Submission(
|
||||||
|
sequence=sequence,
|
||||||
|
started=now,
|
||||||
|
startup=startup,
|
||||||
|
num_tokens=num_tokens,
|
||||||
|
deadline=now + budget,
|
||||||
|
depth=1,
|
||||||
|
)
|
||||||
|
self._condition.notify_all()
|
||||||
|
return sequence
|
||||||
|
|
||||||
|
def track_completion(self, event: Any) -> None:
|
||||||
|
"""Retain a release event until the monitor observes GPU completion."""
|
||||||
|
if not callable(getattr(event, "query", None)):
|
||||||
|
raise TypeError("PLE completion event must provide query()")
|
||||||
|
with self._condition:
|
||||||
|
active = self._submission
|
||||||
|
if active is None:
|
||||||
|
raise RuntimeError("PLE completion has no matching submission")
|
||||||
|
self._pending.append(
|
||||||
|
_PendingCompletion(
|
||||||
|
sequence=active.sequence,
|
||||||
|
started=active.started,
|
||||||
|
startup=active.startup,
|
||||||
|
num_tokens=active.num_tokens,
|
||||||
|
deadline=active.deadline,
|
||||||
|
event=event,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self._condition.notify_all()
|
||||||
|
|
||||||
|
def complete(self) -> None:
|
||||||
|
with self._condition:
|
||||||
|
active = self._submission
|
||||||
|
if active is None or active.depth <= 0:
|
||||||
|
raise RuntimeError("PLE release has no matching transaction")
|
||||||
|
if active.depth == 1:
|
||||||
|
self._submission = None
|
||||||
|
else:
|
||||||
|
self._submission = _Submission(
|
||||||
|
sequence=active.sequence,
|
||||||
|
started=active.started,
|
||||||
|
startup=active.startup,
|
||||||
|
num_tokens=active.num_tokens,
|
||||||
|
deadline=active.deadline,
|
||||||
|
depth=active.depth - 1,
|
||||||
|
)
|
||||||
|
self._condition.notify_all()
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
with self._condition:
|
||||||
|
self._stopping = True
|
||||||
|
self._condition.notify_all()
|
||||||
|
self._thread.join(timeout=1)
|
||||||
|
|
||||||
|
def _run(self) -> None:
|
||||||
|
try:
|
||||||
|
self._monitor()
|
||||||
|
except BaseException as error:
|
||||||
|
ple_fail_closed(
|
||||||
|
where="progress-monitor-thread",
|
||||||
|
rank=self._rank,
|
||||||
|
layer=self._layer,
|
||||||
|
sequence=self._sequence,
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _monitor(self) -> None:
|
||||||
|
while True:
|
||||||
|
with self._condition:
|
||||||
|
if self._stopping:
|
||||||
|
return
|
||||||
|
submission = self._submission
|
||||||
|
pending = self._pending[0] if self._pending else None
|
||||||
|
if submission is None and pending is None:
|
||||||
|
self._condition.wait(timeout=_POLL_INTERVAL_S)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# CUDA completion is observed only here. Never synchronize and never
|
||||||
|
# use the host flag as evidence that the queued reset completed.
|
||||||
|
if pending is not None and pending.event.query():
|
||||||
|
with self._condition:
|
||||||
|
if self._pending and self._pending[0] is pending:
|
||||||
|
self._pending.popleft()
|
||||||
|
self._condition.notify_all()
|
||||||
|
continue
|
||||||
|
|
||||||
|
now = time.monotonic()
|
||||||
|
if pending is not None and now >= pending.deadline:
|
||||||
|
with self._condition:
|
||||||
|
if not self._pending or self._pending[0] is not pending:
|
||||||
|
continue
|
||||||
|
mode = "startup" if pending.startup else "serving"
|
||||||
|
ple_fail_closed(
|
||||||
|
where=(
|
||||||
|
f"progress-deadline:{mode}:gpu-completion:"
|
||||||
|
f"tokens={pending.num_tokens}"
|
||||||
|
),
|
||||||
|
rank=self._rank,
|
||||||
|
layer=self._layer,
|
||||||
|
sequence=pending.sequence,
|
||||||
|
error=TimeoutError("PLE GPU completion made no bounded progress"),
|
||||||
|
)
|
||||||
|
|
||||||
|
if submission is not None and now >= submission.deadline:
|
||||||
|
with self._condition:
|
||||||
|
if self._submission is not submission:
|
||||||
|
continue
|
||||||
|
mode = "startup" if submission.startup else "serving"
|
||||||
|
ple_fail_closed(
|
||||||
|
where=(
|
||||||
|
f"progress-deadline:{mode}:host-submission:"
|
||||||
|
f"tokens={submission.num_tokens}"
|
||||||
|
),
|
||||||
|
rank=self._rank,
|
||||||
|
layer=self._layer,
|
||||||
|
sequence=submission.sequence,
|
||||||
|
error=TimeoutError("PLE host submission made no bounded progress"),
|
||||||
|
)
|
||||||
|
|
||||||
|
deadlines = []
|
||||||
|
if pending is not None:
|
||||||
|
deadlines.append(pending.deadline)
|
||||||
|
if submission is not None:
|
||||||
|
deadlines.append(submission.deadline)
|
||||||
|
timeout = _POLL_INTERVAL_S
|
||||||
|
if deadlines:
|
||||||
|
timeout = min(
|
||||||
|
timeout,
|
||||||
|
max(0.0, min(deadlines) - time.monotonic()),
|
||||||
|
)
|
||||||
|
with self._condition:
|
||||||
|
if self._stopping:
|
||||||
|
return
|
||||||
|
if (
|
||||||
|
self._submission is not submission
|
||||||
|
or (self._pending[0] if self._pending else None) is not pending
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
self._condition.wait(timeout=timeout)
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""IPC message definitions for PLE CPU offload (host-mapped pull design).
|
||||||
|
|
||||||
|
The offload process owns no CUDA context. Every GPU worker owns, per PLE
|
||||||
|
layer, a shared host output buffer and a shared host int32 flag, both
|
||||||
|
page-locked and device-mapped in the worker's context. The offload process
|
||||||
|
waits for ``flag == 0``, writes the layer result into the host buffer and
|
||||||
|
stores ``flag = 1`` with plain CPU stores; the GPU's captured forward waits on
|
||||||
|
the host flag, DMAs host -> GPU on the model stream, and ``release_outputs``
|
||||||
|
resets the flag after the forward.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import msgspec
|
||||||
|
import torch
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# IPC message dataclasses
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PleOffloadRegistration:
|
||||||
|
"""Sent once from each GPU worker during offload setup."""
|
||||||
|
|
||||||
|
worker_id: int
|
||||||
|
tp_rank: int
|
||||||
|
dp_rank: int
|
||||||
|
# Shared CPU tensors per PLE layer, filled by the offload process.
|
||||||
|
host_output_buffers: dict[str, torch.Tensor]
|
||||||
|
# Shared CPU int32 tensors per PLE layer; element 0 is the ready flag.
|
||||||
|
host_flag_tensors: dict[str, torch.Tensor]
|
||||||
|
# CPU tensors are allocated in shared memory and registered once.
|
||||||
|
input_ids_buf: torch.Tensor
|
||||||
|
query_start_loc_buf: torch.Tensor
|
||||||
|
ngram_context_buf: torch.Tensor | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PleOffloadRequest:
|
||||||
|
"""Sent by each DP rank's TP rank zero at every inference step."""
|
||||||
|
|
||||||
|
dp_rank: int
|
||||||
|
num_tokens: int
|
||||||
|
num_reqs: int
|
||||||
|
|
||||||
|
|
||||||
|
_PLE_OFFLOAD_REQUEST_DECODER = msgspec.msgpack.Decoder(PleOffloadRequest)
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Compatibility exports for the top-level test-only fault singleton."""
|
||||||
|
|
||||||
|
from vllm_ple_test_fault import (
|
||||||
|
_FAULT as _FAULT,
|
||||||
|
)
|
||||||
|
from vllm_ple_test_fault import ( # noqa: F401
|
||||||
|
FaultSpec as FaultSpec,
|
||||||
|
)
|
||||||
|
from vllm_ple_test_fault import (
|
||||||
|
_parse_fault_spec as _parse_fault_spec,
|
||||||
|
)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,266 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Per-rank launch-identity log line (log-only; changes no behaviour).
|
||||||
|
|
||||||
|
After model loading -- and after the PLE offload placeholders are armed -- every
|
||||||
|
GPU worker rank logs one line per model role it holds::
|
||||||
|
|
||||||
|
e2-identity[PP<p>_TP<t>]: ple_backend=<class>@<transport> ple_method=<class>
|
||||||
|
ple_weight_dtype=<dtype> breakable=<ON|OFF> compile_mode=<mode>
|
||||||
|
cudagraph_mode=<mode> role=<target|draft>
|
||||||
|
|
||||||
|
(one line in the log). ``role=target`` is logged after model loading;
|
||||||
|
``role=draft`` is logged only where a draft model is loaded (the last PP
|
||||||
|
stage), once the drafter's graph modes are resolved (after the KV cache is
|
||||||
|
initialised -- see below). Every value is read from the live objects:
|
||||||
|
|
||||||
|
* ``ple_backend``: the class of each PLE layer's ``ple_embedding`` module, and
|
||||||
|
the transport that serves its rows on this rank, read from the placeholder's
|
||||||
|
armed state -- ``host-mapped-pull`` (``_is_cpu_offloaded`` with a host output
|
||||||
|
pointer), ``ipc-push`` (offloaded, no host pointer) or ``local:<device>`` (the
|
||||||
|
rank owns the table).
|
||||||
|
* ``ple_method``: the embedding's live ``quant_method``, or, on an offloaded
|
||||||
|
placeholder, the ``_offload_quant_method`` the rank dequantizes with.
|
||||||
|
* ``ple_weight_dtype``: the table's dtype where the rank owns it; on an
|
||||||
|
offloaded placeholder, the dtype of the armed transfer buffer, into which the
|
||||||
|
offload process copies the table's rows unchanged
|
||||||
|
(``get_offload_output_dtype``).
|
||||||
|
* ``breakable``: ``is_breakable_cudagraph_enabled()``, the switch the capture
|
||||||
|
path itself reads.
|
||||||
|
* ``compile_mode`` / ``cudagraph_mode`` for ``role=target``: the engine's
|
||||||
|
``CompilationConfig`` (config level, i.e. before KV-cache-time capture-mode
|
||||||
|
resolution).
|
||||||
|
* ``compile_mode`` / ``cudagraph_mode`` for ``role=draft``: the DRAFTER's own
|
||||||
|
resolved state, not the config. The drafter is built with the target's
|
||||||
|
VllmConfig and there is one ``CompilationConfig`` in the engine, so the
|
||||||
|
config fields name the target's profile (the trap an earlier commit fixed for
|
||||||
|
the draft guard's architectures). What the drafter actually runs is:
|
||||||
|
|
||||||
|
- ``compile_mode``: the mode of the draft model's live
|
||||||
|
``@support_torch_compile`` wrapper(s) (``do_not_compile`` false -- the
|
||||||
|
property the capture path tests, ``gpu/cudagraph_utils.py``
|
||||||
|
``has_compiled_submodule``); ``NONE`` when no wrapper is live.
|
||||||
|
- ``cudagraph_mode``: the mode each speculator cudagraph manager holds after
|
||||||
|
``init_cudagraph_manager`` (V2 runner): ``prefill:<m>,decode:<m>`` for the
|
||||||
|
autoregressive / MTP speculator, the bare mode for a single-manager
|
||||||
|
speculator; for a V1 drafter, its dispatcher's mode. These are the values
|
||||||
|
``e2-guard[runtime]`` checks before capture. ``unresolved`` if a manager
|
||||||
|
does not exist yet.
|
||||||
|
|
||||||
|
Hence the draft line is logged from ``Worker.initialize_from_config`` after
|
||||||
|
``initialize_kv_cache``, not at load time.
|
||||||
|
|
||||||
|
A role with no PLE layer reports ``none`` for the three PLE fields; several
|
||||||
|
distinct values on one rank are joined with ``,``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from torch import nn
|
||||||
|
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
_NONE = "none"
|
||||||
|
|
||||||
|
|
||||||
|
def _join(values: set[str]) -> str:
|
||||||
|
return ",".join(sorted(values)) if values else _NONE
|
||||||
|
|
||||||
|
|
||||||
|
def _enum_name(value: Any) -> str:
|
||||||
|
if value is None:
|
||||||
|
return _NONE
|
||||||
|
return str(getattr(value, "name", value))
|
||||||
|
|
||||||
|
|
||||||
|
def _ple_embeddings(model: nn.Module | None) -> list[nn.Module]:
|
||||||
|
"""The ``ple_embedding`` module of every PLE layer in ``model``."""
|
||||||
|
if model is None:
|
||||||
|
return []
|
||||||
|
found = []
|
||||||
|
for module in model.modules():
|
||||||
|
embedding = module.__dict__.get("_modules", {}).get("ple_embedding")
|
||||||
|
if isinstance(embedding, nn.Module):
|
||||||
|
found.append(embedding)
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def _transport(embedding: nn.Module) -> str:
|
||||||
|
if getattr(embedding, "_is_cpu_offloaded", False):
|
||||||
|
if getattr(embedding, "_host_output_ptr", None) is not None:
|
||||||
|
return "host-mapped-pull"
|
||||||
|
return "ipc-push"
|
||||||
|
weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None)
|
||||||
|
if weight is not None:
|
||||||
|
return f"local:{weight.device.type}"
|
||||||
|
return "local:unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _method(embedding: nn.Module) -> str:
|
||||||
|
table = getattr(embedding, "ngram_embedding", None)
|
||||||
|
method = getattr(table, "quant_method", None)
|
||||||
|
if method is None:
|
||||||
|
method = getattr(embedding, "_offload_quant_method", None)
|
||||||
|
return type(method).__name__ if method is not None else _NONE
|
||||||
|
|
||||||
|
|
||||||
|
def _weight_dtype(embedding: nn.Module) -> str:
|
||||||
|
weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None)
|
||||||
|
if weight is not None:
|
||||||
|
return str(weight.dtype)
|
||||||
|
if getattr(embedding, "_is_cpu_offloaded", False):
|
||||||
|
buffer = getattr(embedding, "_gpu_output_buffer", None)
|
||||||
|
if buffer is not None:
|
||||||
|
return str(buffer.dtype)
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _rank_tag() -> str:
|
||||||
|
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||||
|
|
||||||
|
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
|
||||||
|
|
||||||
|
|
||||||
|
def _draft(model_runner: Any) -> tuple[nn.Module | None, Any]:
|
||||||
|
"""The loaded draft model and its holder (V2 speculator or V1 drafter)."""
|
||||||
|
for name in ("speculator", "drafter"):
|
||||||
|
holder = getattr(model_runner, name, None)
|
||||||
|
model = getattr(holder, "model", None)
|
||||||
|
if isinstance(model, nn.Module):
|
||||||
|
return model, holder
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
#: Written where a draft graph mode has no resolved source yet.
|
||||||
|
UNRESOLVED = "unresolved"
|
||||||
|
#: Speculator cudagraph-manager attributes (the set ``e2-guard[runtime]``
|
||||||
|
#: reads), with the phase label each carries on the draft line.
|
||||||
|
_DRAFT_MANAGER_PHASES = (
|
||||||
|
("prefill_cudagraph_manager", "prefill"),
|
||||||
|
("decode_cudagraph_manager", "decode"),
|
||||||
|
("cudagraph_manager", None),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _draft_compile_mode(model: nn.Module) -> str:
|
||||||
|
"""Compile mode of the draft model's live ``@support_torch_compile`` wrappers.
|
||||||
|
|
||||||
|
``NONE`` when no wrapper is live: the drafter then runs eager, whatever
|
||||||
|
the shared ``CompilationConfig.mode`` says.
|
||||||
|
"""
|
||||||
|
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||||
|
|
||||||
|
modes = {
|
||||||
|
_enum_name(getattr(getattr(m, "compilation_config", None), "mode", None))
|
||||||
|
for m in model.modules()
|
||||||
|
if isinstance(m, TorchCompileWithNoGuardsWrapper)
|
||||||
|
and not getattr(m, "do_not_compile", True)
|
||||||
|
}
|
||||||
|
return _join(modes) if modes else "NONE"
|
||||||
|
|
||||||
|
|
||||||
|
def _draft_cudagraph_mode(holder: Any) -> str:
|
||||||
|
"""The drafter's resolved cudagraph mode(s), read from its managers."""
|
||||||
|
parts = []
|
||||||
|
for attr, phase in _DRAFT_MANAGER_PHASES:
|
||||||
|
if not hasattr(holder, attr):
|
||||||
|
continue
|
||||||
|
manager = getattr(holder, attr)
|
||||||
|
mode = (
|
||||||
|
UNRESOLVED
|
||||||
|
if manager is None
|
||||||
|
else _enum_name(getattr(manager, "cudagraph_mode", None))
|
||||||
|
)
|
||||||
|
parts.append(mode if phase is None else f"{phase}:{mode}")
|
||||||
|
if parts:
|
||||||
|
return ",".join(parts)
|
||||||
|
# V1 drafter: one dispatcher, keyed in _check_and_update_cudagraph_mode.
|
||||||
|
dispatcher = getattr(holder, "cudagraph_dispatcher", None)
|
||||||
|
if dispatcher is not None and getattr(dispatcher, "keys_initialized", False):
|
||||||
|
return _enum_name(getattr(dispatcher, "cudagraph_mode", None))
|
||||||
|
return UNRESOLVED
|
||||||
|
|
||||||
|
|
||||||
|
def identity_line(
|
||||||
|
tag: str,
|
||||||
|
model: nn.Module | None,
|
||||||
|
vllm_config: Any,
|
||||||
|
role: str,
|
||||||
|
*,
|
||||||
|
compile_mode: str | None = None,
|
||||||
|
cudagraph_mode: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""One identity line. ``compile_mode`` / ``cudagraph_mode`` override the
|
||||||
|
config-level values (the draft role passes its resolved state)."""
|
||||||
|
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
|
||||||
|
|
||||||
|
embeddings = _ple_embeddings(model)
|
||||||
|
backends = {f"{type(e).__name__}@{_transport(e)}" for e in embeddings}
|
||||||
|
compilation = getattr(vllm_config, "compilation_config", None)
|
||||||
|
if compile_mode is None:
|
||||||
|
compile_mode = _enum_name(getattr(compilation, "mode", None))
|
||||||
|
if cudagraph_mode is None:
|
||||||
|
cudagraph_mode = _enum_name(getattr(compilation, "cudagraph_mode", None))
|
||||||
|
return (
|
||||||
|
f"e2-identity[{tag}]: "
|
||||||
|
f"ple_backend={_join(backends)} "
|
||||||
|
f"ple_method={_join({_method(e) for e in embeddings})} "
|
||||||
|
f"ple_weight_dtype={_join({_weight_dtype(e) for e in embeddings})} "
|
||||||
|
f"breakable={'ON' if is_breakable_cudagraph_enabled() else 'OFF'} "
|
||||||
|
f"compile_mode={compile_mode} "
|
||||||
|
f"cudagraph_mode={cudagraph_mode} "
|
||||||
|
f"role={role}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def draft_identity_line(tag: str, draft_model: nn.Module, holder: Any) -> str:
|
||||||
|
"""The ``role=draft`` line: PLE fields from the draft modules, graph fields
|
||||||
|
from the drafter's resolved state (never the shared CompilationConfig)."""
|
||||||
|
return identity_line(
|
||||||
|
tag,
|
||||||
|
draft_model,
|
||||||
|
getattr(holder, "vllm_config", None),
|
||||||
|
"draft",
|
||||||
|
compile_mode=_draft_compile_mode(draft_model),
|
||||||
|
cudagraph_mode=_draft_cudagraph_mode(holder),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def log_launch_identity(model_runner: Any, vllm_config: Any) -> None:
|
||||||
|
"""Log this rank's ``role=target`` line after model load. Never raises.
|
||||||
|
|
||||||
|
The ``role=draft`` line comes later, from :func:`log_draft_identity`, once
|
||||||
|
the drafter's graph modes exist.
|
||||||
|
"""
|
||||||
|
tag = "PP?_TP?"
|
||||||
|
try:
|
||||||
|
tag = _rank_tag()
|
||||||
|
logger.info(
|
||||||
|
"%s", identity_line(tag, model_runner.get_model(), vllm_config, "target")
|
||||||
|
)
|
||||||
|
except Exception as error: # log-only: identity must never break a boot
|
||||||
|
logger.warning(
|
||||||
|
"e2-identity[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def log_draft_identity(model_runner: Any) -> None:
|
||||||
|
"""Log this rank's ``role=draft`` line, if it holds a drafter. Never raises.
|
||||||
|
|
||||||
|
Call after ``initialize_kv_cache``: that is where the V2 runner resolves
|
||||||
|
the target's mode and ``init_cudagraph_manager`` derives the drafter's
|
||||||
|
(and where the V1 runner keys the drafter's dispatcher).
|
||||||
|
"""
|
||||||
|
tag = "PP?_TP?"
|
||||||
|
try:
|
||||||
|
draft_model, holder = _draft(model_runner)
|
||||||
|
if draft_model is None:
|
||||||
|
return
|
||||||
|
tag = _rank_tag()
|
||||||
|
logger.info("%s", draft_identity_line(tag, draft_model, holder))
|
||||||
|
except Exception as error: # log-only: identity must never break a boot
|
||||||
|
logger.warning(
|
||||||
|
"e2-identity[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
|
||||||
|
)
|
||||||
@@ -0,0 +1,200 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
"""Debug-only allocator telemetry for E2, BOUNDARY-ONLY. Off unless
|
||||||
|
VLLM_E2_MEMSTATS=<ctl poll seconds> and VLLM_E2_MEMSTATS_DIR=<dir>.
|
||||||
|
|
||||||
|
WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read
|
||||||
|
torch.cuda.memory_stats() and reset_peak_memory_stats() every VLLM_E2_MEMSTATS
|
||||||
|
seconds. Both take the per-device caching-allocator mutex and the Python
|
||||||
|
bindings do not release the GIL, so the old contract's claim that this thread
|
||||||
|
"takes no lock shared with the model path" was FALSE. Three one-boot arms at a
|
||||||
|
4.1e9 KV pin showed a monotone dose-response: sampling at 4 Hz wedged a stage-0
|
||||||
|
worker during a 131K thinking-on row; at 1.1 Hz it survived that row and wedged during
|
||||||
|
a 260K row; with the thread disabled the run completed every row. Causation
|
||||||
|
is not established at n=1 per arm (a known limit of
|
||||||
|
that capacity probe), but polling has no place
|
||||||
|
inside a measured region, and it was never necessary: PyTorch maintains the
|
||||||
|
allocator high-water marks itself, so reading them once per window is exact.
|
||||||
|
|
||||||
|
WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and
|
||||||
|
touches NO allocator state: its tick is a plain read of <dir>/window.ctl, which
|
||||||
|
holds "<seq> <name>". The gate driver crosses a boundary only while the server
|
||||||
|
is quiet (no request running). On a new seq the thread performs these
|
||||||
|
boundary-only allocator reads:
|
||||||
|
1. memory_stats() -> the closing window's exact high-water marks
|
||||||
|
and cumulative retry/OOM counters
|
||||||
|
2. reset_peak_memory_stats() -> immediately after the read, with no file I/O
|
||||||
|
in the gap
|
||||||
|
3. mem_get_info() -> device free/total AT THAT QUIESCENT INSTANT
|
||||||
|
4. host_memory_stats() -> pinned-host allocator current values, outside
|
||||||
|
the read-to-reset gap
|
||||||
|
then writes the "close" record for the old window and the "ack" record for the
|
||||||
|
new one.
|
||||||
|
|
||||||
|
WHAT THIS MEASURES.
|
||||||
|
- A row's allocator peak is EXACT for that row's window (not a sampled
|
||||||
|
maximum; it cannot miss a transient).
|
||||||
|
- Device free/total is observed ONLY at quiescent boundaries. No minimum-free
|
||||||
|
figure exists for the inside of a row, and none may be reported.
|
||||||
|
- Retry and OOM counters are cumulative and never reset; per-row deltas come
|
||||||
|
from consecutive close records.
|
||||||
|
- There is NO interval-level peak. A qualifying interval is narrower
|
||||||
|
than its row and is identified only afterwards from the scheduler log, so
|
||||||
|
the peak attributable to it is the ROW-level high-water mark, which
|
||||||
|
over-includes the prefill ramp. Report this loss of resolution
|
||||||
|
explicitly.
|
||||||
|
|
||||||
|
Records (one JSON per line): kind start | close | ack | error; rank, local,
|
||||||
|
dev, pid; window, seq; and for close: allocated_bytes.all.current,
|
||||||
|
reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free,
|
||||||
|
total, run_alloc_peak, run_res_peak, and a host object containing every
|
||||||
|
host_memory_stats() key ending in ".current". Records are written with
|
||||||
|
synchronous file I/O, never inside the read-to-reset gap. Nothing here raises
|
||||||
|
into the worker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
_SEG = float(os.environ.get("VLLM_E2_MEMSTATS", "0") or 0)
|
||||||
|
_DIR = os.environ.get("VLLM_E2_MEMSTATS_DIR", "")
|
||||||
|
_started = []
|
||||||
|
VERSION = "e2_memstats/5-boundary-current-host"
|
||||||
|
|
||||||
|
|
||||||
|
def _close_metrics(dev):
|
||||||
|
"""Read one quiescent boundary; kept separate for CPU-only testing."""
|
||||||
|
s = torch.cuda.memory_stats(dev)
|
||||||
|
ap = int(s.get("allocated_bytes.all.peak", 0))
|
||||||
|
rp = int(s.get("reserved_bytes.all.peak", 0))
|
||||||
|
torch.cuda.reset_peak_memory_stats(dev) # no file I/O in the gap
|
||||||
|
free, total = torch.cuda.mem_get_info(dev)
|
||||||
|
host_stats = torch.cuda.host_memory_stats()
|
||||||
|
host = {
|
||||||
|
key: int(value) for key, value in host_stats.items() if key.endswith(".current")
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"allocated_bytes.all.current": int(s.get("allocated_bytes.all.current", 0)),
|
||||||
|
"reserved_bytes.all.current": int(s.get("reserved_bytes.all.current", 0)),
|
||||||
|
"alloc_peak_w": ap,
|
||||||
|
"res_peak_w": rp,
|
||||||
|
"retries": int(s.get("num_alloc_retries", 0)),
|
||||||
|
"ooms": int(s.get("num_ooms", 0)),
|
||||||
|
"free": int(free),
|
||||||
|
"total": int(total),
|
||||||
|
"host": host,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def enabled() -> bool:
|
||||||
|
return _SEG > 0 and bool(_DIR)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_started(worker, armed_by: str = "compile_or_warm_up_model") -> None:
|
||||||
|
if _started:
|
||||||
|
return
|
||||||
|
dev = (
|
||||||
|
torch.cuda.current_device()
|
||||||
|
) # captured on the worker's own thread: every call below passes it explicitly
|
||||||
|
t = threading.Thread(
|
||||||
|
target=_loop,
|
||||||
|
name="e2-memstats",
|
||||||
|
daemon=True,
|
||||||
|
args=(
|
||||||
|
dev,
|
||||||
|
getattr(worker, "rank", -1),
|
||||||
|
getattr(worker, "local_rank", -1),
|
||||||
|
armed_by,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
_started.append(t)
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
|
||||||
|
def _ctl():
|
||||||
|
try:
|
||||||
|
with open(os.path.join(_DIR, "window.ctl")) as f:
|
||||||
|
seq, name = f.read().split(None, 1)
|
||||||
|
return int(seq), name.strip()
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _loop(dev, rank, local, armed_by):
|
||||||
|
os.makedirs(_DIR, exist_ok=True)
|
||||||
|
# noqa SIM115: the handle lives as long as the telemetry daemon thread
|
||||||
|
# that writes to it; there is no scope to close it in.
|
||||||
|
f = open( # noqa: SIM115
|
||||||
|
os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"),
|
||||||
|
"a",
|
||||||
|
buffering=1,
|
||||||
|
)
|
||||||
|
base = {"rank": rank, "local": local, "dev": dev, "pid": os.getpid()}
|
||||||
|
|
||||||
|
def w(rec):
|
||||||
|
f.write(json.dumps({**base, **rec}, separators=(",", ":")) + "\n")
|
||||||
|
|
||||||
|
run_ap = run_rp = 0
|
||||||
|
seq, name = _ctl() or (0, "serve")
|
||||||
|
torch.cuda.reset_peak_memory_stats(dev)
|
||||||
|
w(
|
||||||
|
{
|
||||||
|
"kind": "start",
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"armed_by": armed_by,
|
||||||
|
"window": name,
|
||||||
|
"seq": seq,
|
||||||
|
"version": VERSION,
|
||||||
|
"poll_s": _SEG,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
while True:
|
||||||
|
time.sleep(_SEG)
|
||||||
|
try:
|
||||||
|
c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
|
||||||
|
if c is None or c[0] == seq:
|
||||||
|
continue
|
||||||
|
# Boundary. The driver only crosses one while the server is
|
||||||
|
# quiet, so this is the single point at which this thread
|
||||||
|
# touches the allocator at all.
|
||||||
|
metrics = _close_metrics(dev)
|
||||||
|
ap = metrics["alloc_peak_w"]
|
||||||
|
rp = metrics["res_peak_w"]
|
||||||
|
run_ap, run_rp = max(run_ap, ap), max(run_rp, rp)
|
||||||
|
w(
|
||||||
|
{
|
||||||
|
"kind": "close",
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"window": name,
|
||||||
|
"seq": seq,
|
||||||
|
**metrics,
|
||||||
|
"run_alloc_peak": run_ap,
|
||||||
|
"run_res_peak": run_rp,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
seq, name = c
|
||||||
|
w(
|
||||||
|
{
|
||||||
|
"kind": "ack",
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"window": name,
|
||||||
|
"seq": seq,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as e: # noqa: BLE001 - telemetry must never raise into the worker
|
||||||
|
w(
|
||||||
|
{
|
||||||
|
"kind": "error",
|
||||||
|
"t": round(time.time(), 4),
|
||||||
|
"m": round(time.monotonic(), 4),
|
||||||
|
"window": name,
|
||||||
|
"seq": seq,
|
||||||
|
"error": f"{type(e).__name__}: {e}"[:300],
|
||||||
|
}
|
||||||
|
)
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Per-rank determinism-toggle line (log-only; changes no behaviour).
|
||||||
|
|
||||||
|
Why this exists: the effective state of the
|
||||||
|
four determinism toggles and the three persistent_topk switches must be
|
||||||
|
attributable to a RANK and a PID. The kernel's own stderr line
|
||||||
|
(``[merlin] persistent_topk tie policy: ...``, topk.cu report_tie_policy_once)
|
||||||
|
is printed once per process with no tag, and FULL_K / QSA_SORT have no log
|
||||||
|
producer at all. Every GPU worker therefore logs exactly one line::
|
||||||
|
|
||||||
|
e2-toggles[PP<p>_TP<t>]: pid=<pid> MERLIN_FULL_K=<raw> MERLIN_QSA_SORT=<raw>
|
||||||
|
MERLIN_TIE_DET=<raw> MERLIN_TIE_RECENT=<raw> MERLIN_TOPK_EXACT_MEDIUM=<raw>
|
||||||
|
MERLIN_TOPK_EXACT_DECODE=<raw> MERLIN_TOPK_DET_LARGE=<raw>
|
||||||
|
MERLIN_TOPK_SORTED_EMIT=<raw>
|
||||||
|
effective: full_k=<0|1> qsa_sort=<0|1|invalid> tie_det=<0|1> tie_recent=<0|1>
|
||||||
|
tie_recent_effective=<0|1> exact_medium=<0|1> exact_decode=<0|1>
|
||||||
|
det_large=<0|1> sorted_emit=<0|1> source=<libc-getenv|os-environ>
|
||||||
|
cpp-cache: <full_k=... tie_det=... tie_recent=... tie_recent_effective=...
|
||||||
|
exact_medium=... exact_decode=... det_large=... sorted_emit=...
|
||||||
|
source=cpp-cache | n-a reason=<why>>
|
||||||
|
python: qsa_sort=<0|1|n-a> sorted_emit_skip=<0|1|n-a>
|
||||||
|
source=qsa_indexer._QSA_SORT
|
||||||
|
device: switch=<0|1|n-a> switch_large=<0|1|n-a> source=counter-slots-8-9
|
||||||
|
|
||||||
|
(one line in the log). ``<raw>`` is ``unset``, a plain token, or a double-quoted
|
||||||
|
string with every byte outside ``[!-~]`` (and ``"`` / ``\\``) hex-escaped, so
|
||||||
|
the line always splits on whitespace.
|
||||||
|
|
||||||
|
WHAT EACH SECTION IS
|
||||||
|
* ``effective:`` -- the environment at emission time, parsed with the SAME
|
||||||
|
rules the readers use (C++ ``std::getenv`` + ``std::atoi != 0`` for the six
|
||||||
|
C++ names, ``bool(int(...))`` for QSA_SORT; file:line of each below). The
|
||||||
|
C++ names are read through libc ``getenv`` -- the ``environ`` that
|
||||||
|
``std::getenv`` reads -- not ``os.environ``, which can diverge from it after
|
||||||
|
an ``os.putenv``. A MIRROR of the parse, not the kernel's state. It exists
|
||||||
|
to report the rule; nothing may branch on it (vllm/envs.py "NOT A PYTHON
|
||||||
|
READ SITE" still holds: this module decides nothing).
|
||||||
|
* ``cpp-cache:`` -- the kernel's switches as the C++ statics cached them, from
|
||||||
|
``torch.ops._C.persistent_topk_toggle_state`` and
|
||||||
|
``torch.ops._moe_C.marlin_moe_full_k_state`` (query ops that only LOAD a
|
||||||
|
mirror the static initialisers publish; they never fill a cache). ``n-a``
|
||||||
|
with a reason on a build without those ops -- the extensions this release
|
||||||
|
was first cut against (earlier v2.2 builds, not published) have none.
|
||||||
|
* ``python:`` -- ``qsa_indexer._QSA_SORT``, the module constant the QSA sort
|
||||||
|
actually branches on (looked up in ``sys.modules``, never imported here;
|
||||||
|
``n-a`` if the module was never loaded in this process), and
|
||||||
|
``sorted_emit_skip`` -- ``qsa_indexer._kernel_sorted_emit_state``, i.e.
|
||||||
|
whether the Python sort is being SKIPPED because the kernel reported
|
||||||
|
MERLIN_TOPK_SORTED_EMIT through ``persistent_topk_toggle_state`` field 6
|
||||||
|
(``n-a`` until the first QSA top-k has run and latched it).
|
||||||
|
* ``device:`` -- counter slots [8]/[9], which the counter op fills from the
|
||||||
|
C++ ``exact_medium_enabled()`` / ``det_large_enabled()`` statics
|
||||||
|
(topk.cu persistent_topk_exact_counters).
|
||||||
|
|
||||||
|
WHEN. Once per worker, after ``compile_or_warm_up_model`` has run the MRV2
|
||||||
|
kernel warmup and cudagraph capture (``arm()``), i.e. after this process's
|
||||||
|
first persistent_topk and Marlin MoE launches -- NOT at the counter reader's
|
||||||
|
first read: an integration boot logged its first counter line BEFORE
|
||||||
|
the first tie-policy line in the engine log, because the
|
||||||
|
first ``maybe_log`` runs at the head of the warmup's first execute step, before
|
||||||
|
that step's forward. When the cpp-cache ops exist and a static is still
|
||||||
|
uninitialised at ``arm()``, the line is deferred to the head of the first
|
||||||
|
execute step where every static is filled, and forced out at shutdown
|
||||||
|
otherwise (with ``uninit`` values, which the smoke FAILs). It does not depend on
|
||||||
|
``MERLIN_TOPK_EXACT_COUNTERS_LOG``: it fires with the counter log off.
|
||||||
|
|
||||||
|
WHAT THIS DOES NOT PROVE. Without the cpp-cache ops the C++ state is inferred,
|
||||||
|
not read: for FULL_K, TIE_DET, TIE_RECENT and EXACT_DECODE the line is the
|
||||||
|
environment at emission time run through the C++ parser's rules, and the only
|
||||||
|
C++-derived evidence is ``device:`` (EXACT_MEDIUM and DET_LARGE) plus the
|
||||||
|
untagged stderr policy line. The residual gap is an environment mutated
|
||||||
|
between the kernel's first ``getenv`` and this read. Nothing in this tree does
|
||||||
|
that (by grep of csrc/): no ``setenv``/``putenv``/``unsetenv`` under csrc/, and
|
||||||
|
the only runtime ``os.environ.update`` calls in vllm/ are the RunAI streamer's
|
||||||
|
fixed ``RUNAI_STREAMER_*`` keys (model_executor/model_loader/
|
||||||
|
runai_streamer_loader.py:73) and the Ray executor's ``override_env_vars``
|
||||||
|
(v1/executor/ray_utils.py:180), which the multiproc executor never calls. With the
|
||||||
|
cpp-cache ops that gap closes for every C++ name. Even then the line proves the
|
||||||
|
CONFIGURATION each rank ran, not that a switch changed any output: that is the
|
||||||
|
job of an output-equality run (identical sends -> one distinct output).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
# (environment name, effective key, value when unset or empty) -- the C++
|
||||||
|
# readers, each `static const bool enabled = [] { ... }();` once per process:
|
||||||
|
# MERLIN_FULL_K csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu:57-63
|
||||||
|
# MERLIN_TIE_DET csrc/libtorch_stable/topk.cu:60-66
|
||||||
|
# MERLIN_TIE_RECENT csrc/libtorch_stable/topk.cu:52-58
|
||||||
|
# MERLIN_TOPK_EXACT_MEDIUM csrc/libtorch_stable/topk.cu:68-74 (unset -> ON)
|
||||||
|
# MERLIN_TOPK_EXACT_DECODE csrc/libtorch_stable/topk.cu:103-109
|
||||||
|
# MERLIN_TOPK_DET_LARGE csrc/libtorch_stable/topk.cu:86-92 (unset -> ON)
|
||||||
|
# MERLIN_TOPK_SORTED_EMIT csrc/libtorch_stable/topk.cu sorted_emit_enabled
|
||||||
|
CXX_SWITCHES: tuple[tuple[str, str, bool], ...] = (
|
||||||
|
("MERLIN_FULL_K", "full_k", False),
|
||||||
|
("MERLIN_TIE_DET", "tie_det", False),
|
||||||
|
("MERLIN_TIE_RECENT", "tie_recent", False),
|
||||||
|
("MERLIN_TOPK_EXACT_MEDIUM", "exact_medium", True),
|
||||||
|
("MERLIN_TOPK_EXACT_DECODE", "exact_decode", False),
|
||||||
|
("MERLIN_TOPK_DET_LARGE", "det_large", True),
|
||||||
|
("MERLIN_TOPK_SORTED_EMIT", "sorted_emit", False),
|
||||||
|
)
|
||||||
|
QSA_SORT_NAME = "MERLIN_QSA_SORT"
|
||||||
|
RAW_ORDER = (
|
||||||
|
"MERLIN_FULL_K",
|
||||||
|
"MERLIN_QSA_SORT",
|
||||||
|
"MERLIN_TIE_DET",
|
||||||
|
"MERLIN_TIE_RECENT",
|
||||||
|
"MERLIN_TOPK_EXACT_MEDIUM",
|
||||||
|
"MERLIN_TOPK_EXACT_DECODE",
|
||||||
|
"MERLIN_TOPK_DET_LARGE",
|
||||||
|
"MERLIN_TOPK_SORTED_EMIT",
|
||||||
|
)
|
||||||
|
EFFECTIVE_ORDER = (
|
||||||
|
"full_k",
|
||||||
|
"qsa_sort",
|
||||||
|
"tie_det",
|
||||||
|
"tie_recent",
|
||||||
|
"tie_recent_effective",
|
||||||
|
"exact_medium",
|
||||||
|
"exact_decode",
|
||||||
|
"det_large",
|
||||||
|
"sorted_emit",
|
||||||
|
)
|
||||||
|
CPP_ORDER = tuple(k for k in EFFECTIVE_ORDER if k != "qsa_sort")
|
||||||
|
|
||||||
|
_QSA_MODULE = "vllm.models.qwen4_exp.nvidia.ops.qsa_indexer"
|
||||||
|
_C_SPACE = b" \t\n\v\f\r" # isspace() in the "C" locale
|
||||||
|
_LONG_MIN, _LONG_MAX = -(2**63), 2**63 - 1 # LP64 long, the reference host's glibc
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- parsers ----
|
||||||
|
def c_atoi(raw: bytes) -> int:
|
||||||
|
"""glibc ``atoi``: ``(int) strtol(raw, NULL, 10)`` (glibc stdlib/atoi.c).
|
||||||
|
|
||||||
|
Leading C-locale whitespace, one optional sign, then decimal digits up to
|
||||||
|
the first non-digit; no digits -> 0. ``strtol`` clamps to the ``long``
|
||||||
|
range and the ``(int)`` conversion keeps the low 32 bits (GCC's documented
|
||||||
|
modulo behaviour). So ``"true"`` -> 0, ``"1abc"`` -> 1, ``"0x1"`` -> 0,
|
||||||
|
``" +1"`` -> 1, ``"+ 1"`` -> 0 and ``"4294967296"`` -> 0.
|
||||||
|
"""
|
||||||
|
i, n = 0, len(raw)
|
||||||
|
while i < n and raw[i] in _C_SPACE:
|
||||||
|
i += 1
|
||||||
|
negative = False
|
||||||
|
if i < n and raw[i] in b"+-":
|
||||||
|
negative = raw[i] == ord("-")
|
||||||
|
i += 1
|
||||||
|
value, digits = 0, 0
|
||||||
|
while i < n and 0x30 <= raw[i] <= 0x39:
|
||||||
|
value = value * 10 + (raw[i] - 0x30)
|
||||||
|
digits += 1
|
||||||
|
i += 1
|
||||||
|
if digits == 0:
|
||||||
|
return 0
|
||||||
|
if negative:
|
||||||
|
value = -value
|
||||||
|
value = max(_LONG_MIN, min(_LONG_MAX, value))
|
||||||
|
value &= 0xFFFFFFFF
|
||||||
|
return value - (1 << 32) if value >= (1 << 31) else value
|
||||||
|
|
||||||
|
|
||||||
|
def cxx_switch(raw: bytes | None, unset_value: bool) -> bool:
|
||||||
|
"""The C++ getter body, both shapes:
|
||||||
|
default OFF ``(raw != nullptr && raw[0] != '\\0') ? (atoi(raw) != 0) : false``
|
||||||
|
default ON ``(raw == nullptr || raw[0] == '\\0') ? true : (atoi(raw) != 0)``
|
||||||
|
An environment value cannot contain NUL, so "empty" is exactly
|
||||||
|
``raw[0] == '\\0'``."""
|
||||||
|
if raw is None or raw == b"":
|
||||||
|
return unset_value
|
||||||
|
return c_atoi(raw) != 0
|
||||||
|
|
||||||
|
|
||||||
|
def qsa_sort_value(raw: str | None) -> str:
|
||||||
|
"""vllm/envs.py:2372 ``bool(int(os.getenv("MERLIN_QSA_SORT", "0")))``,
|
||||||
|
read once at import by qsa_indexer.py:51 (``_QSA_SORT``). ``int()`` raises
|
||||||
|
on ``"true"`` and on ``""`` -- that engine would not have booted -- which is
|
||||||
|
reported as ``invalid`` rather than guessed. Python's own ``int`` is used,
|
||||||
|
so whitespace, ``+``/``-``, ``1_0`` and non-ASCII digits parse exactly as
|
||||||
|
the reader parses them."""
|
||||||
|
try:
|
||||||
|
return "1" if bool(int("0" if raw is None else raw)) else "0"
|
||||||
|
except ValueError:
|
||||||
|
return "invalid"
|
||||||
|
|
||||||
|
|
||||||
|
def effective_tie_recent(det_ties: bool, tie_recent: bool) -> bool:
|
||||||
|
"""csrc/libtorch_stable/persistent_topk.cuh:65-68: ``det_ties ? tie_recent
|
||||||
|
: true`` -- with TIE_DET off the restart paths keep most-recent-wins."""
|
||||||
|
return tie_recent if det_ties else True
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- sources ----
|
||||||
|
def _libc_getenv() -> tuple[Callable[[str], bytes | None], str]:
|
||||||
|
"""libc ``getenv``: the ``environ`` the C++ ``std::getenv`` calls read.
|
||||||
|
|
||||||
|
Falls back to ``os.environb`` / ``os.environ`` -- and says so in the line's
|
||||||
|
``source=`` -- where there is no C library to ask (an offline run off Linux).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import ctypes
|
||||||
|
|
||||||
|
getenv = ctypes.CDLL(None).getenv
|
||||||
|
getenv.restype = ctypes.c_char_p
|
||||||
|
getenv.argtypes = [ctypes.c_char_p]
|
||||||
|
|
||||||
|
def read(name: str) -> bytes | None:
|
||||||
|
return getenv(name.encode("ascii"))
|
||||||
|
|
||||||
|
return read, "libc-getenv"
|
||||||
|
except (OSError, AttributeError, TypeError):
|
||||||
|
pass
|
||||||
|
environb = getattr(os, "environb", None)
|
||||||
|
if environb is not None:
|
||||||
|
return (lambda name: environb.get(name.encode("ascii"))), "os-environ"
|
||||||
|
|
||||||
|
def read_environ(name: str) -> bytes | None:
|
||||||
|
value = os.environ.get(name)
|
||||||
|
return None if value is None else value.encode("utf-8", "surrogateescape")
|
||||||
|
|
||||||
|
return read_environ, "os-environ"
|
||||||
|
|
||||||
|
|
||||||
|
def _printable(b: int) -> bool:
|
||||||
|
return 0x21 <= b <= 0x7E and b not in b'"\\'
|
||||||
|
|
||||||
|
|
||||||
|
def encode_raw(raw: bytes | None) -> str:
|
||||||
|
"""``unset``, a plain token, or a quoted, hex-escaped string -- never a
|
||||||
|
space, so the line splits on whitespace. The literal value ``unset`` is
|
||||||
|
quoted so it cannot pass for an absent variable."""
|
||||||
|
if raw is None:
|
||||||
|
return "unset"
|
||||||
|
if raw and raw != b"unset" and all(_printable(b) for b in raw):
|
||||||
|
return raw.decode("ascii")
|
||||||
|
return '"' + "".join(chr(b) if _printable(b) else f"\\x{b:02x}" for b in raw) + '"'
|
||||||
|
|
||||||
|
|
||||||
|
def environ_view(
|
||||||
|
cxx_getenv: Callable[[str], bytes | None],
|
||||||
|
py_getenv: Callable[[str], str | None],
|
||||||
|
) -> tuple[dict[str, str], dict[str, str]]:
|
||||||
|
"""(raw value by name, effective map) from one read of the environment."""
|
||||||
|
raw: dict[str, str] = {}
|
||||||
|
flags: dict[str, bool] = {}
|
||||||
|
for name, key, unset_value in CXX_SWITCHES:
|
||||||
|
value = cxx_getenv(name)
|
||||||
|
raw[name] = encode_raw(value)
|
||||||
|
flags[key] = cxx_switch(value, unset_value)
|
||||||
|
qsa_raw = py_getenv(QSA_SORT_NAME)
|
||||||
|
raw[QSA_SORT_NAME] = encode_raw(
|
||||||
|
None if qsa_raw is None else qsa_raw.encode("utf-8", "surrogateescape")
|
||||||
|
)
|
||||||
|
eff = {key: "1" if flags[key] else "0" for key in flags}
|
||||||
|
eff["qsa_sort"] = qsa_sort_value(qsa_raw)
|
||||||
|
eff["tie_recent_effective"] = (
|
||||||
|
"1" if effective_tie_recent(flags["tie_det"], flags["tie_recent"]) else "0"
|
||||||
|
)
|
||||||
|
return raw, {key: eff[key] for key in EFFECTIVE_ORDER}
|
||||||
|
|
||||||
|
|
||||||
|
_TOPK_STATE_FIELDS = (
|
||||||
|
"tie_det",
|
||||||
|
"tie_recent",
|
||||||
|
"tie_recent_effective",
|
||||||
|
"exact_medium",
|
||||||
|
"exact_decode",
|
||||||
|
"det_large",
|
||||||
|
"sorted_emit",
|
||||||
|
)
|
||||||
|
# Known layouts: version -> field count. Layout 2 is layout 1
|
||||||
|
# plus field 6 (MERLIN_TOPK_SORTED_EMIT); fields 0-5 are unchanged.
|
||||||
|
_TOPK_STATE_LAYOUTS = {1: 6, 2: 7}
|
||||||
|
|
||||||
|
|
||||||
|
def decode_topk_state(state: int) -> dict[str, str] | str:
|
||||||
|
"""Unpack ``persistent_topk_toggle_state()``: 2 bits per field (0 off,
|
||||||
|
1 on, 2 not initialised), field count at bits 48-55, layout version at
|
||||||
|
bits 56-62 -- (1, 6) or (2, 7). Any other layout is refused with a reason
|
||||||
|
string. A layout-1 build has no sorted emission, reported as ``n-a``."""
|
||||||
|
version, count = (state >> 56) & 0x7F, (state >> 48) & 0xFF
|
||||||
|
if _TOPK_STATE_LAYOUTS.get(version) != count:
|
||||||
|
return f"unknown-layout-v{version}-{count}-fields"
|
||||||
|
names = {0: "0", 1: "1", 2: "uninit"}
|
||||||
|
out = {
|
||||||
|
key: names.get((state >> (2 * i)) & 3, "invalid")
|
||||||
|
for i, key in enumerate(_TOPK_STATE_FIELDS[:count])
|
||||||
|
}
|
||||||
|
for key in _TOPK_STATE_FIELDS[count:]:
|
||||||
|
out[key] = "n-a"
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def decode_full_k_state(state: int) -> str:
|
||||||
|
"""``marlin_moe_full_k_state()``: -1 not initialised, 0 off, 1 on."""
|
||||||
|
return {-1: "uninit", 0: "0", 1: "1"}.get(int(state), "invalid")
|
||||||
|
|
||||||
|
|
||||||
|
def _op(namespace: str, name: str):
|
||||||
|
try:
|
||||||
|
import torch
|
||||||
|
|
||||||
|
return getattr(getattr(torch.ops, namespace), name, None)
|
||||||
|
except (AttributeError, RuntimeError, ImportError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def cpp_cache() -> dict[str, str] | str:
|
||||||
|
"""The C++ cached switches, or a reason string when the build cannot say."""
|
||||||
|
topk_op = _op("_C", "persistent_topk_toggle_state")
|
||||||
|
full_k_op = _op("_moe_C", "marlin_moe_full_k_state")
|
||||||
|
missing = [
|
||||||
|
name
|
||||||
|
for name, op in (
|
||||||
|
("_C.persistent_topk_toggle_state", topk_op),
|
||||||
|
("_moe_C.marlin_moe_full_k_state", full_k_op),
|
||||||
|
)
|
||||||
|
if op is None
|
||||||
|
]
|
||||||
|
if missing:
|
||||||
|
return "op-absent:" + ",".join(missing)
|
||||||
|
decoded = decode_topk_state(int(topk_op()))
|
||||||
|
if isinstance(decoded, str):
|
||||||
|
return decoded
|
||||||
|
out = dict(decoded, full_k=decode_full_k_state(int(full_k_op())))
|
||||||
|
return {key: out[key] for key in CPP_ORDER}
|
||||||
|
|
||||||
|
|
||||||
|
def qsa_sort_live() -> str:
|
||||||
|
module = sys.modules.get(_QSA_MODULE)
|
||||||
|
if module is None or not hasattr(module, "_QSA_SORT"):
|
||||||
|
return "n-a"
|
||||||
|
return "1" if bool(module._QSA_SORT) else "0"
|
||||||
|
|
||||||
|
|
||||||
|
def sorted_emit_skip_live() -> str:
|
||||||
|
"""``qsa_indexer._kernel_sorted_emit_state``: 1 = the Python sort is being
|
||||||
|
skipped because the kernel reported sorted emission, 0 = not skipped,
|
||||||
|
``n-a`` = not latched yet (or the module was never loaded)."""
|
||||||
|
module = sys.modules.get(_QSA_MODULE)
|
||||||
|
state = getattr(module, "_kernel_sorted_emit_state", None) if module else None
|
||||||
|
if state is None:
|
||||||
|
return "n-a"
|
||||||
|
return "1" if state else "0"
|
||||||
|
|
||||||
|
|
||||||
|
def device_switches() -> dict[str, str]:
|
||||||
|
"""Counter slots [8]/[9] (switch / switch_large). One synchronizing read
|
||||||
|
(topk_counters.read refuses inside a capture), so it is only called from
|
||||||
|
arm() and the head of an execute step."""
|
||||||
|
try:
|
||||||
|
from vllm.models.qwen4_exp.nvidia.ops import topk_counters
|
||||||
|
|
||||||
|
counters = topk_counters.read()
|
||||||
|
except Exception:
|
||||||
|
counters = {}
|
||||||
|
return {
|
||||||
|
key: str(counters[key]) if key in counters else "n-a"
|
||||||
|
for key in ("switch", "switch_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ line ----
|
||||||
|
def toggles_line(
|
||||||
|
tag: str,
|
||||||
|
pid: int,
|
||||||
|
raw: dict[str, str],
|
||||||
|
effective: dict[str, str],
|
||||||
|
env_source: str,
|
||||||
|
cpp: dict[str, str] | str,
|
||||||
|
qsa_live: str,
|
||||||
|
device: dict[str, str],
|
||||||
|
sorted_skip: str = "n-a",
|
||||||
|
) -> str:
|
||||||
|
parts = [f"e2-toggles[{tag}]: pid={pid}"]
|
||||||
|
parts += [f"{name}={raw[name]}" for name in RAW_ORDER]
|
||||||
|
parts.append("effective:")
|
||||||
|
parts += [f"{key}={effective[key]}" for key in EFFECTIVE_ORDER]
|
||||||
|
parts.append(f"source={env_source}")
|
||||||
|
parts.append("cpp-cache:")
|
||||||
|
if isinstance(cpp, str):
|
||||||
|
parts += ["n-a", f"reason={cpp}"]
|
||||||
|
else:
|
||||||
|
parts += [f"{key}={cpp[key]}" for key in CPP_ORDER]
|
||||||
|
parts.append("source=cpp-cache")
|
||||||
|
parts += [
|
||||||
|
"python:",
|
||||||
|
f"qsa_sort={qsa_live}",
|
||||||
|
f"sorted_emit_skip={sorted_skip}",
|
||||||
|
"source=qsa_indexer._QSA_SORT",
|
||||||
|
]
|
||||||
|
parts.append("device:")
|
||||||
|
parts += [f"{key}={device[key]}" for key in ("switch", "switch_large")]
|
||||||
|
parts.append("source=counter-slots-8-9")
|
||||||
|
return " ".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _rank_tag() -> str:
|
||||||
|
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||||
|
|
||||||
|
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- state ----
|
||||||
|
_armed = False
|
||||||
|
_emitted = False
|
||||||
|
_deferral_noted = False
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_for_tests() -> None:
|
||||||
|
global _armed, _emitted, _deferral_noted
|
||||||
|
_armed = _emitted = _deferral_noted = False
|
||||||
|
|
||||||
|
|
||||||
|
def arm(**kwargs) -> str:
|
||||||
|
"""Once per worker, at the end of compile_or_warm_up_model (warmup and
|
||||||
|
capture done): emit now, unless a C++ static is still uninitialised."""
|
||||||
|
global _armed
|
||||||
|
_armed = True
|
||||||
|
return maybe_emit(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def maybe_emit(
|
||||||
|
force: bool = False,
|
||||||
|
*,
|
||||||
|
tag_fn: Callable[[], str] = _rank_tag,
|
||||||
|
getenv_fn: Callable[[], tuple[Callable[[str], bytes | None], str]] = _libc_getenv,
|
||||||
|
cpp_fn: Callable[[], dict[str, str] | str] = cpp_cache,
|
||||||
|
device_fn: Callable[[], dict[str, str]] = device_switches,
|
||||||
|
qsa_fn: Callable[[], str] = qsa_sort_live,
|
||||||
|
sorted_skip_fn: Callable[[], str] = sorted_emit_skip_live,
|
||||||
|
) -> str:
|
||||||
|
"""Emit the line at most once per process. One bool check when already
|
||||||
|
emitted or not yet armed, so it can sit at the head of every execute step.
|
||||||
|
Never raises. Returns the line when this call emitted it, else ""."""
|
||||||
|
global _emitted, _deferral_noted
|
||||||
|
if _emitted or not _armed:
|
||||||
|
return ""
|
||||||
|
tag = "PP?_TP?"
|
||||||
|
try:
|
||||||
|
tag = tag_fn()
|
||||||
|
cpp = cpp_fn()
|
||||||
|
pending = (
|
||||||
|
sorted(k for k, v in cpp.items() if v == "uninit")
|
||||||
|
if isinstance(cpp, dict)
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
if pending and not force:
|
||||||
|
if not _deferral_noted:
|
||||||
|
_deferral_noted = True
|
||||||
|
# Deliberately not 'e2-toggles[': the smoke counts those.
|
||||||
|
logger.info(
|
||||||
|
"e2-toggles deferred on %s: C++ cache not initialised for %s",
|
||||||
|
tag,
|
||||||
|
",".join(pending),
|
||||||
|
)
|
||||||
|
return ""
|
||||||
|
cxx_getenv, env_source = getenv_fn()
|
||||||
|
raw, effective = environ_view(cxx_getenv, os.environ.get)
|
||||||
|
line = toggles_line(
|
||||||
|
tag,
|
||||||
|
os.getpid(),
|
||||||
|
raw,
|
||||||
|
effective,
|
||||||
|
env_source,
|
||||||
|
cpp,
|
||||||
|
qsa_fn(),
|
||||||
|
device_fn(),
|
||||||
|
sorted_skip_fn(),
|
||||||
|
)
|
||||||
|
_emitted = True
|
||||||
|
logger.info("%s", line)
|
||||||
|
return line
|
||||||
|
except Exception as error: # log-only: must never break a boot or a step
|
||||||
|
_emitted = True
|
||||||
|
logger.warning(
|
||||||
|
"e2-toggles[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
|
||||||
|
)
|
||||||
|
return ""
|
||||||
@@ -6,6 +6,7 @@ import torch
|
|||||||
|
|
||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||||
|
from vllm.v1.worker import guard_telemetry
|
||||||
from vllm.v1.worker.gpu.buffer_utils import (
|
from vllm.v1.worker.gpu.buffer_utils import (
|
||||||
FusedStagedWriter,
|
FusedStagedWriter,
|
||||||
StagedWriteTensor,
|
StagedWriteTensor,
|
||||||
@@ -13,6 +14,12 @@ from vllm.v1.worker.gpu.buffer_utils import (
|
|||||||
_load_ptr,
|
_load_ptr,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Bias so a negative row/index survives atomic_max on a zeroed counter
|
||||||
|
# -- it MUST be a tl.constexpr: a plain module-level int is not readable
|
||||||
|
# from a @triton.jit body -- an earlier build shipped one and every guarded kernel died
|
||||||
|
# with "NameError: Cannot access global variable _GUARD_BIAS" on the smoke.
|
||||||
|
_GUARD_BIAS = tl.constexpr(guard_telemetry.GUARD_BIAS)
|
||||||
|
|
||||||
|
|
||||||
class BlockTables:
|
class BlockTables:
|
||||||
# Set by the elastic EP warmup so KV writes land in the null block.
|
# Set by the elastic EP warmup so KV writes land in the null block.
|
||||||
@@ -180,6 +187,9 @@ class BlockTables:
|
|||||||
self.num_blocks.gpu,
|
self.num_blocks.gpu,
|
||||||
self.num_blocks.gpu.stride(0),
|
self.num_blocks.gpu.stride(0),
|
||||||
num_reqs,
|
num_reqs,
|
||||||
|
guard_telemetry.counter(
|
||||||
|
guard_telemetry.SITE_BT_GATHER, idx_mapping.device
|
||||||
|
),
|
||||||
BLOCK_SIZE=1024, # type: ignore
|
BLOCK_SIZE=1024, # type: ignore
|
||||||
)
|
)
|
||||||
return tuple(bt[:num_reqs_padded] for bt in out)
|
return tuple(bt[:num_reqs_padded] for bt in out)
|
||||||
@@ -217,6 +227,10 @@ class BlockTables:
|
|||||||
positions,
|
positions,
|
||||||
self.block_table_ptrs,
|
self.block_table_ptrs,
|
||||||
self.block_table_strides,
|
self.block_table_strides,
|
||||||
|
self.num_blocks.gpu.stride(0),
|
||||||
|
guard_telemetry.counter(
|
||||||
|
guard_telemetry.SITE_SLOT_MAPPING, idx_mapping.device
|
||||||
|
),
|
||||||
self.block_sizes_tensor,
|
self.block_sizes_tensor,
|
||||||
self.kernel_block_sizes_tensor,
|
self.kernel_block_sizes_tensor,
|
||||||
self.slot_mapping_enabled,
|
self.slot_mapping_enabled,
|
||||||
@@ -252,6 +266,7 @@ def _gather_block_tables_kernel(
|
|||||||
num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs]
|
num_blocks_ptr, # [num_kv_cache_groups, max_num_reqs]
|
||||||
num_blocks_stride,
|
num_blocks_stride,
|
||||||
num_reqs, # actual number of requests (for padding)
|
num_reqs, # actual number of requests (for padding)
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
BLOCK_SIZE: tl.constexpr,
|
BLOCK_SIZE: tl.constexpr,
|
||||||
):
|
):
|
||||||
# kv cache group id
|
# kv cache group id
|
||||||
@@ -271,8 +286,39 @@ def _gather_block_tables_kernel(
|
|||||||
return
|
return
|
||||||
|
|
||||||
req_idx = tl.load(batch_idx_to_req_idx + batch_idx)
|
req_idx = tl.load(batch_idx_to_req_idx + batch_idx)
|
||||||
|
# Bounds-guard instrumentation (guard 3): req_idx indexes both num_blocks_ptr and the
|
||||||
|
# persistent source table, which have max_num_reqs rows. num_blocks_stride
|
||||||
|
# IS max_num_reqs, so the bound needs no new argument. Zero the destination
|
||||||
|
# row and return, mirroring the padded-row path above.
|
||||||
|
if (req_idx < 0) or (req_idx >= num_blocks_stride):
|
||||||
|
tl.atomic_add(guard_ctr, 1)
|
||||||
|
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, (req_idx).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
for i in tl.range(0, max_num_blocks, BLOCK_SIZE):
|
||||||
|
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||||
|
tl.store(dst_row_ptr + offset, 0, mask=offset < max_num_blocks)
|
||||||
|
return
|
||||||
|
|
||||||
group_num_blocks_ptr = num_blocks_ptr + group_id * num_blocks_stride
|
group_num_blocks_ptr = num_blocks_ptr + group_id * num_blocks_stride
|
||||||
num_blocks = tl.load(group_num_blocks_ptr + req_idx)
|
num_blocks = tl.load(group_num_blocks_ptr + req_idx)
|
||||||
|
# A row count wider than the table would copy past the end of both rows.
|
||||||
|
#
|
||||||
|
# Clamp UNCONDITIONALLY and record conditionally. Doing the clamp inside
|
||||||
|
# the `if` rebound an int32 name (num_blocks, loaded from num_blocks_ptr)
|
||||||
|
# to int64 (max_num_blocks is `stride`, loaded from block_table_strides),
|
||||||
|
# and Triton requires a name to keep its type across a branch:
|
||||||
|
# "initial value for `num_blocks` is of type int32[], but the then block
|
||||||
|
# redefines it as int64[]"
|
||||||
|
# That killed an early boot in warmup, at the first real gather. The
|
||||||
|
# explicit .to(tl.int32) pins the type rather than relying on whatever the
|
||||||
|
# operands promote to.
|
||||||
|
if (num_blocks < 0) or (num_blocks > max_num_blocks):
|
||||||
|
tl.atomic_add(guard_ctr, 1)
|
||||||
|
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, (num_blocks).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
num_blocks = tl.minimum(
|
||||||
|
tl.maximum(num_blocks, 0), max_num_blocks
|
||||||
|
).to(tl.int32)
|
||||||
|
|
||||||
src_block_table_ptr = _load_ptr(src_block_table_ptrs + group_id, tl.int32)
|
src_block_table_ptr = _load_ptr(src_block_table_ptrs + group_id, tl.int32)
|
||||||
src_row_ptr = src_block_table_ptr + req_idx * stride
|
src_row_ptr = src_block_table_ptr + req_idx * stride
|
||||||
@@ -291,6 +337,8 @@ def _compute_slot_mappings_kernel(
|
|||||||
pos, # [num_tokens]
|
pos, # [num_tokens]
|
||||||
block_table_ptrs, # [num_kv_cache_groups]
|
block_table_ptrs, # [num_kv_cache_groups]
|
||||||
block_table_strides, # [num_kv_cache_groups]
|
block_table_strides, # [num_kv_cache_groups]
|
||||||
|
bt_num_rows, # bounds-guard instrumentation: rows in the persistent block tables
|
||||||
|
guard_ctr, # bounds-guard instrumentation: persistent int64[3]
|
||||||
block_sizes, # [num_kv_cache_groups]
|
block_sizes, # [num_kv_cache_groups]
|
||||||
kernel_block_sizes, # [num_kv_cache_groups]
|
kernel_block_sizes, # [num_kv_cache_groups]
|
||||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||||
@@ -328,9 +376,27 @@ def _compute_slot_mappings_kernel(
|
|||||||
sharded = tl.load(dcp_sharded + group_id)
|
sharded = tl.load(dcp_sharded + group_id)
|
||||||
|
|
||||||
req_state_idx = tl.load(idx_mapping + batch_idx)
|
req_state_idx = tl.load(idx_mapping + batch_idx)
|
||||||
|
# Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table
|
||||||
|
# before any bound is applied. An invalid row writes PAD_ID for this
|
||||||
|
# request rather than reading an arbitrary row.
|
||||||
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
|
# idx_mapping == -1 marks a dummy (or CUDA-graph padding) request that owns
|
||||||
# no blocks: never read its block-table row and emit PAD for its tokens.
|
# no blocks: never read its block-table row and emit PAD for its tokens.
|
||||||
is_real_req = req_state_idx >= 0
|
is_real_req = req_state_idx >= 0
|
||||||
|
|
||||||
|
# Bounds-guard instrumentation (guard 4): req_state_idx indexes the persistent block table
|
||||||
|
# before any bound is applied. An invalid row writes PAD_ID for this
|
||||||
|
# request rather than reading an arbitrary row.
|
||||||
|
if (req_state_idx < 0) or (req_state_idx >= bt_num_rows):
|
||||||
|
tl.atomic_add(guard_ctr, 1)
|
||||||
|
tl.atomic_max(guard_ctr + 1, (batch_idx).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
tl.atomic_max(guard_ctr + 2, (req_state_idx).to(tl.int64) + _GUARD_BIAS)
|
||||||
|
bad_start = tl.load(query_start_loc + batch_idx)
|
||||||
|
bad_end = tl.load(query_start_loc + batch_idx + 1)
|
||||||
|
for i in range(bad_start, bad_end, TRITON_BLOCK_SIZE):
|
||||||
|
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
||||||
|
tl.store(slot_mapping_ptr + offset, PAD_ID, mask=offset < bad_end)
|
||||||
|
return
|
||||||
|
|
||||||
start_idx = tl.load(query_start_loc + batch_idx)
|
start_idx = tl.load(query_start_loc + batch_idx)
|
||||||
end_idx = tl.load(query_start_loc + batch_idx + 1)
|
end_idx = tl.load(query_start_loc + batch_idx + 1)
|
||||||
for i in range(start_idx, end_idx, TRITON_BLOCK_SIZE):
|
for i in range(start_idx, end_idx, TRITON_BLOCK_SIZE):
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import contextlib
|
||||||
import gc
|
import gc
|
||||||
import itertools
|
import itertools
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable, Iterator
|
||||||
from contextlib import ExitStack
|
from contextlib import ExitStack
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
from itertools import groupby, product
|
from itertools import groupby, product
|
||||||
@@ -13,6 +14,7 @@ import torch
|
|||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.compilation.breakable_cudagraph import (
|
from vllm.compilation.breakable_cudagraph import (
|
||||||
BreakableCUDAGraphWrapper,
|
BreakableCUDAGraphWrapper,
|
||||||
is_breakable_cudagraph_enabled,
|
is_breakable_cudagraph_enabled,
|
||||||
@@ -33,6 +35,9 @@ from vllm.distributed.parallel_state import (
|
|||||||
)
|
)
|
||||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
from vllm.model_executor.layers.ple_offload_layer import (
|
||||||
|
observe_ple_copy_geometry,
|
||||||
|
)
|
||||||
from vllm.model_executor.offloader.base import get_offloader
|
from vllm.model_executor.offloader.base import get_offloader
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
@@ -173,6 +178,14 @@ class CudaGraphManager:
|
|||||||
self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map()
|
self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map()
|
||||||
|
|
||||||
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
|
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
|
||||||
|
# FULL graphs bake every PLE wait/copy into their native graph. Metadata
|
||||||
|
# comes from the actual custom-op implementation during native capture.
|
||||||
|
self._ple_full_copy_geometry: dict[
|
||||||
|
BatchExecutionDescriptor, tuple[tuple[str, int, int], ...]
|
||||||
|
] = {}
|
||||||
|
self._expected_ple_copy_layers: tuple[str, ...] = ()
|
||||||
|
# Only the target-model manager is allowed to capture PLE operations.
|
||||||
|
self._allow_ple_full_graph = False
|
||||||
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
|
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
|
||||||
|
|
||||||
self._graphs_captured = False
|
self._graphs_captured = False
|
||||||
@@ -344,7 +357,7 @@ class CudaGraphManager:
|
|||||||
|
|
||||||
# recoverSSM cannot capture a dummy query wider than its workspace.
|
# recoverSSM cannot capture a dummy query wider than its workspace.
|
||||||
if mixed_mode and (
|
if mixed_mode and (
|
||||||
not self.vllm_config.cache_config.use_kda_recoverssm
|
not self.vllm_config.cache_config.use_recoverssm
|
||||||
or num_tokens <= max_decode_tokens
|
or num_tokens <= max_decode_tokens
|
||||||
):
|
):
|
||||||
# for PIECEWISE graphs there is no limit on requests when replaying
|
# for PIECEWISE graphs there is no limit on requests when replaying
|
||||||
@@ -488,6 +501,7 @@ class CudaGraphManager:
|
|||||||
torch.accelerator.synchronize()
|
torch.accelerator.synchronize()
|
||||||
free_before = torch.accelerator.get_memory_info()[0]
|
free_before = torch.accelerator.get_memory_info()[0]
|
||||||
with (
|
with (
|
||||||
|
self.capture_ple_copy_geometry(desc),
|
||||||
capture_pool(self.pool, self.vllm_config) as pool,
|
capture_pool(self.pool, self.vllm_config) as pool,
|
||||||
torch.cuda.graph(
|
torch.cuda.graph(
|
||||||
graph, pool, stream=self._capture_stream(desc)
|
graph, pool, stream=self._capture_stream(desc)
|
||||||
@@ -514,6 +528,78 @@ class CudaGraphManager:
|
|||||||
{desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0}
|
{desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def capture_ple_copy_geometry(
|
||||||
|
self, desc: BatchExecutionDescriptor
|
||||||
|
) -> Iterator[None]:
|
||||||
|
"""Production capture callback for actual PLE custom-op geometry."""
|
||||||
|
with observe_ple_copy_geometry() as geometry:
|
||||||
|
yield
|
||||||
|
|
||||||
|
if geometry and not self._allow_ple_full_graph:
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE wait-copy reached an unsupported FULL graph path; "
|
||||||
|
"speculator/draft FULL graphs may not contain PLE operations"
|
||||||
|
)
|
||||||
|
if geometry:
|
||||||
|
self.record_ple_copy_geometry(desc, geometry)
|
||||||
|
elif self._expected_ple_copy_layers:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE copy geometry was not observed while capturing FULL graph {desc}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def record_ple_copy_geometry(
|
||||||
|
self,
|
||||||
|
desc: BatchExecutionDescriptor,
|
||||||
|
geometry: Iterable[tuple[str, int, int]],
|
||||||
|
) -> None:
|
||||||
|
if desc.cg_mode != CUDAGraphMode.FULL:
|
||||||
|
raise ValueError("PLE capture geometry is recorded only for FULL graphs")
|
||||||
|
observed = tuple(
|
||||||
|
(str(layer), int(rows), int(nbytes))
|
||||||
|
for layer, rows, nbytes in geometry
|
||||||
|
)
|
||||||
|
if not observed:
|
||||||
|
raise RuntimeError(f"PLE FULL graph {desc} observed no wait-copy operations")
|
||||||
|
observed_layers = tuple(layer for layer, _, _ in observed)
|
||||||
|
expected_layers = self._expected_ple_copy_layers
|
||||||
|
if expected_layers and sorted(observed_layers) != sorted(expected_layers):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE FULL graph {desc} observed layers {observed_layers}, "
|
||||||
|
f"expected {expected_layers}"
|
||||||
|
)
|
||||||
|
inconsistent = [
|
||||||
|
(layer, rows, nbytes)
|
||||||
|
for layer, rows, nbytes in observed
|
||||||
|
if rows != desc.num_tokens or nbytes < 0
|
||||||
|
]
|
||||||
|
if inconsistent:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE FULL graph {desc} actual PLE copy geometry "
|
||||||
|
f"{inconsistent} does not match descriptor rows={desc.num_tokens}"
|
||||||
|
)
|
||||||
|
previous = self._ple_full_copy_geometry.get(desc)
|
||||||
|
if previous is not None and previous != observed:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE FULL graph {desc} changed copy geometry from "
|
||||||
|
f"{previous} to {observed}"
|
||||||
|
)
|
||||||
|
self._ple_full_copy_geometry[desc] = observed
|
||||||
|
|
||||||
|
def ple_copy_rows(self, desc: BatchExecutionDescriptor) -> int:
|
||||||
|
try:
|
||||||
|
geometry = self._ple_full_copy_geometry[desc]
|
||||||
|
except KeyError as error:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE copy geometry was not recorded for FULL graph {desc}"
|
||||||
|
) from error
|
||||||
|
rows = {num_rows for _, num_rows, _ in geometry}
|
||||||
|
if len(rows) != 1:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"PLE FULL graph {desc} has inconsistent per-layer rows: {geometry}"
|
||||||
|
)
|
||||||
|
return rows.pop()
|
||||||
|
|
||||||
def dispatch(
|
def dispatch(
|
||||||
self,
|
self,
|
||||||
num_reqs: int,
|
num_reqs: int,
|
||||||
@@ -552,6 +638,11 @@ class CudaGraphManager:
|
|||||||
f"Expected FULL mode, got {desc.cg_mode}"
|
f"Expected FULL mode, got {desc.cg_mode}"
|
||||||
)
|
)
|
||||||
assert desc in self.graphs, f"No cudagraph for {desc}"
|
assert desc in self.graphs, f"No cudagraph for {desc}"
|
||||||
|
if (
|
||||||
|
desc in self._ple_full_copy_geometry
|
||||||
|
and not self._allow_ple_full_graph
|
||||||
|
):
|
||||||
|
raise RuntimeError("refusing unsupported PLE speculator FULL replay")
|
||||||
# Sync offloader before replay - needed when transitioning from
|
# Sync offloader before replay - needed when transitioning from
|
||||||
# eager/piecewise to full cudagraph (e.g., prefill → decode).
|
# eager/piecewise to full cudagraph (e.g., prefill → decode).
|
||||||
# The previous eager iteration's start_prefetch may have queued
|
# The previous eager iteration's start_prefetch may have queued
|
||||||
@@ -597,11 +688,32 @@ class ModelCudaGraphManager(CudaGraphManager):
|
|||||||
varlen_decode=varlen_decode,
|
varlen_decode=varlen_decode,
|
||||||
ubatch_runner=ubatch_runner,
|
ubatch_runner=ubatch_runner,
|
||||||
)
|
)
|
||||||
|
self._allow_ple_full_graph = True
|
||||||
self.hidden_states: torch.Tensor | None = None
|
self.hidden_states: torch.Tensor | None = None
|
||||||
self.aux_hidden_states: list[torch.Tensor] = []
|
self.aux_hidden_states: list[torch.Tensor] = []
|
||||||
self.use_aux_hidden_state_outputs = False
|
self.use_aux_hidden_state_outputs = False
|
||||||
self.intermediate_tensors: IntermediateTensors | None = None
|
self.intermediate_tensors: IntermediateTensors | None = None
|
||||||
|
|
||||||
|
def _prepare_ple_capture(
|
||||||
|
self, model: nn.Module, *, initialize_breakable: bool
|
||||||
|
) -> None:
|
||||||
|
expected_ple_copy_layers = tuple(
|
||||||
|
str(getattr(module, "_ple_offload_name", name))
|
||||||
|
for name, module in model.named_modules()
|
||||||
|
if getattr(module, "_is_cpu_offloaded", False)
|
||||||
|
and getattr(module, "_host_output_ptr", None) is not None
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
"<unbound>" in expected_ple_copy_layers
|
||||||
|
or len(set(expected_ple_copy_layers)) != len(expected_ple_copy_layers)
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"PLE FULL capture requires distinct, bound connector layer identities"
|
||||||
|
)
|
||||||
|
self._expected_ple_copy_layers = expected_ple_copy_layers
|
||||||
|
if initialize_breakable:
|
||||||
|
self.init_breakable_cg_runner(model)
|
||||||
|
|
||||||
def capture(
|
def capture(
|
||||||
self,
|
self,
|
||||||
model: nn.Module,
|
model: nn.Module,
|
||||||
@@ -619,8 +731,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""Capture CUDA graphs for model forward pass."""
|
"""Capture CUDA graphs for model forward pass."""
|
||||||
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
||||||
if self.use_breakable_cg:
|
self._prepare_ple_capture(
|
||||||
self.init_breakable_cg_runner(model)
|
model, initialize_breakable=self.use_breakable_cg
|
||||||
|
)
|
||||||
|
|
||||||
if self.cudagraph_mode.has_piecewise_cudagraphs() and not (
|
if self.cudagraph_mode.has_piecewise_cudagraphs() and not (
|
||||||
self.use_breakable_cg or has_compiled_submodule(model)
|
self.use_breakable_cg or has_compiled_submodule(model)
|
||||||
@@ -926,7 +1039,9 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
|||||||
if manager.use_breakable_cg:
|
if manager.use_breakable_cg:
|
||||||
# Create the breakable runner before the wrapper pool swap so
|
# Create the breakable runner before the wrapper pool swap so
|
||||||
# its pool is covered as well.
|
# its pool is covered as well.
|
||||||
manager.init_breakable_cg_runner(runner.model)
|
manager._prepare_ple_capture(
|
||||||
|
runner.model, initialize_breakable=True
|
||||||
|
)
|
||||||
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||||
BreakableCUDAGraphWrapper._all_instances
|
BreakableCUDAGraphWrapper._all_instances
|
||||||
)
|
)
|
||||||
@@ -1013,12 +1128,46 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
|||||||
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||||
|
|
||||||
|
|
||||||
|
def _profiling_kv_cache_bytes(kv_caches: Iterable[Any]) -> int:
|
||||||
|
"""Bytes of the distinct CUDA storages behind the runner's KV cache views."""
|
||||||
|
storages: dict[int, int] = {}
|
||||||
|
for cache in kv_caches:
|
||||||
|
for t in cache if isinstance(cache, (list, tuple)) else (cache,):
|
||||||
|
if isinstance(t, torch.Tensor) and t.is_cuda:
|
||||||
|
st = t.untyped_storage()
|
||||||
|
storages[st.data_ptr()] = st.nbytes()
|
||||||
|
return sum(storages.values())
|
||||||
|
|
||||||
|
|
||||||
|
def _profiling_kv_still_held(
|
||||||
|
kv_bytes: int, allocated_before: int, allocated_after: int
|
||||||
|
) -> int:
|
||||||
|
"""Lower bound on the profiling KV cache bytes still allocated after the
|
||||||
|
teardown: whatever the teardown freed counts against the cache first, so a
|
||||||
|
positive result means something still references the cache (the RecoverSSM
|
||||||
|
commit-context class of leak, ~26 GiB on H200; against a sub-GiB per-card
|
||||||
|
per-card cushion even a small one matters). One-sided: never a false alarm,
|
||||||
|
may miss a leak masked by other frees."""
|
||||||
|
return max(kv_bytes - max(allocated_before - allocated_after, 0), 0)
|
||||||
|
|
||||||
|
|
||||||
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||||
"""Release the profiling KV cache and captured graphs while keeping model
|
"""Release the profiling KV cache and captured graphs while keeping model
|
||||||
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
|
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
|
||||||
torch.accelerator.synchronize()
|
torch.accelerator.synchronize()
|
||||||
|
kv_bytes = _profiling_kv_cache_bytes(getattr(runner, "kv_caches", ()))
|
||||||
|
allocated_before = torch.cuda.memory_allocated() if kv_bytes else 0
|
||||||
if hasattr(runner.model_state, "_mamba_ctx"):
|
if hasattr(runner.model_state, "_mamba_ctx"):
|
||||||
runner.model_state._mamba_ctx = None
|
runner.model_state._mamba_ctx = None
|
||||||
|
# A PIECEWISE profiling capture records its RecoverSSM step (only FULL
|
||||||
|
# captures skip it), and that step's commit contexts reference the
|
||||||
|
# profiling KV cache.
|
||||||
|
if (recoverssm := getattr(runner.model_state, "recoverssm", None)) is not None:
|
||||||
|
recoverssm.reset()
|
||||||
|
# Under PP the non-last ranks keep detached RecoverSSM steps on the pending
|
||||||
|
# PP entries; they reference the profiling KV cache just the same.
|
||||||
|
if (pp_handler := getattr(runner, "pp_handler", None)) is not None:
|
||||||
|
pp_handler.drop_deferred_states()
|
||||||
# Invalidate the align-mode Mamba group metadata cached from the
|
# Invalidate the align-mode Mamba group metadata cached from the
|
||||||
# profiling KVCacheConfig: the real (e.g. PP-projected) config may
|
# profiling KVCacheConfig: the real (e.g. PP-projected) config may
|
||||||
# place Mamba layers into a different group layout, so it must be
|
# place Mamba layers into a different group layout, so it must be
|
||||||
@@ -1050,3 +1199,22 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
|||||||
runner.maybe_remove_all_loras(runner.lora_config)
|
runner.maybe_remove_all_loras(runner.lora_config)
|
||||||
gc.collect()
|
gc.collect()
|
||||||
torch.accelerator.empty_cache()
|
torch.accelerator.empty_cache()
|
||||||
|
if kv_bytes:
|
||||||
|
# last, after every release above (one-sided check)
|
||||||
|
gc.collect()
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
held = _profiling_kv_still_held(
|
||||||
|
kv_bytes, allocated_before, torch.cuda.memory_allocated()
|
||||||
|
)
|
||||||
|
if held > (1 << 20):
|
||||||
|
message = (
|
||||||
|
"Profiling teardown: at least %.1f MiB of the %.1f MiB profiling KV "
|
||||||
|
"cache is still allocated; something still references it (it will "
|
||||||
|
"be missing from the real KV cache / free-memory cushion)"
|
||||||
|
) % (held / (1 << 20), kv_bytes / (1 << 20))
|
||||||
|
# Fail boot by default: this memory is absent from the real KV
|
||||||
|
# pool and can invalidate capacity planning. Operators may opt out
|
||||||
|
# temporarily while diagnosing with VLLM_FAIL_ON_PROFILING_KV_LEAK=0.
|
||||||
|
if envs.VLLM_FAIL_ON_PROFILING_KV_LEAK:
|
||||||
|
raise RuntimeError(message)
|
||||||
|
logger.warning(message)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user